diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c6429db..acd67fab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -201,7 +201,12 @@ jobs: kvm-smoke-tests: name: KVM Smoke Tests runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 + needs: docker-build + if: ${{ always() && (github.event_name != 'pull_request' || needs.docker-build.result == 'success') }} + permissions: + contents: read + packages: read env: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" @@ -209,8 +214,43 @@ jobs: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@v0.0.9 + - name: Compute image registries + if: github.event_name == 'pull_request' + shell: bash + run: | + set -euo pipefail + owner_lc="$(printf '%s' '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" + ci_registry="ghcr.io/${owner_lc}" + code_registry="$(jq -r '.registry' docker/images.json)" + if [ -z "$code_registry" ] || [ "$code_registry" = "null" ]; then + echo "docker/images.json registry is missing" >&2 + exit 1 + fi + code_registry="${code_registry%/}" + echo "CI_IMAGE_REGISTRY=${ci_registry}" >> "$GITHUB_ENV" + echo "CODE_IMAGE_REGISTRY=${code_registry}" >> "$GITHUB_ENV" + - uses: docker/login-action@v3 + if: github.event_name == 'pull_request' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull and retag images for KVM smoke test + if: github.event_name == 'pull_request' + shell: bash + run: | + SOURCE_IMAGE_REGISTRY="$CI_IMAGE_REGISTRY" \ + TARGET_IMAGE_REGISTRY="$CODE_IMAGE_REGISTRY" \ + SOURCE_IMAGE_TAG="$GITHUB_SHA" \ + ./.github/scripts/prepare_prebuilt_images.sh - name: Run KVM smoke test - run: cargo test -p amber-cli --test kvm_smoke -- --ignored --nocapture --test-threads=1 + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + export AMBER_TEST_USE_PREBUILT_IMAGES=1 + fi + cargo test -p amber-cli --test kvm_smoke -- --ignored --nocapture --test-threads=1 kubernetes-tests: name: Kubernetes Reporter Tests diff --git a/Cargo.lock b/Cargo.lock index 7f0906b4..3d3faf77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,7 @@ dependencies = [ "amber-router", "amber-scenario", "amber-scenario-runner", + "amber-site-controller", "amber-template", "axum", "base64", @@ -133,6 +134,7 @@ dependencies = [ "similar", "strip-ansi-escapes", "tempfile", + "time", "tokio", "tracing", "tracing-subscriber", @@ -178,7 +180,6 @@ dependencies = [ "amber-template", "jsonptr", "jsonschema", - "serde_jcs", "serde_json", ] @@ -186,7 +187,6 @@ dependencies = [ name = "amber-docker-gateway" version = "0.0.0" dependencies = [ - "base64", "bytes", "dashmap", "http-body-util", @@ -196,7 +196,6 @@ dependencies = [ "moka", "serde", "serde_json", - "serde_urlencoded", "serde_with", "tempfile", "thiserror 2.0.18", @@ -348,6 +347,7 @@ name = "amber-router" version = "0.0.0" dependencies = [ "aho-corasick", + "amber-docker-gateway", "amber-mesh", "base64", "bytes", @@ -395,6 +395,32 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "amber-site-controller" +version = "0.0.0" +dependencies = [ + "amber-compiler", + "amber-manifest", + "amber-mesh", + "amber-proxy", + "amber-resolver", + "amber-scenario", + "axum", + "base64", + "libc", + "miette", + "regex", + "reqwest 0.13.2", + "rmcp", + "serde", + "serde_json", + "serde_yaml", + "sha2", + "tempfile", + "tokio", + "url", +] + [[package]] name = "amber-template" version = "0.0.0" @@ -1345,7 +1371,6 @@ checksum = "05669f8e7e2d7badc545c513710f0eba09c2fbef683eb859fd79c46c355048e0" dependencies = [ "bitflags 1.3.2", "byteorder", - "chrono", "log", ] @@ -3131,12 +3156,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "ryu-js" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" - [[package]] name = "same-file" version = "1.0.6" @@ -3273,17 +3292,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_jcs" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cacecf649bc1a7c5f0e299cc813977c6a78116abda2b93b1ee01735b71ead9a8" -dependencies = [ - "ryu-js", - "serde", - "serde_json", -] - [[package]] name = "serde_json" version = "1.0.149" diff --git a/Cargo.toml b/Cargo.toml index 316dc313..c1f29f4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,14 @@ members = [ "runtime/provisioner", "runtime/router", "runtime/scenario-runner", + "runtime/site-controller", ] resolver = "2" [workspace.dependencies] amber-compiler = { path = "compiler" } amber-config = { path = "compiler/config" } +amber-docker-gateway = { path = "runtime/docker-gateway" } amber-images = { path = "images" } amber-json5 = { path = "compiler/json5" } amber-manifest = { path = "compiler/manifest" } @@ -31,6 +33,7 @@ amber-resolver = { path = "compiler/resolver" } amber-router = { path = "runtime/router" } amber-scenario = { path = "compiler/scenario" } amber-scenario-runner = { path = "runtime/scenario-runner" } +amber-site-controller = { path = "runtime/site-controller" } amber-template = { path = "compiler/template" } base64 = "0.22.1" futures = "0.3.31" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d8ded1bb..9de0e090 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" amber-compiler = { workspace = true } amber-config = { workspace = true } amber-helper = { path = "../runtime/helper" } +amber-images = { workspace = true } amber-manifest = { workspace = true } amber-mesh = { workspace = true } amber-proxy = { workspace = true } @@ -17,12 +18,13 @@ amber-resolver = { workspace = true } amber-router = { workspace = true } amber-scenario = { workspace = true } amber-scenario-runner = { workspace = true } +amber-site-controller = { workspace = true } amber-template = { workspace = true } axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } base64 = { workspace = true } clap = { version = "4.5.46", features = ["derive"] } crossterm = "0.29.0" -fatfs = "0.3.6" +fatfs = { version = "=0.3.6", default-features = false, features = ["std", "alloc"] } jsonschema = { workspace = true, default-features = false } libc = "0.2.177" miette = { workspace = true, features = ["fancy"] } @@ -52,3 +54,4 @@ serde_yaml = { workspace = true } similar = "2.7.0" strip-ansi-escapes = "0.2" tempfile = { workspace = true } +time = { version = "0.3.44", features = ["parsing"] } diff --git a/cli/src/command_support.rs b/cli/src/command_support.rs index 70d25e62..fd543eda 100644 --- a/cli/src/command_support.rs +++ b/cli/src/command_support.rs @@ -295,12 +295,39 @@ pub(crate) fn load_compiled_scenario_ir(path: &Path) -> Result Result<()> { + for component in &ir.components { + for (provide_name, provide) in &component.provides { + let kind = provide.decl.kind; + if matches!( + kind, + amber_manifest::CapabilityKind::Component + | amber_manifest::CapabilityKind::Docker + | amber_manifest::CapabilityKind::Kvm + ) { + return Err(miette::miette!( + "Scenario IR input `{}` component `{}` provide `{}` declares framework-owned \ + capability kind `{}`; use `framework.{}` as the source instead of declaring \ + a provider", + path.display(), + component.moniker, + provide_name, + kind, + kind.as_str() + )); + } + } + } + Ok(()) +} + pub(crate) fn is_run_plan_file(path: &Path) -> Result { if !path.is_file() { return Ok(false); @@ -571,7 +598,322 @@ pub(crate) fn write_unmanaged_export_output( unmanaged_export_output_dir_label(kind), &export.files, unmanaged_export_executable_rel_path(kind), - ) + )?; + materialize_standalone_site_controller_export(root, run_plan, &export) +} + +fn materialize_standalone_site_controller_export( + root: &Path, + run_plan: &RunPlan, + export: &amber_compiler::run_plan::UnmanagedExport, +) -> Result<()> { + if !matches!(export.kind, SiteKind::Compose | SiteKind::Kubernetes) { + return Ok(()); + } + let site_plan = run_plan.sites.get(&export.site_id).ok_or_else(|| { + miette::miette!( + "unmanaged export references missing site `{}` in the run plan", + export.site_id + ) + })?; + let Some(controller_port) = crate::mixed_run::site_controller_component_port(site_plan)? else { + return Ok(()); + }; + + const COMPOSE_CONTROLLER_SITE_ROOT: &str = "/amber/site"; + const COMPOSE_CONTROLLER_STATE_ROOT: &str = "/amber/site/.amber/site-controller"; + + let artifact_root = root + .canonicalize() + .into_diagnostic() + .wrap_err_with(|| format!("failed to canonicalize export root `{}`", root.display()))?; + let base_run_id = format!( + "standalone-{}-{}", + unmanaged_export_label(export.kind), + export.site_id + ); + let kubernetes_namespace = if export.kind == SiteKind::Kubernetes { + Some( + amber_site_controller::prepare_kubernetes_artifact_namespace( + &base_run_id, + &export.site_id, + &artifact_root, + )?, + ) + } else { + None + }; + let run_id = kubernetes_namespace + .as_deref() + .unwrap_or(base_run_id.as_str()); + amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_root, run_id)?; + + let state_root = if export.kind == SiteKind::Kubernetes { + standalone_controller_temp_state_root(&export.site_id)? + } else { + artifact_root.join(".amber").join("site-controller") + }; + let site_state_root = state_root.join(&export.site_id); + crate::mixed_run::prepare_site_state_root(&site_state_root, export.kind)?; + let state_path = site_state_root.join("site-controller-state.json"); + let plan_path = crate::mixed_run::site_controller_plan_path(&site_state_root); + let (plan_state_path, plan_run_root, plan_state_root, plan_site_state_root, plan_artifact_root) = + if export.kind == SiteKind::Compose { + let plan_state_root = PathBuf::from(COMPOSE_CONTROLLER_STATE_ROOT); + let plan_site_state_root = plan_state_root.join(&export.site_id); + ( + plan_site_state_root.join("site-controller-state.json"), + PathBuf::from(COMPOSE_CONTROLLER_SITE_ROOT), + plan_state_root, + plan_site_state_root, + PathBuf::from(COMPOSE_CONTROLLER_SITE_ROOT), + ) + } else { + ( + state_path.clone(), + artifact_root.clone(), + state_root.clone(), + site_state_root.clone(), + artifact_root.clone(), + ) + }; + + let mut launch_env = BTreeMap::new(); + launch_env.insert( + amber_mesh::telemetry::SCENARIO_RUN_ID_ENV.to_string(), + run_id.to_string(), + ); + launch_env.insert( + amber_mesh::telemetry::SCENARIO_SCOPE_ENV.to_string(), + run_plan.mesh_scope.clone(), + ); + + let controller_state = amber_site_controller::build_site_controller_state( + run_id, + run_plan, + &export.site_id, + 0, + 1, + )?; + amber_site_controller::write_control_state(&state_path, &controller_state)?; + write_json_file( + &crate::mixed_run::desired_links_path(&site_state_root), + &crate::mixed_run::DesiredLinkState { + schema: crate::mixed_run::DESIRED_LINKS_SCHEMA.to_string(), + version: crate::mixed_run::DESIRED_LINKS_VERSION, + ..Default::default() + }, + )?; + + let listen_addr = SocketAddr::from(([0, 0, 0, 0], controller_port)); + let authority_url = format!( + "http://{}:{}", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME, + controller_port + ); + let local_router_control = + crate::mixed_run::site_controller_local_router_control(export.kind, &artifact_root); + let router_mesh_port = Some(24000); + if let Some(port) = router_mesh_port + && export.kind == SiteKind::Compose + { + amber_site_controller::set_compose_router_published_mesh_port(&artifact_root, port)?; + } + let compose_project = (export.kind == SiteKind::Compose) + .then(|| crate::mixed_run::compose_project_name(run_id, &export.site_id)); + let controller_identity_path = + crate::mixed_run::site_controller_identity_path(export.kind, &artifact_root, None)?; + + let plan = amber_site_controller::write_site_controller_plan( + &plan_path, + run_id, + &run_plan.mesh_scope, + &export.site_id, + export.kind, + listen_addr, + &authority_url, + &site_plan.router_identity_id, + &BTreeMap::new(), + &BTreeMap::new(), + &BTreeMap::new(), + Some(local_router_control.as_str()), + None, + None, + None, + &plan_state_path, + &plan_run_root, + &plan_state_root, + &plan_site_state_root, + &plan_artifact_root, + controller_identity_path.as_deref(), + None, + None, + router_mesh_port, + compose_project.as_deref(), + kubernetes_namespace.as_deref(), + None, + None, + &launch_env, + )?; + + let controller_image = crate::mixed_run::site_controller_image_reference()?; + match export.kind { + SiteKind::Compose => { + let plan_mount_source = + compose_mount_source_relative_to_artifact(&artifact_root, &plan_path)?; + amber_site_controller::inject_compose_site_controller_with_mount_sources( + &artifact_root, + &plan, + ".", + &plan_mount_source, + &controller_image, + )?; + set_compose_environment_values(&artifact_root, &launch_env)?; + } + SiteKind::Kubernetes => { + amber_site_controller::inject_kubernetes_site_controller( + &artifact_root, + &plan, + &controller_image, + )?; + let _ = std::fs::remove_dir_all(&state_root); + } + SiteKind::Direct | SiteKind::Vm => {} + } + + Ok(()) +} + +fn compose_mount_source_relative_to_artifact(artifact_root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(artifact_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to express {} as a path relative to {}", + path.display(), + artifact_root.display() + ) + })?; + let relative = relative + .iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/"); + Ok(format!("./{relative}")) +} + +fn standalone_controller_temp_state_root(site_id: &str) -> Result { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic() + .wrap_err("system clock is before the Unix epoch")? + .as_nanos(); + let mut safe_site = String::new(); + for ch in site_id.chars() { + safe_site.push(if ch.is_ascii_alphanumeric() { ch } else { '-' }); + } + Ok(std::env::temp_dir().join(format!( + "amber-standalone-controller-{safe_site}-{}-{nanos}", + std::process::id() + ))) +} + +fn write_json_file(path: &Path, value: &T) -> Result<()> { + let json = serde_json::to_vec_pretty(value) + .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; + write_artifact(path, &json) +} + +fn set_compose_environment_values( + artifact_root: &Path, + values: &BTreeMap, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&compose_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", compose_path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("failed to parse {}", compose_path.display()))?; + let services = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services", + compose_path.display() + ) + })?; + for service in services.values_mut() { + let service = service.as_mapping_mut().ok_or_else(|| { + miette::miette!( + "compose service in {} is not a mapping", + compose_path.display() + ) + })?; + let environment = service + .entry(yaml_string("environment")) + .or_insert_with(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + set_compose_environment_entries(environment, values, &compose_path)?; + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +fn set_compose_environment_entries( + environment: &mut serde_yaml::Value, + values: &BTreeMap, + compose_path: &Path, +) -> Result<()> { + match environment { + serde_yaml::Value::Mapping(map) => { + for (name, value) in values { + map.insert(yaml_string(name), yaml_string(value)); + } + Ok(()) + } + serde_yaml::Value::Sequence(entries) => { + for (name, value) in values { + let rendered = format!("{name}={value}"); + let mut found = false; + for entry in entries.iter_mut() { + let Some(raw) = entry.as_str() else { + continue; + }; + if raw == name + || raw + .strip_prefix(name) + .is_some_and(|tail| tail.starts_with('=')) + { + *entry = yaml_string(&rendered); + found = true; + } + } + if !found { + entries.push(yaml_string(&rendered)); + } + } + Ok(()) + } + serde_yaml::Value::Null => { + *environment = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); + set_compose_environment_entries(environment, values, compose_path) + } + _ => Err(miette::miette!( + "compose file {} has an unsupported environment shape", + compose_path.display() + )), + } +} + +fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) } pub(crate) fn write_directory_output( diff --git a/cli/src/direct_runtime/preview.rs b/cli/src/direct_runtime/preview.rs index 2f66cc1f..1c2cd073 100644 --- a/cli/src/direct_runtime/preview.rs +++ b/cli/src/direct_runtime/preview.rs @@ -105,9 +105,76 @@ pub(crate) fn component_program_spec( network: ProcessNetwork::Host, }) } + DirectProgramExecutionPlan::InternalSiteController => { + let (program, args) = + split_entrypoint(&internal_site_controller_entrypoint(runtime_root)?)?; + Ok(ProcessSpec { + name: component.program.log_name.clone(), + program, + args, + env: BTreeMap::new(), + work_dir, + sandbox: ProcessSandbox::Unsandboxed, + drop_all_caps: true, + #[cfg(target_os = "linux")] + read_only_mounts: Vec::new(), + writable_dirs, + bind_dirs: Vec::new(), + bind_mounts, + hidden_paths: Vec::new(), + network: ProcessNetwork::Host, + }) + } } } +#[cfg(target_os = "linux")] +pub(crate) fn direct_component_uses_isolated_network(component: &DirectComponentPlan) -> bool { + !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) +} + +pub(crate) fn direct_component_sidecar_network(component: &DirectComponentPlan) -> ProcessNetwork { + #[cfg(target_os = "linux")] + { + if direct_component_uses_isolated_network(component) { + ProcessNetwork::Isolated + } else { + ProcessNetwork::Host + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = component; + ProcessNetwork::Host + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn direct_component_program_joins_sidecar(component: &DirectComponentPlan) -> bool { + direct_component_uses_isolated_network(component) +} + +fn internal_site_controller_plan_path(runtime_root: &Path) -> PathBuf { + let site_state_root = runtime_root.parent().unwrap_or(runtime_root); + mixed_run::site_controller_plan_path(site_state_root) +} + +fn internal_site_controller_entrypoint(runtime_root: &Path) -> Result> { + let command = mixed_run::site_controller_command()?; + let mut argv = vec![command.executable.display().to_string()]; + argv.extend(command.prefix_args.iter().map(|arg| (*arg).to_string())); + argv.push("--plan".to_string()); + argv.push( + internal_site_controller_plan_path(runtime_root) + .display() + .to_string(), + ); + Ok(argv) +} + pub(crate) fn build_direct_site_launch_preview( plan_path: &Path, storage_root: &Path, @@ -120,27 +187,64 @@ pub(crate) fn build_direct_site_launch_preview( direct_plan, mesh_plan, } = load_direct_runtime_inputs(&plan_path)?; - let runtime_state = materialize_direct_runtime( + let mut inspectability_warnings = Vec::new(); + let runtime_state = match materialize_direct_runtime( &plan_root, runtime_root, &direct_plan, &mesh_plan, router_mesh_port, true, - )?; - #[cfg(target_os = "linux")] - configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; - #[cfg(not(target_os = "linux"))] - configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + ) { + Ok(runtime_state) => { + #[cfg(target_os = "linux")] + configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + #[cfg(not(target_os = "linux"))] + configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + runtime_state + } + Err(err) if missing_existing_peer_identity(&err) => { + // Dry-run preview can still show the local process shape before peer-site routers + // exist; the missing identities only block full mesh config materialization. + inspectability_warnings.push(format!( + "preview is missing one or more peer-site router identities, so mesh configs \ + remain unresolved until those sites are running: {err}" + )); + let (preview_peer_identities, preview_peer_ports) = + preview_placeholder_peer_mesh_state(&mesh_plan); + materialize_direct_runtime_with_existing( + &plan_root, + runtime_root, + &direct_plan, + &mesh_plan, + router_mesh_port, + DirectExistingMeshState { + reuse_existing: false, + peer_ports_by_id: &preview_peer_ports, + peer_identities_by_id: &preview_peer_identities, + }, + )? + } + Err(err) => return Err(err), + }; let router_binary = resolve_runtime_binary("amber-router")?; let mut processes = Vec::new(); let mut router_public_key_b64 = None; if let Some(router) = direct_plan.router.as_ref() { - let router_config = read_mesh_config_public(&runtime_root.join(&router.mesh_config_path))?; - router_public_key_b64 = Some( - base64::engine::general_purpose::STANDARD.encode(router_config.identity.public_key), - ); + let router_config_path = runtime_root.join(&router.mesh_config_path); + match read_mesh_config_public(&router_config_path) { + Ok(router_config) => { + router_public_key_b64 = Some( + base64::engine::general_purpose::STANDARD + .encode(router_config.identity.public_key), + ); + } + Err(err) => inspectability_warnings.push(format!( + "failed to inspect direct router mesh config {}: {err}", + router_config_path.display() + )), + } let paths = DirectControlSocketPaths { artifact_link: resolve_direct_artifact_path(&plan_root, &router.control_socket_path), current_link: direct_current_control_socket_path(&plan_root), @@ -228,9 +332,6 @@ pub(crate) fn build_direct_site_launch_preview( ); for passthrough in &component.sidecar.env_passthrough { if let Ok(value) = env::var(passthrough) { - #[cfg(target_os = "linux")] - let value = - rewrite_sidecar_env_passthrough_for_slirp(passthrough.as_str(), value.as_str()); env.insert(passthrough.clone(), value); } } @@ -252,16 +353,7 @@ pub(crate) fn build_direct_site_launch_preview( bind_dirs: Vec::new(), bind_mounts: Vec::new(), hidden_paths: Vec::new(), - network: { - #[cfg(target_os = "linux")] - { - ProcessNetwork::Isolated - } - #[cfg(not(target_os = "linux"))] - { - ProcessNetwork::Host - } - }, + network: direct_component_sidecar_network(component), }, "sidecar", Some(component.moniker.as_str()), @@ -283,18 +375,24 @@ pub(crate) fn build_direct_site_launch_preview( &direct_plan.runtime_addresses, &runtime_state, )?; - spec.hidden_paths.push(runtime_root.join("mesh")); + if !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) { + spec.hidden_paths.push(runtime_root.join("mesh")); + } let resolved_process = match &component.program.execution { DirectProgramExecutionPlan::Direct { .. } => None, DirectProgramExecutionPlan::HelperRunner { .. } => { Some(direct_resolved_process_preview(&spec.env)?) } + DirectProgramExecutionPlan::InternalSiteController => None, }; processes.push(direct_process_preview( spec, "program", Some(component.moniker.as_str()), - Some(direct_program_network_override()), + Some(direct_program_network_override(component)), resolved_process, )); } @@ -302,9 +400,56 @@ pub(crate) fn build_direct_site_launch_preview( Ok(DirectSiteLaunchPreview { router_public_key_b64, processes, + inspectability_warnings, }) } +pub(crate) fn missing_existing_peer_identity(err: &miette::Report) -> bool { + err.to_string() + .contains("mesh provision plan requires existing peer identity") +} + +pub(crate) fn preview_placeholder_peer_mesh_state( + mesh_plan: &MeshProvisionPlan, +) -> (BTreeMap, BTreeMap) { + let target_ids = mesh_plan + .targets + .iter() + .map(|target| target.config.identity.id.as_str()) + .collect::>(); + let mesh_scope = mesh_plan + .targets + .first() + .and_then(|target| target.config.identity.mesh_scope.clone()); + let peer_ids = mesh_plan + .targets + .iter() + .flat_map(|target| target.config.peers.iter()) + .filter(|peer| !target_ids.contains(peer.id.as_str())) + .map(|peer| peer.id.clone()) + .collect::>(); + let peer_identities = peer_ids + .iter() + .map(|peer| { + let identity = MeshIdentity::generate(peer.clone(), mesh_scope.clone()); + ( + peer.clone(), + MeshIdentityPublic { + id: identity.id, + public_key: identity.public_key, + mesh_scope: identity.mesh_scope, + }, + ) + }) + .collect(); + let peer_ports = peer_ids + .into_iter() + .enumerate() + .map(|(index, peer_id)| (peer_id, 39_000 + index as u16)) + .collect(); + (peer_identities, peer_ports) +} + pub(crate) fn load_direct_runtime_inputs(plan_path: &Path) -> Result { let plan_root = plan_path .parent() @@ -409,13 +554,18 @@ pub(crate) fn direct_process_network_label(network: ProcessNetwork) -> String { } } -pub(crate) fn direct_program_network_override() -> &'static str { +pub(crate) fn direct_program_network_override(component: &DirectComponentPlan) -> &'static str { #[cfg(target_os = "linux")] { - "join_component_sidecar" + if direct_component_program_joins_sidecar(component) { + "join_component_sidecar" + } else { + "host" + } } #[cfg(not(target_os = "linux"))] { + let _ = component; "host" } } @@ -525,13 +675,10 @@ pub(crate) async fn wait_for_component_local_targets( if ports.is_empty() { return Ok(()); } - let sidecar_pid = sidecar_pid.ok_or_else(|| { - miette::miette!( - "missing sidecar pid while waiting for component {} local targets", - component.moniker - ) - })?; - let namespace_join = prepare_linux_namespace_join(sidecar_pid)?; + let namespace_join = sidecar_pid + .map(prepare_linux_namespace_join) + .transpose()? + .flatten(); let amber_cli = env::current_exe() .into_diagnostic() .wrap_err("failed to locate current amber binary for direct local probe")?; @@ -809,6 +956,7 @@ pub(crate) fn component_execution_program_path( } Ok(None) } + DirectProgramExecutionPlan::InternalSiteController => Ok(None), } } @@ -1042,9 +1190,9 @@ pub(crate) async fn spawn_managed_process( ProcessSandbox::Sandboxed => sandbox.wrap_command(&spec)?, ProcessSandbox::Unsandboxed => { #[cfg(target_os = "linux")] - if !matches!(spec.network, ProcessNetwork::Host) { + if matches!(spec.network, ProcessNetwork::Isolated) { return Err(miette::miette!( - "unsandboxed direct processes must use host networking" + "unsandboxed direct processes cannot create isolated networking" )); } (spec.program.clone(), spec.args.clone()) @@ -1053,15 +1201,9 @@ pub(crate) async fn spawn_managed_process( #[cfg(target_os = "linux")] let mut args = args; #[cfg(target_os = "linux")] - let namespace_join = if matches!(spec.sandbox, ProcessSandbox::Sandboxed) - && matches!(sandbox, DirectSandbox::Bubblewrap { .. }) - { - match spec.network { - ProcessNetwork::Join(pid) => prepare_linux_namespace_join(pid)?, - _ => None, - } - } else { - None + let namespace_join = match spec.network { + ProcessNetwork::Join(pid) => prepare_linux_namespace_join(pid)?, + _ => None, }; #[cfg(target_os = "linux")] let pid_capture = if matches!(spec.sandbox, ProcessSandbox::Sandboxed) diff --git a/cli/src/direct_runtime/process.rs b/cli/src/direct_runtime/process.rs index dd31ffa5..f8eb5d1c 100644 --- a/cli/src/direct_runtime/process.rs +++ b/cli/src/direct_runtime/process.rs @@ -588,6 +588,10 @@ pub(crate) async fn supervise_children( if child.wrapper_pid == child.managed_pid || !linux_pid_is_alive(child.managed_pid) { + eprintln!( + "direct runtime observed {} exit with status {}", + child.name, status + ); let exit_code = if status.success() { 0 } else { @@ -605,6 +609,10 @@ pub(crate) async fn supervise_children( #[cfg(target_os = "linux")] if child.wrapper.is_none() && !linux_pid_is_alive(child.managed_pid) { let status = synthetic_failure_exit_status(); + eprintln!( + "direct runtime observed {} exit without a wrapper status", + child.name + ); return Ok(( RuntimeExitReason::ChildExited { name: child.name.clone(), @@ -617,6 +625,10 @@ pub(crate) async fn supervise_children( if let Some(wrapper) = child.wrapper.as_mut() && let Some(status) = wrapper.try_wait().into_diagnostic()? { + eprintln!( + "direct runtime observed {} exit with status {}", + child.name, status + ); let exit_code = if status.success() { 0 } else { @@ -636,70 +648,45 @@ pub(crate) async fn supervise_children( } } +#[cfg(all(unix, not(target_os = "linux")))] pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { + let wrapper_pids = children + .iter() + .filter_map(|child| child.wrapper.as_ref().and_then(tokio::process::Child::id)) + .collect::>(); + let _ = + crate::unix_process::terminate_process_roots(&wrapper_pids, DIRECT_SHUTDOWN_GRACE_PERIOD) + .await; for child in children.iter_mut() { - #[cfg(target_os = "linux")] - { - if linux_pid_is_alive(child.managed_pid) { - let _ = send_sigterm(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = send_sigterm(child.wrapper_pid); - } - } - #[cfg(not(target_os = "linux"))] - if let Some(wrapper) = child.wrapper.as_mut() - && wrapper.try_wait().ok().flatten().is_none() - && let Some(pid) = wrapper.id() - { - let _ = send_sigterm(pid); + if let Some(mut wrapper) = child.wrapper.take() { + let _ = wrapper.wait().await; } } +} - let deadline = Instant::now() + DIRECT_SHUTDOWN_GRACE_PERIOD; - loop { - let mut all_exited = true; - for child in children.iter_mut() { - #[cfg(target_os = "linux")] - if let Some(wrapper) = child.wrapper.as_mut() - && wrapper.try_wait().ok().flatten().is_some() - { - child.wrapper = None; - } - #[cfg(target_os = "linux")] - if linux_pid_is_alive(child.managed_pid) { - all_exited = false; - } - #[cfg(target_os = "linux")] - if child.wrapper.is_some() { - all_exited = false; - } - #[cfg(not(target_os = "linux"))] - if child - .wrapper - .as_mut() - .is_some_and(|wrapper| wrapper.try_wait().ok().flatten().is_none()) - { - all_exited = false; - } - } - if all_exited || Instant::now() >= deadline { - break; +#[cfg(target_os = "linux")] +pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { + let root_pids = children + .iter() + .flat_map(|child| { + let managed = (child.managed_pid != child.wrapper_pid).then_some(child.managed_pid); + [Some(child.wrapper_pid), managed] + }) + .flatten() + .collect::>(); + let _ = crate::unix_process::terminate_process_roots(&root_pids, DIRECT_SHUTDOWN_GRACE_PERIOD) + .await; + + for child in children.iter_mut() { + if let Some(mut wrapper) = child.wrapper.take() { + let _ = wrapper.wait().await; } - sleep(DIRECT_CHILD_POLL_INTERVAL).await; } +} +#[cfg(not(unix))] +pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { for child in children.iter_mut() { - #[cfg(target_os = "linux")] - { - if linux_pid_is_alive(child.managed_pid) { - let _ = kill_pid_force(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = kill_pid_force(child.wrapper_pid); - } - } - #[cfg(not(target_os = "linux"))] if let Some(wrapper) = child.wrapper.as_mut() && wrapper.try_wait().ok().flatten().is_none() { @@ -713,13 +700,6 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { } } -#[cfg(unix)] -pub(crate) fn send_sigterm(pid: u32) -> std::result::Result<(), ()> { - let pid = i32::try_from(pid).map_err(|_| ())?; - let rc = unsafe { libc::kill(pid, libc::SIGTERM) }; - if rc == 0 { Ok(()) } else { Err(()) } -} - #[cfg(target_os = "linux")] pub(crate) fn linux_pid_is_alive(pid: u32) -> bool { let Ok(pid) = i32::try_from(pid) else { @@ -732,13 +712,6 @@ pub(crate) fn linux_pid_is_alive(pid: u32) -> bool { std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) } -#[cfg(target_os = "linux")] -pub(crate) fn kill_pid_force(pid: u32) -> std::result::Result<(), ()> { - let pid = i32::try_from(pid).map_err(|_| ())?; - let rc = unsafe { libc::kill(pid, libc::SIGKILL) }; - if rc == 0 { Ok(()) } else { Err(()) } -} - #[cfg(all(target_os = "linux", unix))] pub(crate) fn synthetic_failure_exit_status() -> std::process::ExitStatus { use std::os::unix::process::ExitStatusExt as _; diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 45cff8ee..4c713e4d 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -111,6 +111,8 @@ pub(crate) struct DirectSiteLaunchPreview { pub(crate) router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) processes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) inspectability_warnings: Vec, } #[derive(Debug)] @@ -187,10 +189,12 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { args.existing_peer_ports.as_deref(), "direct existing peer ports", )?; - let existing_peer_identities_by_id = read_existing_peer_identities( + let discovered_peer_identities_by_id = read_existing_peer_identities( args.existing_peer_identities.as_deref(), "direct existing peer identities", )?; + let existing_peer_identities_by_id = + merge_existing_mesh_peer_identities(&mesh_plan, &discovered_peer_identities_by_id)?; let supervision = async { let mut sandbox = DirectSandbox::detect(&runtime_root); @@ -206,7 +210,7 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { "direct runtime requires `slirp4netns` on Linux for isolated component networking" ) })?; - let runtime_state = if args.skip_router || !existing_peer_ports_by_id.is_empty() { + let mut runtime_state = if args.skip_router || !existing_peer_ports_by_id.is_empty() { materialize_direct_runtime_with_existing( &plan_root, &runtime_root, @@ -391,11 +395,6 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { ); for passthrough in &component.sidecar.env_passthrough { if let Ok(value) = env::var(passthrough) { - #[cfg(target_os = "linux")] - let value = rewrite_sidecar_env_passthrough_for_slirp( - passthrough.as_str(), - value.as_str(), - ); env.insert(passthrough.clone(), value); } } @@ -408,6 +407,33 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { work_dir.display() ) })?; + let control_socket_path = direct_component_control_socket_path(&work_dir, component.id); + let control_socket_dir = control_socket_path + .parent() + .ok_or_else(|| miette::miette!("invalid sidecar control socket path"))? + .to_path_buf(); + fs::create_dir_all(&control_socket_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create sidecar control directory {}", + control_socket_dir.display() + ) + })?; + if control_socket_path.exists() { + fs::remove_file(&control_socket_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to remove stale sidecar control socket {}", + control_socket_path.display() + ) + })?; + } + env.insert( + "AMBER_ROUTER_CONTROL_SOCKET_PATH".to_string(), + control_socket_path.display().to_string(), + ); let spec = ProcessSpec { name: component.sidecar.log_name.clone(), program: router_binary.clone(), @@ -422,24 +448,15 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { dest: runtime_root.join("mesh"), }], writable_dirs: Vec::new(), - bind_dirs: Vec::new(), + bind_dirs: vec![control_socket_dir], bind_mounts: Vec::new(), hidden_paths: Vec::new(), - network: { - #[cfg(target_os = "linux")] - { - ProcessNetwork::Isolated - } - #[cfg(not(target_os = "linux"))] - { - ProcessNetwork::Host - } - }, + network: direct_component_sidecar_network(component), }; let sidecar_pid = spawn_managed_process(spec, &mut sandbox, &mut children, &mut log_tasks).await?; #[cfg(target_os = "linux")] - { + if direct_component_uses_isolated_network(component) { let mesh_port = mesh_network .component_mesh_port_by_id .get(&component.id) @@ -478,16 +495,26 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { &direct_plan.runtime_addresses, &runtime_state, )?; - spec.hidden_paths.push(runtime_root.join("mesh")); + if !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) { + spec.hidden_paths.push(runtime_root.join("mesh")); + } #[cfg(target_os = "linux")] { - let pid = component_sidecar_pid_by_id - .get(component_id) - .copied() - .ok_or_else(|| { - miette::miette!("missing sidecar pid for component {}", component.moniker) - })?; - spec.network = ProcessNetwork::Join(pid); + if direct_component_program_joins_sidecar(component) { + let pid = component_sidecar_pid_by_id + .get(component_id) + .copied() + .ok_or_else(|| { + miette::miette!( + "missing sidecar pid for component {}", + component.moniker + ) + })?; + spec.network = ProcessNetwork::Join(pid); + } } let _ = spawn_managed_process(spec, &mut sandbox, &mut children, &mut log_tasks).await?; @@ -495,7 +522,11 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { wait_for_component_local_targets( component, &runtime_root, - component_sidecar_pid_by_id.get(component_id).copied(), + if direct_component_program_joins_sidecar(component) { + component_sidecar_pid_by_id.get(component_id).copied() + } else { + None + }, DIRECT_LOCAL_TARGET_READY_TIMEOUT, ) .await?; @@ -510,6 +541,7 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { } wait_for_direct_mesh_endpoints(&runtime_state, DIRECT_MESH_ENDPOINT_READY_TIMEOUT).await?; + runtime_state.ready = true; write_direct_runtime_state(&plan_root, &runtime_state)?; drop(startup_lock.take()); supervise_children(&mut children).await @@ -634,6 +666,36 @@ pub(crate) fn read_existing_peer_identities( Ok(identities) } +fn merge_existing_mesh_peer_identities( + mesh_plan: &MeshProvisionPlan, + discovered_peer_identities_by_id: &BTreeMap, +) -> Result> { + let mut merged = mesh_plan + .existing_peer_identities + .iter() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect::>(); + + for (peer_id, discovered_identity) in discovered_peer_identities_by_id { + if let Some(embedded_identity) = merged.get(peer_id) { + if embedded_identity.id != discovered_identity.id + || embedded_identity.public_key != discovered_identity.public_key + || embedded_identity.mesh_scope != discovered_identity.mesh_scope + { + return Err(miette::miette!( + "existing peer identity `{peer_id}` from the direct runtime arguments does \ + not match the embedded mesh provision plan" + )); + } + continue; + } + merged.insert(peer_id.clone(), discovered_identity.clone()); + } + + Ok(merged) +} + pub(crate) fn project_existing_direct_peer_identities( runtime_root: &Path, direct_plan: &DirectPlan, @@ -746,6 +808,14 @@ pub(crate) fn direct_runtime_control_socket_path(runtime_root: &Path) -> PathBuf amber_mesh::stable_temp_socket_path("amber-direct-control", "runtime", runtime_root) } +fn direct_component_control_socket_path(work_dir: &Path, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-direct-control", + &format!("sidecar-{component_id}"), + work_dir, + ) +} + #[cfg(unix)] pub(crate) fn ensure_direct_control_socket_link( link: &Path, @@ -830,6 +900,8 @@ pub(crate) fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub(crate) struct DirectRuntimeState { + #[serde(default)] + pub(crate) ready: bool, #[serde(default)] pub(crate) slot_ports_by_component: BTreeMap>, #[serde(default)] @@ -881,11 +953,19 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( } mesh_port_by_peer_id.insert(peer_id.clone(), *port); } + if let Some(port) = fixed_router_mesh_port + && !reserved.insert(port) + { + return Err(miette::miette!( + "router mesh port {} conflicts with an existing direct runtime port", + port + )); + } for component in &direct_plan.components { let path = runtime_root.join(&component.sidecar.mesh_config_path); let mut config = read_mesh_config_public(path.as_path())?; - let mesh_port = allocate_direct_runtime_port(&mut reserved, None)?; + let mesh_port = allocate_direct_runtime_port(&mut reserved)?; if mesh_port_by_peer_id .insert(config.identity.id.clone(), mesh_port) .is_some() @@ -897,7 +977,7 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( } config.mesh_listen = SocketAddr::new(config.mesh_listen.ip(), mesh_port); if let Some(dynamic_caps_listen) = config.dynamic_caps_listen.as_mut() { - let port = allocate_direct_runtime_port(&mut reserved, None)?; + let port = allocate_direct_runtime_port(&mut reserved)?; *dynamic_caps_listen = SocketAddr::new(dynamic_caps_listen.ip(), port); state .dynamic_caps_port_by_component @@ -907,7 +987,7 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( let mut slot_route_ports: BTreeMap> = BTreeMap::new(); for route in &mut config.outbound { let authored_port = route.listen_port; - let port = allocate_direct_runtime_port(&mut reserved, None)?; + let port = allocate_direct_runtime_port(&mut reserved)?; route.listen_port = port; slot_route_ports .entry(route.slot.clone()) @@ -953,7 +1033,10 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( let mut router_config = if let Some(router) = direct_plan.router.as_ref() { let path = runtime_root.join(&router.mesh_config_path); let mut config = read_mesh_config_public(path.as_path())?; - let mesh_port = allocate_direct_runtime_port(&mut reserved, fixed_router_mesh_port)?; + let mesh_port = match fixed_router_mesh_port { + Some(port) => port, + None => allocate_direct_runtime_port(&mut reserved)?, + }; if let Some(existing) = mesh_port_by_peer_id.insert(config.identity.id.clone(), mesh_port) && existing != mesh_port { @@ -1004,21 +1087,9 @@ pub(crate) fn cross_site_router_mesh_bind_ip( } } -pub(crate) fn allocate_direct_runtime_port( - reserved: &mut BTreeSet, - preferred: Option, -) -> Result { - if let Some(preferred) = preferred { - if reserved.insert(preferred) { - return Ok(preferred); - } - return Err(miette::miette!( - "runtime port {} was requested twice in one direct runtime", - preferred - )); - } +pub(crate) fn allocate_direct_runtime_port(reserved: &mut BTreeSet) -> Result { for _ in 0..256 { - let port = pick_free_port()?; + let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { return Ok(port); } @@ -1028,24 +1099,19 @@ pub(crate) fn allocate_direct_runtime_port( )) } -pub(crate) fn pick_free_port() -> Result { - let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)) - .into_diagnostic()?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - pub(crate) fn rewrite_direct_mesh_peer_addrs( config: &mut MeshConfigPublic, mesh_port_by_peer_id: &HashMap, ) -> Result<()> { for route in &mut config.outbound { + let addr = route.peer_addr.parse::().map_err(|err| { + miette::miette!("invalid mesh peer address {}: {err}", route.peer_addr) + })?; let port = mesh_port_by_peer_id .get(route.peer_id.as_str()) .copied() + .or_else(|| (addr.port() != 0).then_some(addr.port())) .ok_or_else(|| miette::miette!("missing mesh port for peer {}", route.peer_id))?; - let addr = route.peer_addr.parse::().map_err(|err| { - miette::miette!("invalid mesh peer address {}: {err}", route.peer_addr) - })?; route.peer_addr = SocketAddr::new(addr.ip(), port).to_string(); } @@ -1054,13 +1120,14 @@ pub(crate) fn rewrite_direct_mesh_peer_addrs( peer_addr, peer_id, .. } = &mut route.target { + let addr = peer_addr + .parse::() + .map_err(|err| miette::miette!("invalid mesh peer address {}: {err}", peer_addr))?; let port = mesh_port_by_peer_id .get(peer_id.as_str()) .copied() + .or_else(|| (addr.port() != 0).then_some(addr.port())) .ok_or_else(|| miette::miette!("missing mesh port for peer {}", peer_id))?; - let addr = peer_addr - .parse::() - .map_err(|err| miette::miette!("invalid mesh peer address {}: {err}", peer_addr))?; peer_addr.clear(); peer_addr.push_str(&SocketAddr::new(addr.ip(), port).to_string()); } @@ -1153,7 +1220,6 @@ pub(crate) fn materialize_direct_runtime( reuse_existing: bool, ) -> Result { let empty_peer_ports = BTreeMap::new(); - let empty_peer_identities = BTreeMap::new(); materialize_direct_runtime_with_existing( plan_root, runtime_root, @@ -1163,7 +1229,7 @@ pub(crate) fn materialize_direct_runtime( DirectExistingMeshState { reuse_existing, peer_ports_by_id: &empty_peer_ports, - peer_identities_by_id: &empty_peer_identities, + peer_identities_by_id: &BTreeMap::new(), }, ) } @@ -1184,13 +1250,18 @@ pub(crate) fn materialize_direct_runtime_with_existing( ) -> Result { let runtime_state_path = direct_runtime_state_path(plan_root); if existing.reuse_existing && runtime_state_path.is_file() { - return read_direct_runtime_state(&runtime_state_path); + let mut runtime_state = read_direct_runtime_state(&runtime_state_path)?; + runtime_state.ready = false; + write_direct_runtime_state(plan_root, &runtime_state)?; + return Ok(runtime_state); } if runtime_state_path.exists() { let _ = fs::remove_file(&runtime_state_path); } + let merged_peer_identities = + merge_existing_mesh_peer_identities(mesh_plan, existing.peer_identities_by_id)?; let existing_mesh_peer_identities = - required_existing_mesh_peer_identities(mesh_plan, existing.peer_identities_by_id)?; + required_existing_mesh_peer_identities(mesh_plan, &merged_peer_identities)?; provision_mesh_filesystem_with_peer_identities( mesh_plan, runtime_root, @@ -1241,14 +1312,16 @@ pub(crate) fn configure_direct_mesh_network( })?; plan.component_mesh_port_by_id .insert(component.id, mesh_port); - config.mesh_listen = rewrite_mesh_listen_for_slirp_guest(config.mesh_listen); + if direct_component_uses_isolated_network(component) { + config.mesh_listen = rewrite_mesh_listen_for_slirp_guest(config.mesh_listen); - for route in &mut config.outbound { - route.peer_addr = rewrite_peer_addr_for_slirp_gateway(route.peer_addr.as_str()); - } - for route in &mut config.inbound { - if let InboundTarget::MeshForward { peer_addr, .. } = &mut route.target { - *peer_addr = rewrite_peer_addr_for_slirp_gateway(peer_addr.as_str()); + for route in &mut config.outbound { + route.peer_addr = rewrite_peer_addr_for_slirp_gateway(route.peer_addr.as_str()); + } + for route in &mut config.inbound { + if let InboundTarget::MeshForward { peer_addr, .. } = &mut route.target { + *peer_addr = rewrite_peer_addr_for_slirp_gateway(peer_addr.as_str()); + } } } @@ -1279,36 +1352,6 @@ pub(crate) fn rewrite_peer_addr_for_slirp_gateway(peer_addr: &str) -> String { SocketAddr::from((Ipv4Addr::new(10, 0, 2, 2), addr.port())).to_string() } -#[cfg(target_os = "linux")] -pub(crate) fn rewrite_sidecar_env_passthrough_for_slirp(name: &str, value: &str) -> String { - if name != amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV { - return value.to_string(); - } - rewrite_loopback_url_for_slirp_gateway(value) -} - -#[cfg(target_os = "linux")] -pub(crate) fn rewrite_loopback_url_for_slirp_gateway(value: &str) -> String { - let Ok(mut url) = Url::parse(value) else { - return value.to_string(); - }; - let Some(host) = url.host_str() else { - return value.to_string(); - }; - let is_loopback = host.eq_ignore_ascii_case("localhost") - || host - .parse::() - .map(|addr| addr.is_loopback()) - .unwrap_or(false); - if !is_loopback { - return value.to_string(); - } - if url.set_host(Some("10.0.2.2")).is_err() { - return value.to_string(); - } - url.to_string() -} - pub(crate) fn read_mesh_config_public(path: &Path) -> Result { let raw = fs::read_to_string(path) .map_err(|err| miette::miette!("failed to read mesh config {}: {err}", path.display()))?; @@ -1324,3 +1367,146 @@ pub(crate) fn write_mesh_config_public(path: &Path, config: &MeshConfigPublic) - .into_diagnostic() .wrap_err_with(|| format!("failed to write mesh config {}", path.display())) } + +#[cfg(test)] +mod tests { + use amber_mesh::{InboundRoute, MeshPeer}; + + use super::*; + + fn mesh_identity_public(id: &str, public_key: [u8; 32]) -> MeshIdentityPublic { + MeshIdentityPublic { + id: id.to_string(), + public_key, + mesh_scope: Some("amber.test".to_string()), + } + } + + #[test] + fn merge_existing_mesh_peer_identities_keeps_embedded_plan_identities() { + let embedded_identity = mesh_identity_public("/site/compose/router", [7; 32]); + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: vec![embedded_identity.clone()], + targets: Vec::new(), + }; + + let merged = + merge_existing_mesh_peer_identities(&mesh_plan, &BTreeMap::new()).expect("merge"); + + let merged_identity = merged + .get("/site/compose/router") + .expect("embedded identity should remain available"); + assert_eq!(merged_identity.id, embedded_identity.id); + assert_eq!(merged_identity.public_key, embedded_identity.public_key); + assert_eq!(merged_identity.mesh_scope, embedded_identity.mesh_scope); + } + + #[test] + fn merge_existing_mesh_peer_identities_rejects_conflicts() { + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: vec![mesh_identity_public("/peer", [1; 32])], + targets: Vec::new(), + }; + let discovered = + BTreeMap::from([("/peer".to_string(), mesh_identity_public("/peer", [2; 32]))]); + + let err = merge_existing_mesh_peer_identities(&mesh_plan, &discovered) + .expect_err("conflicting embedded and discovered identities should fail"); + + assert!( + err.to_string() + .contains("does not match the embedded mesh provision plan"), + "unexpected merge error: {err}", + ); + } + + #[test] + fn rewrite_direct_mesh_peer_addrs_preserves_authored_external_peer_ports() { + let mut config = MeshConfigPublic { + identity: mesh_identity_public("/site/direct/router", [3; 32]), + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeer { + id: "/site/compose/router".to_string(), + public_key: [4; 32], + }], + inbound: vec![InboundRoute { + route_id: "in".to_string(), + capability: "site-controller".to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::MeshForward { + peer_addr: "127.0.0.1:32002".to_string(), + peer_id: "/site/compose/router".to_string(), + route_id: "route".to_string(), + capability: "site-controller".to_string(), + }, + allowed_issuers: Vec::new(), + }], + outbound: vec![OutboundRoute { + route_id: "out".to_string(), + rewrite_route_id: None, + slot: "compose".to_string(), + capability_kind: None, + capability_profile: None, + listen_port: 25000, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:32001".to_string(), + peer_id: "/site/compose/router".to_string(), + capability: "site-controller".to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + }; + + rewrite_direct_mesh_peer_addrs(&mut config, &HashMap::new()).expect("rewrite"); + + assert_eq!(config.outbound[0].peer_addr, "127.0.0.1:32001"); + let InboundTarget::MeshForward { peer_addr, .. } = &config.inbound[0].target else { + panic!("expected mesh forward route"); + }; + assert_eq!(peer_addr, "127.0.0.1:32002"); + } + + #[test] + fn direct_component_control_socket_path_stays_short_on_long_work_dirs() { + let work_dir = Path::new( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + mixed-run-doc-example-detach-very-long/state/runs/run-123/state/direct_local/runtime/\ + work/components/c2-web", + ); + let socket = direct_component_control_socket_path(work_dir, 2); + let rendered = socket.as_os_str().to_string_lossy(); + + assert!( + rendered.len() < 104, + "direct sidecar control socket path must fit within unix socket limits: {rendered}", + ); + } + + #[test] + fn allocate_direct_runtime_port_avoids_shared_loopback_reservations() { + let reserved_port = + amber_site_controller::reserve_loopback_port().expect("shared loopback port"); + let mut reserved = BTreeSet::new(); + + for _ in 0..32 { + let port = allocate_direct_runtime_port(&mut reserved) + .expect("direct runtime port allocation should succeed"); + assert_ne!( + port, reserved_port, + "direct runtime should not reuse loopback ports already reserved by mixed-run \ + infrastructure", + ); + } + } +} diff --git a/cli/src/framework_component/http.rs b/cli/src/framework_component/http.rs deleted file mode 100644 index 3559efcd..00000000 --- a/cli/src/framework_component/http.rs +++ /dev/null @@ -1,733 +0,0 @@ -use super::{ - api::{authorize_capability_instance, capability_instance_record}, - ccs_api::{ - self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, - FrameworkComponentMutateRequest, FrameworkComponentMutateResponse, - }, - control_state_api::{ - self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, - DynamicCapsMutateResponse, - }, - orchestration::*, - planner::*, - state::*, - *, -}; - -pub(crate) async fn run_framework_control_state(plan_path: PathBuf) -> Result<()> { - let plan: FrameworkControlStateServicePlan = - read_json(plan_path.as_path(), "framework control-state plan")?; - let mut control_state: FrameworkControlState = - read_json(Path::new(&plan.state_path), "framework control-state file")?; - persist_control_state(Path::new(&plan.state_path), &mut control_state)?; - let app_state = ControlStateApp { - control_state: Arc::new(Mutex::new(control_state)), - client: ReqwestClient::new(), - state_path: PathBuf::from(&plan.state_path), - run_root: PathBuf::from(&plan.run_root), - state_root: PathBuf::from(&plan.state_root), - mesh_scope: Arc::::from(plan.mesh_scope.clone()), - control_state_auth_token: Arc::::from(plan.auth_token), - authority_locks: Arc::new(Mutex::new(BTreeMap::new())), - bridge_proxies: Arc::new(Mutex::new(BTreeMap::new())), - }; - recover_control_state(&app_state).await?; - let app = Router::new() - .nest_service("/mcp", control_state_mcp::service(app_state.clone())) - .route("/", get(healthz)) - .route("/healthz", get(healthz)) - .route(CONTROL_SERVICE_PATH, get(get_control_state)) - .route("/v1/control-state/children", post(control_create_child)) - .route( - "/v1/control-state/children/{child}/destroy", - post(control_destroy_child), - ) - .route( - "/v1/control-state/dynamic-caps/held", - post(control_dynamic_held_list), - ) - .route( - "/v1/control-state/dynamic-caps/held/detail", - post(control_dynamic_held_detail), - ) - .route( - "/v1/control-state/dynamic-caps/share", - post(control_dynamic_share), - ) - .route( - "/v1/control-state/dynamic-caps/inspect-ref", - post(control_dynamic_inspect_ref), - ) - .route( - "/v1/control-state/dynamic-caps/revoke", - post(control_dynamic_revoke), - ) - .route( - "/v1/control-state/dynamic-caps/resolve-origin", - post(control_dynamic_resolve_origin), - ) - .with_state(app_state.clone()); - let listener = TcpListener::bind(plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to bind framework control-state service on {}", - plan.listen_addr - ) - })?; - let serve_result = axum::serve(listener, app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) - .await - .into_diagnostic(); - let cleanup_result = cleanup_dynamic_bridge_proxies(&app_state).await; - match (serve_result, cleanup_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(err), Ok(())) => Err(err).wrap_err("framework control-state service failed"), - (Ok(()), Err(err)) => { - Err(err).wrap_err("framework control-state service failed to stop bridge proxies") - } - (Err(serve_err), Err(cleanup_err)) => Err(miette::miette!( - "framework control-state service failed: {serve_err}\nbridge proxy cleanup failed: \ - {cleanup_err}" - )), - } -} - -pub(crate) async fn run_framework_ccs(plan_path: PathBuf) -> Result<()> { - let plan: FrameworkCcsPlan = read_json(plan_path.as_path(), "framework CCS plan")?; - let app_state = CcsApp { - client: ReqwestClient::new(), - site_state_root: PathBuf::from(&plan.site_state_root), - control_state_url: Arc::::from(plan.control_state_url), - router_auth_token: Arc::::from(plan.router_auth_token), - control_state_auth_token: Arc::::from(plan.control_state_auth_token), - }; - let app = Router::new() - .nest_service("/mcp", mcp::service(app_state.clone())) - .route("/", get(healthz)) - .route("/healthz", get(healthz)) - .route("/v1/templates", get(ccs_list_templates)) - .route("/v1/templates/{template}", get(ccs_describe_template)) - .route( - "/v1/templates/{template}/resolve", - post(ccs_resolve_template), - ) - .route( - "/v1/children", - get(ccs_list_children).post(ccs_create_child), - ) - .route( - "/v1/children/{child}", - get(ccs_describe_child).delete(ccs_destroy_child), - ) - .route("/v1/snapshot", post(ccs_snapshot)) - .route( - "/v1/internal/dynamic-caps/origins/publish", - post(ccs_publish_dynamic_capability_origin), - ) - .with_state(app_state); - let listener = TcpListener::bind(plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind framework CCS on {}", plan.listen_addr))?; - axum::serve(listener, app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) - .await - .into_diagnostic() - .wrap_err("framework CCS failed") -} - -pub(super) async fn healthz() -> Json { - Json(json!({ "ok": true })) -} - -pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { - let mut bridge_proxies = { - let mut guard = app.bridge_proxies.lock().await; - std::mem::take(&mut *guard) - }; - stop_bridge_proxies(&mut bridge_proxies).await -} - -pub(super) async fn get_control_state( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - Ok(Json(app.control_state.lock().await.clone())) -} - -pub(super) async fn control_create_child( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; - Ok(Json( - execute_create_child(&app, authority_realm_id, request.request).await?, - )) -} - -pub(super) async fn control_destroy_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, - Json(request): Json, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; - execute_destroy_child(&app, authority_realm_id, &child).await?; - Ok(StatusCode::NO_CONTENT) -} - -pub(super) async fn control_dynamic_held_list( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::HeldList(request), - ) - .await? - { - DynamicCapsInspectResponse::HeldList(response) => Ok(Json(response)), - _ => unreachable!("held_list should return held list"), - } -} - -pub(super) async fn control_dynamic_held_detail( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::HeldDetail(request), - ) - .await? - { - DynamicCapsInspectResponse::HeldDetail(response) => Ok(Json(response)), - _ => unreachable!("held_detail should return held detail"), - } -} - -pub(super) async fn control_dynamic_share( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_mutate( - &app, - DynamicCapsMutateRequest::Share(request), - ) - .await? - { - DynamicCapsMutateResponse::Share(response) => Ok(Json(response)), - _ => unreachable!("share should return share response"), - } -} - -pub(super) async fn control_dynamic_inspect_ref( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::InspectRef(request), - ) - .await? - { - DynamicCapsInspectResponse::InspectRef(response) => Ok(Json(response)), - _ => unreachable!("inspect_ref should return inspect response"), - } -} - -pub(super) async fn control_dynamic_revoke( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_mutate( - &app, - DynamicCapsMutateRequest::Revoke(request), - ) - .await? - { - DynamicCapsMutateResponse::Revoke(response) => Ok(Json(response)), - _ => unreachable!("revoke should return revoke response"), - } -} - -pub(super) async fn control_dynamic_resolve_origin( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> -{ - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::ResolveOrigin(request), - ) - .await? - { - DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(Json(response)), - _ => unreachable!("resolve_origin should return origin resolution"), - } -} - -pub(super) async fn ccs_list_templates( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ListTemplates, - ) - .await? - { - FrameworkComponentInspectResponse::ListTemplates(response) => Ok(Json(response)), - _ => unreachable!("list_templates should return template list"), - } -} - -pub(super) async fn ccs_describe_template( - State(app): State, - headers: HeaderMap, - AxumPath(template): AxumPath, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetTemplate { template }, - ) - .await? - { - FrameworkComponentInspectResponse::GetTemplate(response) => Ok(Json(response)), - _ => unreachable!("get_template should return template description"), - } -} - -pub(super) async fn ccs_resolve_template( - State(app): State, - headers: HeaderMap, - AxumPath(template): AxumPath, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ResolveTemplate { template, request }, - ) - .await? - { - FrameworkComponentInspectResponse::ResolveTemplate(response) => Ok(Json(response)), - _ => unreachable!("resolve_template should return resolved template"), - } -} - -pub(super) async fn ccs_list_children( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ListChildren, - ) - .await? - { - FrameworkComponentInspectResponse::ListChildren(response) => Ok(Json(response)), - _ => unreachable!("list_children should return child list"), - } -} - -pub(super) async fn ccs_create_child( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - let (record, _) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_mutate( - &app, - &record.cap_instance_id, - FrameworkComponentMutateRequest::CreateChild(request), - ) - .await? - { - FrameworkComponentMutateResponse::CreateChild(response) => Ok(Json(response)), - _ => unreachable!("create_child should return create response"), - } -} - -pub(super) async fn ccs_describe_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetChild { child }, - ) - .await? - { - FrameworkComponentInspectResponse::GetChild(response) => Ok(Json(response)), - _ => unreachable!("get_child should return child description"), - } -} - -pub(super) async fn ccs_snapshot( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetSnapshot, - ) - .await? - { - FrameworkComponentInspectResponse::GetSnapshot(response) => Ok(Json(response)), - _ => unreachable!("get_snapshot should return snapshot"), - } -} - -pub(super) async fn ccs_publish_dynamic_capability_origin( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> -{ - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - Ok(Json( - publish_dynamic_capability_origin_local(&app, request).await?, - )) -} - -pub(super) async fn ccs_destroy_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, -) -> std::result::Result { - let (record, _) = authorize_request(&app, &headers).await?; - let _ = ccs_api::execute_framework_component_mutate( - &app, - &record.cap_instance_id, - FrameworkComponentMutateRequest::DestroyChild { child }, - ) - .await?; - Ok(StatusCode::NO_CONTENT) -} - -pub(super) async fn authorize_request( - app: &CcsApp, - headers: &HeaderMap, -) -> std::result::Result<(CapabilityInstanceRecord, FrameworkControlState), ProtocolApiError> { - authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; - let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; - let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; - let state = fetch_control_state(app).await?; - let record = authorize_capability_instance(&state, &route_id, &peer_id) - .map_err(ProtocolApiError::from)? - .clone(); - Ok((record, state)) -} - -pub(super) fn authorize_framework_auth_header( - headers: &HeaderMap, - expected: &str, -) -> std::result::Result<(), ProtocolApiError> { - let actual = required_header(headers, FRAMEWORK_AUTH_HEADER)?; - if actual != expected { - return Err(ProtocolApiError::unauthorized( - "invalid authenticated framework request header", - )); - } - Ok(()) -} - -pub(super) fn required_header( - headers: &HeaderMap, - name: &str, -) -> std::result::Result { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) - .ok_or_else(|| { - ProtocolApiError::unauthorized(format!( - "missing authenticated framework request header `{name}`" - )) - }) -} - -pub(super) async fn fetch_control_state( - app: &CcsApp, -) -> std::result::Result { - let url = format!( - "{}{}", - app.control_state_url.trim_end_matches('/'), - CONTROL_SERVICE_PATH - ); - let response = app - .client - .get(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - if !response.status().is_success() { - return Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {}", - response.status() - ))); - } - response.json().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned invalid JSON: {err}" - )) - }) -} - -pub(super) async fn forward_create_child( - app: &CcsApp, - cap_instance_id: &str, - request: CreateChildRequest, -) -> std::result::Result { - let url = format!( - "{}/v1/control-state/children", - app.control_state_url.trim_end_matches('/') - ); - let response = app - .client - .post(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .json(&ControlCreateChildRequest { - cap_instance_id: cap_instance_id.to_string(), - request, - }) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - parse_control_service_json(response).await -} - -pub(super) async fn forward_destroy_child( - app: &CcsApp, - cap_instance_id: &str, - child: &str, -) -> std::result::Result<(), ProtocolApiError> { - let url = format!( - "{}/v1/control-state/children/{child}/destroy", - app.control_state_url.trim_end_matches('/') - ); - let response = app - .client - .post(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .json(&ControlDestroyChildRequest { - cap_instance_id: cap_instance_id.to_string(), - }) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - parse_control_service_empty(response).await -} - -pub(super) async fn parse_control_service_json Deserialize<'de>>( - response: reqwest::Response, -) -> std::result::Result { - if response.status().is_success() { - return response.json().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned invalid JSON: {err}" - )) - }); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to read authoritative control-state error response: {err}" - )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {status}" - ))) -} - -pub(super) async fn parse_control_service_empty( - response: reqwest::Response, -) -> std::result::Result<(), ProtocolApiError> { - if response.status().is_success() { - return Ok(()); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to read authoritative control-state error response: {err}" - )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {status}" - ))) -} - -pub(super) async fn shutdown_signal() { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("framework service should install Ctrl-C handler"); - }; - - #[cfg(unix)] - let terminate = async { - use tokio::signal::unix::{SignalKind, signal}; - - signal(SignalKind::terminate()) - .expect("framework service should install SIGTERM handler") - .recv() - .await; - }; - - #[cfg(unix)] - tokio::select! { - _ = ctrl_c => {} - _ = terminate => {} - } - - #[cfg(not(unix))] - ctrl_c.await; -} - -pub(super) fn write_json(path: &Path, value: &T) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - let bytes = serde_json::to_vec_pretty(value) - .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; - write_bytes_atomic(path, &bytes) -} - -pub(super) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> Result<()> { - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("tmp"); - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - let tmp_path = path.with_file_name(format!(".{file_name}.tmp-{}-{nonce}", std::process::id())); - - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp_path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", tmp_path.display()))?; - if let Err(err) = file.write_all(bytes) { - let _ = fs::remove_file(&tmp_path); - return Err(miette::miette!( - "failed to write {}: {err}", - tmp_path.display() - )); - } - if let Err(err) = file.sync_all() { - let _ = fs::remove_file(&tmp_path); - return Err(miette::miette!( - "failed to sync {}: {err}", - tmp_path.display() - )); - } - drop(file); - - fs::rename(&tmp_path, path) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to replace {} with {}", - path.display(), - tmp_path.display() - ) - })?; - sync_parent_directory(path)?; - Ok(()) -} - -pub(super) fn sync_parent_directory(path: &Path) -> Result<()> { - #[cfg(unix)] - { - if let Some(parent) = path.parent() { - fs::File::open(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to open parent directory {}", parent.display()))? - .sync_all() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to sync parent directory {}", parent.display()) - })?; - } - } - #[cfg(not(unix))] - { - let _ = path; - } - Ok(()) -} - -pub(super) fn read_json Deserialize<'de>>(path: &Path, label: &str) -> Result { - let bytes = fs::read(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {label} {}", path.display()))?; - serde_json::from_slice(&bytes) - .map_err(|err| miette::miette!("invalid {label} {}: {err}", path.display())) -} diff --git a/cli/src/main.rs b/cli/src/main.rs index d103b54f..04dbcfc0 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,13 +1,13 @@ mod command_support; mod direct_runtime; mod docs; -mod framework_component; mod mixed_run; mod run_inputs; mod run_logs; mod scenario_runner; mod site_proxy_metadata; mod tcp_readiness; +mod unix_process; mod vm_runtime; #[cfg(target_os = "linux")] @@ -22,7 +22,7 @@ use std::{ future::Future, io, io::{IsTerminal as _, Write as _}, - net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream}, + net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}, path::{Path, PathBuf}, process::{Child, Stdio}, sync::Arc, @@ -41,7 +41,7 @@ use amber_compiler::{ dot::DotReporter, metadata::MetadataReporter, scenario_ir::ScenarioIrReporter, - vm::VM_PLAN_FILENAME, + vm::{VM_PLAN_FILENAME, VM_RUN_SCRIPT_FILENAME, VmReporter}, }, run_plan::{ PlacementFile, RUN_PLAN_SCHEMA, RunPlan, SiteKind, build_homogeneous_export_run_plan, @@ -429,12 +429,8 @@ enum Command { RunVmInit(RunVmInitArgs), #[command(hide = true, name = "run-site-supervisor")] RunSiteSupervisor(RunSiteSupervisorArgs), - #[command(hide = true, name = "run-site-actuator")] - RunSiteActuator(RunSiteActuatorArgs), - #[command(hide = true, name = "run-framework-control-state")] - RunFrameworkControlState(RunFrameworkControlStateArgs), - #[command(hide = true, name = "run-framework-ccs")] - RunFrameworkCcs(RunFrameworkCcsArgs), + #[command(hide = true, name = "run-site-controller")] + RunSiteController(RunSiteControllerArgs), #[command(hide = true, name = "run-detached-coordinator")] RunDetachedCoordinator(RunDetachedCoordinatorArgs), #[command(hide = true, name = "run-observability-sink")] @@ -711,22 +707,8 @@ struct RunSiteSupervisorArgs { } #[derive(Args)] -struct RunSiteActuatorArgs { - /// Path to a mixed-site site-actuator plan JSON file. - #[arg(long = "plan", value_name = "FILE")] - plan: PathBuf, -} - -#[derive(Args)] -struct RunFrameworkControlStateArgs { - /// Path to a framework control-state service plan JSON file. - #[arg(long = "plan", value_name = "FILE")] - plan: PathBuf, -} - -#[derive(Args)] -struct RunFrameworkCcsArgs { - /// Path to a framework CCS plan JSON file. +struct RunSiteControllerArgs { + /// Path to a mixed-site site-controller plan JSON file. #[arg(long = "plan", value_name = "FILE")] plan: PathBuf, } @@ -905,12 +887,8 @@ async fn main() -> Result<()> { .await } Command::RunSiteSupervisor(args) => mixed_run::run_site_supervisor(args.plan).await, - Command::RunSiteActuator(args) => mixed_run::run_site_actuator(args.plan).await, - Command::RunFrameworkControlState(args) => { - framework_component::run_framework_control_state(args.plan).await - } - Command::RunFrameworkCcs(args) => { - framework_component::run_framework_ccs(args.plan).await + Command::RunSiteController(args) => { + amber_site_controller::run_site_controller_default(args.plan).await } Command::RunDetachedCoordinator(args) => run_detached_coordinator(args).await, Command::RunObservabilitySink(args) => { @@ -1188,10 +1166,16 @@ async fn compile(args: CompileArgs) -> Result<()> { } if let Some(vm_dest) = outputs.vm { - let run_plan = build_homogeneous_export_run_plan(&compiled, SiteKind::Vm) - .into_diagnostic() - .wrap_err("failed to build homogeneous vm export plan")?; - write_unmanaged_export_output(&vm_dest, &run_plan, SiteKind::Vm)?; + let artifact = VmReporter + .emit(&compiled) + .map_err(miette::Report::new) + .wrap_err("failed to build vm artifact")?; + write_directory_output( + &vm_dest, + "vm output directory", + &artifact.files, + Some(Path::new(VM_RUN_SCRIPT_FILENAME)), + )?; } if let Some(metadata_dest) = outputs.metadata { diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index a9b65e97..acd590e3 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -1,8 +1,6 @@ use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - env, - fmt::Write as _, - fs, + collections::{BTreeMap, BTreeSet}, + env, fs, future::Future, io::{Read as _, Write as _}, net::{SocketAddr, TcpListener, TcpStream}, @@ -17,71 +15,41 @@ use std::{ use amber_compiler::{ mesh::ProxyMetadata, - reporter::{ - direct::{DirectPlan, DirectRuntimeUrlSource}, - vm::{VM_PLAN_FILENAME, VmPlan}, - }, + reporter::vm::{VM_PLAN_FILENAME, VmPlan}, run_plan::{RunLink, RunPlan, RunSitePlan, SiteKind}, }; use amber_manifest::{CapabilityKind, CapabilityTransport, NetworkProtocol}; use amber_mesh::{ - InboundRoute, InboundTarget, MESH_CONFIG_FILENAME, MESH_IDENTITY_FILENAME, MeshConfig, - MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, MeshProtocol, - MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTargetKind, OutboundRoute, - TransportConfig, component_route_id, http_route_plugins_for_capability_kind, - router_dynamic_export_route_id, router_export_route_id, + InboundRoute, InboundTarget, MeshConfig, MeshConfigPublic, MeshIdentity, MeshIdentityPublic, + MeshPeer, MeshProtocol, OutboundRoute, TransportConfig, component_route_id, + router_export_route_id, telemetry::{SCENARIO_RUN_ID_ENV, SCENARIO_SCOPE_ENV}, }; use amber_proxy::{ - ControlEndpoint, RouterDiscovery, apply_route_overlay_with_retry, discover_router_for_output, - fetch_router_identity, load_output_proxy_metadata, register_export_peer_with_retry, - register_external_slot_with_retry, revoke_route_overlay_with_retry, + ControlEndpoint, RouterDiscovery, discover_router_for_output, fetch_router_identity, + load_output_proxy_metadata, register_export_peer_with_retry, register_external_slot_with_retry, }; -use axum::{ - Router, - extract::{Path as AxumPath, State}, - http::StatusCode, - routing::{get, post}, +use amber_site_controller::{ + assign_compose_egress_network_subnets, cleanup_dynamic_site_children, + observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, walk_files, }; use base64::Engine as _; use miette::{Context as _, IntoDiagnostic as _, Result}; -use opentelemetry_proto::tonic::{ - collector::logs::v1::ExportLogsServiceRequest, - common::v1::{AnyValue as OtlpAnyValue, KeyValue as OtlpKeyValue, any_value}, - logs::v1::LogRecord as OtlpLogRecord, -}; -use prost::Message as _; use serde::{Deserialize, Serialize}; -use serde_json::{Map as JsonMap, Value as JsonValue}; -use sha2::{Digest as _, Sha256}; -use tokio::{ - io::{AsyncReadExt as _, AsyncWriteExt as _}, - net::TcpListener as TokioTcpListener, - sync::Mutex as AsyncMutex, - time::{Instant, sleep}, -}; +use serde_json::Value as JsonValue; +use tokio::time::{Instant, sleep}; use url::Url; use crate::{ DirectLaunchProcessPreview, DirectSiteLaunchPreview, build_direct_site_launch_preview, - direct_current_control_socket_path, - direct_runtime::ensure_direct_control_socket_link, - direct_runtime_state_path, - framework_component::{ - DynamicInputRouteRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - DynamicSitePlanRecord, SiteActuatorDestroyRequest, SiteActuatorPrepareRequest, - SiteActuatorPublishRequest, - }, + direct_current_control_socket_path, direct_runtime_state_path, run_inputs::{collect_run_interface, validate_export_bindings, validate_slot_bindings}, site_proxy_metadata::load_site_proxy_metadata, - tcp_readiness::{ - endpoint_accepts_stable_connection, endpoint_returns_http_response, wait_for_http_response, - wait_for_stable_endpoint, - }, + tcp_readiness::{wait_for_http_response, wait_for_stable_endpoint}, + unix_process::{pid_is_alive, terminate_process_roots}, vm_runtime::{ TCG_VM_STARTUP_TIMEOUT, VmLaunchPreview, VmRuntimeState, VmSiteLaunchPreview, - build_vm_site_launch_preview, ensure_control_socket_link, vm_current_control_socket_path, - vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, write_vm_runtime_state, + build_vm_site_launch_preview, vm_current_control_socket_path, vm_uses_tcg_accel, }, }; mod launch_bundle; @@ -91,31 +59,15 @@ mod supervisor; pub(crate) use self::{launch_bundle::*, outside_proxy::*, supervisor::*}; const RECEIPT_SCHEMA: &str = "amber.run.receipt"; -const RECEIPT_VERSION: u32 = 3; +const RECEIPT_VERSION: u32 = 4; const LAUNCH_BUNDLE_SCHEMA: &str = "amber.run.launch_bundle"; -const DYNAMIC_COMPOSE_CHILD_SCHEMA: &str = "amber.run.dynamic_compose_child"; -const DYNAMIC_COMPOSE_CHILD_VERSION: u32 = 1; -const DYNAMIC_COMPOSE_MESH_ROOT: &str = ".amber/mesh"; -const DYNAMIC_ROUTE_OVERLAY_FILENAME: &str = "site-router-overlay.json"; -const DYNAMIC_PROXY_EXPORTS_FILENAME: &str = "proxy-exports.json"; -const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; -const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; -const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; -const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; -const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; -const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; -const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; -const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; -const KUBERNETES_ROUTER_COMPONENT_NAME: &str = "amber-router"; const LAUNCH_BUNDLE_VERSION: u32 = 1; const SITE_STATE_SCHEMA: &str = "amber.run.site_state"; -const SITE_STATE_VERSION: u32 = 2; +const SITE_STATE_VERSION: u32 = 3; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; -const SITE_PLAN_VERSION: u32 = 1; -const SITE_ACTUATOR_PLAN_SCHEMA: &str = "amber.run.site_actuator_plan"; -const SITE_ACTUATOR_PLAN_VERSION: u32 = 1; -const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; -const DESIRED_LINKS_VERSION: u32 = 1; +const SITE_PLAN_VERSION: u32 = 2; +pub(crate) const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; +pub(crate) const DESIRED_LINKS_VERSION: u32 = 1; const OTLP_SINK_PLAN_SCHEMA: &str = "amber.run.observability_sink"; const OTLP_SINK_PLAN_VERSION: u32 = 1; const OTELCOL_UPSTREAM_ENV: &str = "AMBER_OTEL_UPSTREAM_OTLP_HTTP_ENDPOINT"; @@ -124,9 +76,6 @@ const OUTSIDE_PROXY_PLAN_SCHEMA: &str = "amber.run.outside_proxy"; const OUTSIDE_PROXY_PLAN_VERSION: u32 = 1; const OUTSIDE_PROXY_STATE_SCHEMA: &str = "amber.run.outside_proxy_state"; const OUTSIDE_PROXY_STATE_VERSION: u32 = 1; -const PERSISTED_TRACE_EVENT_SCHEMA: &str = "amber.trace.event"; -const PERSISTED_TRACE_EVENT_VERSION: u32 = 1; - const ROUTER_CONTROL_TIMEOUT: Duration = Duration::from_secs(30); const SUPERVISOR_POLL_INTERVAL: Duration = Duration::from_millis(500); const RESTART_BACKOFF: Duration = Duration::from_secs(1); @@ -136,8 +85,11 @@ const PROCESS_SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); const SITE_SUPERVISOR_STOP_TIMEOUT: Duration = Duration::from_secs(60); const FORCED_SUPERVISOR_EXIT_GRACE_PERIOD: Duration = Duration::from_secs(5); const KUBERNETES_NAMESPACE_READY_TIMEOUT: Duration = Duration::from_secs(60); +const KUBERNETES_NAMESPACE_DELETE_TIMEOUT: Duration = Duration::from_secs(30); +const KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD: Duration = Duration::from_secs(10); const KUBERNETES_WORKLOAD_READY_TIMEOUT: Duration = Duration::from_secs(120); const KUBERNETES_SITE_READY_BUFFER: Duration = Duration::from_secs(30); +const KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL: Duration = Duration::from_millis(250); const VM_LOCAL_TARGET_READY_TIMEOUT: Duration = Duration::from_secs(1); const DEFAULT_EXTERNAL_ENV_FILE: &str = "router-external.env"; @@ -147,6 +99,112 @@ const CONTAINER_HOST_ALIAS: &str = "host.docker.internal"; static MANAGER_OBSERVABILITY_ENDPOINT: OnceLock>> = OnceLock::new(); static KUBERNETES_CONTAINER_HOST_IP: OnceLock> = OnceLock::new(); +pub(crate) fn amber_cli_executable() -> Result { + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve amber executable path")?; + amber_cli_executable_from(¤t) +} + +fn amber_cli_executable_from(current: &Path) -> Result { + workspace_executable_from(current, "CARGO_BIN_EXE_amber", "amber", true) +} + +#[derive(Clone, Debug)] +pub(crate) struct SiteControllerCommand { + pub(crate) executable: PathBuf, + pub(crate) prefix_args: Vec<&'static str>, +} + +pub(crate) fn site_controller_command() -> Result { + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve site controller executable path")?; + site_controller_command_from(¤t) +} + +fn site_controller_command_from(current: &Path) -> Result { + if let Some(executable) = cargo_binary_override("CARGO_BIN_EXE_amber-site-controller") { + return Ok(SiteControllerCommand { + executable, + prefix_args: Vec::new(), + }); + } + let amber_executable_name = format!("amber{}", std::env::consts::EXE_SUFFIX); + if current.file_name().and_then(|name| name.to_str()) == Some(amber_executable_name.as_str()) { + return Ok(SiteControllerCommand { + executable: current.to_path_buf(), + prefix_args: vec!["run-site-controller"], + }); + } + if launched_from_cargo_test_binary(current) { + return Ok(SiteControllerCommand { + executable: amber_cli_executable_from(current)?, + prefix_args: vec!["run-site-controller"], + }); + } + match workspace_executable_from( + current, + "CARGO_BIN_EXE_amber-site-controller", + "amber-site-controller", + false, + ) { + Ok(executable) => Ok(SiteControllerCommand { + executable, + prefix_args: Vec::new(), + }), + Err(_) => Ok(SiteControllerCommand { + executable: amber_cli_executable()?, + prefix_args: vec!["run-site-controller"], + }), + } +} + +fn workspace_executable_from( + current: &Path, + env_var: &str, + binary_name: &str, + allow_current: bool, +) -> Result { + if let Some(path) = cargo_binary_override(env_var) { + return Ok(path); + } + + let executable_name = format!("{binary_name}{}", std::env::consts::EXE_SUFFIX); + if allow_current + && current.file_name().and_then(|name| name.to_str()) == Some(executable_name.as_str()) + { + return Ok(current.to_path_buf()); + } + if let Some(bin_dir) = current.parent() { + for dir in [Some(bin_dir), bin_dir.parent()].into_iter().flatten() { + let candidate = dir.join(&executable_name); + if candidate.is_file() { + return Ok(candidate); + } + } + } + Err(miette::miette!( + "failed to locate the `{binary_name}` executable; expected {env_var} or a sibling binary \ + next to {}", + current.display() + )) +} + +fn cargo_binary_override(env_var: &str) -> Option { + env::var_os(env_var) + .map(PathBuf::from) + .filter(|path| path.is_file()) +} + +fn launched_from_cargo_test_binary(current: &Path) -> bool { + current + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + == Some("deps") +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RunReceipt { pub(crate) schema: String, @@ -158,27 +216,10 @@ pub(crate) struct RunReceipt { pub(crate) source_plan_path: Option, pub(crate) run_root: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) framework_control_state: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) observability: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) bridge_proxies: Vec, pub(crate) sites: BTreeMap, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlStateReceipt { - pub(crate) pid: u32, - pub(crate) url: String, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct BridgeProxyReceipt { - pub(crate) export_name: String, - pub(crate) pid: u32, - pub(crate) listen: String, -} - #[derive(Clone, Debug, Serialize, Deserialize)] struct LaunchBundleManifest { schema: String, @@ -323,9 +364,15 @@ pub(crate) struct SiteReceipt { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) site_controller_url: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -363,10 +410,16 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] last_error: Option, } @@ -401,99 +454,11 @@ pub(crate) struct SiteSupervisorPlan { #[serde(default, skip_serializing_if = "Option::is_none")] observability_endpoint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - framework_ccs_plan_path: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - site_actuator_plan_path: Option, + site_controller_url: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) run_id: String, - pub(crate) mesh_scope: String, - pub(crate) run_root: String, - pub(crate) site_id: String, - pub(crate) kind: SiteKind, - pub(crate) router_identity_id: String, - pub(crate) artifact_dir: String, - pub(crate) site_state_root: String, - pub(crate) listen_addr: SocketAddr, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) storage_root: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) runtime_root: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) router_mesh_port: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) compose_project: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) kubernetes_namespace: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) context: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) observability_endpoint: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) launch_env: BTreeMap, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteActuatorState { - schema: String, - version: u32, - run_id: String, - site_id: String, - kind: SiteKind, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - children: BTreeMap, -} - -#[derive(Clone, Debug)] -pub(crate) struct LiveComponentRuntimeMetadata { - pub(crate) moniker: String, - pub(crate) host_mesh_addr: String, - pub(crate) mesh_config: MeshConfigPublic, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteActuatorChildRecord { - child_id: u64, - artifact_root: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - assigned_components: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - proxy_exports: BTreeMap, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - routed_inputs: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - process_pid: Option, - published: bool, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct DynamicComposeChildMetadata { - schema: String, - version: u32, - services: Vec, - readiness_services: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct StoredRouteOverlayPayload { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - peers: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - inbound_routes: Vec, -} - -#[derive(Debug, Deserialize)] -struct KubernetesSecretPayload { - #[serde(default)] - data: BTreeMap, -} - struct SupervisorPlanInput<'a> { run_root: &'a Path, run_id: &'a str, @@ -503,8 +468,7 @@ struct SupervisorPlanInput<'a> { artifact_dir: &'a Path, site_state_root: &'a Path, observability_endpoint: Option<&'a str>, - framework_ccs_plan_path: Option<&'a Path>, - site_actuator_plan_path: Option<&'a Path>, + site_controller_url: Option<&'a str>, } #[derive(Clone, Debug)] @@ -514,12 +478,11 @@ struct MaterializedObservability { } #[derive(Clone, Debug)] -struct MaterializedFrameworkControlState { - plan_path: PathBuf, - receipt: FrameworkControlStateReceipt, - router_auth_token: String, - control_state_auth_token: String, - dynamic_caps_token_verify_key_b64: String, +pub(crate) struct MaterializedSiteController { + pub(crate) state_path: PathBuf, + pub(crate) plan_path: PathBuf, + pub(crate) listen_addr: SocketAddr, + pub(crate) url: String, } #[derive(Clone, Debug)] @@ -527,13 +490,13 @@ pub(crate) struct MaterializedSite { pub(crate) site_plan: RunSitePlan, pub(crate) artifact_dir: PathBuf, pub(crate) site_state_root: PathBuf, + pub(crate) controller: Option, pub(crate) base_supervisor_plan: SiteSupervisorPlan, } #[derive(Clone, Debug)] struct MaterializedLaunchBundle { run_plan_path: PathBuf, - framework_control_state: Option, observability: Option, sites: BTreeMap, } @@ -670,6 +633,8 @@ pub(crate) struct ResolvedRunProxyTarget { #[derive(Debug, Deserialize)] struct DirectRuntimeStateView { + #[serde(default)] + ready: bool, #[serde(default)] router_mesh_port: Option, } @@ -679,36 +644,11 @@ struct SupervisorChild { child: Child, } -pub(crate) struct BridgeProxyHandle { - pub(crate) child: Child, - pub(crate) export_name: String, - pub(crate) listen: SocketAddr, -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) struct BridgeProxyKey { - pub(crate) provider_output_dir: String, - pub(crate) export_name: String, - pub(crate) consumer_kind: SiteKind, -} - -pub(crate) async fn stop_bridge_proxies( - bridge_proxies: &mut BTreeMap, -) -> Result<()> { - for bridge in bridge_proxies.values_mut() { - supervisor::stop_child(&mut bridge.child).await?; - } - bridge_proxies.clear(); - Ok(()) -} - #[derive(Debug)] struct SupervisorRuntime { site_process: Option, site_started: bool, port_forward: Option, - framework_ccs: Option, - site_actuator: Option, last_start_attempt: Option, last_stitch_refresh: Option, ready_since: Option, @@ -747,12 +687,37 @@ fn site_state_from_receipt( context: site.context.clone(), router_control: site.router_control.clone(), router_mesh_addr: site.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: site.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: site.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: site.router_identity_id.clone(), router_public_key_b64: site.router_public_key_b64.clone(), + site_controller_url: site.site_controller_url.clone(), last_error, } } +fn published_container_consumer_router_mesh_addrs( + site_kind: SiteKind, + router_mesh_addr: Option<&str>, +) -> (Option, Option) { + let Some(router_mesh_addr) = router_mesh_addr else { + return (None, None); + }; + let compose = amber_site_controller::router_mesh_addr_for_consumer( + site_kind, + SiteKind::Compose, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute compose consumer mesh addr: {err}")); + let kubernetes = amber_site_controller::router_mesh_addr_for_consumer( + site_kind, + SiteKind::Kubernetes, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute kubernetes consumer mesh addr: {err}")); + (Some(compose), Some(kubernetes)) +} + enum SiteSupervisorStopStatus { Graceful { shutdown_failed: bool }, Exited, @@ -865,8 +830,6 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { site_process: None, site_started: false, port_forward: None, - framework_ccs: None, - site_actuator: None, last_start_attempt: None, last_stitch_refresh: None, ready_since: None, @@ -887,6 +850,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { format!("stopped site supervisor {}", plan.site_id), &[("amber.run_id", plan.run_id.clone())], ); + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, None); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -906,8 +871,11 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { context: plan.context.clone(), router_control: None, router_mesh_addr: None, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, + site_controller_url: None, last_error: None, }, )?; @@ -918,6 +886,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { && coordinator_has_exited(&run_root, plan.coordinator_pid)? { cleanup_site(&plan, &mut runtime).await?; + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, None); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -937,8 +907,11 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { context: plan.context.clone(), router_control: None, router_mesh_addr: None, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, + site_controller_url: None, last_error: Some("coordinator exited before commit".to_string()), }, )?; @@ -1024,6 +997,12 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { let public_key_b64 = base64::engine::general_purpose::STANDARD .encode(discovery.router_identity.public_key); + let router_mesh_addr = discovery.router_addr.map(|addr| addr.to_string()); + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs( + plan.kind, + router_mesh_addr.as_deref(), + ); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -1042,9 +1021,12 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { port_forward_pid: runtime.port_forward.as_ref().map(Child::id), context: plan.context.clone(), router_control: Some(discovery.control_endpoint.to_string()), - router_mesh_addr: discovery.router_addr.map(|addr| addr.to_string()), + router_mesh_addr, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: Some(discovery.router_identity.id), router_public_key_b64: Some(public_key_b64), + site_controller_url: None, last_error: None, }, )?; @@ -1115,11 +1097,9 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } match listener.accept() { - Ok((mut stream, _)) => handle_otlp_connection( - &mut stream, - Path::new(&plan.requests_log), - Path::new(&plan.events_ndjson), - )?, + Ok((mut stream, _)) => { + handle_otlp_connection(&mut stream, Path::new(&plan.requests_log))? + } Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { std::thread::sleep(Duration::from_millis(100)); } @@ -1130,5755 +1110,274 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } } -pub(crate) async fn run_site_actuator(plan_path: PathBuf) -> Result<()> { - let plan: SiteActuatorPlan = read_json(&plan_path, "site actuator plan")?; - let state_path = site_actuator_state_path(Path::new(&plan.site_state_root)); - let initial_state = if state_path.is_file() { - read_json(&state_path, "site actuator state")? - } else { - let state = SiteActuatorState { - schema: "amber.run.site_actuator_state".to_string(), - version: 1, - run_id: plan.run_id.clone(), - site_id: plan.site_id.clone(), - kind: plan.kind, - children: BTreeMap::new(), - }; - write_json(&state_path, &state)?; - state - }; - let app = SiteActuatorApp { - plan, - state_path, - state: Arc::new(AsyncMutex::new(initial_state)), - }; - let stop_requested = install_signal_flag(); - let listener = TokioTcpListener::bind(app.plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind site actuator on {}", app.plan.listen_addr))?; - let router = Router::new() - .route("/healthz", get(site_actuator_healthz)) - .route( - "/v1/children/{child_id}/prepare", - post(site_actuator_prepare), - ) - .route( - "/v1/children/{child_id}/publish", - post(site_actuator_publish), - ) - .route( - "/v1/children/{child_id}/rollback", - post(site_actuator_rollback), - ) - .route( - "/v1/children/{child_id}/destroy", - post(site_actuator_destroy), - ) - .with_state(app); - axum::serve(listener, router.into_make_service()) - .with_graceful_shutdown(async move { - while !stop_requested.load(Ordering::SeqCst) { - sleep(Duration::from_millis(100)).await; - } - }) - .await - .into_diagnostic() - .wrap_err("site actuator failed") +pub(crate) fn reserve_loopback_port() -> Result { + amber_site_controller::reserve_loopback_port() } -#[derive(Clone)] -struct SiteActuatorApp { - plan: SiteActuatorPlan, - state_path: PathBuf, - state: Arc>, +pub(crate) fn reserve_host_port() -> Result { + amber_site_controller::reserve_host_port() } -type ActuatorHttpResult = std::result::Result; +pub(crate) fn new_run_id() -> String { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("run-{millis:x}-{:x}", std::process::id()) +} -async fn site_actuator_healthz() -> StatusCode { - StatusCode::NO_CONTENT +fn receipt_path(run_root: &Path) -> PathBuf { + run_root.join("receipt.json") } -async fn site_actuator_prepare( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_prepare_child(&app, child_id, request.site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +fn run_plan_path(run_root: &Path) -> PathBuf { + run_root.join("run-plan.json") } -async fn site_actuator_publish( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_publish_child(&app, child_id, request.site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +fn launch_bundle_manifest_path(run_root: &Path) -> PathBuf { + run_root.join("launch-bundle.json") } -async fn site_actuator_rollback( - State(app): State, - AxumPath(child_id): AxumPath, -) -> ActuatorHttpResult { - actuator_rollback_child(&app, child_id) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +pub(crate) fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { + state_root.join(site_id).join("manager-state.json") } -async fn site_actuator_destroy( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_destroy_child(&app, child_id, request.desired_site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +fn state_path_for_plan(plan: &SiteSupervisorPlan) -> PathBuf { + Path::new(&plan.site_state_root).join("manager-state.json") } -fn actuator_error(err: miette::Report) -> (StatusCode, String) { - (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) +fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-supervisor-plan.json") } -fn site_actuator_state_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-actuator-state.json") +pub(crate) fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-plan.json") } -fn site_actuator_child_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - Path::new(&plan.site_state_root) - .join("framework-component") - .join("children") - .join(child_id.to_string()) +pub(crate) fn site_existing_peer_ports_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("existing-peer-ports.json") } -pub(super) fn site_actuator_child_root_for_site(site_state_root: &Path, child_id: u64) -> PathBuf { - site_state_root - .join("framework-component") - .join("children") - .join(child_id.to_string()) +pub(crate) fn site_existing_peer_identities_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("existing-peer-identities.json") } -fn site_actuator_child_artifact_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("artifact") +pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("desired-links.json") } -fn site_actuator_child_runtime_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("runtime") +fn commit_marker_path(run_root: &Path) -> PathBuf { + run_root.join("committed") } -fn site_actuator_child_storage_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("storage") +fn coordinator_lock_path(run_root: &Path) -> PathBuf { + run_root.join("coordinator.lock") } -fn site_actuator_child_peer_ports_path(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("existing-peer-ports.json") +fn stop_marker_path(run_root: &Path) -> PathBuf { + run_root.join("stop-requested") } -fn site_actuator_child_peer_identities_path(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("existing-peer-identities.json") +fn kubectl_command_with_bin(kubectl_bin: &Path, context: Option<&str>) -> Command { + let mut cmd = Command::new(kubectl_bin); + if let Some(context) = context { + cmd.arg("--context").arg(context); + } + cmd } -fn dynamic_compose_child_metadata_path(artifact_root: &Path) -> PathBuf { - artifact_root.join(".amber").join("compose-child.json") +fn kubectl_stderr(output: &std::process::Output) -> String { + String::from_utf8_lossy(&output.stderr).trim().to_string() } -fn dynamic_route_overlay_path(artifact_root: &Path) -> PathBuf { - artifact_root - .join(".amber") - .join(DYNAMIC_ROUTE_OVERLAY_FILENAME) +fn kubectl_stdout(output: &std::process::Output) -> String { + String::from_utf8_lossy(&output.stdout).trim().to_string() } -fn write_dynamic_route_overlay_payload( - artifact_root: &Path, - payload: &StoredRouteOverlayPayload, -) -> Result<()> { - let path = dynamic_route_overlay_path(artifact_root); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; +fn kubectl_detail(output: &std::process::Output) -> String { + let stdout = kubectl_stdout(output); + let stderr = kubectl_stderr(output); + match (stdout.is_empty(), stderr.is_empty()) { + (false, false) => format!("stdout:\n{stdout}\nstderr:\n{stderr}"), + (false, true) => format!("stdout:\n{stdout}"), + (true, false) => format!("stderr:\n{stderr}"), + (true, true) => format!("status {}", output.status), } - write_json(&path, payload) } -fn dynamic_proxy_exports_path(artifact_root: &Path) -> PathBuf { - artifact_root - .join(".amber") - .join(DYNAMIC_PROXY_EXPORTS_FILENAME) +fn kubectl_is_not_found(output: &std::process::Output) -> bool { + let stderr = kubectl_stderr(output); + stderr.contains("(NotFound)") || stderr.contains("not found") } -fn write_dynamic_proxy_exports_metadata( - artifact_root: &Path, - proxy_exports: &BTreeMap, +fn kubectl_force_delete_namespace_pods_with_bin( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, ) -> Result<()> { - if proxy_exports.is_empty() { - return Ok(()); - } - let path = dynamic_proxy_exports_path(artifact_root); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + let output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "-n", + namespace, + "delete", + "pods", + "--all", + "--ignore-not-found", + "--force", + "--grace-period=0", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to force-delete remaining pods in kubernetes namespace `{namespace}`") + })?; + if !output.status.success() && !kubectl_is_not_found(&output) { + return Err(miette::miette!( + "force-delete of remaining pods in kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&output) + )); } - write_json(&path, proxy_exports) + Ok(()) } -fn load_dynamic_proxy_exports_metadata( - artifact_root: &Path, -) -> Result> { - let path = dynamic_proxy_exports_path(artifact_root); - if !path.is_file() { - return Ok(BTreeMap::new()); +fn kubectl_list_namespace_pods_with_bin( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, +) -> Result> { + let output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "-n", + namespace, + "get", + "pods", + "--ignore-not-found", + "-o", + "name", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to list remaining pods in kubernetes namespace `{namespace}`") + })?; + if !output.status.success() { + if kubectl_is_not_found(&output) { + return Ok(Vec::new()); + } + return Err(miette::miette!( + "listing remaining pods in kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&output) + )); } - read_json(&path, "dynamic proxy exports") + Ok(kubectl_stdout(&output) + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect()) } -pub(super) fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> Result<()> { - let state_path = site_actuator_state_path(site_state_root); - if !state_path.is_file() { - return Ok(()); - } - let mut state: SiteActuatorState = read_json(&state_path, "site actuator state")?; - let compose_supervisor_plan = matches!(kind, SiteKind::Compose) - .then(|| { - read_json::( - &site_supervisor_plan_path(site_state_root), - "site supervisor plan", - ) - }) - .transpose()?; - for child in state.children.values() { - if let Some(plan) = compose_supervisor_plan.as_ref() { - cleanup_dynamic_compose_child(plan, child)?; - } - if let Some(pid) = child.process_pid { - terminate_pid(pid, site_ready_timeout_for_kind(kind))?; - } - remove_dir_if_exists(&site_actuator_child_root_for_site( - site_state_root, - child.child_id, - ))?; - } - if state.children.is_empty() { - return Ok(()); - } - state.children.clear(); - write_json(&state_path, &state) +pub(super) fn stop_kubernetes_namespace( + context: Option<&str>, + namespace: &str, + timeout: Duration, +) -> Result<()> { + stop_kubernetes_namespace_with_kubectl(Path::new("kubectl"), context, namespace, timeout) } -fn cleanup_dynamic_compose_child( - plan: &SiteSupervisorPlan, - child: &SiteActuatorChildRecord, +fn stop_kubernetes_namespace_with_kubectl( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, + timeout: Duration, ) -> Result<()> { - let artifact_root = Path::new(&child.artifact_root); - if !dynamic_compose_child_metadata_path(artifact_root).is_file() { - return Ok(()); - } - let compose_project = plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - plan.site_id - ) - })?; - let status = compose_command(Some(compose_project), artifact_root) - .envs(plan.launch_env.clone()) - .arg("down") - .arg("-v") - .status() + let delete_output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "delete", + "namespace", + namespace, + "--ignore-not-found", + "--wait=false", + ]) + .output() .into_diagnostic() .wrap_err_with(|| { - format!( - "failed to clean up dynamic compose child for site `{}`", - plan.site_id - ) + format!("failed to start deletion of kubernetes namespace `{namespace}`") })?; - if !status.success() { + if !delete_output.status.success() && !kubectl_is_not_found(&delete_output) { return Err(miette::miette!( - "dynamic compose child cleanup on site `{}` failed with status {status}", - plan.site_id + "starting deletion of kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&delete_output) )); } - Ok(()) -} -fn rewrite_dynamic_proxy_metadata( - artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, -) -> Result<()> { - if site_plan.proxy_exports.is_empty() { - return Ok(()); - } - if site_plan.kind == SiteKind::Compose { - let path = artifact_root.join("compose.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + let mut deadline = Instant::now() + timeout; + let mut forced_pod_cleanup = false; + let mut last_namespace_detail = None::; + loop { + let get_output = kubectl_command_with_bin(kubectl_bin, context) + .args(["get", "namespace", namespace, "-o", "json"]) + .output() .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); - let Some(x_amber) = root.get_mut(&x_amber_key) else { - return Err(miette::miette!( - "compose file {} is missing x-amber metadata", - path.display() - )); - }; - let exports = site_plan - .proxy_exports - .iter() - .map(|(name, export)| { - Ok(( - serde_yaml::Value::String(name.clone()), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([ - ( - serde_yaml::Value::String("component".to_string()), - serde_yaml::Value::String(export.component.clone()), - ), - ( - serde_yaml::Value::String("provide".to_string()), - serde_yaml::Value::String(export.provide.clone()), - ), - ( - serde_yaml::Value::String("protocol".to_string()), - serde_yaml::Value::String(export.protocol.clone()), - ), - ( - serde_yaml::Value::String("router_mesh_port".to_string()), - serde_yaml::Value::Number(0u64.into()), - ), - ( - serde_yaml::Value::String("route_id".to_string()), - serde_yaml::Value::String(dynamic_proxy_export_route_id(name, export)?), - ), - ])), - )) - }) - .collect::>()?; - let mut metadata = x_amber.as_mapping().cloned().ok_or_else(|| { - miette::miette!( - "compose file {} has non-mapping x-amber metadata", - path.display() - ) - })?; - metadata.insert( - serde_yaml::Value::String("exports".to_string()), - serde_yaml::Value::Mapping(exports), - ); - *x_amber = serde_yaml::Value::Mapping(metadata); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) - } else { - let exports = site_plan - .proxy_exports - .iter() - .map(|(name, export)| { - Ok(( - name.clone(), - JsonValue::Object(JsonMap::from_iter([ - ( - "component".to_string(), - JsonValue::String(export.component.clone()), - ), - ( - "provide".to_string(), - JsonValue::String(export.provide.clone()), - ), - ( - "protocol".to_string(), - JsonValue::String(export.protocol.clone()), - ), - ("router_mesh_port".to_string(), JsonValue::from(0u64)), - ( - "route_id".to_string(), - JsonValue::String(dynamic_proxy_export_route_id(name, export)?), - ), - ])), - )) - }) - .collect::>>()?; - let path = artifact_root.join("amber-proxy.json"); - let mut metadata: JsonValue = read_json(&path, "proxy metadata")?; - let Some(object) = metadata.as_object_mut() else { - return Err(miette::miette!( - "proxy metadata {} is not a JSON object", - path.display() - )); - }; - object.insert("exports".to_string(), JsonValue::Object(exports)); - write_json(&path, &metadata) - } -} - -fn rewrite_dynamic_compose_proxy_metadata( - artifact_root: &Path, - compose_project: &str, -) -> Result<()> { - let path = artifact_root.join("compose.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); - let Some(x_amber) = root.get_mut(&x_amber_key) else { - return Err(miette::miette!( - "compose file {} is missing x-amber metadata", - path.display() - )); - }; - let mut metadata: ProxyMetadata = serde_yaml::from_value(x_amber.clone()) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "compose file {} has invalid x-amber metadata", - path.display() - ) - })?; - if let Some(router) = metadata.router.as_mut() { - router.compose_project = Some(compose_project.to_string()); - } - *x_amber = serde_yaml::to_value(&metadata) - .into_diagnostic() - .wrap_err("failed to serialize compose proxy metadata")?; - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn yaml_string(value: &str) -> serde_yaml::Value { - serde_yaml::Value::String(value.to_string()) -} - -fn compose_services<'a>( - document: &'a serde_yaml::Value, - path: &Path, -) -> Result<&'a serde_yaml::Mapping> { - document - .as_mapping() - .and_then(|root| root.get(yaml_string("services"))) - .and_then(serde_yaml::Value::as_mapping) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing a services mapping", - path.display() - ) - }) -} - -fn compose_services_mut<'a>( - document: &'a mut serde_yaml::Value, - path: &Path, -) -> Result<&'a mut serde_yaml::Mapping> { - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("services"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing a services mapping", - path.display() - ) - }) -} - -fn compose_networks_mut<'a>( - document: &'a mut serde_yaml::Value, - path: &Path, -) -> Result> { - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a mapping", - path.display() - )); - }; - Ok(root - .get_mut(yaml_string("networks")) - .and_then(serde_yaml::Value::as_mapping_mut)) -} - -fn assign_compose_egress_network_subnets( - artifact_dir: &Path, - run_id: &str, - site_id: &str, -) -> Result<()> { - let compose_path = artifact_dir.join("compose.yaml"); - if !compose_path.is_file() { - return Ok(()); - } - - let mut document = read_compose_document(&compose_path)?; - let Some(networks) = compose_networks_mut(&mut document, &compose_path)? else { - return Ok(()); - }; - - let mut used_subnets = networks - .values() - .filter_map(compose_network_subnet) - .map(str::to_string) - .collect::>(); - let mut changed = false; - - for (name, network) in networks.iter_mut() { - let Some(name) = name.as_str() else { - continue; - }; - if !name.starts_with("amber_egress_") || compose_network_subnet(network).is_some() { - continue; - } - let subnet = next_compose_egress_subnet(run_id, site_id, name, &used_subnets); - set_compose_network_subnet(network, &subnet)?; - used_subnets.insert(subnet); - changed = true; - } - - if !changed { - return Ok(()); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display())) -} - -fn compose_network_subnet(network: &serde_yaml::Value) -> Option<&str> { - network - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("ipam"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|ipam| ipam.get(yaml_string("config"))) - .and_then(serde_yaml::Value::as_sequence) - .and_then(|configs| configs.first()) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|config| config.get(yaml_string("subnet"))) - .and_then(serde_yaml::Value::as_str) -} - -fn set_compose_network_subnet(network: &mut serde_yaml::Value, subnet: &str) -> Result<()> { - let Some(mapping) = network.as_mapping_mut() else { - return Err(miette::miette!( - "compose network definition is not a mapping" - )); - }; - mapping.insert( - yaml_string("ipam"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("config"), - serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( - serde_yaml::Mapping::from_iter([(yaml_string("subnet"), yaml_string(subnet))]), - )]), - )])), - ); - Ok(()) -} - -fn next_compose_egress_subnet( - run_id: &str, - site_id: &str, - network_name: &str, - used_subnets: &BTreeSet, -) -> String { - let digest = Sha256::digest(format!("{run_id}:{site_id}:{network_name}")); - let mut bytes = [0u8; 4]; - bytes.copy_from_slice(&digest[..4]); - let base = u32::from_be_bytes(bytes) % COMPOSE_EGRESS_SUBNET_COUNT; - for offset in 0..COMPOSE_EGRESS_SUBNET_COUNT { - let candidate = - compose_egress_subnet_from_index((base + offset) % COMPOSE_EGRESS_SUBNET_COUNT); - if !used_subnets.contains(&candidate) { - return candidate; - } - } - unreachable!("compose egress subnet pool exhausted") -} - -const COMPOSE_EGRESS_SUBNET_COUNT: u32 = 1 << 18; - -fn compose_egress_subnet_from_index(index: u32) -> String { - let second_octet = 64 + ((index >> 12) & 0x3f); - let third_octet = (index >> 4) & 0xff; - let fourth_octet = (index & 0x0f) << 4; - format!("100.{second_octet}.{third_octet}.{fourth_octet}/28") -} - -fn compose_service_names(document: &serde_yaml::Value, path: &Path) -> Result> { - Ok(compose_services(document, path)? - .keys() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()) -} - -fn compose_service_dependency_names(service: &serde_yaml::Value) -> Result> { - let Some(mapping) = service.as_mapping() else { - return Err(miette::miette!( - "compose service definition is not a mapping" - )); - }; - let Some(depends_on) = mapping.get(yaml_string("depends_on")) else { - return Ok(Vec::new()); - }; - if let Some(sequence) = depends_on.as_sequence() { - return Ok(sequence - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()); - } - if let Some(depends_on_map) = depends_on.as_mapping() { - return Ok(depends_on_map - .keys() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()); - } - Err(miette::miette!( - "compose service has invalid depends_on declaration" - )) -} - -fn retain_compose_service_dependencies( - service: &mut serde_yaml::Value, - keep_services: &BTreeSet, -) -> Result<()> { - let Some(mapping) = service.as_mapping_mut() else { - return Err(miette::miette!( - "compose service definition is not a mapping" - )); - }; - let depends_on_key = yaml_string("depends_on"); - let Some(depends_on) = mapping.get_mut(&depends_on_key) else { - return Ok(()); - }; - match depends_on { - serde_yaml::Value::Sequence(sequence) => { - sequence.retain(|value| { - value - .as_str() - .is_some_and(|name| keep_services.contains(name)) - }); - if sequence.is_empty() { - mapping.remove(&depends_on_key); - } - } - serde_yaml::Value::Mapping(depends_on_map) => { - depends_on_map.retain(|key, _| { - key.as_str() - .is_some_and(|name| keep_services.contains(name)) - }); - if depends_on_map.is_empty() { - mapping.remove(&depends_on_key); - } - } - _ => { - return Err(miette::miette!( - "compose service has invalid depends_on declaration" - )); - } - } - Ok(()) -} - -fn compose_dynamic_root_service_names( - artifact_root: &Path, - assigned_components: &[String], -) -> Result> { - let compose_path = artifact_root.join("compose.yaml"); - let document = read_compose_document(&compose_path)?; - let service_names = compose_service_names(&document, &compose_path)?; - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut out = BTreeSet::new(); - for target in plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) - || !assigned.contains(target.config.identity.id.as_str()) - { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = target.output else { - return Err(miette::miette!( - "compose component {} does not use filesystem mesh output", - target.config.identity.id - )); - }; - let sidecar = Path::new(&dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose component {} has invalid mesh output dir {}", - target.config.identity.id, - dir - ) - })? - .to_string(); - if service_names.contains(&sidecar) { - out.insert(sidecar.clone()); - } - if let Some(program) = sidecar.strip_suffix("-net") - && service_names.contains(program) - { - out.insert(program.to_string()); - } - } - if out.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not contain services for {:?}", - compose_path.display(), - assigned_components - )); - } - Ok(out) -} - -fn compose_live_service_names( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], -) -> Result> { - let site_compose_path = Path::new(&plan.artifact_dir).join("compose.yaml"); - let site_document = read_compose_document(&site_compose_path)?; - let mut live_services = compose_service_names(&site_document, &site_compose_path)?; - for child in published_children { - let child_compose_path = Path::new(&child.artifact_root).join("compose.yaml"); - let child_document = read_compose_document(&child_compose_path)?; - live_services.extend(compose_service_names(&child_document, &child_compose_path)?); - } - Ok(live_services) -} - -fn compose_service_closure( - document: &serde_yaml::Value, - compose_path: &Path, - roots: &BTreeSet, -) -> Result> { - let services = compose_services(document, compose_path)?; - let mut closure = BTreeSet::new(); - let mut queue = roots.iter().cloned().collect::>(); - while let Some(service_name) = queue.pop() { - if !closure.insert(service_name.clone()) { - continue; - } - let service = services.get(yaml_string(&service_name)).ok_or_else(|| { - miette::miette!( - "compose file {} is missing service {}", - compose_path.display(), - service_name - ) - })?; - for dependency in compose_service_dependency_names(service)? { - queue.push(dependency); - } - } - Ok(closure) -} - -fn dynamic_compose_mesh_dir(service_name: &str) -> String { - format!("{DYNAMIC_COMPOSE_MESH_ROOT}/{service_name}") -} - -fn filter_dynamic_router_target( - router_target: &mut amber_mesh::MeshProvisionTarget, - kept_component_ids: &BTreeSet, -) { - router_target - .config - .peers - .retain(|peer| kept_component_ids.contains(&peer.id)); - router_target - .config - .inbound - .retain(|route| match &route.target { - InboundTarget::MeshForward { peer_id, .. } => kept_component_ids.contains(peer_id), - _ => route - .allowed_issuers - .iter() - .any(|issuer| kept_component_ids.contains(issuer)), - }); -} - -fn dynamic_proxy_export_mesh_protocol(export: &DynamicProxyExportRecord) -> Result { - let protocol = export - .protocol - .parse::() - .map_err(|err| miette::miette!("invalid dynamic proxy export protocol: {err}"))?; - mesh_protocol(protocol) -} - -fn dynamic_proxy_export_route_id( - export_name: &str, - export: &DynamicProxyExportRecord, -) -> Result { - Ok(router_dynamic_export_route_id( - &export.component, - export_name, - dynamic_proxy_export_mesh_protocol(export)?, - )) -} - -fn dynamic_input_route_mesh_protocol(input: &DynamicInputRouteRecord) -> Result { - let protocol = input - .protocol - .parse::() - .map_err(|err| miette::miette!("invalid dynamic routed-input protocol: {err}"))?; - mesh_protocol(protocol) -} - -fn dynamic_input_route_route_id(input: &DynamicInputRouteRecord, protocol: MeshProtocol) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => { - component_route_id(&input.provider_component, provide, protocol) - } - } -} - -fn dynamic_input_route_capability(input: &DynamicInputRouteRecord) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), - } -} - -fn is_compose_component_sidecar_service(service_name: &str) -> bool { - service_name.ends_with("-net") -} - -fn overlay_peer_addr_map_from_ports(ports: &BTreeMap) -> BTreeMap { - ports - .iter() - .map(|(component, port)| (component.clone(), format!("127.0.0.1:{port}"))) - .collect() -} - -fn overlay_issuer_sets( - routed_inputs: &[DynamicInputRouteRecord], -) -> Result>> { - dynamic_route_issuer_grants(&[SiteActuatorChildRecord { - child_id: 0, - artifact_root: String::new(), - assigned_components: Vec::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: routed_inputs.to_vec(), - process_pid: None, - published: false, - }]) -} - -fn overlay_upsert_peer( - peers: &mut Vec, - peer_identities: &BTreeMap, - peer_id: &str, -) -> Result<()> { - if peers.iter().any(|peer| peer.id == peer_id) { - return Ok(()); - } - let identity = peer_identities.get(peer_id).ok_or_else(|| { - miette::miette!("dynamic route overlay is missing mesh identity for peer {peer_id}") - })?; - peers.push(MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }); - Ok(()) -} - -fn overlay_upsert_route(routes: &mut Vec, route: InboundRoute) { - if let Some(existing) = routes - .iter_mut() - .find(|existing| existing.route_id == route.route_id) - { - *existing = route; - } else { - routes.push(route); - } -} - -fn routed_input_overlay_route( - input: &DynamicInputRouteRecord, - provider_peer_addr: &str, - allowed_issuers: Vec, -) -> Result { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let (target_route_id, capability) = match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => ( - component_route_id(&input.provider_component, provide, protocol), - provide.clone(), - ), - }; - Ok(InboundRoute { - route_id: dynamic_input_route_route_id(input, protocol), - capability: dynamic_input_route_capability(input), - capability_kind: Some(input.capability_kind.clone()), - capability_profile: input.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(input.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::MeshForward { - peer_addr: provider_peer_addr.to_string(), - peer_id: input.provider_component.clone(), - route_id: target_route_id, - capability, - }, - allowed_issuers, - }) -} - -fn augment_route_overlay_payload( - payload: &mut StoredRouteOverlayPayload, - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, - allowed_issuers_by_route: Option<&BTreeMap>>, - skip_missing_providers: bool, -) -> Result<()> { - for export in proxy_exports.values() { - overlay_upsert_peer(&mut payload.peers, peer_identities, &export.component)?; - } - add_dynamic_proxy_export_overlay_routes( - &mut payload.inbound_routes, - proxy_exports, - |export| { - provider_peer_addrs - .get(&export.component) - .cloned() - .ok_or_else(|| { - miette::miette!( - "dynamic route overlay is missing a live peer address for {}", - export.component - ) - }) - }, - )?; - - for input in routed_inputs { - let Some(provider_peer_addr) = provider_peer_addrs.get(&input.provider_component) else { - if skip_missing_providers { - continue; - } - return Err(miette::miette!( - "dynamic route overlay is missing a live peer address for {}", - input.provider_component - )); - }; - overlay_upsert_peer( - &mut payload.peers, - peer_identities, - &input.provider_component, - )?; - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - let allowed_issuers = allowed_issuers_by_route - .and_then(|issuers| issuers.get(&route_id)) - .map(|issuers| issuers.iter().cloned().collect()) - .unwrap_or_default(); - overlay_upsert_route( - &mut payload.inbound_routes, - routed_input_overlay_route(input, provider_peer_addr, allowed_issuers)?, - ); - } - - Ok(()) -} - -fn routed_input_router_peer_addr(kind: SiteKind, router_mesh_port: Option) -> Result { - let router_mesh_port = router_mesh_port.ok_or_else(|| { - miette::miette!("site {kind:?} is missing its router mesh port for routed child inputs") - })?; - Ok(match kind { - SiteKind::Direct | SiteKind::Vm => format!("127.0.0.1:{router_mesh_port}"), - SiteKind::Compose => format!("{COMPOSE_ROUTER_SERVICE_NAME}:{router_mesh_port}"), - SiteKind::Kubernetes => { - format!("{KUBERNETES_ROUTER_COMPONENT_NAME}:{router_mesh_port}") - } - }) -} - -fn router_mesh_port_from_plan(mesh_plan: &MeshProvisionPlan, artifact_kind: &str) -> Result { - mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .map(|target| target.config.mesh_listen.port()) - .ok_or_else(|| { - miette::miette!("{artifact_kind} mesh provision plan is missing its router mesh target") - }) -} - -fn rewrite_dynamic_routed_inputs( - mesh_plan: &mut MeshProvisionPlan, - routed_inputs: &[DynamicInputRouteRecord], - kind: SiteKind, - router_identity_id: &str, - router_mesh_port: Option, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - - let router_peer_addr = routed_input_router_peer_addr(kind, router_mesh_port)?; - for input in routed_inputs { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let component_target = mesh_plan - .targets - .iter_mut() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == input.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic routed input {}.{} is missing component {} in the mesh provision plan", - input.component, - input.slot, - input.component - ) - })?; - if !component_target - .config - .peers - .iter() - .any(|peer| peer.id == router_identity_id) - { - component_target - .config - .peers - .push(amber_mesh::MeshPeerTemplate { - id: router_identity_id.to_string(), - }); - } - - let route_id = dynamic_input_route_route_id(input, protocol); - let capability = dynamic_input_route_capability(input); - let mut matched = false; - for route in component_target - .config - .outbound - .iter_mut() - .filter(|route| route.slot == input.slot) - { - matched = true; - route.route_id = route_id.clone(); - route.protocol = protocol; - route.peer_addr = router_peer_addr.clone(); - route.peer_id = router_identity_id.to_string(); - route.capability = capability.clone(); - route.capability_kind = Some(input.capability_kind.clone()); - route.capability_profile = input.capability_profile.clone(); - route.http_plugins = http_route_plugins_for_capability_kind( - Some(input.capability_kind.as_str()), - protocol, - ); - } - if !matched { - return Err(miette::miette!( - "dynamic routed input {}.{} is missing an outbound route in the mesh provision \ - plan", - input.component, - input.slot - )); - } - } - - Ok(()) -} - -fn rewrite_dynamic_routed_inputs_in_artifact( - artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, - router_mesh_port: Option, -) -> Result<()> { - if site_plan.routed_inputs.is_empty() { - return Ok(()); - } - let path = artifact_root.join("mesh-provision-plan.json"); - let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - &site_plan.routed_inputs, - site_plan.kind, - &site_plan.router_identity_id, - router_mesh_port, - )?; - write_json(&path, &mesh_plan) -} - -fn build_filesystem_route_overlay_base( - artifact_root: &Path, - assigned_components: &[String], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result { - let mesh_plan: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - let kept_component_ids = assigned_components.iter().cloned().collect::>(); - let mut router_target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "dynamic artifact {} is missing a router mesh target", - artifact_root.display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = provider_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - peer_identities.get(&peer.id).map(|identity| MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>() - .ok_or_else(|| { - miette::miette!( - "dynamic artifact {} is missing a live mesh identity for one of its route peers", - artifact_root.display() - ) - })?; - Ok(StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }) -} - -fn write_direct_vm_startup_route_overlay_payload( - artifact_root: &Path, - _artifact_kind: &str, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; - let mut payload = StoredRouteOverlayPayload { - peers: Vec::new(), - inbound_routes: Vec::new(), - }; - augment_route_overlay_payload( - &mut payload, - &BTreeMap::new(), - routed_inputs, - provider_peer_addrs, - existing_site_peer_identities, - Some(&allowed_issuers), - true, - )?; - if payload.inbound_routes.is_empty() { - return Ok(()); - } - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn write_direct_vm_live_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result<()> { - let mut payload = build_filesystem_route_overlay_base( - artifact_root, - assigned_components, - provider_peer_addrs, - peer_identities, - )?; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - provider_peer_addrs, - peer_identities, - None, - false, - )?; - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn ensure_dynamic_proxy_export_component_routes( - mesh_plan: &mut MeshProvisionPlan, - proxy_exports: &BTreeMap, - router_identity_id: &str, -) -> Result<()> { - for export in proxy_exports.values() { - let protocol = dynamic_proxy_export_mesh_protocol(export)?; - let route_id = component_route_id(&export.component, &export.provide, protocol); - let component_target = mesh_plan - .targets - .iter_mut() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the mesh provision plan", - export.component - ) - })?; - if !component_target - .config - .peers - .iter() - .any(|peer| peer.id == router_identity_id) - { - component_target - .config - .peers - .push(amber_mesh::MeshPeerTemplate { - id: router_identity_id.to_string(), - }); - } - if let Some(route) = component_target - .config - .inbound - .iter_mut() - .find(|route| route.route_id == route_id) - { - if !route - .allowed_issuers - .iter() - .any(|issuer| issuer == router_identity_id) - { - route.allowed_issuers.push(router_identity_id.to_string()); - route.allowed_issuers.sort(); - route.allowed_issuers.dedup(); - } - continue; - } - component_target.config.inbound.push(InboundRoute { - route_id, - capability: export.provide.clone(), - capability_kind: Some(export.capability_kind.clone()), - capability_profile: export.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(export.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::Local { - port: export.target_port, - }, - allowed_issuers: vec![router_identity_id.to_string()], - }); - } - Ok(()) -} - -fn ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root: &Path, - proxy_exports: &BTreeMap, - router_identity_id: &str, -) -> Result<()> { - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mut mesh_plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; - ensure_dynamic_proxy_export_component_routes( - &mut mesh_plan, - proxy_exports, - router_identity_id, - )?; - write_json(&plan_path, &mesh_plan) -} - -fn add_dynamic_proxy_export_overlay_routes( - inbound_routes: &mut Vec, - proxy_exports: &BTreeMap, - mut peer_addr_for_export: impl FnMut(&DynamicProxyExportRecord) -> Result, -) -> Result<()> { - for (export_name, export) in proxy_exports { - let protocol = dynamic_proxy_export_mesh_protocol(export)?; - let route_id = router_dynamic_export_route_id(&export.component, export_name, protocol); - let route = InboundRoute { - route_id, - capability: export_name.clone(), - capability_kind: Some(export.capability_kind.clone()), - capability_profile: export.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(export.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::MeshForward { - peer_addr: peer_addr_for_export(export)?, - peer_id: export.component.clone(), - route_id: component_route_id(&export.component, &export.provide, protocol), - capability: export.provide.clone(), - }, - allowed_issuers: Vec::new(), - }; - if let Some(existing) = inbound_routes - .iter_mut() - .find(|existing| existing.route_id == route.route_id) - { - *existing = route; - } else { - inbound_routes.push(route); - } - } - Ok(()) -} - -struct DynamicComposeMeshPlan { - mesh_plan: MeshProvisionPlan, - mesh_dirs: BTreeMap, - component_mesh_dirs: BTreeMap, -} - -fn build_dynamic_compose_mesh_plan( - artifact_root: &Path, - assigned_components: &[String], -) -> Result { - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut filtered_targets = Vec::new(); - let mut mesh_dirs = BTreeMap::new(); - let mut component_mesh_dirs = BTreeMap::new(); - - for mut target in plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - if !assigned.contains(target.config.identity.id.as_str()) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &mut target.output else { - return Err(miette::miette!( - "compose component {} does not use filesystem mesh output", - target.config.identity.id - )); - }; - let sidecar = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose component {} has invalid mesh output dir {}", - target.config.identity.id, - dir - ) - })? - .to_string(); - let relative_dir = dynamic_compose_mesh_dir(&sidecar); - *dir = relative_dir.clone(); - component_mesh_dirs.insert(target.config.identity.id.clone(), relative_dir.clone()); - mesh_dirs.insert(sidecar, relative_dir); - filtered_targets.push(target); - } - MeshProvisionTargetKind::Router => {} - } - } - - if filtered_targets.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not contain assigned child mesh targets", - artifact_root.join("compose.yaml").display() - )); - } - - Ok(DynamicComposeMeshPlan { - mesh_plan: MeshProvisionPlan { - version: plan.version, - identity_seed: plan.identity_seed, - existing_peer_identities: Vec::new(), - targets: filtered_targets, - }, - mesh_dirs, - component_mesh_dirs, - }) -} - -fn compose_component_mesh_peer_addr( - artifact_root: &Path, - component_id: &str, - output: &MeshProvisionOutput, - mesh_port: u16, -) -> Result { - let MeshProvisionOutput::Filesystem { dir } = output else { - return Err(miette::miette!( - "compose artifact {} component {} does not use filesystem mesh output", - artifact_root.display(), - component_id - )); - }; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose artifact {} component {} has invalid mesh output dir {}", - artifact_root.display(), - component_id, - dir - ) - })?; - Ok(format!("{service_name}:{mesh_port}")) -} - -fn kubernetes_component_mesh_peer_addr( - artifact_root: &Path, - component_id: &str, - output: &MeshProvisionOutput, - mesh_port: u16, -) -> Result { - let MeshProvisionOutput::KubernetesSecret { name, .. } = output else { - return Err(miette::miette!( - "kubernetes artifact {} component {} does not use a kubernetes secret mesh output", - artifact_root.display(), - component_id - )); - }; - let service_name = name.strip_suffix("-mesh").ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} component {} uses invalid mesh secret name {}", - artifact_root.display(), - component_id, - name - ) - })?; - Ok(format!("{service_name}:{mesh_port}")) -} - -fn build_dynamic_compose_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - component_mesh_dirs: &BTreeMap, - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - existing_site_peer_identities: &BTreeMap, -) -> Result { - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let kept_component_ids = plan - .targets - .iter() - .filter(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && assigned.contains(target.config.identity.id.as_str()) - }) - .map(|target| target.config.identity.id.clone()) - .collect::>(); - let mut router_target = plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "compose child artifact {} is missing a router mesh target", - artifact_root.join("compose.yaml").display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - let component_peer_addrs = plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - Ok(( - target.config.identity.id.clone(), - compose_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - target.config.mesh_listen.port(), - )?, - )) - }) - .collect::>>()?; - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = component_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - - let component_mesh_scopes = plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - ( - target.config.identity.id.clone(), - target.config.identity.mesh_scope.clone(), - ) - }) - .collect::>(); - let mut peer_identities = existing_site_peer_identities.clone(); - for (component, relative_dir) in component_mesh_dirs { - let identity: MeshIdentitySecret = read_json( - &artifact_root - .join(relative_dir) - .join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?; - peer_identities.insert( - component.clone(), - MeshIdentityPublic { - id: identity.id.clone(), - public_key: identity.public_key().into_diagnostic()?, - mesh_scope: component_mesh_scopes.get(component).cloned().flatten(), - }, - ); - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - let identity = peer_identities.get(&peer.id).ok_or_else(|| { - miette::miette!( - "compose child router overlay peer {} is missing a live mesh identity", - peer.id - ) - })?; - Ok(MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>()?; - - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; - let mut payload = StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - &component_peer_addrs, - &peer_identities, - Some(&allowed_issuers), - false, - )?; - Ok(payload) -} - -fn rewrite_compose_mesh_bind_mounts( - artifact_root: &Path, - mesh_dirs: &BTreeMap, -) -> Result<()> { - let compose_path = artifact_root.join("compose.yaml"); - let mut document = read_compose_document(&compose_path)?; - let services = compose_services_mut(&mut document, &compose_path)?; - for (service_name, relative_dir) in mesh_dirs { - let service = services.get_mut(yaml_string(service_name)).ok_or_else(|| { - miette::miette!( - "compose file {} is missing child sidecar service {}", - compose_path.display(), - service_name - ) - })?; - let Some(service_mapping) = service.as_mapping_mut() else { - return Err(miette::miette!( - "compose service {} is not a mapping", - service_name - )); - }; - let volumes_key = yaml_string("volumes"); - let volumes = service_mapping - .get_mut(&volumes_key) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!( - "compose child sidecar {} is missing a volumes list", - service_name - ) - })?; - let expected_prefix = format!("{service_name}-mesh:/amber/mesh"); - let replacement = serde_yaml::Value::String(format!("./{relative_dir}:/amber/mesh:ro")); - let mut replaced = false; - for volume in volumes.iter_mut() { - if volume - .as_str() - .is_some_and(|value| value.starts_with(&expected_prefix)) - { - *volume = replacement.clone(); - replaced = true; - } - } - if !replaced { - volumes.push(replacement); - } - } - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display())) -} - -fn load_dynamic_compose_child_metadata( - artifact_root: &Path, -) -> Result { - read_json( - &dynamic_compose_child_metadata_path(artifact_root), - "dynamic compose child metadata", - ) -} - -fn load_running_site_router_identity(plan: &SiteActuatorPlan) -> Result { - let state: SiteManagerState = read_json( - &Path::new(&plan.site_state_root).join("manager-state.json"), - "site manager state", - )?; - let router_identity_id = state.router_identity_id.ok_or_else(|| { - miette::miette!( - "site `{}` manager state is missing router identity id", - plan.site_id - ) - })?; - let router_public_key_b64 = state.router_public_key_b64.ok_or_else(|| { - miette::miette!( - "site `{}` manager state is missing router public key", - plan.site_id - ) - })?; - Ok(MeshIdentityPublic { - id: router_identity_id, - public_key: decode_public_key(&router_public_key_b64)?, - mesh_scope: Some(plan.mesh_scope.clone()), - }) -} - -fn filesystem_component_peer_identities_for_artifact( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - artifact_kind: &str, - compose_project: Option<&str>, -) -> Result> { - let mut peers = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "{artifact_kind} artifact {} has non-filesystem mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let config: MeshConfigPublic = if Path::new(dir).is_absolute() { - let compose_project = compose_project.ok_or_else(|| { - miette::miette!( - "{artifact_kind} artifact {} uses absolute mesh output {} without a compose \ - project", - artifact_root.display(), - dir - ) - })?; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "{artifact_kind} artifact {} has invalid absolute mesh output {}", - artifact_root.display(), - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name)? - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )? - }; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn read_compose_volume_mesh_config( - compose_project: &str, - service_name: &str, -) -> Result { - let volume_name = format!("{compose_project}_{service_name}-mesh"); - let output = Command::new("docker") - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{volume_name}:/amber/mesh:ro")) - .arg("busybox:1.36.1") - .arg("cat") - .arg(format!("/amber/mesh/{MESH_CONFIG_FILENAME}")) - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to read compose mesh config from docker volume {volume_name}") - })?; - if !output.status.success() { - return Err(miette::miette!( - "failed to read compose mesh config from docker volume \ - {volume_name}\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - } - serde_json::from_slice(&output.stdout) - .into_diagnostic() - .wrap_err_with(|| format!("docker volume {volume_name} returned invalid mesh config json")) -} - -fn compose_peer_identities_for_artifact( - artifact_root: &Path, - compose_project: Option<&str>, -) -> Result> { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - filesystem_component_peer_identities_for_artifact( - artifact_root, - &mesh_plan, - "compose", - compose_project, - ) -} - -fn local_compose_peer_identities( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], -) -> Result> { - let mut peers = BTreeMap::new(); - let router = load_running_site_router_identity(plan)?; - peers.insert(router.id.clone(), router); - peers.extend(compose_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - plan.compose_project.as_deref(), - )?); - for child in published_children { - peers.extend(compose_peer_identities_for_artifact( - Path::new(&child.artifact_root), - plan.compose_project.as_deref(), - )?); - } - Ok(peers) -} - -fn kubernetes_peer_identities_for_artifact( - plan: &SiteActuatorPlan, - artifact_root: &Path, -) -> Result> { - let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let mut peers = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes artifact {} has non-secret mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn local_kubernetes_peer_identities( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], -) -> Result> { - let mut peers = BTreeMap::new(); - let router = load_running_site_router_identity(plan)?; - peers.insert(router.id.clone(), router); - peers.extend(kubernetes_peer_identities_for_artifact( - plan, - Path::new(&plan.artifact_dir), - )?); - for child in published_children { - peers.extend(kubernetes_peer_identities_for_artifact( - plan, - Path::new(&child.artifact_root), - )?); - } - Ok(peers) -} - -pub(crate) fn collect_live_component_runtime_metadata( - plan: &SiteActuatorPlan, -) -> Result> { - let state: SiteActuatorState = read_json( - &site_actuator_state_path(Path::new(&plan.site_state_root)), - "site actuator state", - )?; - let published_children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - let mut components = match plan.kind { - SiteKind::Direct => collect_direct_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?), - )?, - SiteKind::Vm => collect_vm_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?), - )?, - SiteKind::Compose => collect_compose_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - plan.compose_project.as_deref(), - )?, - SiteKind::Kubernetes => { - collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&plan.artifact_dir))? - } - }; - for child in &published_children { - let child_components = match plan.kind { - SiteKind::Direct => collect_direct_artifact_runtime_metadata( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?, - SiteKind::Vm => collect_vm_artifact_runtime_metadata( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?, - SiteKind::Compose => collect_compose_artifact_runtime_metadata( - Path::new(&child.artifact_root), - plan.compose_project.as_deref(), - )?, - SiteKind::Kubernetes => { - collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&child.artifact_root))? - } - }; - components.extend(child_components); - } - Ok(components) -} - -pub(crate) fn load_live_site_router_mesh_config( - plan: &SiteActuatorPlan, -) -> Result { - let artifact_root = Path::new(&plan.artifact_dir); - match plan.kind { - SiteKind::Direct => { - let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let direct_plan: DirectPlan = - read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let router = direct_plan.router.ok_or_else(|| { - miette::miette!("direct site `{}` is missing its router plan", plan.site_id) - })?; - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") - } - SiteKind::Vm => { - let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let vm_plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let router = vm_plan.router.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router plan", plan.site_id) - })?; - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") - } - SiteKind::Compose => { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - let target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing a router mesh target", - plan.site_id - ) - })?; - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "compose site `{}` has non-filesystem mesh output for router {}", - plan.site_id, - target.config.identity.id - )); - }; - if Path::new(dir).is_absolute() { - let compose_project = plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project", - plan.site_id - ) - })?; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose site `{}` has invalid router mesh output {}", - plan.site_id, - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name) - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - ) - } - } - SiteKind::Kubernetes => { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .ok_or_else(|| { - miette::miette!( - "kubernetes site `{}` is missing a router mesh target", - plan.site_id - ) - })?; - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes site `{}` has non-secret mesh output for router {}", - plan.site_id, - target.config.identity.id - )); - }; - load_kubernetes_mesh_config_public(plan, name, namespace.as_deref()) - } - } -} - -fn collect_direct_artifact_runtime_metadata( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state: crate::direct_runtime::DirectRuntimeState = read_json( - &direct_runtime_state_path(artifact_root), - "direct runtime state", - )?; - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut components = BTreeMap::new(); - for component in &plan.components { - let mesh_config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - let mesh_port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "direct runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - components.insert( - component.moniker.clone(), - LiveComponentRuntimeMetadata { - moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_vm_artifact_runtime_metadata( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut components = BTreeMap::new(); - for component in &plan.components { - let mesh_config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - let mesh_port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "vm runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - components.insert( - component.moniker.clone(), - LiveComponentRuntimeMetadata { - moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_compose_artifact_runtime_metadata( - artifact_root: &Path, - compose_project: Option<&str>, -) -> Result> { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - let mut components = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "compose artifact {} has non-filesystem mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose artifact {} has invalid mesh output {} for component {}", - artifact_root.display(), - dir, - target.config.identity.id - ) - })?; - let mesh_config = if Path::new(dir).is_absolute() { - let compose_project = compose_project.ok_or_else(|| { - miette::miette!( - "compose artifact {} requires a compose project to resolve mesh output {}", - artifact_root.display(), - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name)? - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )? - }; - components.insert( - target.config.identity.id.clone(), - LiveComponentRuntimeMetadata { - moniker: target.config.identity.id.clone(), - host_mesh_addr: compose_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - mesh_config.mesh_listen.port(), - )?, - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_kubernetes_artifact_runtime_metadata( - plan: &SiteActuatorPlan, - artifact_root: &Path, -) -> Result> { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let mut components = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes artifact {} has non-secret mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - name.strip_suffix("-mesh").ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} component {} uses invalid mesh secret name {}", - artifact_root.display(), - target.config.identity.id, - name - ) - })?; - let mesh_config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; - components.insert( - target.config.identity.id.clone(), - LiveComponentRuntimeMetadata { - moniker: target.config.identity.id.clone(), - host_mesh_addr: kubernetes_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - mesh_config.mesh_listen.port(), - )?, - mesh_config, - }, - ); - } - Ok(components) -} - -fn prepare_dynamic_compose_child_artifact( - plan: &SiteActuatorPlan, - site_plan: &DynamicSitePlanRecord, - artifact_root: &Path, - published_children: &[SiteActuatorChildRecord], - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; - let compose_path = artifact_root.join("compose.yaml"); - let mut desired_document = read_compose_document(&compose_path)?; - let root_services = - compose_dynamic_root_service_names(artifact_root, &site_plan.assigned_components)?; - let service_closure = - compose_service_closure(&desired_document, &compose_path, &root_services)?; - let live_services = compose_live_service_names(plan, published_children)?; - let mut kept_services = service_closure - .difference(&live_services) - .cloned() - .collect::>(); - kept_services.remove(COMPOSE_PROVISIONER_SERVICE_NAME); - if kept_services.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not retain any child-owned services after filtering", - compose_path.display() - )); - } - - let services = compose_services_mut(&mut desired_document, &compose_path)?; - services.retain(|name, _| { - name.as_str() - .is_some_and(|service_name| kept_services.contains(service_name)) - }); - for service in services.values_mut() { - retain_compose_service_dependencies(service, &kept_services)?; - } - let rendered = serde_yaml::to_string(&desired_document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display()))?; - - let DynamicComposeMeshPlan { - mesh_plan, - mesh_dirs, - component_mesh_dirs, - } = build_dynamic_compose_mesh_plan(artifact_root, &site_plan.assigned_components)?; - let mut mesh_plan = mesh_plan; - let router_mesh_port = router_mesh_port_from_plan( - &read_embedded_compose_mesh_provision_plan(artifact_root)?, - "compose", - )?; - ensure_dynamic_proxy_export_component_routes( - &mut mesh_plan, - &site_plan.proxy_exports, - &plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - &site_plan.routed_inputs, - SiteKind::Compose, - &plan.router_identity_id, - Some(router_mesh_port), - )?; - let existing_peer_identities = crate::direct_runtime::required_existing_mesh_peer_identities( - &mesh_plan, - existing_site_peer_identities, - )?; - mesh_plan.existing_peer_identities = existing_peer_identities.values().cloned().collect(); - write_json(&artifact_root.join("mesh-provision-plan.json"), &mesh_plan)?; - crate::direct_runtime::provision_mesh_filesystem_with_peer_identities( - &mesh_plan, - artifact_root, - &existing_peer_identities, - )?; - for relative_dir in mesh_dirs.values() { - crate::direct_runtime::project_existing_peer_identities_into_mesh_config( - &artifact_root.join(relative_dir).join(MESH_CONFIG_FILENAME), - &existing_peer_identities, - )?; - } - rewrite_compose_mesh_bind_mounts(artifact_root, &mesh_dirs)?; - let overlay_payload = build_dynamic_compose_route_overlay_payload( - artifact_root, - &site_plan.assigned_components, - &component_mesh_dirs, - &site_plan.proxy_exports, - &site_plan.routed_inputs, - existing_site_peer_identities, - )?; - write_json(&dynamic_route_overlay_path(artifact_root), &overlay_payload)?; - write_embedded_compose_mesh_provision_plan(artifact_root, &mesh_plan)?; - - write_json( - &dynamic_compose_child_metadata_path(artifact_root), - &DynamicComposeChildMetadata { - schema: DYNAMIC_COMPOSE_CHILD_SCHEMA.to_string(), - version: DYNAMIC_COMPOSE_CHILD_VERSION, - services: kept_services.iter().cloned().collect(), - readiness_services: root_services - .into_iter() - .filter(|service| kept_services.contains(service)) - .collect(), - }, - ) -} - -fn read_compose_document(path: &Path) -> Result { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display())) -} - -fn read_embedded_compose_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root.join("compose.yaml"); - let document = read_compose_document(&path)?; - let Some(root) = document.as_mapping() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let configs_key = serde_yaml::Value::String("configs".to_string()); - let config_name = serde_yaml::Value::String("amber-mesh-provision-plan".to_string()); - let content_key = serde_yaml::Value::String("content".to_string()); - let content = root - .get(&configs_key) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|configs| configs.get(&config_name)) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|config| config.get(&content_key)) - .and_then(serde_yaml::Value::as_str) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing configs.amber-mesh-provision-plan.content", - path.display() - ) - })?; - serde_json::from_str(content).map_err(|err| { - miette::miette!( - "compose file {} has invalid embedded mesh provision plan: {err}", - path.display() - ) - }) -} - -fn write_embedded_compose_mesh_provision_plan( - artifact_root: &Path, - plan: &MeshProvisionPlan, -) -> Result<()> { - let path = artifact_root.join("compose.yaml"); - let mut document = read_compose_document(&path)?; - let plan_json = serde_json::to_string_pretty(plan) - .into_diagnostic() - .wrap_err("failed to serialize compose mesh provision plan")?; - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("configs"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .and_then(|configs| configs.get_mut(yaml_string("amber-mesh-provision-plan"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing configs.amber-mesh-provision-plan", - path.display() - ) - })? - .insert(yaml_string("content"), serde_yaml::Value::String(plan_json)); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn read_embedded_kubernetes_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root - .join("01-configmaps") - .join("amber-mesh-provision.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let document: serde_yaml::Value = - serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "invalid kubernetes mesh provision configmap {}", - path.display() - ) - })?; - let mesh_plan = document - .as_mapping() - .and_then(|root| root.get(yaml_string("data"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|data| data.get(yaml_string("mesh-plan.json"))) - .and_then(serde_yaml::Value::as_str) - .ok_or_else(|| { - miette::miette!( - "kubernetes mesh provision configmap {} is missing data.mesh-plan.json", - path.display() - ) - })?; - serde_json::from_str(mesh_plan).map_err(|err| { - miette::miette!( - "kubernetes mesh provision configmap {} has invalid mesh plan: {err}", - path.display() - ) - }) -} - -fn read_kubernetes_runtime_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root.join("mesh-provision-plan.json"); - if path.is_file() { - return read_json(&path, "mesh provision plan"); - } - read_embedded_kubernetes_mesh_provision_plan(artifact_root) -} - -fn write_embedded_kubernetes_mesh_provision_plan( - artifact_root: &Path, - plan: &MeshProvisionPlan, -) -> Result<()> { - let path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "invalid kubernetes mesh provision configmap {}", - path.display() - ) - })?; - let plan_json = serde_json::to_string_pretty(plan) - .into_diagnostic() - .wrap_err("failed to serialize kubernetes mesh provision plan")?; - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("data"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "kubernetes mesh provision configmap {} is missing a data mapping", - path.display() - ) - })? - .insert( - yaml_string("mesh-plan.json"), - serde_yaml::Value::String(plan_json), - ); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { - document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) -} - -fn kubernetes_dynamic_apply_resource_kept_from_contents( - resource: &str, - raw: &str, - child_component_labels: &BTreeSet, -) -> Result { - if matches!( - resource, - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH - | KUBERNETES_PROVISIONER_JOB_PATH - | KUBERNETES_PROVISIONER_ROLE_PATH - | KUBERNETES_PROVISIONER_ROLEBINDING_PATH - | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH - ) || resource.starts_with("03-persistentvolumeclaims/") - { - return Ok(true); - } - - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - Ok(document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("labels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|component_id| child_component_labels.contains(component_id))) -} - -pub(crate) fn project_kubernetes_dynamic_child_artifact_files( - artifact_files: &BTreeMap, - component_ids: &[usize], -) -> Result> { - let child_component_labels = component_ids - .iter() - .map(|component_id| format!("c{component_id}")) - .collect::>(); - let kustomization_path = "kustomization.yaml"; - let raw = artifact_files.get(kustomization_path).ok_or_else(|| { - miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") - })?; - let mut document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is not a YAML mapping") - })?; - let resources = root - .get_mut(yaml_string("resources")) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is missing a resources sequence") - })?; - let mut projected = artifact_files - .iter() - .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) - .map(|(path, contents)| (path.clone(), contents.clone())) - .collect::>(); - let mut kept_resources = Vec::new(); - let mut kept_resource_names = BTreeSet::new(); - for resource in resources - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_owned) - { - let raw = artifact_files - .get(&resource) - .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; - if !kubernetes_dynamic_apply_resource_kept_from_contents( - &resource, - raw, - &child_component_labels, - )? { - continue; - } - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - if let Some(name) = kubernetes_resource_name(&document) { - kept_resource_names.insert(name.to_string()); - } - projected.insert(resource.clone(), raw.clone()); - kept_resources.push(serde_yaml::Value::String(resource)); - } - *resources = kept_resources; - - if let Some(generators) = root - .get_mut(yaml_string("secretGenerator")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - generators.retain(|generator| { - generator - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) - }); - } - - if let Some(replacements) = root - .get_mut(yaml_string("replacements")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - replacements.retain_mut(|replacement| { - let Some(targets) = replacement - .as_mapping_mut() - .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) - .and_then(serde_yaml::Value::as_sequence_mut) - else { - return false; - }; - targets.retain(|target| { - target - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("select"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|select| select.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|name| kept_resource_names.contains(name)) - }); - !targets.is_empty() - }); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; - projected.insert(kustomization_path.to_string(), rendered); - Ok(projected) -} - -fn project_kubernetes_dynamic_child_destroy_artifact_files( - artifact_files: &BTreeMap, -) -> Result> { - let kustomization_path = "kustomization.yaml"; - let raw = artifact_files.get(kustomization_path).ok_or_else(|| { - miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") - })?; - let mut document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is not a YAML mapping") - })?; - let resources = root - .get_mut(yaml_string("resources")) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is missing a resources sequence") - })?; - let mut projected = artifact_files - .iter() - .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) - .map(|(path, contents)| (path.clone(), contents.clone())) - .collect::>(); - let shared_paths = [ - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH, - KUBERNETES_PROVISIONER_ROLE_PATH, - KUBERNETES_PROVISIONER_ROLEBINDING_PATH, - KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, - ] - .into_iter() - .collect::>(); - let mut kept_resources = Vec::new(); - for resource in resources - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_owned) - { - if shared_paths.contains(resource.as_str()) { - continue; - } - let raw = artifact_files - .get(&resource) - .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; - projected.insert(resource.clone(), raw.clone()); - kept_resources.push(serde_yaml::Value::String(resource)); - } - *resources = kept_resources; - - if let Some(generators) = root - .get_mut(yaml_string("secretGenerator")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - generators.retain(|generator| { - generator - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) - }); - } - - if let Some(replacements) = root - .get_mut(yaml_string("replacements")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - replacements.clear(); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; - projected.insert(kustomization_path.to_string(), rendered); - Ok(projected) -} - -fn read_artifact_snapshot(root: &Path) -> Result> { - walk_files(root)? - .into_iter() - .map(|path| { - let relative = path - .strip_prefix(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; - let relative = path_to_forward_slash_string(relative); - let contents = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - Ok((relative, contents)) - }) - .collect() -} - -fn rewrite_dynamic_kubernetes_apply_bundle( - artifact_root: &Path, - component_ids: &[usize], -) -> Result<()> { - let files = read_artifact_snapshot(artifact_root)?; - let projected = project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; - replace_artifact_snapshot(artifact_root, &projected) -} - -fn prepare_dynamic_kubernetes_child_artifact( - plan: &SiteActuatorPlan, - site_plan: &DynamicSitePlanRecord, - artifact_root: &Path, - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let router_mesh_port = router_mesh_port_from_plan(&mesh_plan, "kubernetes")?; - let assigned = site_plan - .assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut kept_component_ids = BTreeSet::new(); - let mut router_target = None; - let mut overlay_targets = Vec::with_capacity(mesh_plan.targets.len()); - let mut provision_targets = Vec::new(); - for target in mesh_plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - if assigned.contains(target.config.identity.id.as_str()) { - kept_component_ids.insert(target.config.identity.id.clone()); - provision_targets.push(target.clone()); - overlay_targets.push(target); - } - } - MeshProvisionTargetKind::Router => { - router_target = Some(target); - } - } - } - - let Some(mut router_target) = router_target else { - return Err(miette::miette!( - "dynamic mesh provision plan {} is missing a router target", - plan_path.display() - )); - }; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - overlay_targets.push(router_target); - let mut overlay_plan = MeshProvisionPlan { - version: mesh_plan.version.clone(), - identity_seed: mesh_plan.identity_seed.clone(), - existing_peer_identities: Vec::new(), - targets: overlay_targets, - }; - let overlay_existing_peer_identities = - crate::direct_runtime::required_existing_mesh_peer_identities( - &overlay_plan, - existing_site_peer_identities, - )?; - overlay_plan.existing_peer_identities = - overlay_existing_peer_identities.values().cloned().collect(); - write_json(&plan_path, &overlay_plan)?; - let mut provision_plan = MeshProvisionPlan { - version: mesh_plan.version, - identity_seed: mesh_plan.identity_seed, - existing_peer_identities: Vec::new(), - targets: provision_targets, - }; - ensure_dynamic_proxy_export_component_routes( - &mut provision_plan, - &site_plan.proxy_exports, - &plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs( - &mut provision_plan, - &site_plan.routed_inputs, - SiteKind::Kubernetes, - &plan.router_identity_id, - Some(router_mesh_port), - )?; - let provision_existing_peer_identities = - crate::direct_runtime::required_existing_mesh_peer_identities( - &provision_plan, - existing_site_peer_identities, - )?; - provision_plan.existing_peer_identities = provision_existing_peer_identities - .values() - .cloned() - .collect(); - write_embedded_kubernetes_mesh_provision_plan(artifact_root, &provision_plan)?; - project_dynamic_kubernetes_proxy_export_resources( - artifact_root, - &provision_plan, - &site_plan.proxy_exports, - )?; - rewrite_dynamic_kubernetes_apply_bundle(artifact_root, &site_plan.component_ids) -} - -fn kubernetes_peer_addrs_for_artifact(artifact_root: &Path) -> Result> { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - mesh_plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - Ok(( - target.config.identity.id.clone(), - kubernetes_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - target.config.mesh_listen.port(), - )?, - )) - }) - .collect() -} - -fn build_kubernetes_route_overlay_base( - artifact_root: &Path, - assigned_components: &[String], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let kept_component_ids = assigned_components.iter().cloned().collect::>(); - let mut router_target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} is missing a router mesh target", - artifact_root.display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = provider_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - peer_identities.get(&peer.id).map(|identity| MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>() - .ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} is missing a live mesh identity for one of its route peers", - artifact_root.display() - ) - })?; - Ok(StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }) -} - -fn write_kubernetes_live_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - peer_identities: &BTreeMap, -) -> Result<()> { - let provider_peer_addrs = kubernetes_peer_addrs_for_artifact(artifact_root)?; - let mut payload = build_kubernetes_route_overlay_base( - artifact_root, - assigned_components, - &provider_peer_addrs, - peer_identities, - )?; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - &provider_peer_addrs, - peer_identities, - None, - false, - )?; - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn kubernetes_network_policy_paths_by_component_label( - artifact_root: &Path, -) -> Result> { - let netpol_root = artifact_root.join("05-networkpolicies"); - if !netpol_root.is_dir() { - return Ok(BTreeMap::new()); - } - let mut netpol_paths = BTreeMap::new(); - for path in walk_files(&netpol_root)? { - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; - let Some(root) = document.as_mapping() else { - continue; - }; - if root - .get(yaml_string("kind")) - .and_then(serde_yaml::Value::as_str) - != Some("NetworkPolicy") - { - continue; - } - let Some(component_label) = root - .get(yaml_string("metadata")) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("labels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) - .and_then(serde_yaml::Value::as_str) - else { - continue; - }; - netpol_paths.insert(component_label.to_string(), path); - } - Ok(netpol_paths) -} - -fn project_dynamic_kubernetes_proxy_export_resources( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - proxy_exports: &BTreeMap, -) -> Result<()> { - if proxy_exports.is_empty() { - return Ok(()); - } - - let netpol_paths = kubernetes_network_policy_paths_by_component_label(artifact_root)?; - let exported_mesh_ports = proxy_exports - .values() - .map(|export| { - let component_target = mesh_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the kubernetes mesh plan", - export.component - ) - })?; - Ok(( - format!("c{}", export.component_id), - component_target.config.mesh_listen.port(), - )) - }) - .collect::>>()?; - - for (component_label, mesh_port) in exported_mesh_ports { - let path = netpol_paths.get(&component_label).ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {component_label} is missing a kubernetes network \ - policy in {}", - artifact_root.join("05-networkpolicies").display() - ) - })?; - ensure_kubernetes_network_policy_router_ingress(path, mesh_port)?; - } - - Ok(()) -} - -fn ensure_kubernetes_network_policy_router_ingress(path: &Path, mesh_port: u16) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} is not a YAML mapping", - path.display() - ) - })?; - let spec = root - .get_mut(yaml_string("spec")) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} is missing a spec mapping", - path.display() - ) - })?; - let ingress = spec - .entry(yaml_string("ingress")) - .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) - .as_sequence_mut() - .ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} has non-sequence spec.ingress", - path.display() - ) - })?; - - let router_peer = kubernetes_router_network_policy_peer(); - let mut matched_port_rule = false; - let mut changed = false; - for rule in ingress.iter_mut() { - let Some(rule_mapping) = rule.as_mapping_mut() else { - continue; - }; - let matches_port = rule_mapping - .get(yaml_string("ports")) - .and_then(serde_yaml::Value::as_sequence) - .is_some_and(|ports| { - ports - .iter() - .any(|port| network_policy_port_matches(port, mesh_port)) - }); - if !matches_port { - continue; - } - matched_port_rule = true; - let Some(from) = rule_mapping - .get_mut(yaml_string("from")) - .and_then(serde_yaml::Value::as_sequence_mut) - else { - break; - }; - if from.iter().any(network_policy_peer_is_router) { - break; - } - from.push(router_peer.clone()); - changed = true; - break; - } - - if !matched_port_rule { - ingress.push(serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter( - [ - ( - yaml_string("from"), - serde_yaml::Value::Sequence(vec![router_peer.clone()]), - ), - ( - yaml_string("ports"), - serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( - serde_yaml::Mapping::from_iter([ - (yaml_string("protocol"), yaml_string("TCP")), - ( - yaml_string("port"), - serde_yaml::Value::Number(u64::from(mesh_port).into()), - ), - ]), - )]), - ), - ], - ))); - changed = true; - } - - if !changed { - return Ok(()); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn kubernetes_router_network_policy_peer() -> serde_yaml::Value { - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("podSelector"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("matchLabels"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("amber.io/component"), - yaml_string(KUBERNETES_ROUTER_COMPONENT_NAME), - )])), - )])), - )])) -} - -fn network_policy_port_matches(port: &serde_yaml::Value, expected_port: u16) -> bool { - port.as_mapping() - .and_then(|port| port.get(yaml_string("port"))) - .and_then(|value| { - value - .as_u64() - .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) - }) - == Some(u64::from(expected_port)) -} - -fn network_policy_peer_is_router(peer: &serde_yaml::Value) -> bool { - peer.as_mapping() - .and_then(|peer| peer.get(yaml_string("podSelector"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|selector| selector.get(yaml_string("matchLabels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component"))) - .and_then(serde_yaml::Value::as_str) - == Some(KUBERNETES_ROUTER_COMPONENT_NAME) -} - -fn dynamic_proxy_export_kubernetes_peer_addr( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - export: &DynamicProxyExportRecord, -) -> Result { - let component_target = mesh_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the kubernetes mesh plan", - export.component - ) - })?; - kubernetes_component_mesh_peer_addr( - artifact_root, - &export.component, - &component_target.output, - component_target.config.mesh_listen.port(), - ) -} - -fn project_dynamic_child_mesh_scope(artifact_root: &Path, mesh_scope: Option<&str>) -> Result<()> { - let Some(mesh_scope) = mesh_scope else { - return Ok(()); - }; - let path = artifact_root.join("mesh-provision-plan.json"); - if path.is_file() { - let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_json(&path, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - let compose_path = artifact_root.join("compose.yaml"); - if compose_path.is_file() { - let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); - if configmap_path.is_file() { - let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - Err(miette::miette!( - "dynamic artifact {} is missing a mesh provision plan", - artifact_root.display() - )) -} - -fn project_mesh_provision_plan_scope(plan: &mut MeshProvisionPlan, mesh_scope: &str) -> bool { - let mut changed = false; - for target in &mut plan.targets { - if target.config.identity.mesh_scope.as_deref() == Some(mesh_scope) { - continue; - } - target.config.identity.mesh_scope = Some(mesh_scope.to_string()); - changed = true; - } - changed -} - -fn mesh_provision_plan_scopes(plan: &MeshProvisionPlan) -> BTreeSet { - let mut scopes = BTreeSet::new(); - for target in &plan.targets { - if let Some(scope) = target.config.identity.mesh_scope.as_deref() { - scopes.insert(scope.to_string()); - } - } - for identity in &plan.existing_peer_identities { - if let Some(scope) = identity.mesh_scope.as_deref() { - scopes.insert(scope.to_string()); - } - } - scopes -} - -fn rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root: &Path, - existing_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let rewrite_scopes = existing_scopes - .iter() - .filter(|scope| scope.as_str() != mesh_scope) - .cloned() - .collect::>(); - if rewrite_scopes.is_empty() { - return Ok(()); - } - - for path in walk_files(artifact_root)? { - match path.extension().and_then(|extension| extension.to_str()) { - Some("json") => rewrite_json_scope_literals(&path, &rewrite_scopes, mesh_scope)?, - Some("yaml" | "yml") => { - rewrite_yaml_scope_literals(&path, &rewrite_scopes, mesh_scope)? - } - Some("env") => rewrite_env_scope_literals(&path, &rewrite_scopes, mesh_scope)?, - _ => {} - } - } - Ok(()) -} - -fn rewrite_json_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_json::Value = serde_json::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid json {}", path.display()))?; - if !rewrite_scope_json_value(&mut document, rewrite_scopes, mesh_scope) { - return Ok(()); - } - let rendered = serde_json::to_string_pretty(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_yaml_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid yaml {}", path.display()))?; - if !rewrite_scope_yaml_value(&mut document, rewrite_scopes, mesh_scope) { - return Ok(()); - } - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_env_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut changed = false; - let mut rendered = raw - .lines() - .map(|line| { - let Some((name, value)) = line.split_once('=') else { - return line.to_string(); - }; - if !rewrite_scopes.contains(value) { - return line.to_string(); - } - changed = true; - format!("{name}={mesh_scope}") - }) - .collect::>() - .join("\n"); - if !changed { - return Ok(()); - } - if raw.ends_with('\n') { - rendered.push('\n'); - } - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_scope_json_value( - value: &mut serde_json::Value, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - match value { - serde_json::Value::String(string) => { - rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) - } - serde_json::Value::Array(values) => { - let mut changed = false; - for value in values { - changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); - } - changed - } - serde_json::Value::Object(map) => { - let mut changed = false; - for value in map.values_mut() { - changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); - } - changed - } - _ => false, - } -} - -fn rewrite_scope_yaml_value( - value: &mut serde_yaml::Value, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - match value { - serde_yaml::Value::String(string) => { - rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) - } - serde_yaml::Value::Sequence(values) => { - let mut changed = false; - for value in values { - changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); - } - changed - } - serde_yaml::Value::Mapping(map) => { - let mut changed = false; - for (_, value) in map.iter_mut() { - changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); - } - changed - } - _ => false, - } -} - -fn rewrite_scope_string_value( - string: &mut String, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - if rewrite_scopes.contains(string) { - *string = mesh_scope.to_string(); - return true; - } - let Some((name, value)) = string.split_once('=') else { - return false; - }; - if !rewrite_scopes.contains(value) { - return false; - } - *string = format!("{name}={mesh_scope}"); - true -} - -fn reconcile_artifact_files(site_plan: &DynamicSitePlanRecord) -> &BTreeMap { - if site_plan.desired_artifact_files.is_empty() { - &site_plan.artifact_files - } else { - &site_plan.desired_artifact_files - } -} - -fn load_kubernetes_mesh_secret_payload( - plan: &SiteActuatorPlan, - name: &str, - namespace: Option<&str>, -) -> Result<(String, KubernetesSecretPayload)> { - let namespace = namespace - .or(plan.kubernetes_namespace.as_deref()) - .ok_or_else(|| { - miette::miette!( - "kubernetes site `{}` is missing its namespace", - plan.site_id - ) - })? - .to_string(); - let output = kubectl_command(plan.context.as_deref()) - .arg("-n") - .arg(&namespace) - .arg("get") - .arg("secret") - .arg(name) - .arg("-o") - .arg("json") - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to read kubernetes mesh secret {} in namespace {}", - name, namespace - ) - })?; - if !output.status.success() { - return Err(miette::miette!( - "failed to read kubernetes mesh secret {} in namespace {}: {}", - name, - namespace, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - let payload: KubernetesSecretPayload = - serde_json::from_slice(&output.stdout).map_err(|err| { - miette::miette!( - "invalid kubernetes secret payload for {} in namespace {}: {err}", - name, - namespace - ) - })?; - Ok((namespace, payload)) -} - -fn decode_kubernetes_mesh_secret_json( - payload: &KubernetesSecretPayload, - namespace: &str, - name: &str, - key: &str, - description: &str, -) -> Result -where - T: for<'de> Deserialize<'de>, -{ - let encoded = payload.data.get(key).ok_or_else(|| { - miette::miette!( - "kubernetes mesh secret {} in namespace {} is missing {}", - name, - namespace, - key - ) - })?; - let bytes = base64::engine::general_purpose::STANDARD - .decode(encoded.as_bytes()) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to decode kubernetes {description} {} in namespace {}", - name, namespace - ) - })?; - serde_json::from_slice(&bytes).map_err(|err| { - miette::miette!( - "invalid kubernetes {description} {} in namespace {}: {err}", - name, - namespace - ) - }) -} - -fn load_kubernetes_mesh_identity_secret( - plan: &SiteActuatorPlan, - name: &str, - namespace: Option<&str>, -) -> Result { - let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; - decode_kubernetes_mesh_secret_json( - &payload, - &namespace, - name, - MESH_IDENTITY_FILENAME, - "mesh identity", - ) -} - -fn load_kubernetes_mesh_config_public( - plan: &SiteActuatorPlan, - name: &str, - namespace: Option<&str>, -) -> Result { - let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; - decode_kubernetes_mesh_secret_json( - &payload, - &namespace, - name, - MESH_CONFIG_FILENAME, - "mesh config", - ) -} - -fn reconcile_site_proxy_metadata( - site_artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, -) -> Result<()> { - let Some(proxy_metadata) = reconcile_artifact_files(site_plan).get("amber-proxy.json") else { - return Ok(()); - }; - let path = site_artifact_root.join("amber-proxy.json"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&path, proxy_metadata) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn project_dynamic_direct_router_surface( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, -) -> Result<()> { - let state_path = direct_runtime_state_path(Path::new(&child.artifact_root)); - let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its router mesh port", - plan.site_id - ) - })?; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - let mut state: crate::direct_runtime::DirectRuntimeState = - read_json(&state_path, "direct runtime state")?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - state.router_mesh_port = Some(router_mesh_port); - write_json(&state_path, &state)?; - std::thread::sleep(Duration::from_millis(100)); - let state: crate::direct_runtime::DirectRuntimeState = - read_json(&state_path, "direct runtime state")?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - if std::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out projecting direct child router mesh port into {}", - state_path.display() - )); - } - } - - let site_current = direct_current_control_socket_path(Path::new(&plan.artifact_dir)); - let child_current = direct_current_control_socket_path(Path::new(&child.artifact_root)); - if let Some(parent) = child_current.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_direct_control_socket_link( - &child_current, - &site_current, - "dynamic child router control symlink", - )?; - let artifact_link = Path::new(&child.artifact_root) - .join(".amber") - .join("control") - .join("router-control.sock"); - if let Some(parent) = artifact_link.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_direct_control_socket_link( - &artifact_link, - &child_current, - "dynamic child router control artifact symlink", - ) -} - -fn project_dynamic_vm_router_surface( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, -) -> Result<()> { - let state_path = Path::new(&child.artifact_root) - .join(".amber") - .join("vm-runtime.json"); - let runtime_root = site_actuator_child_runtime_root(plan, child.child_id); - let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) - })?; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - let mut state = - load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - state.router_mesh_port = Some(router_mesh_port); - write_vm_runtime_state(Path::new(&child.artifact_root), &state)?; - std::thread::sleep(Duration::from_millis(100)); - let state = - load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - if std::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out projecting vm child router mesh port into {}", - state_path.display() - )); - } - } - - let site_current = vm_current_control_socket_path(Path::new(&plan.artifact_dir)); - let child_current = vm_current_control_socket_path(Path::new(&child.artifact_root)); - if let Some(parent) = child_current.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_control_socket_link( - &child_current, - &site_current, - "dynamic child vm router control symlink", - )?; - let artifact_link = Path::new(&child.artifact_root) - .join(".amber") - .join("control") - .join("router-control.sock"); - if let Some(parent) = artifact_link.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_control_socket_link( - &artifact_link, - &child_current, - "dynamic child vm router control artifact symlink", - ) -} - -fn dynamic_child_route_overlay_id(plan: &SiteActuatorPlan, child_id: u64) -> String { - format!("framework-child:{}:{child_id}", plan.site_id) -} - -fn site_router_control_endpoint(plan: &SiteActuatorPlan) -> Result { - let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); - if state_path.is_file() { - let state: SiteManagerState = read_json(&state_path, "site manager state")?; - if let Some(raw) = state.router_control { - return parse_control_endpoint(&raw); - } - } - - match plan.kind { - SiteKind::Direct => Ok(ControlEndpoint::Unix(direct_current_control_socket_path( - Path::new(&plan.artifact_dir), - ))), - SiteKind::Vm => Ok(ControlEndpoint::Unix(vm_current_control_socket_path( - Path::new(&plan.artifact_dir), - ))), - SiteKind::Compose | SiteKind::Kubernetes => Err(miette::miette!( - "site `{}` manager state is missing router control endpoint", - plan.site_id - )), - } -} - -fn child_router_overlay_payload( - plan: &SiteActuatorPlan, - artifact_root: &Path, - runtime_root: &Path, -) -> Result<(Vec, Vec)> { - let overlay_path = dynamic_route_overlay_path(artifact_root); - if overlay_path.is_file() { - let payload: StoredRouteOverlayPayload = read_json(&overlay_path, "site router overlay")?; - return Ok((payload.peers, payload.inbound_routes)); - } - let provision: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - let Some(router_target) = provision - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - else { - return Ok((Vec::new(), Vec::new())); - }; - let proxy_exports = load_dynamic_proxy_exports_metadata(artifact_root)?; - let mut overlay_peer_ids = router_target - .config - .peers - .iter() - .map(|peer| peer.id.clone()) - .collect::>(); - overlay_peer_ids.extend( - proxy_exports - .values() - .map(|export| export.component.clone()), - ); - - let mut peers = Vec::new(); - let mut peer_addr_by_id = BTreeMap::new(); - for peer_id in &overlay_peer_ids { - let component_target = provision - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == *peer_id - }) - .ok_or_else(|| { - miette::miette!( - "router overlay peer {} is missing from mesh provision plan", - peer_id - ) - })?; - let (identity, runtime_config) = match &component_target.output { - MeshProvisionOutput::Filesystem { dir } => { - let identity: MeshIdentitySecret = read_json( - &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?; - let runtime_config: MeshConfigPublic = read_json( - &runtime_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )?; - (identity, Some(runtime_config)) - } - MeshProvisionOutput::KubernetesSecret { name, namespace } => ( - load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())?, - None, - ), - }; - let public_key = identity.public_key().into_diagnostic()?; - if let Some(runtime_config) = runtime_config { - peer_addr_by_id.insert( - runtime_config.identity.id.clone(), - runtime_config.mesh_listen, - ); - } - peers.push(MeshPeer { - id: identity.id, - public_key, - }); - } - let mut inbound_routes = router_target.config.inbound.clone(); - for route in &mut inbound_routes { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = peer_addr_by_id.get(peer_id) - { - *peer_addr = resolved.to_string(); - } - } - if !proxy_exports.is_empty() { - add_dynamic_proxy_export_overlay_routes(&mut inbound_routes, &proxy_exports, |export| { - match plan.kind { - SiteKind::Kubernetes => { - dynamic_proxy_export_kubernetes_peer_addr(artifact_root, &provision, export) - } - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => peer_addr_by_id - .get(&export.component) - .map(ToString::to_string) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing a live mesh address", - export.component - ) - }), - } - })?; - } - Ok((peers, inbound_routes)) -} - -fn child_overlay_runtime_root(plan: &SiteActuatorPlan, child: &SiteActuatorChildRecord) -> PathBuf { - match plan.kind { - SiteKind::Direct | SiteKind::Vm => site_actuator_child_runtime_root(plan, child.child_id), - SiteKind::Compose | SiteKind::Kubernetes => PathBuf::from(&child.artifact_root), - } -} - -fn dynamic_route_issuer_grants( - children: &[SiteActuatorChildRecord], -) -> Result>> { - let mut issuers_by_route_id = BTreeMap::>::new(); - for child in children { - for input in &child.routed_inputs { - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - issuers_by_route_id - .entry(route_id) - .or_default() - .insert(input.component.clone()); - } - } - Ok(issuers_by_route_id) -} - -fn load_published_component_peers( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], -) -> Result> { - let mut component_peers = BTreeMap::new(); - for child in published_children { - if child.assigned_components.is_empty() { - continue; - } - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(plan, child); - let provision: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - for component in &child.assigned_components { - let target = provision - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == *component - }) - .ok_or_else(|| { - miette::miette!( - "published child {} is missing component {} in its mesh provision plan", - child.child_id, - component - ) - })?; - let identity = match &target.output { - MeshProvisionOutput::Filesystem { dir } => read_json( - &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?, - MeshProvisionOutput::KubernetesSecret { name, namespace } => { - load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())? - } - }; - component_peers.insert( - component.clone(), - MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key().into_diagnostic()?, - }, - ); - } - } - Ok(component_peers) -} - -fn apply_dynamic_route_issuer_grants( - peers: &mut Vec, - inbound_routes: &mut [InboundRoute], - issuers_by_route_id: &BTreeMap>, - component_peers: &BTreeMap, -) -> Result<()> { - let mut known_peer_ids = peers - .iter() - .map(|peer| peer.id.clone()) - .collect::>(); - for route in inbound_routes { - let Some(issuers) = issuers_by_route_id.get(&route.route_id) else { - continue; - }; - route.allowed_issuers = issuers.iter().cloned().collect(); - for issuer in issuers { - if known_peer_ids.contains(issuer) { - continue; - } - let peer = component_peers.get(issuer).ok_or_else(|| { - miette::miette!( - "dynamic route {} references published issuer {} with no live mesh peer", - route.route_id, - issuer - ) - })?; - peers.push(peer.clone()); - known_peer_ids.insert(issuer.clone()); - } - } - Ok(()) -} - -async fn reconcile_dynamic_site_router_overlays(app: &SiteActuatorApp) -> Result<()> { - let published_children = { - let state = app.state.lock().await; - state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>() - }; - reconcile_dynamic_site_router_overlays_for_children( - app, - &published_children, - &published_children, - ) - .await -} - -async fn reconcile_dynamic_site_router_overlays_for_children( - app: &SiteActuatorApp, - overlay_children: &[SiteActuatorChildRecord], - issuer_children: &[SiteActuatorChildRecord], -) -> Result<()> { - if overlay_children.is_empty() { - return Ok(()); - } - - let issuers_by_route_id = dynamic_route_issuer_grants(issuer_children)?; - let component_peers = load_published_component_peers(&app.plan, issuer_children)?; - for child in overlay_children { - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(&app.plan, child); - let (mut peers, mut inbound_routes) = - child_router_overlay_payload(&app.plan, artifact_root, &runtime_root)?; - apply_dynamic_route_issuer_grants( - &mut peers, - &mut inbound_routes, - &issuers_by_route_id, - &component_peers, - )?; - if inbound_routes.is_empty() { - continue; - } - let endpoint = site_router_control_endpoint(&app.plan)?; - apply_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(&app.plan, child.child_id), - &peers, - &inbound_routes, - Duration::from_secs(30), - ) - .await?; - } - - Ok(()) -} - -async fn apply_dynamic_site_router_overlay( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, -) -> Result<()> { - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(plan, child); - let (peers, inbound_routes) = child_router_overlay_payload(plan, artifact_root, &runtime_root)?; - if inbound_routes.is_empty() { - return Ok(()); - } - let endpoint = site_router_control_endpoint(plan)?; - apply_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(plan, child.child_id), - &peers, - &inbound_routes, - Duration::from_secs(30), - ) - .await -} - -async fn revoke_dynamic_site_router_overlay( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, -) -> Result<()> { - let endpoint = site_router_control_endpoint(plan)?; - revoke_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(plan, child.child_id), - Duration::from_secs(30), - ) - .await -} - -async fn actuator_prepare_child( - app: &SiteActuatorApp, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let artifact_root = site_actuator_child_artifact_root(&app.plan, child_id); - let published_children = { - let state = app.state.lock().await; - state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>() - }; - replace_artifact_snapshot(&artifact_root, &site_plan.artifact_files)?; - project_dynamic_child_mesh_scope(&artifact_root, Some(&app.plan.mesh_scope))?; - if site_plan.kind == SiteKind::Kubernetes { - let _ = prepare_kubernetes_artifact_namespace( - &app.plan.run_id, - &app.plan.site_id, - &artifact_root, - )?; - } - patch_site_artifacts( - &artifact_root, - &app.plan.run_id, - &app.plan.site_id, - site_plan.kind, - &app.plan.launch_env, - app.plan.observability_endpoint.as_deref(), - )?; - match site_plan.kind { - SiteKind::Direct => { - filter_direct_stage_plan(&artifact_root, &site_plan.component_ids)?; - ensure_dynamic_proxy_export_component_routes_in_artifact( - &artifact_root, - &site_plan.proxy_exports, - &app.plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &site_plan, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "direct", - &site_plan.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_direct_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_direct_peer_identities_for_children(&app.plan, &published_children)?, - )?; - } - SiteKind::Vm => { - filter_vm_stage_plan(&artifact_root, &site_plan.component_ids)?; - ensure_dynamic_proxy_export_component_routes_in_artifact( - &artifact_root, - &site_plan.proxy_exports, - &app.plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &site_plan, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "vm", - &site_plan.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_vm_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_vm_peer_identities_for_children(&app.plan, &published_children)?, - )?; - } - SiteKind::Compose => { - let existing_peer_identities = - local_compose_peer_identities(&app.plan, &published_children)?; - prepare_dynamic_compose_child_artifact( - &app.plan, - &site_plan, - &artifact_root, - &published_children, - &existing_peer_identities, - )? - } - SiteKind::Kubernetes => { - let existing_peer_identities = - local_kubernetes_peer_identities(&app.plan, &published_children)?; - prepare_dynamic_kubernetes_child_artifact( - &app.plan, - &site_plan, - &artifact_root, - &existing_peer_identities, - )? - } - } - write_dynamic_proxy_exports_metadata(&artifact_root, &site_plan.proxy_exports)?; - rewrite_dynamic_proxy_metadata(&artifact_root, &site_plan)?; - if site_plan.kind == SiteKind::Compose - && let Some(compose_project) = app.plan.compose_project.as_deref() - { - rewrite_dynamic_compose_proxy_metadata(&artifact_root, compose_project)?; - } - let mut state = app.state.lock().await; - state.children.insert( - child_id, - SiteActuatorChildRecord { - child_id, - artifact_root: artifact_root.display().to_string(), - assigned_components: site_plan.assigned_components.clone(), - proxy_exports: site_plan.proxy_exports.clone(), - routed_inputs: site_plan.routed_inputs.clone(), - process_pid: None, - published: false, - }, - ); - write_json(&app.state_path, &*state) -} - -async fn actuator_publish_child( - app: &SiteActuatorApp, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let (child, published_children) = { - let state = app.state.lock().await; - let child = state - .children - .get(&child_id) - .cloned() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; - let published_children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - (child, published_children) - }; - if child.published { - return Ok(()); - } - - match app.plan.kind { - SiteKind::Direct => { - let state = app.state.lock().await; - let existing_peer_ports = local_direct_peer_ports(&app.plan, &state)?; - let existing_peer_identities = local_direct_peer_identities(&app.plan, &state)?; - drop(state); - write_json( - &site_actuator_child_peer_ports_path(&app.plan, child_id), - &existing_peer_ports, - )?; - write_json( - &site_actuator_child_peer_identities_path(&app.plan, child_id), - &existing_peer_identities, - )?; - project_dynamic_child_mesh_scope( - Path::new(&child.artifact_root), - existing_peer_identities - .get(&app.plan.router_identity_id) - .and_then(|identity| identity.mesh_scope.as_deref()), - )?; - if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - } - let runtime_root = site_actuator_child_runtime_root(&app.plan, child_id); - let storage_root = site_actuator_child_storage_root(&app.plan, child_id); - let child_root = site_actuator_child_root(&app.plan, child_id); - fs::create_dir_all(&runtime_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; - fs::create_dir_all(&storage_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-direct-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("direct-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_actuator_child_peer_ports_path(&app.plan, child_id)) - .arg("--existing-peer-identities") - .arg(site_actuator_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; - wait_for_detached_child_runtime_state( - process.id(), - &direct_runtime_state_path(Path::new(&child.artifact_root)), - site_ready_timeout_for_kind(SiteKind::Direct), - &child_root.join("site.log"), - ) - .await?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_ports = overlay_peer_addr_map_from_ports( - &local_direct_peer_ports_for_children(&app.plan, &live_children)?, - ); - let live_peer_identities = - local_direct_peer_identities_for_children(&app.plan, &live_children)?; - write_direct_vm_live_route_overlay_payload( - Path::new(&child.artifact_root), - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_ports, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - { - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - record.published = true; - } - write_json(&app.state_path, &*state)?; - drop(state); - project_dynamic_direct_router_surface(&app.plan, &child)?; - reconcile_dynamic_site_router_overlays(app).await?; - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_plan)?; - } - SiteKind::Vm => { - let state = app.state.lock().await; - let existing_peer_ports = local_vm_peer_ports(&app.plan, &state)?; - let existing_peer_identities = local_vm_peer_identities(&app.plan, &state)?; - drop(state); - write_json( - &site_actuator_child_peer_ports_path(&app.plan, child_id), - &existing_peer_ports, - )?; - write_json( - &site_actuator_child_peer_identities_path(&app.plan, child_id), - &existing_peer_identities, - )?; - project_dynamic_child_mesh_scope( - Path::new(&child.artifact_root), - existing_peer_identities - .get(&app.plan.router_identity_id) - .and_then(|identity| identity.mesh_scope.as_deref()), - )?; - if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - } - let runtime_root = site_actuator_child_runtime_root(&app.plan, child_id); - let storage_root = site_actuator_child_storage_root(&app.plan, child_id); - let child_root = site_actuator_child_root(&app.plan, child_id); - fs::create_dir_all(&runtime_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; - fs::create_dir_all(&storage_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-vm-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("vm-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_actuator_child_peer_ports_path(&app.plan, child_id)) - .arg("--existing-peer-identities") - .arg(site_actuator_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; - wait_for_detached_child_runtime_state( - process.id(), - &Path::new(&child.artifact_root) - .join(".amber") - .join("vm-runtime.json"), - vm_endpoint_forward_ready_timeout(), - &child_root.join("site.log"), - ) - .await?; - wait_for_detached_vm_child_endpoints_ready( - process.id(), - Path::new(&child.artifact_root), - &runtime_root, - vm_endpoint_forward_ready_timeout(), - &child_root.join("site.log"), - )?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_ports = overlay_peer_addr_map_from_ports( - &local_vm_peer_ports_for_children(&app.plan, &live_children)?, - ); - let live_peer_identities = - local_vm_peer_identities_for_children(&app.plan, &live_children)?; - write_direct_vm_live_route_overlay_payload( - Path::new(&child.artifact_root), - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_ports, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - { - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - record.published = true; - } - write_json(&app.state_path, &*state)?; - drop(state); - project_dynamic_vm_router_surface(&app.plan, &child)?; - reconcile_dynamic_site_router_overlays(app).await?; - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_plan)?; - } - SiteKind::Compose => { - let mut issuer_children = published_children.clone(); - issuer_children.push(child.clone()); - reconcile_dynamic_site_router_overlays_for_children( - app, - &published_children, - &issuer_children, - ) - .await?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; - let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - app.plan.site_id - ) - })?; - let (sidecar_services, workload_services): (Vec<_>, Vec<_>) = metadata - .services - .iter() - .cloned() - .partition(|service| is_compose_component_sidecar_service(service)); - if !sidecar_services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("up") - .arg("-d") - .args(&sidecar_services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to publish compose child sidecars on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child sidecar publish on site `{}` failed with status {status}", - app.plan.site_id - )); - } - wait_for_compose_services_running( - compose_project, - Path::new(&child.artifact_root), - &sidecar_services, - site_ready_timeout_for_kind(SiteKind::Compose), - ) - .await?; - } - if !workload_services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("up") - .arg("-d") - .args(&workload_services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to publish compose child workloads on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child workload publish on site `{}` failed with status {status}", - app.plan.site_id - )); - } - } - wait_for_compose_services_running( - compose_project, - Path::new(&child.artifact_root), - &metadata.readiness_services, - site_ready_timeout_for_kind(SiteKind::Compose), - ) - .await?; - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.published = true; - write_json(&app.state_path, &*state)?; - drop(state); - reconcile_dynamic_site_router_overlays(app).await?; - } - SiteKind::Kubernetes => { - let artifact_root = Path::new(&child.artifact_root); - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; - ensure_kubernetes_namespace(&supervisor_plan)?; - let status = kubectl_command(app.plan.context.as_deref()) - .current_dir(artifact_root) - .arg("apply") - .arg("-k") - .arg(".") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to publish kubernetes site `{}`", app.plan.site_id) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes site `{}` publish failed with status {status}", - app.plan.site_id - )); - } - ensure_kubernetes_workloads_ready(&supervisor_plan)?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_identities = local_kubernetes_peer_identities(&app.plan, &live_children)?; - write_kubernetes_live_route_overlay_payload( - artifact_root, - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.published = true; - write_json(&app.state_path, &*state)?; - drop(state); - reconcile_dynamic_site_router_overlays(app).await?; - wait_for_kubernetes_site_router_ready( - &app.plan, - site_ready_timeout_for_kind(SiteKind::Kubernetes), - ) - .await?; - } - } - - Ok(()) -} - -async fn actuator_rollback_child(app: &SiteActuatorApp, child_id: u64) -> Result<()> { - let child = { - let mut state = app.state.lock().await; - let removed = state.children.remove(&child_id); - write_json(&app.state_path, &*state)?; - removed - }; - let Some(child) = child else { - return Ok(()); - }; - if let Some(pid) = child.process_pid { - terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; - } - remove_dir_if_exists(&site_actuator_child_root(&app.plan, child_id)) -} - -async fn actuator_destroy_child( - app: &SiteActuatorApp, - child_id: u64, - desired_site_plan: Option, -) -> Result<()> { - let child = { - let state = app.state.lock().await; - state.children.get(&child_id).cloned() - }; - if child.as_ref().is_some_and(|child| child.published) - && matches!( - app.plan.kind, - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes - ) - { - revoke_dynamic_site_router_overlay( - &app.plan, - child - .as_ref() - .expect("published child should be available for overlay revoke"), - ) - .await?; - } - if let Some(pid) = child.as_ref().and_then(|child| child.process_pid) { - terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; - } - - match app.plan.kind { - SiteKind::Compose => { - let _ = desired_site_plan; - let child = child - .as_ref() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; - let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; - let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - app.plan.site_id - ) - })?; - if !metadata.services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("rm") - .arg("--stop") - .arg("--force") - .arg("-v") - .args(&metadata.services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to destroy compose child on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child destroy on site `{}` failed with status {status}", - app.plan.site_id - )); - } - } - } - SiteKind::Kubernetes => { - let _ = desired_site_plan; - let child = child - .as_ref() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; - let artifact_root = Path::new(&child.artifact_root); - let files = read_artifact_snapshot(artifact_root)?; - let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files(&files)?; - replace_artifact_snapshot(artifact_root, &destroy_bundle)?; - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; - let workloads = supervisor::kubernetes_expected_workloads(artifact_root)?; - let status = kubectl_command(app.plan.context.as_deref()) - .current_dir(artifact_root) - .arg("delete") - .arg("-k") - .arg(".") - .arg("--ignore-not-found=true") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to destroy kubernetes child on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes child destroy on site `{}` failed with status {status}", - app.plan.site_id - )); - } - wait_for_kubernetes_artifact_workloads_deleted( - app.plan.context.as_deref(), - supervisor_plan - .kubernetes_namespace - .as_deref() - .expect("kubernetes supervisor plan should include a namespace"), - &workloads, - &app.plan.site_id, - )?; - wait_for_kubernetes_site_router_ready( - &app.plan, - site_ready_timeout_for_kind(SiteKind::Kubernetes), - ) - .await?; - } - SiteKind::Direct | SiteKind::Vm => { - if let Some(desired) = desired_site_plan.as_ref() { - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), desired)?; - } - } - } - - let mut state = app.state.lock().await; - state.children.remove(&child_id); - write_json(&app.state_path, &*state)?; - drop(state); - if matches!( - app.plan.kind, - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes - ) { - reconcile_dynamic_site_router_overlays(app).await?; - } - remove_dir_if_exists(&site_actuator_child_root(&app.plan, child_id)) -} - -fn replace_artifact_snapshot(root: &Path, files: &BTreeMap) -> Result<()> { - fs::create_dir_all(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", root.display()))?; - let expected = files.keys().map(PathBuf::from).collect::>(); - for path in walk_files(root)? { - let relative = path - .strip_prefix(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to relativize {}", path.display()))? - .to_path_buf(); - if !expected.contains(&relative) { - fs::remove_file(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to remove stale artifact {}", path.display()))?; - } - } - for (relative, contents) in files { - let path = root.join(relative); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&path, contents) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display()))?; - } - Ok(()) -} - -fn wait_for_named_kubernetes_resource_deleted( - context: Option<&str>, - namespace: &str, - resource_kind: &str, - name: &str, - label: &str, -) -> Result<()> { - let resource = format!("{resource_kind}/{name}"); - let timeout = format!("{}s", KUBERNETES_WORKLOAD_READY_TIMEOUT.as_secs().max(1)); - let output = kubectl_command(context) - .args([ - "-n", - namespace, - "wait", - "--for=delete", - "--timeout", - timeout.as_str(), - resource.as_str(), - ]) - .output() - .into_diagnostic() - .wrap_err_with(|| format!("{label}: wait for {resource} deletion"))?; - if output.status.success() { - return Ok(()); - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.contains("not found") { - return Ok(()); - } - let detail = if stderr.is_empty() { - format!("status {}", output.status) - } else { - stderr - }; - Err(miette::miette!("{label} failed: {detail}")) -} - -fn wait_for_kubernetes_artifact_workloads_deleted( - context: Option<&str>, - namespace: &str, - workloads: &supervisor::KubernetesArtifactWorkloads, - site_id: &str, -) -> Result<()> { - let label = format!("wait for kubernetes child workload deletion on site `{site_id}`"); - for job in &workloads.jobs { - wait_for_named_kubernetes_resource_deleted(context, namespace, "job", job, &label)?; - } - for deployment in &workloads.deployments { - wait_for_named_kubernetes_resource_deleted( - context, - namespace, - "deployment", - deployment, - &label, - )?; - } - Ok(()) -} - -async fn wait_for_kubernetes_site_router_ready( - plan: &SiteActuatorPlan, - timeout: Duration, -) -> Result<()> { - debug_assert_eq!(plan.kind, SiteKind::Kubernetes); - - let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); - let deadline = Instant::now() + timeout; - loop { - if state_path.is_file() { - let state: SiteManagerState = read_json(&state_path, "site manager state")?; - if matches!(state.status, SiteLifecycleStatus::Failed) { - return Err(miette::miette!( - "kubernetes site `{}` failed while waiting for router recovery: {}", - plan.site_id, - state - .last_error - .unwrap_or_else(|| "unknown failure".to_string()) - )); - } - if matches!(state.status, SiteLifecycleStatus::Running) - && let (Some(control), Some(mesh_addr)) = ( - state.router_control.as_deref(), - state.router_mesh_addr.as_deref(), - ) - { - let control_addr: SocketAddr = - control.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router control addr `{control}`") - })?; - let mesh_addr: SocketAddr = - mesh_addr.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router mesh addr `{mesh_addr}`") - })?; - if probe_kubernetes_router_identity(control_addr, Duration::from_millis(250)) - .await? - && router_mesh_listener_ready(mesh_addr).await - { - return Ok(()); - } - } - } - if Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for kubernetes site `{}` router recovery", - plan.site_id - )); - } - sleep(Duration::from_millis(200)).await; - } -} - -async fn probe_kubernetes_router_identity(addr: SocketAddr, timeout: Duration) -> Result { - let mut stream = match tokio::time::timeout(timeout, tokio::net::TcpStream::connect(addr)).await - { - Ok(Ok(stream)) => stream, - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionRefused - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to connect to kubernetes router control at {addr}: {err}" - )); - } - Err(_) => return Ok(false), - }; - - let request = b"GET /identity HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; - match tokio::time::timeout(timeout, stream.write_all(request)).await { - Ok(Ok(())) => {} - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to write kubernetes router identity probe to {addr}: {err}" - )); - } - Err(_) => return Ok(false), - } - - let deadline = Instant::now() + timeout; - let mut buf = Vec::new(); - loop { - if let Some(end) = find_header_end(&buf) - && let Some(content_length) = router_identity_probe_content_length(&buf[..end])? - { - let body_len = buf.len().saturating_sub(end + 4); - if body_len >= content_length { - return parse_router_identity_probe_response(&buf, end); - } - } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - let mut chunk = [0u8; 1024]; - match tokio::time::timeout(remaining, stream.read(&mut chunk)).await { - Ok(Ok(0)) => return Ok(false), - Ok(Ok(read)) => buf.extend_from_slice(&chunk[..read]), - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to read kubernetes router identity probe from {addr}: {err}" - )); - } - Err(_) => return Ok(false), - } - } -} - -fn router_identity_probe_content_length(header: &[u8]) -> Result> { - let header = std::str::from_utf8(header) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - Ok(header.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-length") - .then_some(value.trim()) - .and_then(|value| value.parse::().ok()) - })) -} - -fn parse_router_identity_probe_response(buf: &[u8], header_end: usize) -> Result { - let header = std::str::from_utf8(&buf[..header_end]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - let status = header - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|code| code.parse::().ok()); - if status != Some(200) { - return Ok(false); - } - let body = std::str::from_utf8(&buf[header_end + 4..]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 body")?; - Ok(serde_json::from_str::(body.trim()).is_ok()) -} - -fn filter_direct_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { - let keep = component_ids.iter().copied().collect::>(); - let plan_path = artifact_root.join("direct-plan.json"); - let mut plan: DirectPlan = read_json(&plan_path, "direct plan")?; - plan.components - .retain(|component| keep.contains(&component.id)); - plan.startup_order - .retain(|component_id| keep.contains(component_id)); - filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); - let keep_mesh_output_dirs = plan - .components - .iter() - .filter_map(|component| { - Path::new(&component.sidecar.mesh_config_path) - .parent() - .map(path_to_forward_slash_string) - }) - .collect::>(); - plan.router = None; - write_json(&plan_path, &plan)?; - filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) -} - -fn filter_vm_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { - let keep = component_ids.iter().copied().collect::>(); - let plan_path = artifact_root.join("vm-plan.json"); - let mut plan: VmPlan = read_json(&plan_path, "vm plan")?; - plan.components - .retain(|component| keep.contains(&component.id)); - plan.startup_order - .retain(|component_id| keep.contains(component_id)); - filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); - let keep_mesh_output_dirs = plan - .components - .iter() - .filter_map(|component| { - Path::new(&component.mesh_config_path) - .parent() - .map(path_to_forward_slash_string) - }) - .collect::>(); - plan.router = None; - write_json(&plan_path, &plan)?; - filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) -} - -fn filter_dynamic_runtime_addresses( - runtime_addresses: &mut amber_compiler::reporter::direct::DirectRuntimeAddressPlan, - keep: &BTreeSet, -) { - runtime_addresses - .slots_by_scope - .retain(|scope, _| keep.contains(scope)); - for sources in runtime_addresses.slots_by_scope.values_mut() { - sources.retain(|_, source| keep.contains(&dynamic_runtime_source_component_id(source))); - } - runtime_addresses - .slot_items_by_scope - .retain(|scope, _| keep.contains(scope)); - for sources in runtime_addresses.slot_items_by_scope.values_mut() { - sources.retain(|_, items| { - items.retain(|source| keep.contains(&dynamic_runtime_source_component_id(source))); - !items.is_empty() - }); - } -} - -fn dynamic_runtime_source_component_id(source: &DirectRuntimeUrlSource) -> usize { - match source { - DirectRuntimeUrlSource::Slot { component_id, .. } - | DirectRuntimeUrlSource::SlotItem { component_id, .. } => *component_id, - } -} - -fn path_to_forward_slash_string(path: &Path) -> String { - path.components() - .fold(String::new(), |mut rendered, component| { - if !rendered.is_empty() { - rendered.push('/'); - } - rendered.push_str(&component.as_os_str().to_string_lossy()); - rendered - }) -} - -fn filter_dynamic_mesh_provision_plan( - artifact_root: &Path, - keep_component_output_dirs: &BTreeSet, -) -> Result<()> { - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mut plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; - let mut kept_component_ids = BTreeSet::new(); - let mut router_target = None; - let mut filtered_targets = Vec::with_capacity(plan.targets.len()); - for target in plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "dynamic mesh target {} does not use filesystem output", - target.config.identity.id - )); - }; - if keep_component_output_dirs.contains(dir) { - kept_component_ids.insert(target.config.identity.id.clone()); - filtered_targets.push(target); - } - } - MeshProvisionTargetKind::Router => { - router_target = Some(target); - } - } - } - - let Some(mut router_target) = router_target else { - return Err(miette::miette!( - "dynamic mesh provision plan {} is missing a router target", - plan_path.display() - )); - }; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - filtered_targets.push(router_target); - plan.targets = filtered_targets; - write_json(&plan_path, &plan) -} - -async fn wait_for_detached_child_runtime_state( - pid: u32, - state_path: &Path, - timeout: Duration, - log_path: &Path, -) -> Result<()> { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if state_path.is_file() { - return Ok(()); - } - if !pid_is_alive(pid) { - let log = fs::read_to_string(log_path).unwrap_or_default(); - return Err(miette::miette!( - "dynamic child runtime exited before becoming ready\nlog ({}):\n{}", - log_path.display(), - log - )); - } - sleep(Duration::from_millis(100)).await; - } - let log = fs::read_to_string(log_path).unwrap_or_default(); - Err(miette::miette!( - "timed out waiting for dynamic child runtime state {}\nlog ({}):\n{}", - state_path.display(), - log_path.display(), - log - )) -} - -fn wait_for_detached_vm_child_endpoints_ready( - pid: u32, - artifact_root: &Path, - runtime_root: &Path, - timeout: Duration, - log_path: &Path, -) -> Result<()> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let deadline = Instant::now() + timeout; - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - for route in config.inbound { - let InboundTarget::Local { port: host_port } = route.target else { - continue; - }; - let addr = SocketAddr::from(([127, 0, 0, 1], host_port)); - loop { - let ready = match route.protocol { - MeshProtocol::Http => endpoint_returns_http_response( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - ), - MeshProtocol::Tcp => endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - ), - }; - if ready { - break; - } - if !pid_is_alive(pid) { - let log = fs::read_to_string(log_path).unwrap_or_default(); - return Err(miette::miette!( - "dynamic vm child runtime exited before component {} endpoint {} became \ - ready\nlog ({}):\n{}", - component.moniker, - addr, - log_path.display(), - log - )); - } - if Instant::now() >= deadline { - let log = fs::read_to_string(log_path).unwrap_or_default(); - let protocol = match route.protocol { - MeshProtocol::Http => "http", - MeshProtocol::Tcp => "tcp", - }; - return Err(miette::miette!( - "timed out waiting for dynamic vm child component {} {} endpoint {} to \ - become ready\nlog ({}):\n{}", - component.moniker, - protocol, - addr, - log_path.display(), - log - )); - } - std::thread::sleep(Duration::from_millis(50)); - } - } - } - Ok(()) -} - -fn local_direct_peer_ports( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_direct_peer_ports_for_children(plan, &children) -} - -fn local_direct_peer_ports_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - peers.insert( - plan.router_identity_id.clone(), - plan.router_mesh_port.ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its router mesh port", - plan.site_id - ) - })?, - ); - peers.extend(direct_peer_ports_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(direct_peer_ports_for_artifact( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn local_direct_peer_identities( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_direct_peer_identities_for_children(plan, &children) -} - -fn local_direct_peer_identities_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - let router = - direct_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? - .ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its live router identity", - plan.site_id - ) - })?; - peers.insert(router.id.clone(), router); - peers.extend(direct_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(direct_peer_identities_for_artifact( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn direct_peer_ports_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state: crate::direct_runtime::DirectRuntimeState = read_json( - &direct_runtime_state_path(artifact_root), - "direct runtime state", - )?; - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - let port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "direct runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - peers.insert(config.identity.id, port); - } - Ok(peers) -} - -fn direct_peer_identities_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn direct_router_identity_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let Some(router) = plan.router.as_ref() else { - return Ok(None); - }; - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - Ok(Some(config.identity)) -} - -fn local_vm_peer_ports( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_vm_peer_ports_for_children(plan, &children) -} - -fn local_vm_peer_ports_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - peers.insert( - plan.router_identity_id.clone(), - plan.router_mesh_port.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) - })?, - ); - peers.extend(vm_peer_ports_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(vm_peer_ports_for_artifact( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn local_vm_peer_identities( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_vm_peer_identities_for_children(plan, &children) -} - -fn local_vm_peer_identities_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - let router = vm_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? - .ok_or_else(|| { - miette::miette!( - "vm site `{}` is missing its live router identity", - plan.site_id - ) - })?; - peers.insert(router.id.clone(), router); - peers.extend(vm_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(vm_peer_identities_for_artifact( - Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn vm_peer_ports_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - let port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "vm runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - peers.insert(config.identity.id, port); - } - Ok(peers) -} - -fn load_vm_runtime_state_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result { - let state_path = artifact_root.join(".amber").join("vm-runtime.json"); - if state_path.is_file() { - return read_json(&state_path, "vm runtime state"); - } - - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut state = VmRuntimeState::default(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - state - .component_mesh_port_by_id - .insert(component.id, config.mesh_listen.port()); - } - if let Some(router) = &plan.router { - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - state.router_mesh_port = Some(config.mesh_listen.port()); - } - write_vm_runtime_state(artifact_root, &state)?; - Ok(state) -} - -fn vm_peer_identities_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn vm_router_identity_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let Some(router) = plan.router.as_ref() else { - return Ok(None); - }; - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - Ok(Some(config.identity)) -} - -fn terminate_pid(pid: u32, timeout: Duration) -> Result<()> { - #[cfg(unix)] - { - terminate_detached_runtime(pid, timeout) - } - - #[cfg(not(unix))] - { - send_sigterm(pid); - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if !pid_is_alive(pid) { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - Ok(()) - } -} - -#[cfg(unix)] -fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { - let mut tracked = process_tree_postorder(root_pid)? - .into_iter() - .filter(|pid| pid_is_alive(*pid)) - .collect::>(); - if tracked.is_empty() { - return Ok(()); - } - - send_signal_to_process_group(root_pid, libc::SIGTERM); - send_signal_to_pids(&tracked, libc::SIGTERM); - - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - tracked.retain(|pid| pid_is_alive(*pid)); - if tracked.is_empty() { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - - send_signal_to_process_group(root_pid, libc::SIGKILL); - send_signal_to_pids(&tracked, libc::SIGKILL); - - let force_deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < force_deadline { - tracked.retain(|pid| pid_is_alive(*pid)); - if tracked.is_empty() { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - - Err(miette::miette!( - "failed to terminate detached runtime rooted at pid {root_pid}; surviving processes: {}", - tracked - .iter() - .map(u32::to_string) - .collect::>() - .join(", ") - )) -} - -fn remove_dir_if_exists(path: &Path) -> Result<()> { - if !path.exists() { - return Ok(()); - } - fs::remove_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to remove {}", path.display())) -} - -fn prepare_kubernetes_artifact_namespace( - run_id: &str, - site_id: &str, - artifact_dir: &Path, -) -> Result { - let namespace = kubernetes_namespace_name(run_id, site_id); - let kustomization = artifact_dir.join("kustomization.yaml"); - let contents = fs::read_to_string(&kustomization) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", kustomization.display()))?; - - let mut saw_namespace = false; - let mut out = String::new(); - for line in contents.lines() { - if line.trim_start().starts_with("namespace:") { - saw_namespace = true; - out.push_str("namespace: "); - out.push_str(&namespace); - } else { - out.push_str(line); - } - out.push('\n'); - } - if !saw_namespace { - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); - } - out.push_str("namespace: "); - out.push_str(&namespace); - out.push('\n'); - } - if out != contents { - fs::write(&kustomization, out) - .into_diagnostic() - .wrap_err_with(|| format!("failed to update {}", kustomization.display()))?; - } - - Ok(namespace) -} - -fn site_supervisor_plan_for_actuator( - plan: &SiteActuatorPlan, - artifact_dir: &Path, - kubernetes_namespace: Option, -) -> SiteSupervisorPlan { - SiteSupervisorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: plan.run_id.clone(), - mesh_scope: plan.mesh_scope.clone(), - run_root: plan.run_root.clone(), - coordinator_pid: 0, - site_id: plan.site_id.clone(), - kind: plan.kind, - artifact_dir: artifact_dir.display().to_string(), - site_state_root: plan.site_state_root.clone(), - storage_root: plan.storage_root.clone(), - runtime_root: plan.runtime_root.clone(), - router_mesh_port: plan.router_mesh_port, - compose_project: plan.compose_project.clone(), - kubernetes_namespace, - context: plan.context.clone(), - port_forward_mesh_port: None, - port_forward_control_port: None, - observability_endpoint: plan.observability_endpoint.clone(), - framework_ccs_plan_path: None, - site_actuator_plan_path: None, - launch_env: plan.launch_env.clone(), - } -} - -fn prepare_kubernetes_artifact_for_apply( - plan: &SiteActuatorPlan, - artifact_dir: &Path, -) -> Result { - debug_assert_eq!(plan.kind, SiteKind::Kubernetes); - let namespace = - prepare_kubernetes_artifact_namespace(&plan.run_id, &plan.site_id, artifact_dir)?; - patch_site_artifacts( - artifact_dir, - &plan.run_id, - &plan.site_id, - plan.kind, - &plan.launch_env, - plan.observability_endpoint.as_deref(), - )?; - Ok(site_supervisor_plan_for_actuator( - plan, - artifact_dir, - Some(namespace), - )) -} - -fn kubernetes_namespace_name(run_id: &str, site_id: &str) -> String { - let raw = format!("amber-{run_id}-{site_id}"); - let mut out = String::with_capacity(raw.len().min(63)); - let mut last_was_dash = false; - - for ch in raw.chars() { - let next = if ch.is_ascii_alphanumeric() { - ch.to_ascii_lowercase() - } else { - '-' - }; - if next == '-' { - if out.is_empty() || last_was_dash { - continue; - } - last_was_dash = true; - } else { - last_was_dash = false; - } - out.push(next); - if out.len() == 63 { - break; - } - } - while out.ends_with('-') { - out.pop(); - } - if out.is_empty() { - "amber".to_string() - } else { - out - } -} - -async fn wait_for_compose_services_running( - compose_project: &str, - artifact_root: &Path, - services: &[String], - timeout: Duration, -) -> Result<()> { - if services.is_empty() { - return Ok(()); - } - let deadline = Instant::now() + timeout; - loop { - let output = compose_command(Some(compose_project), artifact_root) - .arg("ps") - .arg("--services") - .arg("--status") - .arg("running") - .args(services) - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to query compose child services in {}", - artifact_root.display() - ) - })?; - if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); - let running = stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .collect::>(); - if services - .iter() - .all(|service| running.contains(service.as_str())) - { + .wrap_err_with(|| format!("failed to query kubernetes namespace `{namespace}`"))?; + if !get_output.status.success() { + if kubectl_is_not_found(&get_output) { return Ok(()); } + last_namespace_detail = Some(kubectl_detail(&get_output)); + } else { + last_namespace_detail = Some(kubectl_stdout(&get_output)); } + if Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for compose child services {:?} in {}", - services, - artifact_root.display() - )); - } - sleep(Duration::from_millis(200)).await; - } -} + if !forced_pod_cleanup { + kubectl_force_delete_namespace_pods_with_bin(kubectl_bin, context, namespace)?; + let retry_output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "delete", + "namespace", + namespace, + "--ignore-not-found", + "--wait=false", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to retry deletion of kubernetes namespace `{namespace}`") + })?; + if !retry_output.status.success() && !kubectl_is_not_found(&retry_output) { + return Err(miette::miette!( + "retrying deletion of kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&retry_output) + )); + } + forced_pod_cleanup = true; + deadline = Instant::now() + KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD; + std::thread::sleep(KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL); + continue; + } -fn walk_files(root: &Path) -> Result> { - let mut files = Vec::new(); - let mut stack = vec![root.to_path_buf()]; - while let Some(path) = stack.pop() { - for entry in fs::read_dir(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to list {}", path.display()))? - { - let entry = entry.into_diagnostic()?; - let entry_path = entry.path(); - if entry_path.is_dir() { - stack.push(entry_path); + let remaining_pods = + kubectl_list_namespace_pods_with_bin(kubectl_bin, context, namespace)?; + let pod_detail = if remaining_pods.is_empty() { + "remaining pods: none".to_string() } else { - files.push(entry_path); - } + format!("remaining pods: {}", remaining_pods.join(", ")) + }; + let namespace_detail = last_namespace_detail + .as_deref() + .filter(|detail| !detail.is_empty()) + .map(|detail| format!("last namespace response:\n{detail}")) + .unwrap_or_else(|| "last namespace response: ".to_string()); + return Err(miette::miette!( + "timed out waiting for kubernetes namespace `{namespace}` to delete after \ + {}s\n{pod_detail}\n{namespace_detail}", + timeout.as_secs() + KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD.as_secs() + )); } - } - Ok(files) -} - -fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result { - if !matches!(kind, SiteKind::Compose | SiteKind::Kubernetes) { - return Ok(endpoint.to_string()); - } - let mut url = Url::parse(endpoint) - .into_diagnostic() - .wrap_err_with(|| format!("invalid observability endpoint {endpoint}"))?; - let should_rewrite = url.host_str().is_some_and(|host| { - host.eq_ignore_ascii_case("localhost") - || host - .parse::() - .map(|addr| addr.is_loopback() || addr.is_unspecified()) - .unwrap_or(false) - }); - if should_rewrite { - let host = container_host_for_consumer(SiteKind::Direct, kind); - url.set_host(Some(&host)) - .map_err(|_| miette::miette!("failed to rewrite observability endpoint {endpoint}"))?; + std::thread::sleep(KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL); } - Ok(url.to_string()) -} - -pub(crate) fn reserve_loopback_port() -> Result { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .into_diagnostic() - .wrap_err("failed to allocate a loopback port")?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - -pub(crate) fn new_run_id() -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(); - format!("run-{millis:x}-{:x}", std::process::id()) -} - -fn receipt_path(run_root: &Path) -> PathBuf { - run_root.join("receipt.json") -} - -fn run_plan_path(run_root: &Path) -> PathBuf { - run_root.join("run-plan.json") -} - -fn launch_bundle_manifest_path(run_root: &Path) -> PathBuf { - run_root.join("launch-bundle.json") -} - -pub(crate) fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { - state_root.join(site_id).join("manager-state.json") -} - -fn state_path_for_plan(plan: &SiteSupervisorPlan) -> PathBuf { - Path::new(&plan.site_state_root).join("manager-state.json") -} - -fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-supervisor-plan.json") -} - -fn site_actuator_plan_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-actuator-plan.json") -} - -pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("desired-links.json") -} - -fn commit_marker_path(run_root: &Path) -> PathBuf { - run_root.join("committed") -} - -fn coordinator_lock_path(run_root: &Path) -> PathBuf { - run_root.join("coordinator.lock") -} - -fn stop_marker_path(run_root: &Path) -> PathBuf { - run_root.join("stop-requested") } async fn wait_for_stop_request(stop_requested: &AtomicBool, run_root: &Path) { @@ -7168,23 +1667,6 @@ pub(crate) fn decode_public_key(value: &str) -> Result<[u8; 32]> { .map_err(|_| miette::miette!("invalid router public key length")) } -fn pid_is_alive(pid: u32) -> bool { - #[cfg(unix)] - { - let alive = unsafe { - libc::kill(pid as i32, 0) == 0 - || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) - }; - alive && process_status_code(pid) != Some('Z') - } - - #[cfg(not(unix))] - { - let _ = pid; - true - } -} - fn send_sigterm(pid: u32) { #[cfg(unix)] { @@ -7197,24 +1679,6 @@ fn send_sigterm(pid: u32) { } } -#[cfg(unix)] -fn send_signal_to_pids(pids: &[u32], signal: i32) { - for pid in pids { - let Some(pid) = i32::try_from(*pid).ok() else { - continue; - }; - let _ = unsafe { libc::kill(pid, signal) }; - } -} - -#[cfg(unix)] -fn send_signal_to_process_group(root_pid: u32, signal: i32) { - let Some(root_pid) = i32::try_from(root_pid).ok() else { - return; - }; - let _ = unsafe { libc::kill(-root_pid, signal) }; -} - fn install_signal_flag() -> Arc { let flag = Arc::new(AtomicBool::new(false)); @@ -7283,15 +1747,7 @@ fn emit_manager_event(event_name: &'static str, body: String, attributes: &[(&st let _ = send_manager_observability(&endpoint, "/v1/logs", &payload); } -fn handle_otlp_connection( - stream: &mut TcpStream, - requests_log: &Path, - events_ndjson: &Path, -) -> Result<()> { - stream - .set_nonblocking(false) - .into_diagnostic() - .wrap_err("failed to configure observability stream blocking mode")?; +fn handle_otlp_connection(stream: &mut TcpStream, requests_log: &Path) -> Result<()> { stream .set_read_timeout(Some(Duration::from_secs(5))) .into_diagnostic() @@ -7322,15 +1778,6 @@ fn handle_otlp_connection( .and_then(|line| line.split_whitespace().nth(1)) .unwrap_or("/") .to_string(); - let content_type = header - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-type") - .then_some(value.trim().to_ascii_lowercase()) - }) - .unwrap_or_default(); let content_length = header .lines() .find_map(|line| { @@ -7368,17 +1815,6 @@ fn handle_otlp_connection( writeln!(log, "{}\t{}", path, body_len) .into_diagnostic() .wrap_err_with(|| format!("failed to append {}", requests_log.display()))?; - if path == "/v1/logs" { - let body = &buf[body_start..]; - if content_type.contains("application/json") || body.starts_with(b"{") { - let _ = serde_json::from_slice::(body); - } else { - match persist_otlp_log_records(body) { - Ok(events) => append_jsonl(events_ndjson, &events)?, - Err(err) => eprintln!("warning: failed to decode OTLP log payload: {err}"), - } - } - } stream .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") .into_diagnostic() @@ -7386,369 +1822,38 @@ fn handle_otlp_connection( Ok(()) } -fn append_jsonl(path: &Path, events: &[PersistedTraceEvent]) -> Result<()> { - if events.is_empty() { - return Ok(()); - } - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - let mut log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to open {}", path.display()))?; - for event in events { - serde_json::to_writer(&mut log, event) - .into_diagnostic() - .wrap_err_with(|| format!("failed to append {}", path.display()))?; - writeln!(log) - .into_diagnostic() - .wrap_err_with(|| format!("failed to append {}", path.display()))?; - } - Ok(()) -} - -fn persist_otlp_log_records(body: &[u8]) -> Result> { - let request = ExportLogsServiceRequest::decode(body) - .into_diagnostic() - .wrap_err("invalid OTLP log payload")?; - let mut events = Vec::new(); - for resource_logs in &request.resource_logs { - let resource_attrs = resource_logs - .resource - .as_ref() - .map(|resource| resource.attributes.as_slice()) - .unwrap_or(&[]); - for scope_logs in &resource_logs.scope_logs { - for record in &scope_logs.log_records { - if let Some(event) = persist_otlp_log_record(resource_attrs, record) { - events.push(event); - } - } - } - } - Ok(events) -} - -fn persist_otlp_log_record( - resource_attrs: &[OtlpKeyValue], - record: &OtlpLogRecord, -) -> Option { - let entity_kind = otlp_attribute(resource_attrs, &record.attributes, "amber_entity_kind"); - let edge_ref = otlp_attribute(resource_attrs, &record.attributes, "amber_edge_ref"); - if entity_kind.as_deref() != Some("binding") { - return None; - } - if edge_ref - .as_deref() - .is_some_and(|edge_ref| edge_ref.starts_with("/site/")) - { - return None; - } - - let body = record.body.as_ref().map(otlp_any_value_to_text)?; - let body = body.trim(); - if body.is_empty() { - return None; - } - - Some(PersistedTraceEvent { - schema: PERSISTED_TRACE_EVENT_SCHEMA.to_string(), - version: PERSISTED_TRACE_EVENT_VERSION, - observed_at_unix_nano: nonzero_u64(record.time_unix_nano) - .or_else(|| nonzero_u64(record.observed_time_unix_nano)), - message: body.to_string(), - severity: nonempty(record.severity_text.trim()), - trace_id: otlp_trace_id(&record.trace_id), - attributes: merged_otlp_attributes(resource_attrs, &record.attributes), - }) -} - -fn merged_otlp_attributes( - resource_attrs: &[OtlpKeyValue], - record_attrs: &[OtlpKeyValue], -) -> BTreeMap { - let mut attributes = BTreeMap::new(); - for attr in resource_attrs { - if let Some(value) = attr.value.as_ref() { - attributes.insert(attr.key.clone(), otlp_any_value_to_json(value)); - } - } - for attr in record_attrs { - if let Some(value) = attr.value.as_ref() { - attributes.insert(attr.key.clone(), otlp_any_value_to_json(value)); - } - } - attributes -} - -fn nonzero_u64(value: u64) -> Option { - (value != 0).then_some(value) -} - -fn nonempty(value: &str) -> Option { - (!value.is_empty()).then_some(value.to_string()) -} - -fn otlp_attribute( - resource_attrs: &[OtlpKeyValue], - record_attrs: &[OtlpKeyValue], - key: &str, -) -> Option { - if let Some(value) = otlp_attribute_from(record_attrs, key) { - return Some(value); - } - otlp_attribute_from(resource_attrs, key) -} - -fn otlp_attribute_from(attrs: &[OtlpKeyValue], key: &str) -> Option { - attrs.iter().find_map(|attr| { - (attr.key == key) - .then(|| attr.value.as_ref().map(otlp_any_value_to_text)) - .flatten() - }) -} - -fn otlp_any_value_to_text(value: &OtlpAnyValue) -> String { - match value.value.as_ref() { - Some(any_value::Value::StringValue(value)) => value.clone(), - Some(any_value::Value::BoolValue(value)) => value.to_string(), - Some(any_value::Value::IntValue(value)) => value.to_string(), - Some(any_value::Value::DoubleValue(value)) => value.to_string(), - Some(any_value::Value::BytesValue(value)) => format!("0x{}", hex_bytes(value)), - Some(any_value::Value::ArrayValue(values)) => values - .values - .iter() - .map(otlp_any_value_to_text) - .collect::>() - .join(", "), - Some(any_value::Value::KvlistValue(values)) => values - .values - .iter() - .map(|entry| { - format!( - "{}={}", - entry.key, - entry - .value - .as_ref() - .map(otlp_any_value_to_text) - .unwrap_or_default() - ) - }) - .collect::>() - .join(", "), - None => String::new(), - } -} - -fn otlp_any_value_to_json(value: &OtlpAnyValue) -> JsonValue { - match value.value.as_ref() { - Some(any_value::Value::StringValue(value)) => JsonValue::String(value.clone()), - Some(any_value::Value::BoolValue(value)) => JsonValue::Bool(*value), - Some(any_value::Value::IntValue(value)) => JsonValue::Number((*value).into()), - Some(any_value::Value::DoubleValue(value)) => serde_json::Number::from_f64(*value) - .map(JsonValue::Number) - .unwrap_or(JsonValue::Null), - Some(any_value::Value::BytesValue(value)) => { - JsonValue::String(format!("0x{}", hex_bytes(value))) - } - Some(any_value::Value::ArrayValue(values)) => JsonValue::Array( - values - .values - .iter() - .map(otlp_any_value_to_json) - .collect::>(), - ), - Some(any_value::Value::KvlistValue(values)) => JsonValue::Object( - values - .values - .iter() - .filter_map(|entry| { - Some(( - entry.key.clone(), - otlp_any_value_to_json(entry.value.as_ref()?), - )) - }) - .collect::>(), - ), - None => JsonValue::Null, - } -} - -fn otlp_trace_id(trace_id: &[u8]) -> Option { - if trace_id.len() != 16 || trace_id.iter().all(|byte| *byte == 0) { - return None; - } - Some(hex_bytes(trace_id)) -} - -fn hex_bytes(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len() * 2); - for byte in bytes { - write!(&mut out, "{byte:02x}").expect("writing to an in-memory string should not fail"); - } - out -} - -#[cfg(unix)] -fn send_sigkill(pid: u32) { - let _ = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; -} - #[cfg(unix)] async fn shutdown_recorded_processes(site: &SiteReceipt) -> Result<()> { - let mut roots = Vec::new(); - if let Some(pid) = site.process_pid { - roots.push(pid); - } - if let Some(pid) = site.port_forward_pid { - roots.push(pid); - } - terminate_recorded_processes(&roots).await + terminate_recorded_processes(&recorded_process_roots(site)).await } #[cfg(not(unix))] async fn shutdown_recorded_processes(site: &SiteReceipt) -> Result<()> { - if let Some(pid) = site.process_pid { + for pid in recorded_process_roots(site) { send_sigterm(pid); } - if let Some(pid) = site.port_forward_pid { - send_sigterm(pid); - } - Ok(()) -} - -#[cfg(unix)] -async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { - let mut seen = BTreeSet::new(); - let mut ordered = Vec::new(); - for root_pid in root_pids { - for pid in process_tree_postorder(*root_pid)? { - if seen.insert(pid) { - ordered.push(pid); - } - } - } - if ordered.is_empty() { - return Ok(()); - } - for pid in &ordered { - send_sigterm(*pid); - } - wait_for_pids_exit(&ordered, PROCESS_SHUTDOWN_GRACE_PERIOD).await; - let survivors = ordered - .iter() - .copied() - .filter(|pid| pid_is_alive(*pid)) - .collect::>(); - for pid in &survivors { - send_sigkill(*pid); - } - wait_for_pids_exit(&survivors, Duration::from_secs(2)).await; Ok(()) } -#[cfg(unix)] -fn process_status_code(pid: u32) -> Option { - let output = Command::new("ps") - .arg("-o") - .arg("stat=") - .arg("-p") - .arg(pid.to_string()) - .output() - .ok()?; - if !output.status.success() { - return None; +fn recorded_process_roots(site: &SiteReceipt) -> Vec { + let mut roots = vec![site.supervisor_pid]; + if let Some(pid) = site.process_pid { + roots.push(pid); } - parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) -} - -#[cfg(unix)] -fn parse_process_status_code(raw: &str) -> Option { - raw.split_whitespace() - .next()? - .chars() - .next() - .map(|state| state.to_ascii_uppercase()) -} - -#[cfg(unix)] -async fn wait_for_pids_exit(pids: &[u32], timeout: Duration) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if pids.iter().all(|pid| !pid_is_alive(*pid)) { - return; - } - sleep(Duration::from_millis(100)).await; + if let Some(pid) = site.port_forward_pid { + roots.push(pid); } + roots } #[cfg(unix)] -fn process_tree_postorder(root_pid: u32) -> Result> { - let output = Command::new("ps") - .arg("-axo") - .arg("pid=,ppid=") - .output() - .into_diagnostic() - .wrap_err("failed to enumerate process tree")?; - if !output.status.success() { - return Err(miette::miette!( - "failed to enumerate process tree: status {}", - output.status - )); - } - - let parent_by_pid = parse_process_table(&String::from_utf8_lossy(&output.stdout))?; - let mut children_by_parent = HashMap::>::new(); - for (pid, ppid) in parent_by_pid { - children_by_parent.entry(ppid).or_default().push(pid); - } - - let mut ordered = Vec::new(); - collect_process_tree_postorder(root_pid, &children_by_parent, &mut ordered); - Ok(ordered) +async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { + terminate_process_roots(root_pids, PROCESS_SHUTDOWN_GRACE_PERIOD).await } #[cfg(unix)] -fn collect_process_tree_postorder( - pid: u32, - children_by_parent: &HashMap>, - ordered: &mut Vec, -) { - if let Some(children) = children_by_parent.get(&pid) { - for child in children { - collect_process_tree_postorder(*child, children_by_parent, ordered); - } - } - ordered.push(pid); -} - -fn parse_process_table(raw: &str) -> Result> { - let mut parent_by_pid = HashMap::new(); - for line in raw.lines() { - let mut fields = line.split_whitespace(); - let Some(pid) = fields.next() else { - continue; - }; - let Some(ppid) = fields.next() else { - continue; - }; - let pid = pid - .parse::() - .into_diagnostic() - .wrap_err_with(|| format!("invalid process table pid `{pid}`"))?; - let ppid = ppid - .parse::() - .into_diagnostic() - .wrap_err_with(|| format!("invalid process table parent pid `{ppid}`"))?; - parent_by_pid.insert(pid, ppid); - } - Ok(parent_by_pid) +fn send_sigkill(pid: u32) { + let _ = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; } fn find_header_end(buf: &[u8]) -> Option { diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 975fb065..9def352f 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -1,5 +1,164 @@ use super::*; +pub(super) fn site_controller_image_reference_from_overrides( + overrides: &BTreeMap, +) -> String { + overrides + .get("site_controller") + .map(|tag| amber_images::override_reference(&amber_images::AMBER_SITE_CONTROLLER, tag)) + .unwrap_or_else(|| amber_images::AMBER_SITE_CONTROLLER.reference.to_string()) +} + +pub(crate) fn site_controller_image_reference() -> Result { + let overrides = + amber_images::parse_dev_image_tag_overrides(amber_images::INTERNAL_IMAGE_OVERRIDE_KEYS) + .map_err(|err| miette::miette!(err))?; + Ok(site_controller_image_reference_from_overrides(&overrides)) +} + +pub(crate) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { + match kind { + SiteKind::Direct => format!( + "unix://{}", + direct_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Vm => format!( + "unix://{}", + vm_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Compose => "unix:///amber/control/router-control.sock".to_string(), + SiteKind::Kubernetes => "amber-router:24100".to_string(), + } +} + +pub(crate) fn site_controller_component_port(site_plan: &RunSitePlan) -> Result> { + let scenario = + amber_scenario::Scenario::try_from(site_plan.scenario_ir.clone()).map_err(|err| { + miette::miette!( + "failed to parse site scenario for router `{}`: {err}", + site_plan.router_identity_id + ) + })?; + let mut controller_ports = Vec::new(); + for (_, component) in scenario.components_iter() { + if amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_none() + { + continue; + } + let port = component + .program + .as_ref() + .and_then(|program| program.network()) + .and_then(|network| network.endpoints.first()) + .map(|endpoint| endpoint.port) + .ok_or_else(|| { + miette::miette!( + "synthetic framework.component controller `{}` on site `{}` is missing its \ + endpoint", + component.moniker, + site_plan.router_identity_id + ) + })?; + controller_ports.push((component.moniker.clone(), port)); + } + match controller_ports.as_slice() { + [] => Ok(None), + [(_, port)] => Ok(Some(*port)), + _ => Err(miette::miette!( + "site `{}` has multiple synthetic framework.component controllers: {}", + site_plan.router_identity_id, + controller_ports + .iter() + .map(|(moniker, _)| moniker.as_str()) + .collect::>() + .join(", ") + )), + } +} + +pub(crate) fn site_controller_identity_path( + kind: SiteKind, + artifact_dir: &Path, + runtime_root: Option<&str>, +) -> Result> { + match kind { + SiteKind::Compose | SiteKind::Kubernetes => Ok(Some( + amber_site_controller::SITE_CONTROLLER_MESH_IDENTITY_PATH.to_string(), + )), + SiteKind::Direct => { + let Some(runtime_root) = runtime_root else { + return Ok(None); + }; + let direct_plan_path = + artifact_dir.join(amber_compiler::reporter::direct::DIRECT_PLAN_FILENAME); + let direct_plan: amber_compiler::reporter::direct::DirectPlan = + read_json(&direct_plan_path, "direct plan")?; + let controller = direct_plan + .components + .iter() + .find(|component| { + matches!( + component.program.execution, + amber_compiler::reporter::direct::DirectProgramExecutionPlan::InternalSiteController + ) + }) + .ok_or_else(|| { + miette::miette!( + "direct site controller plan is missing its internal site controller \ + component" + ) + })?; + Ok(Some( + Path::new(runtime_root) + .join(&controller.sidecar.mesh_identity_path) + .display() + .to_string(), + )) + } + SiteKind::Vm => Ok(None), + } +} + +pub(crate) fn prepare_site_state_root(site_state_root: &Path, kind: SiteKind) -> Result<()> { + fs::create_dir_all(site_state_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create site state directory {}", + site_state_root.display() + ) + })?; + + #[cfg(unix)] + if kind == SiteKind::Compose { + use std::os::unix::fs::PermissionsExt as _; + + let mut permissions = fs::metadata(site_state_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to read permissions for site state directory {}", + site_state_root.display() + ) + })? + .permissions(); + permissions.set_mode(0o700); + fs::set_permissions(site_state_root, permissions) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to restrict compose site state directory {}", + site_state_root.display() + ) + })?; + } + + Ok(()) +} + pub(crate) fn dry_run_run_plan( source_plan_path: Option<&Path>, run_plan: &RunPlan, @@ -52,9 +211,7 @@ pub(super) fn materialize_launch_bundle( let run_plan_path = run_plan_path(bundle_root); write_json(&run_plan_path, run_plan)?; - - let framework_control_state = - materialize_framework_control_state(run_plan, &state_root, run_id)?; + let site_controller_image = site_controller_image_reference()?; let observability = materialize_observability(bundle_root, run_id, &run_plan.mesh_scope, observability)?; @@ -63,49 +220,38 @@ pub(super) fn materialize_launch_bundle( .map(|materialized| materialized.receipt.endpoint.as_str()); let mut sites = BTreeMap::new(); - for (site_id, site_plan) in &run_plan.sites { + for (site_index, (site_id, site_plan)) in run_plan.sites.iter().enumerate() { let artifact_dir = materialize_site_artifacts(&sites_root, site_id, site_plan)?; + amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_dir, run_id)?; let site_state_root = state_root.join(site_id); - let mut framework_env = BTreeMap::new(); - let framework_ccs_plan_path = if let Some(control_state) = framework_control_state.as_ref() - { - let port = reserve_loopback_port()?; - let listen_addr = - crate::framework_component::ccs_listen_addr_for_site(site_plan.site.kind, port); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), - crate::framework_component::ccs_url_for_site(site_plan.site.kind, port), - ); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV.to_string(), - control_state.router_auth_token.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV.to_string(), - control_state.receipt.url.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV.to_string(), - control_state.control_state_auth_token.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV.to_string(), - control_state.dynamic_caps_token_verify_key_b64.clone(), - ); - let plan_path = site_state_root.join("framework-ccs-plan.json"); - crate::framework_component::write_framework_ccs_plan( - &plan_path, - site_id, - &site_state_root, + prepare_site_state_root(&site_state_root, site_plan.site.kind)?; + let controller = site_controller_component_port(site_plan)?.map(|controller_port| { + let listen_addr = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => { + host_service_bind_addr_for_consumer(site_plan.site.kind, controller_port) + } + SiteKind::Compose | SiteKind::Kubernetes => { + SocketAddr::from(([0, 0, 0, 0], controller_port)) + } + }; + let url = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => { + amber_site_controller::authority_url_for_listen_addr(listen_addr) + } + SiteKind::Compose | SiteKind::Kubernetes => format!( + "http://{}:{}", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME, + controller_port + ), + }; + MaterializedSiteController { + state_path: site_state_root.join("site-controller-state.json"), + plan_path: site_controller_plan_path(&site_state_root), listen_addr, - control_state.receipt.url.as_str(), - &control_state.router_auth_token, - &control_state.control_state_auth_token, - )?; - Some(plan_path) - } else { - None - }; + url, + } + }); + let framework_env = BTreeMap::new(); patch_site_artifacts( &artifact_dir, run_id, @@ -114,16 +260,6 @@ pub(super) fn materialize_launch_bundle( &merge_env_maps(runtime_env, &framework_env), observability_endpoint, )?; - let site_actuator_plan_path = if framework_control_state.is_some() - && run_plan - .dynamic_enabled_sites - .iter() - .any(|active| active == site_id) - { - Some(site_actuator_plan_path(&site_state_root)) - } else { - None - }; let launch_env = launch_env( run_id, &run_plan.mesh_scope, @@ -142,41 +278,30 @@ pub(super) fn materialize_launch_bundle( artifact_dir: &artifact_dir, site_state_root: &site_state_root, observability_endpoint, - framework_ccs_plan_path: framework_ccs_plan_path.as_deref(), - site_actuator_plan_path: site_actuator_plan_path.as_deref(), + site_controller_url: controller + .as_ref() + .map(|controller| controller.url.as_str()), }, launch_env.clone(), )?; - if let Some(site_actuator_plan_path) = site_actuator_plan_path.as_ref() { - write_site_actuator_plan( - site_actuator_plan_path, - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, - run_id: run_id.to_string(), - mesh_scope: run_plan.mesh_scope.clone(), - run_root: bundle_root.display().to_string(), - site_id: site_id.clone(), - kind: site_plan.site.kind, - router_identity_id: site_plan.router_identity_id.clone(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], reserve_loopback_port()?)), - storage_root: base_supervisor_plan.storage_root.clone(), - runtime_root: base_supervisor_plan.runtime_root.clone(), - router_mesh_port: base_supervisor_plan.router_mesh_port, - compose_project: base_supervisor_plan.compose_project.clone(), - kubernetes_namespace: base_supervisor_plan.kubernetes_namespace.clone(), - context: base_supervisor_plan.context.clone(), - observability_endpoint: base_supervisor_plan.observability_endpoint.clone(), - launch_env: launch_env.clone(), - }, + if site_plan.site.kind == SiteKind::Compose + && let Some(router_mesh_port) = base_supervisor_plan.router_mesh_port + { + amber_site_controller::set_compose_router_published_mesh_port( + &artifact_dir, + router_mesh_port, )?; } - write_json( - &site_supervisor_plan_path(&site_state_root), - &base_supervisor_plan, - )?; + if let Some(controller) = &controller { + let controller_state = amber_site_controller::build_site_controller_state( + run_id, + run_plan, + site_id, + site_index, + run_plan.sites.len(), + )?; + amber_site_controller::write_control_state(&controller.state_path, &controller_state)?; + } write_json( &desired_links_path(&site_state_root), &DesiredLinkState { @@ -194,11 +319,179 @@ pub(super) fn materialize_launch_bundle( site_plan: site_plan.clone(), artifact_dir, site_state_root, + controller, base_supervisor_plan, }, ); } + let planned_router_mesh_addrs = sites + .iter() + .map(|(site_id, site)| { + Ok(( + site_id.clone(), + planned_router_mesh_addr(&site.base_supervisor_plan)?, + )) + }) + .collect::>>()?; + let controller_sites = sites + .iter() + .filter(|(_, site)| site.controller.is_some()) + .map(|(site_id, site)| { + ( + site_id.clone(), + site.site_plan.router_identity_id.clone(), + site.site_plan.site.kind, + ) + }) + .collect::>(); + + for (site_id, site) in &mut sites { + if let Some(controller) = &site.controller { + let allowed_issuers = controller_sites + .iter() + .filter(|(peer_site_id, _, _)| peer_site_id != site_id) + .map(|(_, router_identity_id, _)| router_identity_id.clone()) + .collect::>(); + let route_listen_addr = + site_controller_route_listen_addr(site.site_plan.site.kind).to_string(); + let mut peer_site_router_urls = BTreeMap::new(); + let mut peer_router_identities = BTreeMap::new(); + let mut peer_router_mesh_addrs = BTreeMap::new(); + let mut controller_routes = Vec::new(); + for (peer_site_id, peer_router_identity_id, peer_site_kind) in &controller_sites { + if peer_site_id == site_id { + continue; + } + let route_port = reserve_loopback_port()?; + peer_site_router_urls.insert( + peer_site_id.clone(), + amber_site_controller::site_controller_peer_router_url( + site.site_plan.site.kind, + route_port, + ), + ); + peer_router_identities.insert( + peer_site_id.clone(), + planned_router_identity(run_id, &run_plan.mesh_scope, peer_router_identity_id), + ); + let peer_router_mesh_addr = planned_router_mesh_addrs + .get(peer_site_id) + .expect("planned router mesh addr should exist for every site"); + peer_router_mesh_addrs.insert(peer_site_id.clone(), peer_router_mesh_addr.clone()); + let peer_addr = amber_site_controller::router_mesh_addr_for_consumer( + *peer_site_kind, + site.site_plan.site.kind, + peer_router_mesh_addr, + )?; + controller_routes.push(amber_site_controller::SiteControllerPeerRouterRoute { + site_id: peer_site_id.clone(), + peer_router: planned_router_identity( + run_id, + &run_plan.mesh_scope, + peer_router_identity_id, + ), + peer_addr, + listen_addr: route_listen_addr.clone(), + listen_port: route_port, + }); + } + + write_json( + &site_existing_peer_ports_path(&site.site_state_root), + &peer_router_ports_by_identity(&peer_router_identities, &peer_router_mesh_addrs)?, + )?; + write_json( + &site_existing_peer_identities_path(&site.site_state_root), + &peer_router_identities_by_id(&peer_router_identities), + )?; + + amber_site_controller::inject_site_controller_peer_router_routes( + &site.artifact_dir, + site_id, + &allowed_issuers, + &controller_routes, + )?; + let local_router_control = + site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); + let published_router_mesh_addr = + planned_router_mesh_addrs.get(site_id).map(String::as_str); + let compose_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Compose, + addr, + ) + }) + .transpose()?; + let kubernetes_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Kubernetes, + addr, + ) + }) + .transpose()?; + + amber_site_controller::write_site_controller_plan( + &controller.plan_path, + run_id, + &run_plan.mesh_scope, + site_id, + site.site_plan.site.kind, + controller.listen_addr, + &controller.url, + &site.site_plan.router_identity_id, + &peer_site_router_urls, + &peer_router_identities, + &peer_router_mesh_addrs, + Some(local_router_control.as_str()), + published_router_mesh_addr, + compose_consumer_router_mesh_addr.as_deref(), + kubernetes_consumer_router_mesh_addr.as_deref(), + &controller.state_path, + bundle_root, + &state_root, + &site.site_state_root, + &site.artifact_dir, + site_controller_identity_path( + site.site_plan.site.kind, + &site.artifact_dir, + site.base_supervisor_plan.runtime_root.as_deref(), + )? + .as_deref(), + site.base_supervisor_plan.storage_root.as_deref(), + site.base_supervisor_plan.runtime_root.as_deref(), + site.base_supervisor_plan.router_mesh_port, + site.base_supervisor_plan.compose_project.as_deref(), + site.base_supervisor_plan.kubernetes_namespace.as_deref(), + site.base_supervisor_plan.context.as_deref(), + site.base_supervisor_plan.observability_endpoint.as_deref(), + &site.base_supervisor_plan.launch_env, + )?; + match site.site_plan.site.kind { + SiteKind::Compose => amber_site_controller::inject_compose_site_controller( + &site.artifact_dir, + &read_json(&controller.plan_path, "site controller plan")?, + &controller.plan_path, + site_controller_image.as_str(), + )?, + SiteKind::Kubernetes => amber_site_controller::inject_kubernetes_site_controller( + &site.artifact_dir, + &read_json(&controller.plan_path, "site controller plan")?, + site_controller_image.as_str(), + )?, + SiteKind::Direct | SiteKind::Vm => {} + } + } + write_json( + &site_supervisor_plan_path(&site.site_state_root), + &site.base_supervisor_plan, + )?; + } + let manifest = build_launch_bundle_manifest( run_id, source_plan_path, @@ -212,60 +505,78 @@ pub(super) fn materialize_launch_bundle( Ok(MaterializedLaunchBundle { run_plan_path, - framework_control_state, observability, sites, }) } -fn materialize_framework_control_state( - run_plan: &RunPlan, - state_root: &Path, - run_id: &str, -) -> Result> { - let control_state = crate::framework_component::build_control_state(run_id, run_plan)?; - let root = state_root.join("framework-component"); - let state_path = root.join("control-state.json"); - crate::framework_component::write_control_state(&state_path, &control_state)?; - let router_auth_token = - crate::framework_component::generate_framework_auth_token(&run_plan.mesh_scope, "router"); - let control_state_auth_token = crate::framework_component::generate_framework_auth_token( - &run_plan.mesh_scope, - "control-state", - ); +fn planned_router_mesh_addr(plan: &SiteSupervisorPlan) -> Result { + match plan.kind { + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => plan + .router_mesh_port + .map(|port| format!("127.0.0.1:{port}")) + .ok_or_else(|| { + miette::miette!("site `{}` is missing its router mesh port", plan.site_id) + }), + SiteKind::Kubernetes => plan + .port_forward_mesh_port + .map(|port| format!("127.0.0.1:{port}")) + .ok_or_else(|| { + miette::miette!( + "site `{}` is missing its kubernetes router mesh forward port", + plan.site_id + ) + }), + } +} - let listen_addr = SocketAddr::from(([127, 0, 0, 1], reserve_loopback_port()?)); - let plan_path = root.join("control-state-plan.json"); - crate::framework_component::write_control_state_service_plan( - &plan_path, - listen_addr, - &state_path, - state_root.parent().unwrap_or(state_root), - state_root, - &run_plan.mesh_scope, - &control_state_auth_token, - )?; - let dynamic_caps_token_verify_key_b64 = amber_mesh::dynamic_caps::verify_key_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &control_state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { - miette::miette!( - "framework control-state dynamic capability signing seed is invalid: {err}" - ) - })?, - ); +fn site_controller_route_listen_addr(kind: SiteKind) -> &'static str { + match kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1", + SiteKind::Compose | SiteKind::Kubernetes => "0.0.0.0", + } +} - Ok(Some(MaterializedFrameworkControlState { - plan_path, - receipt: FrameworkControlStateReceipt { - pid: 0, - url: crate::framework_component::control_state_service_url(listen_addr), - }, - router_auth_token, - control_state_auth_token, - dynamic_caps_token_verify_key_b64, - })) +fn planned_router_identity( + identity_seed: &str, + mesh_scope: &str, + router_identity_id: &str, +) -> amber_mesh::MeshIdentityPublic { + amber_mesh::MeshIdentityPublic::from_identity(&amber_mesh::MeshIdentity::derive( + router_identity_id, + Some(mesh_scope.to_string()), + identity_seed, + )) +} + +fn peer_router_ports_by_identity( + peer_router_identities: &BTreeMap, + peer_router_mesh_addrs: &BTreeMap, +) -> Result> { + peer_router_identities + .iter() + .map(|(site_id, identity)| { + let addr = peer_router_mesh_addrs.get(site_id).ok_or_else(|| { + miette::miette!("missing planned router mesh address for peer site `{site_id}`") + })?; + let port = addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid planned router mesh address `{addr}`"))? + .port(); + Ok((identity.id.clone(), port)) + }) + .collect() +} + +fn peer_router_identities_by_id( + peer_router_identities: &BTreeMap, +) -> BTreeMap { + peer_router_identities + .values() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect() } pub(super) fn build_launch_bundle_manifest( @@ -384,7 +695,7 @@ pub(super) fn site_launch_preview(plan: &SiteSupervisorPlan) -> Result { @@ -516,10 +827,9 @@ pub(super) fn preview_external_slot_url( } pub(super) fn site_launch_commands(plan: &SiteSupervisorPlan) -> Result> { - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; - Ok(match plan.kind { + let exe = super::amber_cli_executable()?; + let mut commands = Vec::new(); + let site_commands = match plan.kind { SiteKind::Direct => { let mut argv = vec![ exe.display().to_string(), @@ -657,7 +967,9 @@ pub(super) fn site_launch_commands(plan: &SiteSupervisorPlan) -> Result, -) -> Result> { - let Some(framework_control_state) = framework_control_state else { - return Ok(None); - }; - - let mut child = spawn_detached_child( - run_root, - &run_root - .join("state") - .join("framework-component") - .join("control-state.log"), - |cmd| { - cmd.arg("run-framework-control-state") - .arg("--plan") - .arg(&framework_control_state.plan_path); - }, - )?; - let listen_addr = framework_control_state - .receipt - .url - .strip_prefix("http://") - .ok_or_else(|| { - miette::miette!( - "framework control-state receipt url `{}` is not an http:// socket address", - framework_control_state.receipt.url - ) - })? - .parse::() - .into_diagnostic() - .wrap_err("invalid framework control-state listen address")?; - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - if let Some(status) = child.try_wait().into_diagnostic()? { - return Err(miette::miette!( - "framework control-state service exited before becoming ready with status {status}" - )); - } - if wait_for_http_response(listen_addr, Duration::from_millis(250)).is_ok() { - let mut receipt = framework_control_state.receipt.clone(); - receipt.pid = child.id(); - return Ok(Some(receipt)); - } - sleep(Duration::from_millis(100)).await; - } - Err(miette::miette!( - "timed out waiting for framework control-state service" - )) -} - pub(super) fn prepare_site_launch( site: &MaterializedSite, runtime_env: &BTreeMap, @@ -949,11 +1207,9 @@ pub(crate) async fn run_run_plan_with_id( ], ); - let mut framework_control_state_receipt = None; let mut launched_by_site = BTreeMap::::new(); let mut started_site_receipts = BTreeMap::::new(); let mut supervisor_children = BTreeMap::::new(); - let mut bridge_proxies = BTreeMap::::new(); let test_wave_delay = test_wave_delay()?; let result = async { @@ -992,7 +1248,6 @@ pub(crate) async fn run_run_plan_with_id( &launched_by_site, &run_root, &state_root, - &mut bridge_proxies, ) .await?; @@ -1010,13 +1265,6 @@ pub(crate) async fn run_run_plan_with_id( sleep(delay).await; } } - - framework_control_state_receipt = start_materialized_framework_control_state( - &run_root, - launch_bundle.framework_control_state.as_ref(), - ) - .await?; - write_commit_marker(&run_root)?; emit_manager_event( "amber.run.committed", @@ -1032,16 +1280,7 @@ pub(crate) async fn run_run_plan_with_id( plan_path: launch_bundle.run_plan_path.display().to_string(), source_plan_path: source_plan_path.map(|path| path.display().to_string()), run_root: run_root.display().to_string(), - framework_control_state: framework_control_state_receipt.clone(), observability: observability_receipt.clone(), - bridge_proxies: bridge_proxies - .values() - .map(|proxy| BridgeProxyReceipt { - export_name: proxy.export_name.clone(), - pid: proxy.child.id(), - listen: proxy.listen.to_string(), - }) - .collect(), sites: launched_by_site .into_iter() .map(|(site_id, launched)| (site_id, launched.receipt)) @@ -1054,15 +1293,9 @@ pub(crate) async fn run_run_plan_with_id( if result.is_err() { let _ = write_stop_marker(&run_root); - for bridge in bridge_proxies.values_mut() { - send_sigterm(bridge.child.id()); - } for supervisor in supervisor_children.values_mut() { send_sigterm(supervisor.child.id()); } - for bridge in bridge_proxies.values_mut() { - let _ = wait_for_child_exit(&mut bridge.child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; - } for supervisor in supervisor_children.values_mut() { let _ = wait_for_child_exit(&mut supervisor.child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; } @@ -1098,8 +1331,15 @@ pub(crate) async fn run_run_plan_with_id( context: receipt.context.clone(), router_control: receipt.router_control.clone(), router_mesh_addr: receipt.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: receipt + .compose_consumer_router_mesh_addr + .clone(), + kubernetes_consumer_router_mesh_addr: receipt + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: receipt.router_identity_id.clone(), router_public_key_b64: receipt.router_public_key_b64.clone(), + site_controller_url: receipt.site_controller_url.clone(), last_error: Some("coordinator cleanup after failed startup".to_string()), }, ); @@ -1110,12 +1350,6 @@ pub(crate) async fn run_run_plan_with_id( { send_sigterm(pid); } - if let Some(pid) = framework_control_state_receipt - .as_ref() - .map(|value| value.pid) - { - send_sigterm(pid); - } } result @@ -1216,10 +1450,6 @@ pub(crate) async fn stop_run(run_id: &str, storage_root_override: Option<&Path>) { send_sigterm(pid); } - if let Some(framework_control_state) = receipt.framework_control_state.as_ref() { - send_sigterm(framework_control_state.pid); - } - if let Some(observability) = receipt.observability.as_ref() && let Some(pid) = observability.sink_pid && !wait_for_pid_exit(pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await @@ -1229,28 +1459,6 @@ pub(crate) async fn stop_run(run_id: &str, storage_root_override: Option<&Path>) PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() )); } - if let Some(framework_control_state) = receipt.framework_control_state.as_ref() - && !wait_for_pid_exit(framework_control_state.pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await - { - shutdown_failures.push(format!( - "framework control-state service (pid {}) did not stop within {}s", - framework_control_state.pid, - PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() - )); - } - for proxy in &receipt.bridge_proxies { - send_sigterm(proxy.pid); - } - for proxy in &receipt.bridge_proxies { - if !wait_for_pid_exit(proxy.pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await { - shutdown_failures.push(format!( - "bridge proxy `{}` (pid {}) did not stop within {}s", - proxy.export_name, - proxy.pid, - PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() - )); - } - } if !shutdown_failures.is_empty() { return Err(miette::miette!( diff --git a/cli/src/mixed_run/outside_proxy.rs b/cli/src/mixed_run/outside_proxy.rs index b7ab1a42..aa3ebf50 100644 --- a/cli/src/mixed_run/outside_proxy.rs +++ b/cli/src/mixed_run/outside_proxy.rs @@ -344,22 +344,13 @@ pub(super) async fn stop_site_from_receipt( SiteKind::Compose => { if let Some(project_name) = site.compose_project.as_deref() { let launch_env = read_compose_launch_env(run_root, site_id)?; - let status = compose_command(Some(project_name), Path::new(&site.artifact_dir)) - .envs(launch_env) - .arg("down") - .arg("-v") - .arg("--remove-orphans") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop orphaned compose site `{project_name}`") - })?; - if !status.success() { - return Err(miette::miette!( - "orphaned compose site `{project_name}` failed to stop with status \ - {status}" - )); - } + stop_compose_site_with_docker( + Path::new("docker"), + project_name, + Path::new(&site.artifact_dir), + &launch_env, + Duration::from_secs(30), + )?; } } SiteKind::Kubernetes => { @@ -367,22 +358,14 @@ pub(super) async fn stop_site_from_receipt( send_sigterm(pid); } if let Some(namespace) = site.kubernetes_namespace.as_deref() { - let status = kubectl_command(site.context.as_deref()) - .arg("delete") - .arg("namespace") - .arg(namespace) - .arg("--ignore-not-found") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop orphaned kubernetes site `{namespace}`") - })?; - if !status.success() { - return Err(miette::miette!( - "orphaned kubernetes site `{namespace}` failed to stop with status \ - {status}" - )); - } + stop_kubernetes_namespace( + site.context.as_deref(), + namespace, + KUBERNETES_NAMESPACE_DELETE_TIMEOUT, + ) + .wrap_err_with(|| { + format!("failed to stop orphaned kubernetes site `{namespace}`") + })?; } } } @@ -390,6 +373,74 @@ pub(super) async fn stop_site_from_receipt( Ok(()) } +fn stop_compose_site_with_docker( + docker_bin: &Path, + project_name: &str, + artifact_dir: &Path, + launch_env: &BTreeMap, + timeout: Duration, +) -> Result<()> { + let status = compose_command_with_docker_bin(docker_bin, Some(project_name), artifact_dir) + .envs(launch_env) + .arg("down") + .arg("-v") + .arg("--remove-orphans") + .status() + .into_diagnostic() + .wrap_err_with(|| format!("failed to stop orphaned compose site `{project_name}`"))?; + if !status.success() { + return Err(miette::miette!( + "orphaned compose site `{project_name}` failed to stop with status {status}" + )); + } + + let deadline = Instant::now() + timeout; + loop { + let output = compose_command_with_docker_bin(docker_bin, Some(project_name), artifact_dir) + .envs(launch_env) + .arg("ps") + .arg("-q") + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to verify orphaned compose site `{project_name}` is stopped") + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to verify orphaned compose site `{project_name}` is \ + stopped\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + if String::from_utf8_lossy(&output.stdout).trim().is_empty() { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for orphaned compose site `{project_name}` to fully stop" + )); + } + std::thread::sleep(Duration::from_millis(200)); + } +} + +fn compose_command_with_docker_bin( + docker_bin: &Path, + project_name: Option<&str>, + artifact_dir: &Path, +) -> Command { + let mut cmd = Command::new(docker_bin); + cmd.arg("compose") + .arg("-f") + .arg(artifact_dir.join("compose.yaml")); + if let Some(project_name) = project_name { + cmd.arg("-p").arg(project_name); + } + cmd.current_dir(artifact_dir); + cmd +} + pub(super) fn resolve_proxy_run_root( run_root: &Path, site_id: Option<&str>, @@ -641,3 +692,64 @@ pub(super) async fn wait_for_socket_listener(addr: SocketAddr) -> Result<()> { } Err(miette::miette!("timed out waiting for listener {}", addr)) } + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + + #[test] + fn stop_compose_site_waits_for_ps_to_empty_after_down_returns() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_dir = temp.path().join("artifact"); + fs::create_dir_all(&artifact_dir).expect("artifact dir"); + fs::write(artifact_dir.join("compose.yaml"), "services: {}\n").expect("compose file"); + + let docker = temp.path().join("docker"); + let log_path = temp.path().join("docker.log"); + let count_path = temp.path().join("ps-count"); + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\ncount_path='{}'\nprintf '%s\\n' \"$*\" >> \ + \"$log_path\"\nif [ \"${{1:-}}\" = compose ] && [ \"${{6:-}}\" = down ]; \ + then\nexit 0\nfi\nif [ \"${{1:-}}\" = compose ] && [ \"${{6:-}}\" = ps ] && [ \ + \"${{7:-}}\" = -q ]; then\ncount=0\nif [ -f \"$count_path\" ]; then\ncount=$(cat \ + \"$count_path\")\nfi\ncount=$((count + 1))\nprintf '%s' \"$count\" > \ + \"$count_path\"\nif [ \"$count\" -lt 3 ]; then\nprintf \ + 'container-id\\n'\nfi\nexit 0\nfi\necho \"unexpected docker invocation: $*\" \ + >&2\nexit 1\n", + log_path.display(), + count_path.display(), + ), + ) + .expect("docker stub"); + let mut perms = fs::metadata(&docker) + .expect("docker metadata") + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&docker, perms).expect("docker chmod"); + + stop_compose_site_with_docker( + &docker, + "amber-test-project", + &artifact_dir, + &BTreeMap::new(), + Duration::from_secs(1), + ) + .expect("compose stop should wait for project removal"); + + let log = fs::read_to_string(&log_path).expect("docker log"); + let ps_polls = log.lines().filter(|line| line.contains(" ps -q")).count(); + assert!( + log.lines() + .any(|line| line.contains(" down -v --remove-orphans")), + "expected compose down invocation in log:\n{log}" + ); + assert!( + ps_polls >= 3, + "expected repeated compose ps polling after down, got {ps_polls} polls:\n{log}" + ); + } +} diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 86356cc1..b66f719d 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -170,6 +170,18 @@ pub(super) fn launch_env( observability_endpoint: Option<&str>, ) -> Result> { let mut env = merge_env_maps(runtime_env, external_env); + if let Some(overrides) = std::env::var_os(amber_images::DEV_IMAGE_TAGS_ENV) + && let Some(overrides) = overrides.to_str() + { + env.entry(amber_images::DEV_IMAGE_TAGS_ENV.to_string()) + .or_insert_with(|| overrides.to_string()); + } + if let Some(force_tcg) = std::env::var_os("AMBER_VM_FORCE_TCG") + && let Some(force_tcg) = force_tcg.to_str() + { + env.entry("AMBER_VM_FORCE_TCG".to_string()) + .or_insert_with(|| force_tcg.to_string()); + } env.insert(SCENARIO_RUN_ID_ENV.to_string(), run_id.to_string()); env.insert(SCENARIO_SCOPE_ENV.to_string(), mesh_scope.to_string()); if let Some(endpoint) = observability_endpoint { @@ -228,8 +240,11 @@ pub(super) fn build_supervisor_plan( .then(|| input.site_state_root.join("storage").display().to_string()), runtime_root: matches!(input.site_plan.site.kind, SiteKind::Direct | SiteKind::Vm) .then(|| input.site_state_root.join("runtime").display().to_string()), - router_mesh_port: if matches!(input.site_plan.site.kind, SiteKind::Direct | SiteKind::Vm) { - Some(reserve_loopback_port()?) + router_mesh_port: if matches!( + input.site_plan.site.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose + ) { + Some(reserve_host_port()?) } else { None }, @@ -256,20 +271,11 @@ pub(super) fn build_supervisor_plan( None }, observability_endpoint: input.observability_endpoint.map(ToOwned::to_owned), - framework_ccs_plan_path: input - .framework_ccs_plan_path - .map(|path| path.display().to_string()), - site_actuator_plan_path: input - .site_actuator_plan_path - .map(|path| path.display().to_string()), + site_controller_url: input.site_controller_url.map(ToOwned::to_owned), launch_env, }) } -pub(super) fn write_site_actuator_plan(path: &Path, plan: &SiteActuatorPlan) -> Result<()> { - write_json(path, plan) -} - pub(super) fn spawn_site_supervisor(site_state_root: &Path) -> Result { let child = spawn_detached_child( site_state_root, @@ -345,9 +351,10 @@ pub(super) async fn wait_for_site_ready( )); } if Instant::now() >= deadline { + let diagnostics = + site_startup_timeout_diagnostics(site_plan, site_state_root, &state_path); return Err(miette::miette!( - "timed out waiting for site `{site_id}` to become ready{}", - site_startup_context(site_state_root, &state_path) + "timed out waiting for site `{site_id}` to become ready{diagnostics}" )); } sleep(Duration::from_millis(200)).await; @@ -423,7 +430,6 @@ pub(super) async fn register_new_site_links( launched_by_site: &BTreeMap, run_root: &Path, state_root: &Path, - bridge_proxies: &mut BTreeMap, ) -> Result<()> { for link in links { if link.consumer_site == site_id { @@ -436,14 +442,8 @@ pub(super) async fn register_new_site_links( link.provider_site )); }; - let external_url = resolve_link_external_url( - provider, - link, - launched.receipt.kind, - run_root, - bridge_proxies, - ) - .await?; + let external_url = + resolve_link_external_url(provider, link, launched.receipt.kind, run_root).await?; let consumer_key = base64::engine::general_purpose::STANDARD .encode(launched.router_identity.public_key); @@ -494,14 +494,8 @@ pub(super) async fn register_new_site_links( let Some(consumer) = launched_by_site.get(&link.consumer_site) else { continue; }; - let external_url = resolve_link_external_url( - launched, - link, - consumer.receipt.kind, - run_root, - bridge_proxies, - ) - .await?; + let external_url = + resolve_link_external_url(launched, link, consumer.receipt.kind, run_root).await?; let consumer_key = base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); @@ -582,68 +576,6 @@ pub(crate) fn update_desired_links_for_provider( write_json(&path, &state) } -pub(crate) fn update_desired_overlay_for_consumer( - site_state_root: &Path, - overlay_id: &str, - overlay: DesiredExternalSlotOverlay, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - empty_desired_link_state() - }; - state - .external_slot_overlays - .insert(overlay_id.to_string(), overlay); - write_json(&path, &state) -} - -pub(crate) fn update_desired_overlay_for_provider( - site_state_root: &Path, - overlay_id: &str, - overlay: DesiredExportPeerOverlay, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - empty_desired_link_state() - }; - state - .export_peer_overlays - .insert(overlay_id.to_string(), overlay); - write_json(&path, &state) -} - -pub(crate) fn clear_desired_overlay_for_consumer( - site_state_root: &Path, - overlay_id: &str, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - return Ok(()); - }; - state.external_slot_overlays.remove(overlay_id); - write_json(&path, &state) -} - -pub(crate) fn clear_desired_overlay_for_provider( - site_state_root: &Path, - overlay_id: &str, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - return Ok(()); - }; - state.export_peer_overlays.remove(overlay_id); - write_json(&path, &state) -} - fn empty_desired_link_state() -> DesiredLinkState { DesiredLinkState { schema: DESIRED_LINKS_SCHEMA.to_string(), @@ -698,8 +630,13 @@ pub(super) fn launched_site_from_state( context: state.context.clone(), router_control: state.router_control.clone(), router_mesh_addr: state.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: state.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: state + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), + site_controller_url: state.site_controller_url.clone(), }, router_control, router_identity, @@ -751,40 +688,16 @@ pub(super) async fn ensure_site_running( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.framework_ccs)?; - reap_child(&mut runtime.site_actuator)?; - - if runtime.framework_ccs.is_none() - && let Some(plan_path) = plan.framework_ccs_plan_path.as_deref() - { - runtime.framework_ccs = Some(spawn_runtime_process( - &PathBuf::from(&plan.site_state_root), - "framework-ccs.log", - &plan.launch_env, - |cmd| { - cmd.arg("run-framework-ccs").arg("--plan").arg(plan_path); - }, - )?); - } - - if runtime.site_actuator.is_none() - && let Some(plan_path) = plan.site_actuator_plan_path.as_deref() - { - runtime.site_actuator = Some(spawn_runtime_process( - &PathBuf::from(&plan.site_state_root), - "site-actuator.log", - &plan.launch_env, - |cmd| { - cmd.arg("run-site-actuator").arg("--plan").arg(plan_path); - }, - )?); - } match plan.kind { SiteKind::Direct => { if runtime.site_process.is_none() { runtime.last_start_attempt = Some(Instant::now()); runtime.ready_since = None; + let existing_peer_ports_path = + site_existing_peer_ports_path(Path::new(&plan.site_state_root)); + let existing_peer_identities_path = + site_existing_peer_identities_path(Path::new(&plan.site_state_root)); runtime.site_process = Some(spawn_runtime_process( &PathBuf::from(&plan.site_state_root), "site.log", @@ -804,6 +717,14 @@ pub(super) async fn ensure_site_running( if let Some(port) = plan.router_mesh_port { cmd.arg("--router-mesh-port").arg(port.to_string()); } + if existing_peer_ports_path.is_file() { + cmd.arg("--existing-peer-ports") + .arg(&existing_peer_ports_path); + } + if existing_peer_identities_path.is_file() { + cmd.arg("--existing-peer-identities") + .arg(&existing_peer_identities_path); + } }, )?); } @@ -812,6 +733,10 @@ pub(super) async fn ensure_site_running( if runtime.site_process.is_none() { runtime.last_start_attempt = Some(Instant::now()); runtime.ready_since = None; + let existing_peer_ports_path = + site_existing_peer_ports_path(Path::new(&plan.site_state_root)); + let existing_peer_identities_path = + site_existing_peer_identities_path(Path::new(&plan.site_state_root)); runtime.site_process = Some(spawn_runtime_process( &PathBuf::from(&plan.site_state_root), "site.log", @@ -831,6 +756,14 @@ pub(super) async fn ensure_site_running( if let Some(port) = plan.router_mesh_port { cmd.arg("--router-mesh-port").arg(port.to_string()); } + if existing_peer_ports_path.is_file() { + cmd.arg("--existing-peer-ports") + .arg(&existing_peer_ports_path); + } + if existing_peer_identities_path.is_file() { + cmd.arg("--existing-peer-identities") + .arg(&existing_peer_identities_path); + } }, )?); } @@ -904,7 +837,11 @@ pub(super) async fn try_discover_site( try_discover_kubernetes_site(plan, runtime, stop_requested, run_root).await } }?; - if discovery.is_none() && plan.kind == SiteKind::Compose { + if discovery.is_none() + && plan.kind == SiteKind::Compose + && compose_site_requires_controller(plan) + && !compose_site_controller_started(plan)? + { runtime.site_started = false; } Ok(discovery) @@ -930,6 +867,9 @@ pub(super) async fn try_discover_direct_site( &direct_runtime_state_path(&artifact_dir), "direct runtime state", )?; + if !state.ready { + return Ok(None); + } let Some(router_mesh_port) = state.router_mesh_port else { return Ok(None); }; @@ -946,6 +886,9 @@ pub(super) async fn try_discover_direct_site( Ok(None) | Err(_) => return Ok(None), }; let router_addr = SocketAddr::from(([127, 0, 0, 1], router_mesh_port)); + if !local_site_controller_ready(plan, VM_LOCAL_TARGET_READY_TIMEOUT)? { + return Ok(None); + } return Ok(Some(RouterDiscovery { control_endpoint, router_identity, @@ -989,6 +932,9 @@ pub(super) async fn try_discover_vm_site( if !vm_component_targets_ready(plan, &artifact_dir)? { return Ok(None); } + if !local_site_controller_ready(plan, VM_LOCAL_TARGET_READY_TIMEOUT)? { + return Ok(None); + } Ok(Some(RouterDiscovery { control_endpoint, router_identity, @@ -1042,18 +988,254 @@ pub(super) fn mesh_config_local_targets_ready(path: &Path, timeout: Duration) -> Ok(true) } +pub(super) fn local_site_controller_ready( + plan: &SiteSupervisorPlan, + timeout: Duration, +) -> Result { + let Some(addr) = local_site_controller_addr(plan)? else { + return Ok(true); + }; + Ok(wait_for_http_response(addr, timeout).is_ok()) +} + +pub(super) fn local_site_controller_addr(plan: &SiteSupervisorPlan) -> Result> { + let Some(url) = plan.site_controller_url.as_deref() else { + return Ok(None); + }; + let url = Url::parse(url) + .into_diagnostic() + .wrap_err_with(|| format!("invalid site controller url `{url}`"))?; + if url.scheme() != "http" { + return Ok(None); + } + let host = url.host_str().ok_or_else(|| { + miette::miette!( + "site controller url `{url}` for site `{}` is missing a host", + plan.site_id + ) + })?; + let ip = match host { + "localhost" => std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + _ => host.parse().into_diagnostic().wrap_err_with(|| { + format!( + "site controller url `{url}` for site `{}` has a non-IP host `{host}`", + plan.site_id + ) + })?, + }; + if !ip.is_loopback() { + return Ok(None); + } + let port = url.port_or_known_default().ok_or_else(|| { + miette::miette!( + "site controller url `{url}` for site `{}` is missing a port", + plan.site_id + ) + })?; + Ok(Some(SocketAddr::new(ip, port))) +} + +fn compose_site_requires_controller(plan: &SiteSupervisorPlan) -> bool { + plan.kind == SiteKind::Compose && plan.site_controller_url.is_some() +} + +pub(super) fn compose_site_controller_container_name(plan: &SiteSupervisorPlan) -> Option { + compose_site_requires_controller(plan) + .then_some(plan.compose_project.as_deref()?) + .map(|project| { + format!( + "{project}-{}-1", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME + ) + }) +} + +pub(super) fn parse_container_runtime_status(raw: &str) -> Option<(&str, Option<&str>)> { + let mut parts = raw.split_whitespace(); + let status = parts.next()?; + let health = parts.next(); + Some((status, health)) +} + +fn tail_lines(raw: &str, keep: usize) -> String { + let lines = raw.lines().collect::>(); + let start = lines.len().saturating_sub(keep); + lines[start..].join("\n") +} + +fn timeout_file_section(label: &str, path: &Path, keep_lines: usize) -> Option { + let raw = fs::read_to_string(path).ok()?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(format!( + "{label} ({}):\n{}", + path.display(), + tail_lines(trimmed, keep_lines) + )) +} + +fn timeout_command_section(label: &str, command: &mut Command) -> Option { + let output = command.output().ok()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let body = if output.status.success() { + stdout.trim().to_string() + } else if stdout.trim().is_empty() { + stderr.trim().to_string() + } else if stderr.trim().is_empty() { + stdout.trim().to_string() + } else { + format!("stdout:\n{}\n\nstderr:\n{}", stdout.trim(), stderr.trim()) + }; + if body.is_empty() { + return None; + } + Some(format!("{label}:\n{body}")) +} + +fn compose_site_timeout_sections(plan: &SiteSupervisorPlan) -> Vec { + let mut sections = Vec::new(); + if let Ok(status) = inspect_compose_site_controller_status(plan) { + let summary = status + .map(|(status, health)| match health { + Some(health) => format!("{status} {health}"), + None => status, + }) + .unwrap_or_else(|| "unavailable".to_string()); + sections.push(format!( + "compose site controller status (`{}`): {summary}", + compose_site_controller_container_name(plan).unwrap_or_else(|| "".to_string()) + )); + } + if let Some(compose_ps) = timeout_command_section( + "docker compose ps --all", + compose_command( + plan.compose_project.as_deref(), + Path::new(&plan.artifact_dir), + ) + .arg("ps") + .arg("--all"), + ) { + sections.push(compose_ps); + } + if let Some(container_name) = compose_site_controller_container_name(plan) { + if let Some(inspect_state) = timeout_command_section( + &format!("docker inspect state for `{container_name}`"), + Command::new("docker") + .arg("inspect") + .arg("--format") + .arg("{{json .State}}") + .arg(&container_name), + ) { + sections.push(inspect_state); + } + if let Some(container_logs) = timeout_command_section( + &format!("docker logs `{container_name}`"), + Command::new("docker") + .arg("logs") + .arg("--tail") + .arg("200") + .arg(&container_name), + ) { + sections.push(container_logs); + } + } + sections +} + +fn site_startup_timeout_diagnostics( + site_plan: &RunSitePlan, + site_state_root: &Path, + state_path: &Path, +) -> String { + let mut sections = Vec::new(); + if let Some(section) = timeout_file_section("manager state", state_path, 200) { + sections.push(section); + } + if let Some(section) = timeout_file_section( + "supervisor log", + &site_state_root.join("supervisor.log"), + 200, + ) { + sections.push(section); + } + if let Some(section) = timeout_file_section("site log", &site_state_root.join("site.log"), 200) + { + sections.push(section); + } + if site_plan.site.kind == SiteKind::Compose + && let Ok(plan) = read_json::( + &site_supervisor_plan_path(site_state_root), + "site supervisor plan", + ) + { + sections.extend(compose_site_timeout_sections(&plan)); + } + if sections.is_empty() { + String::new() + } else { + format!("\nstartup diagnostics:\n{}", sections.join("\n\n")) + } +} + +fn inspect_compose_site_controller_status( + plan: &SiteSupervisorPlan, +) -> Result)>> { + let Some(container_name) = compose_site_controller_container_name(plan) else { + return Ok(None); + }; + let output = Command::new("docker") + .arg("inspect") + .arg("--format") + .arg("{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + .arg(&container_name) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to inspect compose site controller container `{container_name}`") + })?; + if !output.status.success() { + return Ok(None); + } + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(parse_container_runtime_status(&stdout) + .map(|(status, health)| (status.to_string(), health.map(str::to_string)))) +} + +fn compose_site_controller_started(plan: &SiteSupervisorPlan) -> Result { + Ok(inspect_compose_site_controller_status(plan)? + .is_some_and(|(status, _)| matches!(status.as_str(), "created" | "running" | "restarting"))) +} + +fn compose_site_controller_ready(plan: &SiteSupervisorPlan) -> Result { + Ok( + inspect_compose_site_controller_status(plan)?.is_some_and(|(status, health)| { + status == "running" && health.as_deref().is_none_or(|health| health == "healthy") + }), + ) +} + pub(super) async fn try_discover_compose_site( plan: &SiteSupervisorPlan, stop_requested: &AtomicBool, run_root: &Path, ) -> Result> { - run_until_stop( + let Some(discovery) = run_until_stop( run_root, stop_requested, discover_router_for_output(&plan.artifact_dir, plan.compose_project.as_deref(), true), ) .await - .wrap_err_with(|| format!("compose router discovery for site `{}`", plan.site_id)) + .wrap_err_with(|| format!("compose router discovery for site `{}`", plan.site_id))? + else { + return Ok(None); + }; + if compose_site_requires_controller(plan) && !compose_site_controller_ready(plan)? { + return Ok(None); + } + Ok(Some(discovery)) } pub(super) async fn try_discover_kubernetes_site( @@ -1183,26 +1365,15 @@ pub(super) async fn cleanup_site( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.framework_ccs)?; - reap_child(&mut runtime.site_actuator)?; - if let Some(child) = runtime.site_process.as_mut() { stop_child(child).await?; } if let Some(child) = runtime.port_forward.as_mut() { stop_child(child).await?; } - if let Some(child) = runtime.framework_ccs.as_mut() { - stop_child(child).await?; - } - if let Some(child) = runtime.site_actuator.as_mut() { - stop_child(child).await?; - } runtime.site_process = None; runtime.site_started = false; runtime.port_forward = None; - runtime.framework_ccs = None; - runtime.site_actuator = None; match plan.kind { SiteKind::Compose => { @@ -1225,22 +1396,17 @@ pub(super) async fn cleanup_site( } SiteKind::Kubernetes => { if let Some(namespace) = plan.kubernetes_namespace.as_deref() { - let status = kubectl_command(plan.context.as_deref()) - .arg("delete") - .arg("namespace") - .arg(namespace) - .arg("--ignore-not-found") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop kubernetes site `{}`", plan.site_id) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes site `{}` failed to stop with status {status}", + stop_kubernetes_namespace( + plan.context.as_deref(), + namespace, + KUBERNETES_NAMESPACE_DELETE_TIMEOUT, + ) + .wrap_err_with(|| { + format!( + "failed to stop kubernetes site `{}` in namespace `{namespace}`", plan.site_id - )); - } + ) + })?; } } SiteKind::Direct | SiteKind::Vm => {} @@ -1270,6 +1436,8 @@ pub(super) fn build_site_state( } else { (None, None, None, None) }; + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, router_mesh_addr.as_deref()); SiteManagerState { schema: SITE_STATE_SCHEMA.to_string(), version: SITE_STATE_VERSION, @@ -1286,8 +1454,11 @@ pub(super) fn build_site_state( context: plan.context.clone(), router_control, router_mesh_addr, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id, router_public_key_b64, + site_controller_url: plan.site_controller_url.clone(), last_error, } } @@ -1322,13 +1493,55 @@ pub(super) fn persist_site_state( context: launched.receipt.context.clone(), router_control: launched.receipt.router_control.clone(), router_mesh_addr: launched.receipt.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: launched + .receipt + .compose_consumer_router_mesh_addr + .clone(), + kubernetes_consumer_router_mesh_addr: launched + .receipt + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: launched.receipt.router_identity_id.clone(), router_public_key_b64: launched.receipt.router_public_key_b64.clone(), + site_controller_url: launched.receipt.site_controller_url.clone(), last_error, }, ) } +fn published_container_consumer_router_mesh_addrs( + provider_kind: SiteKind, + router_mesh_addr: Option<&str>, +) -> (Option, Option) { + let Some(router_mesh_addr) = router_mesh_addr else { + return (None, None); + }; + let compose = amber_site_controller::router_mesh_addr_for_consumer( + provider_kind, + SiteKind::Compose, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute compose consumer mesh addr: {err}")); + let kubernetes = amber_site_controller::router_mesh_addr_for_consumer( + provider_kind, + SiteKind::Kubernetes, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute kubernetes consumer mesh addr: {err}")); + (Some(compose), Some(kubernetes)) +} + +fn published_router_mesh_addr_for_consumer_kind( + receipt: &SiteReceipt, + consumer_kind: SiteKind, +) -> Option<&str> { + match consumer_kind { + SiteKind::Compose => receipt.compose_consumer_router_mesh_addr.as_deref(), + SiteKind::Kubernetes => receipt.kubernetes_consumer_router_mesh_addr.as_deref(), + SiteKind::Direct | SiteKind::Vm => None, + } +} + pub(super) fn write_site_state(path: &Path, state: SiteManagerState) -> Result<()> { write_json(path, &state) } @@ -1669,7 +1882,7 @@ fn wait_for_named_kubernetes_resource( } } -pub(super) fn compose_project_name(run_id: &str, site_id: &str) -> String { +pub(crate) fn compose_project_name(run_id: &str, site_id: &str) -> String { let mut out = String::from("amber_"); for ch in format!("{run_id}_{site_id}").chars() { if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { @@ -1689,9 +1902,7 @@ pub(crate) fn spawn_detached_child( #[cfg(unix)] use std::os::unix::process::CommandExt as _; - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; + let exe = super::amber_cli_executable()?; let log = fs::File::create(log_path) .into_diagnostic() .wrap_err_with(|| format!("failed to create log {}", log_path.display()))?; @@ -1728,18 +1939,37 @@ pub(super) fn spawn_runtime_process( extra_env: &BTreeMap, build: impl FnOnce(&mut Command), ) -> Result { - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; + let exe = super::amber_cli_executable()?; + spawn_runtime_process_with_executable(&exe, site_state_root, log_name, extra_env, build) +} + +pub(super) fn spawn_runtime_process_with_executable( + executable: &Path, + site_state_root: &Path, + log_name: &str, + extra_env: &BTreeMap, + build: impl FnOnce(&mut Command), +) -> Result { let log_path = site_state_root.join(log_name); - let log = fs::File::create(&log_path) + let mut log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", log_path.display()))?; + writeln!(log, "\n--- starting runtime process ---") + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to write runtime start marker to {}", + log_path.display() + ) + })?; let log_err = log .try_clone() .into_diagnostic() .wrap_err("failed to clone site log")?; - let mut cmd = Command::new(exe); + let mut cmd = Command::new(executable); cmd.envs(extra_env); cmd.stdout(Stdio::from(log)); cmd.stderr(Stdio::from(log_err)); @@ -1775,9 +2005,8 @@ pub(super) fn spawn_port_forward(plan: &SiteSupervisorPlan) -> Result { .arg("0.0.0.0") .arg("deploy/amber-router") .arg(format!("{mesh_port}:24000")) - .arg(format!("{control_port}:24100")) - .stdout(Stdio::from(log)) - .stderr(Stdio::from(log_err)); + .arg(format!("{control_port}:24100")); + cmd.stdout(Stdio::from(log)).stderr(Stdio::from(log_err)); cmd.spawn() .into_diagnostic() .wrap_err("failed to spawn kubectl port-forward") @@ -1858,7 +2087,6 @@ pub(crate) async fn resolve_link_external_url( link: &RunLink, consumer_kind: SiteKind, run_root: &Path, - bridge_proxies: &mut BTreeMap, ) -> Result { resolve_link_external_url_for_output( provider, @@ -1866,7 +2094,6 @@ pub(crate) async fn resolve_link_external_url( link, consumer_kind, run_root, - bridge_proxies, ) .await } @@ -1876,130 +2103,16 @@ pub(crate) async fn resolve_link_external_url_for_output( provider_output_dir: &Path, link: &RunLink, consumer_kind: SiteKind, - run_root: &Path, - bridge_proxies: &mut BTreeMap, -) -> Result { - if !link_needs_bridge_proxy(provider.receipt.kind, consumer_kind) { - return external_slot_url(provider, provider_output_dir, link, consumer_kind); - } - - let port = ensure_bridge_proxy( - run_root, - provider, - provider_output_dir, - &link.export_name, - consumer_kind, - bridge_proxies, - ) - .await?; - bridge_proxy_external_url(port, link.protocol, consumer_kind) -} - -pub(super) fn link_needs_bridge_proxy(provider_kind: SiteKind, consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) - && provider_kind != SiteKind::Kubernetes -} - -pub(super) async fn ensure_bridge_proxy( - run_root: &Path, - provider: &LaunchedSite, - provider_output_dir: &Path, - export_name: &str, - consumer_kind: SiteKind, - bridge_proxies: &mut BTreeMap, -) -> Result { - let key = BridgeProxyKey { - provider_output_dir: provider_output_dir.display().to_string(), - export_name: export_name.to_string(), - consumer_kind, - }; - if let Some(proxy) = bridge_proxies.get_mut(&key) - && proxy.child.try_wait().into_diagnostic()?.is_none() - { - return Ok(proxy.listen.port()); - } - - let listen = bridge_proxy_bind_addr(consumer_kind, reserve_loopback_port()?); - let child = spawn_bridge_proxy(run_root, provider, provider_output_dir, export_name, listen)?; - wait_for_socket_listener(bridge_proxy_probe_addr(listen)).await?; - bridge_proxies.insert( - key, - BridgeProxyHandle { - child, - export_name: export_name.to_string(), - listen, - }, - ); - Ok(listen.port()) -} - -pub(super) fn spawn_bridge_proxy( - run_root: &Path, - provider: &LaunchedSite, - provider_output_dir: &Path, - export_name: &str, - listen: SocketAddr, -) -> Result { - let logs_root = run_root.join("bridge-proxies"); - fs::create_dir_all(&logs_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", logs_root.display()))?; - let log_path = logs_root.join(format!("{export_name}.log")); - spawn_detached_child(run_root, &log_path, |cmd| { - cmd.arg("proxy") - .arg(provider_output_dir) - .arg("--export") - .arg(bridge_proxy_export_binding(export_name, listen)); - if provider.receipt.kind == SiteKind::Kubernetes { - let control = provider.router_control.to_string(); - cmd.arg("--router-addr") - .arg(provider.router_addr.to_string()) - .arg("--router-control-addr") - .arg(control); - } - }) -} - -pub(super) fn bridge_proxy_export_binding(export_name: &str, listen: SocketAddr) -> String { - format!("{export_name}={}:{}", listen.ip(), listen.port()) -} - -pub(super) fn bridge_proxy_bind_addr(consumer_kind: SiteKind, port: u16) -> SocketAddr { - host_service_bind_addr_for_consumer(consumer_kind, port) -} - -pub(super) fn bridge_proxy_probe_addr(listen: SocketAddr) -> SocketAddr { - listener_probe_addr(listen) -} - -pub(super) fn bridge_proxy_external_url( - port: u16, - protocol: NetworkProtocol, - consumer_kind: SiteKind, + _run_root: &Path, ) -> Result { - let host = host_service_host_for_consumer(consumer_kind); - Ok(match protocol { - NetworkProtocol::Http | NetworkProtocol::Https => format!("http://{host}:{port}"), - NetworkProtocol::Tcp => format!("tcp://{host}:{port}"), - _ => { - return Err(miette::miette!( - "mixed-site bridge proxy does not support protocol `{protocol}`" - )); - } - }) -} - -pub(crate) fn host_service_host_for_consumer(consumer_kind: SiteKind) -> String { - match consumer_kind { - SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), - SiteKind::Direct | SiteKind::Vm | SiteKind::Kubernetes => { - container_host_for_consumer(SiteKind::Direct, consumer_kind) - } - } + external_slot_url(provider, provider_output_dir, link, consumer_kind) } pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) + matches!( + consumer_kind, + SiteKind::Compose | SiteKind::Kubernetes | SiteKind::Vm + ) || (cfg!(target_os = "linux") && matches!(consumer_kind, SiteKind::Direct)) } pub(crate) fn host_service_bind_addr_for_consumer( @@ -2009,37 +2122,6 @@ pub(crate) fn host_service_bind_addr_for_consumer( host_proxy_bind_addr(consumer_needs_host_wide_listener(consumer_kind), port) } -pub(crate) fn router_mesh_addr_for_consumer( - provider_kind: SiteKind, - consumer_kind: SiteKind, - router_mesh_addr: &str, -) -> Result { - match consumer_kind { - SiteKind::Compose | SiteKind::Kubernetes => { - let addr = router_mesh_addr - .parse::() - .into_diagnostic() - .wrap_err_with(|| { - format!("invalid live router mesh address `{router_mesh_addr}`") - })?; - let host = container_host_for_consumer(provider_kind, consumer_kind); - Ok(format!("{host}:{}", addr.port())) - } - SiteKind::Direct | SiteKind::Vm => { - #[cfg(target_os = "linux")] - { - Ok(crate::direct_runtime::rewrite_peer_addr_for_slirp_gateway( - router_mesh_addr, - )) - } - #[cfg(not(target_os = "linux"))] - { - Ok(router_mesh_addr.to_string()) - } - } - } -} - pub(super) fn host_proxy_bind_addr(needs_host_wide_listener: bool, port: u16) -> SocketAddr { if needs_host_wide_listener { SocketAddr::from(([0, 0, 0, 0], port)) @@ -2062,9 +2144,23 @@ pub(super) fn external_slot_url( link: &RunLink, consumer_kind: SiteKind, ) -> Result { - let host = container_host_for_consumer(provider.receipt.kind, consumer_kind); + let router_mesh_addr = if let Some(router_mesh_addr) = + published_router_mesh_addr_for_consumer_kind(&provider.receipt, consumer_kind) + { + router_mesh_addr.to_string() + } else { + amber_site_controller::router_mesh_addr_for_consumer( + provider.receipt.kind, + consumer_kind, + provider + .receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("provider site is missing router mesh addr"))?, + )? + }; let route_id = provider_export_route_id(provider_output_dir, link)?; - let mut mesh_url = Url::parse(&format!("mesh://{}:{}", host, provider.router_addr.port())) + let mut mesh_url = Url::parse(&format!("mesh://{router_mesh_addr}")) .into_diagnostic() .wrap_err("failed to build mesh link url")?; let peer_key = @@ -2101,21 +2197,13 @@ pub(super) fn container_host_for_consumer( } pub(super) fn container_host_from_resolved_ip( - provider_kind: SiteKind, + _provider_kind: SiteKind, consumer_kind: SiteKind, container_host_ip: Option<&str>, ) -> String { match consumer_kind { SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), - SiteKind::Compose => { - if provider_kind == SiteKind::Kubernetes { - container_host_ip - .unwrap_or(CONTAINER_HOST_ALIAS) - .to_string() - } else { - CONTAINER_HOST_ALIAS.to_string() - } - } + SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), SiteKind::Kubernetes => container_host_ip .unwrap_or(CONTAINER_HOST_ALIAS) .to_string(), @@ -2186,3 +2274,220 @@ pub(super) fn mesh_protocol(protocol: NetworkProtocol) -> Result { } }) } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, fs, net::SocketAddr}; + + use amber_compiler::mesh::{PROXY_METADATA_FILENAME, PROXY_METADATA_VERSION}; + use amber_mesh::{MeshIdentity, MeshIdentityPublic, router_export_route_id}; + + use super::*; + + #[test] + fn launch_env_includes_process_dev_image_tag_overrides() { + unsafe { + std::env::set_var( + amber_images::DEV_IMAGE_TAGS_ENV, + "router=dev-tag,helper=dev-tag", + ); + } + let env = launch_env( + "run-test", + "scope", + SiteKind::Kubernetes, + &BTreeMap::new(), + &BTreeMap::new(), + None, + ) + .expect("launch env should build"); + assert_eq!( + env.get(amber_images::DEV_IMAGE_TAGS_ENV) + .map(String::as_str), + Some("router=dev-tag,helper=dev-tag"), + ); + unsafe { + std::env::remove_var(amber_images::DEV_IMAGE_TAGS_ENV); + } + } + + #[test] + fn launch_env_includes_vm_tcg_override() { + let previous = std::env::var_os("AMBER_VM_FORCE_TCG"); + unsafe { + std::env::set_var("AMBER_VM_FORCE_TCG", "1"); + } + let env = launch_env( + "run-test", + "scope", + SiteKind::Kubernetes, + &BTreeMap::new(), + &BTreeMap::new(), + None, + ) + .expect("launch env should build"); + assert_eq!(env.get("AMBER_VM_FORCE_TCG").map(String::as_str), Some("1")); + match previous { + Some(value) => unsafe { std::env::set_var("AMBER_VM_FORCE_TCG", value) }, + None => unsafe { std::env::remove_var("AMBER_VM_FORCE_TCG") }, + } + } + + #[test] + fn cross_site_container_consumers_resolve_links_as_mesh_router_urls() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + for consumer_kind in [SiteKind::Compose, SiteKind::Kubernetes] { + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + consumer_kind, + temp.path(), + )) + .expect("container consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://"), + "cross-site {consumer_kind:?} consumers should route through mesh, got {resolved}" + ); + assert!( + !resolved.starts_with("http://"), + "cross-site {consumer_kind:?} consumers should not bounce through an HTTP bridge \ + proxy: {resolved}" + ); + assert!(resolved.contains("peer_id=%2Fsite%2Fcompose_local%2Frouter")); + } + } + + #[test] + fn vm_site_consumers_keep_router_mesh_loopback_for_cross_site_links() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-vm/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "vm_local".to_string(), + provider_component: "/job-vm/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-vm/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + SiteKind::Vm, + temp.path(), + )) + .expect("vm site consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://127.0.0.1:24000"), + "vm site consumers should keep the live router mesh address, got {resolved}" + ); + } +} diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index ef932fe0..30b72a59 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -1,3931 +1,573 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; use std::{ - collections::{BTreeMap, BTreeSet}, - io::{Read as _, Write as _}, - net::{TcpListener, TcpStream}, -}; - -use base64::Engine as _; -use opentelemetry_proto::tonic::{ - collector::logs::v1::ExportLogsServiceRequest, - common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value}, - logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, - resource::v1::Resource, + fs, + net::{SocketAddr, TcpListener}, + thread, + time::Duration, }; -use prost::Message as _; -use tempfile::tempdir; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - -use super::*; -use crate::framework_component::DynamicProxyExportRecord; - -fn test_dynamic_proxy_export_record( - component_id: usize, - component: &str, - provide: &str, - protocol: &str, - capability_kind: &str, - target_port: u16, -) -> DynamicProxyExportRecord { - DynamicProxyExportRecord { - component_id, - component: component.to_string(), - provide: provide.to_string(), - protocol: protocol.to_string(), - capability_kind: capability_kind.to_string(), - capability_profile: None, - target_port, - } -} - -fn test_site_receipt( - kind: SiteKind, - artifact_dir: &Path, - router_control: Option<&str>, - router_mesh_addr: Option<&str>, -) -> SiteReceipt { - SiteReceipt { - kind, - artifact_dir: artifact_dir.display().to_string(), - supervisor_pid: 100, - process_pid: None, - compose_project: None, - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: router_control.map(str::to_string), - router_mesh_addr: router_mesh_addr.map(str::to_string), - router_identity_id: None, - router_public_key_b64: None, - } -} - -fn otlp_string_value(value: &str) -> AnyValue { - AnyValue { - value: Some(any_value::Value::StringValue(value.to_string())), - } -} - -#[test] -fn compose_start_failure_explains_missing_docker_daemon() { - let output = std::process::Command::new("sh") - .arg("-c") - .arg( - "echo 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the \ - docker daemon running?' 1>&2; exit 1", - ) - .output() - .expect("shell output"); - - let err = compose_start_failure("compose_local", &output); - let message = err.to_string(); - assert!(message.contains("could not reach Docker")); - assert!(message.contains("Start Docker Desktop or the Docker daemon")); - assert!(message.contains("Cannot connect to the Docker daemon")); -} - -#[test] -fn compose_start_failure_surfaces_specific_compose_stderr() { - let output = std::process::Command::new("sh") - .arg("-c") - .arg( - "echo 'Network amber_boundary Creating' 1>&2; echo 'services.web.image must be a \ - string' 1>&2; exit 1", - ) - .output() - .expect("shell output"); - - let err = compose_start_failure("compose_local", &output); - let message = err.to_string(); - assert!(message.contains("compose site `compose_local` failed to start")); - assert!(message.contains("services.web.image must be a string")); - assert!(!message.contains("Network amber_boundary Creating")); -} - -fn test_site_state( - run_id: &str, - site_id: &str, - kind: SiteKind, - artifact_dir: &Path, - router_control: Option<&str>, - router_mesh_addr: Option<&str>, -) -> SiteManagerState { - SiteManagerState { - schema: "amber.run.site-state".to_string(), - version: 1, - run_id: run_id.to_string(), - site_id: site_id.to_string(), - kind, - status: SiteLifecycleStatus::Running, - artifact_dir: artifact_dir.display().to_string(), - supervisor_pid: 101, - process_pid: None, - compose_project: None, - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: router_control.map(str::to_string), - router_mesh_addr: router_mesh_addr.map(str::to_string), - router_identity_id: None, - router_public_key_b64: None, - last_error: None, - } -} - -fn test_site_actuator_plan( - kind: SiteKind, - artifact_dir: &Path, - site_state_root: &Path, -) -> SiteActuatorPlan { - SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "run-test".to_string(), - mesh_scope: "test-scope".to_string(), - run_root: site_state_root.display().to_string(), - site_id: "direct_local".to_string(), - kind, - router_identity_id: "/site/direct_local/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - } -} - -#[test] -fn forwarded_endpoint_ready_accepts_open_connection() { - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - let handle = std::thread::spawn(move || { - let (_stream, _) = listener.accept().expect("listener should accept"); - std::thread::sleep(Duration::from_millis(500)); - }); - - assert!(crate::tcp_readiness::endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - )); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn forwarded_endpoint_ready_rejects_reset_connection() { - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - let handle = std::thread::spawn(move || { - let (stream, _) = listener.accept().expect("listener should accept"); - drop(stream); - }); - - assert!(!crate::tcp_readiness::endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - )); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn desired_link_overlays_are_owned_per_overlay_id() { - let temp = tempdir().expect("tempdir"); - let site_state_root = temp.path(); - - super::supervisor::update_desired_overlay_for_consumer( - site_state_root, - "overlay-a", - DesiredExternalSlotOverlay { - slot_name: "api".to_string(), - url: "http://provider".to_string(), - }, - ) - .expect("first consumer overlay should persist"); - super::supervisor::update_desired_overlay_for_consumer( - site_state_root, - "overlay-b", - DesiredExternalSlotOverlay { - slot_name: "api".to_string(), - url: "http://provider".to_string(), - }, - ) - .expect("second consumer overlay should persist"); - super::supervisor::update_desired_overlay_for_provider( - site_state_root, - "provider-a", - DesiredExportPeerOverlay { - export_name: "amber_export_shared".to_string(), - peer_id: "consumer-a".to_string(), - peer_key_b64: "a2V5".to_string(), - protocol: "http".to_string(), - route_id: Some("route-a".to_string()), - }, - ) - .expect("first provider overlay should persist"); - super::supervisor::update_desired_overlay_for_provider( - site_state_root, - "provider-b", - DesiredExportPeerOverlay { - export_name: "amber_export_shared".to_string(), - peer_id: "consumer-b".to_string(), - peer_key_b64: "a2V5".to_string(), - protocol: "http".to_string(), - route_id: Some("route-b".to_string()), - }, - ) - .expect("second provider overlay should persist"); - - let path = desired_links_path(site_state_root); - let desired: DesiredLinkState = read_json(&path, "desired links").expect("desired links"); - assert_eq!(desired.external_slot_overlays.len(), 2); - assert_eq!(desired.export_peer_overlays.len(), 2); - - super::supervisor::clear_desired_overlay_for_consumer(site_state_root, "overlay-a") - .expect("consumer overlay removal should persist"); - super::supervisor::clear_desired_overlay_for_provider(site_state_root, "provider-a") - .expect("provider overlay removal should persist"); - - let desired: DesiredLinkState = read_json(&path, "desired links").expect("desired links"); - assert_eq!(desired.external_slot_overlays.len(), 1); - assert_eq!(desired.export_peer_overlays.len(), 1); - assert!(desired.external_slot_overlays.contains_key("overlay-b")); - assert!(desired.export_peer_overlays.contains_key("provider-b")); -} - -#[test] -fn site_router_control_endpoint_prefers_manager_state_for_local_sites() { - let temp = tempdir().expect("tempdir"); - let artifact_dir = temp.path().join("artifact"); - let site_state_root = temp.path().join("state"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - let manager_state_path = site_state_root.join("manager-state.json"); - - for kind in [SiteKind::Direct, SiteKind::Vm] { - write_json( - &manager_state_path, - &test_site_state( - "run-test", - "direct_local", - kind, - &artifact_dir, - Some("unix:///tmp/router-from-manager.sock"), - Some("127.0.0.1:24000"), - ), - ) - .expect("manager state should be written"); - - let endpoint = site_router_control_endpoint(&test_site_actuator_plan( - kind, - &artifact_dir, - &site_state_root, - )) - .expect("router control endpoint should resolve"); - - assert_eq!( - endpoint.to_string(), - "unix:///tmp/router-from-manager.sock", - "local site actuator should use the live manager-state endpoint for {kind:?}", - ); - } -} - -fn test_local_mesh_config(path: &Path, protocol: MeshProtocol, port: u16) -> Result<()> { - write_json( - path, - &MeshConfigPublic { - identity: MeshIdentityPublic { - id: "/site/test/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("test-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 24000)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 24100))), - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: vec![InboundRoute { - route_id: "route".to_string(), - capability: "http".to_string(), - capability_kind: None, - capability_profile: None, - protocol, - http_plugins: Vec::new(), - target: InboundTarget::Local { port }, - allowed_issuers: Vec::new(), - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - ) -} - -#[test] -fn mesh_config_local_targets_ready_accepts_http_inbound_routes() { - let temp = tempdir().expect("tempdir should be created"); - let config_path = temp.path().join("mesh-config.json"); - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - test_local_mesh_config(&config_path, MeshProtocol::Http, addr.port()) - .expect("mesh config should be written"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("listener should accept"); - let mut request = [0u8; 256]; - let _ = stream.read(&mut request); - let _ = stream.write_all( - b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - ); - }); - - assert!( - mesh_config_local_targets_ready(&config_path, Duration::from_secs(1)) - .expect("mesh config should be readable") - ); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn mesh_config_local_targets_ready_rejects_unreachable_http_inbound_routes() { - let temp = tempdir().expect("tempdir should be created"); - let config_path = temp.path().join("mesh-config.json"); - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - drop(listener); - test_local_mesh_config(&config_path, MeshProtocol::Http, addr.port()) - .expect("mesh config should be written"); - - assert!( - !mesh_config_local_targets_ready(&config_path, Duration::from_millis(100)) - .expect("mesh config should be readable") - ); -} - -#[test] -fn read_compose_launch_env_returns_saved_launch_env() { - let temp = tempdir().expect("tempdir should be created"); - let run_root = temp.path().join("run-root"); - let state_root = run_root.join("state").join("compose_local"); - write_json( - &site_supervisor_plan_path(&state_root), - &SiteSupervisorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "test.scope".to_string(), - run_root: run_root.display().to_string(), - coordinator_pid: 1, - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - artifact_dir: temp.path().join("artifact").display().to_string(), - site_state_root: state_root.display().to_string(), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - port_forward_mesh_port: None, - port_forward_control_port: None, - observability_endpoint: None, - framework_ccs_plan_path: None, - site_actuator_plan_path: None, - launch_env: BTreeMap::from([ - ("AMBER_CONFIG_TENANT".to_string(), "acme-local".to_string()), - ( - "AMBER_CONFIG_CATALOG_TOKEN".to_string(), - "demo-token".to_string(), - ), - ]), - }, - ) - .expect("site supervisor plan should be written"); - - assert_eq!( - read_compose_launch_env(&run_root, "compose_local") - .expect("compose launch env should be readable"), - BTreeMap::from([ - ( - "AMBER_CONFIG_CATALOG_TOKEN".to_string(), - "demo-token".to_string() - ), - ("AMBER_CONFIG_TENANT".to_string(), "acme-local".to_string()), - ]) - ); -} - -#[test] -fn rewrite_dynamic_proxy_metadata_updates_compose_x_amber_exports() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - fs::write( - artifact_root.join("compose.yaml"), - r#" -services: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 - exports: - stale: - component: /stale - provide: old - protocol: http - router_mesh_port: 24000 -"#, - ) - .expect("compose artifact should be written"); - - rewrite_dynamic_proxy_metadata( - artifact_root, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - routed_inputs: Vec::new(), - }, - ) - .expect("compose proxy metadata should rewrite"); - - let raw = fs::read_to_string(artifact_root.join("compose.yaml")) - .expect("compose artifact should be readable"); - let document: serde_yaml::Value = - serde_yaml::from_str(&raw).expect("compose yaml should remain valid"); - let exports = document["x-amber"]["exports"] - .as_mapping() - .expect("compose x-amber exports should be a mapping"); - assert_eq!(exports.len(), 1, "stale compose exports should be replaced"); - let export = exports - .get(serde_yaml::Value::String("http".to_string())) - .expect("expected rewritten compose export"); - assert_eq!(export["component"].as_str(), Some("/job/root")); - assert_eq!(export["provide"].as_str(), Some("http")); - assert_eq!(export["protocol"].as_str(), Some("http")); -} - -#[test] -fn prepare_dynamic_compose_child_artifact_keeps_only_child_owned_services() { - let temp = tempdir().expect("tempdir should be created"); - let site_artifact = temp.path().join("site-artifact"); - let child_artifact = temp.path().join("child-artifact"); - let site_state_root = temp.path().join("state").join("compose_local"); - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/compose_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/compose_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c1-compose-admin-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c8-root-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/compose_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 24000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/compose_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id( - "compose_admin_http", - MeshProtocol::Http, - ), - capability: "compose_admin_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c1-compose-admin-net:23000".to_string(), - peer_id: "/compose_admin".to_string(), - route_id: component_route_id( - "/compose_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id("http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c8-root-net:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id( - "/job/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/compose_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/amber-router".to_string(), - }, - }, - ], - }; - - fs::create_dir_all(&site_artifact).expect("site artifact dir should exist"); - fs::create_dir_all(&child_artifact).expect("child artifact dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - - fs::write( - site_artifact.join("compose.yaml"), - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - c1-compose-admin: - image: python:3.13-alpine - depends_on: - - c1-compose-admin-net - c1-compose-admin-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c1-compose-admin-net-mesh:/amber/mesh:ro -volumes: - c1-compose-admin-net-mesh: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"#, - ) - .expect("static compose artifact should be written"); - - let mesh_plan_json = serde_json::to_string(&mesh_plan).expect("mesh plan should serialize"); - fs::write( - child_artifact.join("compose.yaml"), - format!( - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - amber-provisioner: - image: ghcr.io/rdi-foundation/amber-provisioner:v0.1 - c1-compose-admin: - image: python:3.13-alpine - depends_on: - - c1-compose-admin-net - c1-compose-admin-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c1-compose-admin-net-mesh:/amber/mesh:ro - c8-root: - image: python:3.13-alpine - depends_on: - - c8-root-net - c8-root-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - depends_on: - - amber-provisioner - environment: - - AMBER_SCENARIO_SCOPE=scope - networks: - amber_mesh: {{}} - amber_egress_c8-root-net: {{}} - volumes: - - c8-root-net-mesh:/amber/mesh:ro -configs: - amber-mesh-provision-plan: - content: '{mesh_plan_json}' -volumes: - c1-compose-admin-net-mesh: {{}} - c8-root-net-mesh: {{}} -networks: - amber_mesh: - driver: bridge - internal: true - amber_egress_c8-root-net: - driver: bridge - internal: false -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"# - ), - ) - .expect("dynamic compose artifact should be written"); - - write_json( - &site_state_root.join("manager-state.json"), - &SiteManagerState { - schema: SITE_STATE_SCHEMA.to_string(), - version: SITE_STATE_VERSION, - run_id: "run-123".to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - status: SiteLifecycleStatus::Running, - artifact_dir: site_artifact.display().to_string(), - supervisor_pid: 123, - process_pid: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: Some("http://127.0.0.1:24100".to_string()), - router_mesh_addr: Some("127.0.0.1:24000".to_string()), - router_identity_id: Some("/site/compose_local/router".to_string()), - router_public_key_b64: Some( - base64::engine::general_purpose::STANDARD.encode([7u8; 32]), - ), - last_error: None, - }, - ) - .expect("site manager state should be written"); - - supervisor::patch_site_artifacts( - &child_artifact, - "run-123", - "compose_local", - SiteKind::Compose, - &BTreeMap::new(), - None, - ) - .expect("compose child artifact should get explicit egress subnets"); - let mut patched_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let initial_subnet = { - let patched_networks = - compose_networks_mut(&mut patched_document, &child_artifact.join("compose.yaml")) - .expect("compose networks") - .expect("compose networks should exist"); - assert!( - compose_network_subnet( - patched_networks - .get(yaml_string("amber_mesh")) - .expect("mesh network should exist"), - ) - .is_none(), - "mesh network should not get an egress subnet", - ); - compose_network_subnet( - patched_networks - .get(yaml_string("amber_egress_c8-root-net")) - .expect("child egress network should exist"), - ) - .expect("child egress network should get a subnet") - .to_string() - }; - supervisor::patch_site_artifacts( - &child_artifact, - "run-123", - "compose_local", - SiteKind::Compose, - &BTreeMap::new(), - None, - ) - .expect("compose subnet assignment should be idempotent"); - - prepare_dynamic_compose_child_artifact( - &SiteActuatorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - artifact_dir: site_artifact.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: vec![8], - assigned_components: vec!["/job/root".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - &child_artifact, - &[], - &BTreeMap::from([( - "/site/compose_local/router".to_string(), - MeshIdentityPublic { - id: "/site/compose_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh-scope-test".to_string()), - }, - )]), - ) - .expect("dynamic compose child artifact should be prepared"); - - let child_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let child_services = - compose_services(&child_document, &child_artifact.join("compose.yaml")).expect("services"); - assert_eq!( - child_services - .keys() - .filter_map(serde_yaml::Value::as_str) - .collect::>(), - vec!["c8-root", "c8-root-net"] - ); - - let sidecar_volumes = child_services - .get(yaml_string("c8-root-net")) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|service| service.get(yaml_string("volumes"))) - .and_then(serde_yaml::Value::as_sequence) - .expect("child sidecar volumes should be present"); - assert_eq!( - sidecar_volumes, - &vec![serde_yaml::Value::String( - "./.amber/mesh/c8-root-net:/amber/mesh:ro".to_string() - )] - ); - let child_compose_yaml = fs::read_to_string(child_artifact.join("compose.yaml")) - .expect("child compose artifact should be readable"); - assert!( - child_compose_yaml.contains("AMBER_SCENARIO_SCOPE=mesh-scope-test"), - "dynamic compose artifact should rewrite sidecar scenario scope onto the live run scope", - ); - assert!( - !child_compose_yaml.contains("AMBER_SCENARIO_SCOPE=scope"), - "dynamic compose artifact should not retain the compiled child scope", - ); - let mut child_network_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let child_networks = compose_networks_mut( - &mut child_network_document, - &child_artifact.join("compose.yaml"), - ) - .expect("compose networks") - .expect("compose networks should exist"); - assert_eq!( - compose_network_subnet( - child_networks - .get(yaml_string("amber_egress_c8-root-net")) - .expect("child egress network should remain present"), - ), - Some(initial_subnet.as_str()), - "dynamic compose filtering should preserve the assigned egress subnet", - ); - - let metadata = load_dynamic_compose_child_metadata(&child_artifact) - .expect("dynamic compose child metadata should be readable"); - assert_eq!( - metadata.services, - vec!["c8-root".to_string(), "c8-root-net".to_string()] - ); - assert_eq!( - metadata.readiness_services, - vec!["c8-root".to_string(), "c8-root-net".to_string()] - ); - - let filtered_plan: MeshProvisionPlan = read_json( - &child_artifact.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should exist"); - assert_eq!(filtered_plan.targets.len(), 1); - let component = filtered_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered component target should exist"); - let MeshProvisionOutput::Filesystem { dir } = &component.output else { - panic!("filtered component mesh output should be filesystem"); - }; - assert_eq!(dir, ".amber/mesh/c8-root-net"); - assert_eq!( - component - .config - .inbound - .iter() - .map(|route| route.route_id.as_str()) - .collect::>(), - vec!["component:/job/root:http:http"] - ); - assert_eq!( - component.config.inbound[0].allowed_issuers, - vec!["/site/compose_local/router".to_string()] - ); - - let mesh_config: MeshConfigPublic = read_json( - &child_artifact - .join(".amber/mesh/c8-root-net") - .join("mesh-config.json"), - "mesh config", - ) - .expect("child sidecar mesh config should exist"); - let router_peer = mesh_config - .peers - .iter() - .find(|peer| peer.id == "/site/compose_local/router") - .expect("child sidecar should peer with the site router"); - assert_eq!(router_peer.public_key, [7u8; 32]); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&child_artifact), - "site router overlay", - ) - .expect("compose child router overlay should exist"); - assert_eq!( - overlay - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job/root"] - ); - assert_eq!( - overlay - .inbound_routes - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("http", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); -} - -#[test] -fn prepare_dynamic_compose_child_artifact_rewrites_same_site_static_provider_inputs() { - let temp = tempdir().expect("tempdir should be created"); - let site_artifact = temp.path().join("site-artifact"); - let child_artifact = temp.path().join("child-artifact"); - fs::create_dir_all(&site_artifact).expect("site artifact dir should exist"); - fs::create_dir_all(&child_artifact).expect("child artifact dir should exist"); - - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/provider".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c4-provider-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/sibling".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/provider".to_string(), - }], - inbound: Vec::new(), - outbound: vec![OutboundRoute { - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - rewrite_route_id: None, - slot: "upstream".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - listen_port: 20000, - listen_addr: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - peer_addr: "c4-provider-net:23000".to_string(), - peer_id: "/provider".to_string(), - capability: "http".to_string(), - }], - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c5-sibling-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/compose_local/router".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 24000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/provider".to_string(), - }], - inbound: vec![InboundRoute { - route_id: router_export_route_id("provider_http", MeshProtocol::Http), - capability: "provider_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c4-provider-net:23000".to_string(), - peer_id: "/provider".to_string(), - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/amber-router".to_string(), - }, - }, - ], - }; - let mesh_plan_json = serde_json::to_string(&mesh_plan).expect("mesh plan should serialize"); - fs::write( - child_artifact.join("compose.yaml"), - format!( - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - amber-provisioner: - image: ghcr.io/rdi-foundation/amber-provisioner:v0.1 - c4-provider: - image: python:3.13-alpine - depends_on: - - c4-provider-net - c4-provider-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c4-provider-net-mesh:/amber/mesh:ro - c5-sibling: - image: python:3.13-alpine - depends_on: - - c5-sibling-net - - c4-provider - c5-sibling-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - depends_on: - - amber-provisioner - environment: - - AMBER_SCENARIO_SCOPE=compiled-scope - volumes: - - c5-sibling-net-mesh:/amber/mesh:ro -configs: - amber-mesh-provision-plan: - content: '{mesh_plan_json}' -volumes: - c4-provider-net-mesh: {{}} - c5-sibling-net-mesh: {{}} -networks: - amber_mesh: - driver: bridge - amber_boundary: - driver: bridge -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"# - ), - ) - .expect("dynamic compose artifact should be written"); - - fs::write( - site_artifact.join("compose.yaml"), - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - c4-provider: - image: python:3.13-alpine - depends_on: - - c4-provider-net - c4-provider-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c4-provider-net-mesh:/amber/mesh:ro -volumes: - c4-provider-net-mesh: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"#, - ) - .expect("site compose artifact should be written"); - - prepare_dynamic_compose_child_artifact( - &SiteActuatorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "live-scope".to_string(), - run_root: temp.path().display().to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - artifact_dir: site_artifact.display().to_string(), - site_state_root: temp.path().display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: vec![5], - assigned_components: vec!["/sibling".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: vec![DynamicInputRouteRecord { - component: "/sibling".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - }, - &child_artifact, - &[], - &BTreeMap::from([ - ( - "/site/compose_local/router".to_string(), - MeshIdentityPublic { - id: "/site/compose_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("live-scope".to_string()), - }, - ), - ( - "/provider".to_string(), - MeshIdentityPublic { - id: "/provider".to_string(), - public_key: [8; 32], - mesh_scope: Some("live-scope".to_string()), - }, - ), - ]), - ) - .expect("dynamic compose child artifact should be prepared"); - - let filtered_plan: MeshProvisionPlan = read_json( - &child_artifact.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should exist"); - assert_eq!(filtered_plan.targets.len(), 1); - let component = filtered_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered component target should exist"); - assert_eq!(component.config.identity.id, "/sibling"); - assert!( - component - .config - .peers - .iter() - .any(|peer| peer.id == "/site/compose_local/router"), - "rewritten child mesh plan should peer with the site router", - ); - assert_eq!(component.config.outbound.len(), 1); - let outbound = &component.config.outbound[0]; - assert_eq!(outbound.slot, "upstream"); - assert_eq!( - outbound.route_id, - component_route_id("/provider", "http", MeshProtocol::Http) - ); - assert_eq!( - outbound.peer_addr, - format!("{COMPOSE_ROUTER_SERVICE_NAME}:24000") - ); - assert_eq!(outbound.peer_id, "/site/compose_local/router"); - assert_eq!(outbound.capability, "http"); - - let embedded_plan = read_embedded_compose_mesh_provision_plan(&child_artifact) - .expect("embedded compose mesh plan should be readable"); - let embedded_component = embedded_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == "/sibling" - }) - .expect("embedded child component target should exist"); - assert_eq!(embedded_component.config.outbound.len(), 1); - assert_eq!( - embedded_component.config.outbound[0].peer_addr, - format!("{COMPOSE_ROUTER_SERVICE_NAME}:24000") - ); - assert_eq!( - embedded_component.config.outbound[0].peer_id, - "/site/compose_local/router" - ); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&child_artifact), - "site router overlay", - ) - .expect("compose child router overlay should exist"); - assert!( - overlay.peers.iter().any(|peer| peer.id == "/provider"), - "compose routed-input overlay should include the provider peer", - ); - let routed_input_route = overlay - .inbound_routes - .iter() - .find(|route| route.route_id == component_route_id("/provider", "http", MeshProtocol::Http)) - .expect("compose routed-input overlay should include the provider route"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &routed_input_route.target - else { - panic!("compose routed-input overlay should forward across mesh"); - }; - assert_eq!(peer_addr, "c4-provider-net:23000"); - assert_eq!(peer_id, "/provider"); - assert_eq!(route_id, "component:/provider:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn dynamic_route_issuer_grants_include_component_provide_inputs() { - let issuers = dynamic_route_issuer_grants(&[SiteActuatorChildRecord { - child_id: 7, - artifact_root: "/tmp/child".to_string(), - assigned_components: vec!["/sibling".to_string()], - proxy_exports: BTreeMap::new(), - routed_inputs: vec![DynamicInputRouteRecord { - component: "/sibling".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - process_pid: None, - published: true, - }]) - .expect("component-provide routed inputs should produce issuer grants"); - - assert_eq!( - issuers.get("component:/provider:http:http"), - Some(&BTreeSet::from(["/sibling".to_string()])) - ); -} - -#[test] -fn direct_startup_route_overlay_does_not_require_materialized_child_mesh_files() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path().join("child-artifact"); - fs::create_dir_all(&artifact_root).expect("child artifact dir should exist"); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/consumer".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/direct_local/router".to_string(), - }], - inbound: Vec::new(), - outbound: vec![OutboundRoute { - route_id: "placeholder".to_string(), - rewrite_route_id: None, - slot: "upstream".to_string(), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - listen_port: 21000, - listen_addr: None, - peer_addr: "127.0.0.1:1".to_string(), - peer_id: "/provider".to_string(), - http_plugins: Vec::new(), - }], - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c1-consumer".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/direct_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "direct", - &[DynamicInputRouteRecord { - component: "/consumer".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - &BTreeMap::from([("/provider".to_string(), "127.0.0.1:25000".to_string())]), - &BTreeMap::from([( - "/provider".to_string(), - MeshIdentityPublic { - id: "/provider".to_string(), - public_key: [9; 32], - mesh_scope: Some("scope".to_string()), - }, - )]), - ) - .expect("startup overlay should be written without child mesh files in the artifact"); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&artifact_root), - "site router overlay", - ) - .expect("startup overlay should exist"); - assert_eq!(overlay.peers.len(), 1); - assert_eq!(overlay.peers[0].id, "/provider"); - let route = overlay - .inbound_routes - .iter() - .find(|route| route.route_id == component_route_id("/provider", "http", MeshProtocol::Http)) - .expect("startup overlay should include the provider route"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &route.target - else { - panic!("startup overlay should forward routed inputs over mesh"); - }; - assert_eq!(peer_addr, "127.0.0.1:25000"); - assert_eq!(peer_id, "/provider"); - assert_eq!(route_id, "component:/provider:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn filter_dynamic_mesh_provision_plan_keeps_only_child_owned_router_routes() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/vm_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/vm_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/vm_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c4-vm_admin".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/vm_helper/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/vm_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id( - "/job-compose/vm_helper/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c9-root".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/vm_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/vm_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job-compose/vm_helper/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id( - "vm_admin_http", - MeshProtocol::Http, - ), - capability: "vm_admin_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "127.0.0.1:23001".to_string(), - peer_id: "/vm_admin".to_string(), - route_id: component_route_id( - "/vm_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id( - "amber_export_916351bacd5bad90", - MeshProtocol::Http, - ), - capability: "amber_export_916351bacd5bad90".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "127.0.0.1:23002".to_string(), - peer_id: "/job-compose/vm_helper/root".to_string(), - route_id: component_route_id( - "/job-compose/vm_helper/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/vm_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job-compose/vm_helper/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - - filter_dynamic_mesh_provision_plan( - artifact_root, - &BTreeSet::from(["mesh/components/c9-root".to_string()]), - ) - .expect("dynamic mesh provision plan should be filtered"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should be readable"); - assert_eq!(filtered.targets.len(), 2); - - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered child component target should remain"); - assert_eq!(component.config.identity.id, "/job-compose/vm_helper/root"); - - let router = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .expect("filtered router target should remain"); - assert_eq!( - router - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job-compose/vm_helper/root"] - ); - assert_eq!( - router - .config - .inbound - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("amber_export_916351bacd5bad90", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); -} - -#[test] -fn ensure_dynamic_proxy_export_component_routes_in_artifact_adds_provider_route() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "admin", MeshProtocol::Http), - capability: "admin".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 9090 }, - allowed_issuers: Vec::new(), - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }], - }, - ) - .expect("mesh provision plan should be written"); - - ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - "/site/direct_local/router", - ) - .expect("dynamic provider routes should be projected into the child mesh plan"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("mesh provision plan should be readable"); - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("component target should remain"); - assert_eq!( - component - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/site/direct_local/router"] - ); - let admin_route = component - .config - .inbound - .iter() - .find(|route| route.capability == "admin") - .expect("existing local admin route should remain"); - assert!( - admin_route.allowed_issuers.is_empty(), - "rewriting a dynamic export must not broaden unrelated local routes", - ); - let provider_route = component - .config - .inbound - .iter() - .find(|route| route.capability == "http") - .expect("dynamic export provider route should be added"); - assert_eq!( - provider_route.allowed_issuers, - vec!["/site/direct_local/router".to_string()] - ); -} - -#[test] -fn ensure_dynamic_proxy_export_component_routes_in_artifact_leaves_local_routes_closed_without_exports() - { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: Some(SocketAddr::from(([127, 0, 0, 1], 19000))), - control_allow: None, - peers: Vec::new(), - inbound: vec![ - InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: Vec::new(), - }, - InboundRoute { - route_id: "external".to_string(), - capability: "external".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "UPSTREAM".to_string(), - optional: false, - }, - allowed_issuers: Vec::new(), - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }], - }, - ) - .expect("mesh provision plan should be written"); - - ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root, - &BTreeMap::new(), - "/site/direct_local/router", - ) - .expect("empty dynamic exports should leave existing routes unchanged"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("mesh provision plan should be readable"); - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("component target should remain"); - assert!( - component.config.peers.is_empty(), - "the router should not be peered when there are no dynamic exports to expose", - ); - assert_eq!( - component.config.inbound[0].allowed_issuers, - Vec::::new() - ); - assert!( - component.config.inbound[1].allowed_issuers.is_empty(), - "non-local routes should also remain unchanged" - ); -} - -#[test] -fn compose_component_mesh_peer_addr_uses_service_name() { - let temp = tempdir().expect("tempdir should be created"); - let addr = compose_component_mesh_peer_addr( - temp.path(), - "/provider", - &MeshProvisionOutput::Filesystem { - dir: "mesh/components/c4-provider-net".to_string(), - }, - 23000, - ) - .expect("compose peer addr should resolve"); - assert_eq!(addr, "c4-provider-net:23000"); -} - -#[test] -fn kubernetes_component_mesh_peer_addr_uses_service_name() { - let temp = tempdir().expect("tempdir should be created"); - let addr = kubernetes_component_mesh_peer_addr( - temp.path(), - "/provider", - &MeshProvisionOutput::KubernetesSecret { - name: "c4-provider-net-mesh".to_string(), - namespace: Some("ns".to_string()), - }, - 23000, - ) - .expect("kubernetes peer addr should resolve"); - assert_eq!(addr, "c4-provider-net:23000"); -} - -#[test] -fn add_dynamic_proxy_export_overlay_routes_rewrites_existing_export_route() { - let mut inbound_routes = vec![InboundRoute { - route_id: router_dynamic_export_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "stale-service:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }]; - - add_dynamic_proxy_export_overlay_routes( - &mut inbound_routes, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - |_| Ok("current-service:23000".to_string()), - ) - .expect("dynamic export routes should be rewritten from the current artifact state"); - - assert_eq!(inbound_routes.len(), 1); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &inbound_routes[0].target - else { - panic!("export route should remain a mesh forward"); - }; - assert_eq!(peer_addr, "current-service:23000"); - assert_eq!(peer_id, "/job/root"); - assert_eq!(route_id, "component:/job/root:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn child_router_overlay_payload_synthesizes_dynamic_proxy_export_routes_for_direct_children() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path().join("artifact"); - let runtime_root = temp.path().join("runtime"); - fs::create_dir_all(&artifact_root).expect("artifact root should be created"); - fs::create_dir_all(runtime_root.join("mesh/components/c8-root")) - .expect("runtime mesh dir should be created"); - - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/direct_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/direct_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 24000)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 24100))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }], - inbound: vec![InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - write_dynamic_proxy_exports_metadata( - &artifact_root, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - ) - .expect("dynamic proxy export metadata should be written"); - let identity = MeshIdentity::generate("/job/root", Some("scope".to_string())); - write_json( - &runtime_root - .join("mesh/components/c8-root") - .join(MESH_IDENTITY_FILENAME), - &MeshIdentitySecret::from_identity(&identity), - ) - .expect("mesh identity should be written"); - write_json( - &runtime_root - .join("mesh/components/c8-root") - .join(MESH_CONFIG_FILENAME), - &MeshConfigPublic { - identity: MeshIdentityPublic { - id: "/job/root".to_string(), - public_key: identity.public_key, - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 26000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - ) - .expect("mesh config should be written"); - - let (_peers, inbound_routes) = child_router_overlay_payload( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - artifact_dir: artifact_root.display().to_string(), - site_state_root: temp.path().join("state").display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: Some(temp.path().join("storage").display().to_string()), - runtime_root: Some(runtime_root.display().to_string()), - router_mesh_port: Some(24000), - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &artifact_root, - &runtime_root, - ) - .expect("child router overlay payload should be synthesized"); - - let export_route = inbound_routes - .iter() - .find(|route| { - route.route_id - == router_dynamic_export_route_id("/job/root", "http", MeshProtocol::Http) - }) - .expect("dynamic export route should be synthesized"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &export_route.target - else { - panic!("dynamic export route should forward to the child provider"); - }; - assert_eq!(peer_addr, "127.0.0.1:26000"); - assert_eq!(peer_id, "/job/root"); - assert_eq!(route_id, "component:/job/root:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn prepare_dynamic_kubernetes_child_artifact_keeps_router_overlay_local() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - let rbac_dir = temp.path().join("02-rbac"); - let deployments_dir = temp.path().join("03-deployments"); - let services_dir = temp.path().join("04-services"); - let netpol_dir = temp.path().join("05-networkpolicies"); - let site_state_root = temp.path().join("state").join("kind_local"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - fs::create_dir_all(&rbac_dir).expect("rbac dir should exist"); - fs::create_dir_all(&deployments_dir).expect("deployments dir should exist"); - fs::create_dir_all(&services_dir).expect("services dir should exist"); - fs::create_dir_all(&netpol_dir).expect("network policies dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - - let full_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/kind_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/kind_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/kind_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c3-kind-admin-mesh".to_string(), - namespace: None, - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23007)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/kind_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/kind_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/kind_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id("kind_http", MeshProtocol::Http), - capability: "kind_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c3-kind-admin:23000".to_string(), - peer_id: "/kind_admin".to_string(), - route_id: component_route_id( - "/kind_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id( - "amber_export_5062ceb53a8ac5c2", - MeshProtocol::Http, - ), - capability: "amber_export_5062ceb53a8ac5c2".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c7-kind-helper:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id( - "/job/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/kind_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "amber-router-external".to_string(), - namespace: None, - }, - }, - ], - }; - - let full_plan_json = - serde_json::to_string_pretty(&full_plan).expect("mesh plan should serialize"); - let indented_json = full_plan_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - for (path, body) in - [ - ( - temp.path().join("kustomization.yaml"), - r#"apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- 01-configmaps/amber-mesh-provision.yaml -- 02-rbac/amber-provisioner-job.yaml -- 02-rbac/amber-provisioner-role.yaml -- 02-rbac/amber-provisioner-rolebinding.yaml -- 02-rbac/amber-provisioner-sa.yaml -- 03-deployments/amber-router.yaml -- 03-deployments/c3-kind-admin.yaml -- 03-deployments/c7-kind-helper.yaml -- 04-services/amber-router.yaml -- 04-services/c3-kind-admin.yaml -- 04-services/c7-kind-helper.yaml -- 05-networkpolicies/amber-router-netpol.yaml -- 05-networkpolicies/c3-kind-admin-netpol.yaml -- 05-networkpolicies/c7-kind-helper-netpol.yaml -secretGenerator: -- name: amber-router-external - envs: - - router-external.env -"#, - ), - ( - temp.path().join("router-external.env"), - "AMBER_EXTERNAL_SLOT_API_URL=\n", - ), - ( - rbac_dir.join("amber-provisioner-job.yaml"), - "apiVersion: batch/v1\nkind: Job\nmetadata:\n name: amber-provisioner-bdcf48c3\n", - ), - ( - rbac_dir.join("amber-provisioner-role.yaml"), - "apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: \ - amber-provisioner\n", - ), - ( - rbac_dir.join("amber-provisioner-rolebinding.yaml"), - "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: \ - amber-provisioner\n", - ), - ( - rbac_dir.join("amber-provisioner-sa.yaml"), - "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: amber-provisioner\n", - ), - ( - deployments_dir.join("amber-router.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: amber-router\n \ - labels:\n amber.io/component-id: router\n", - ), - ( - deployments_dir.join("c3-kind-admin.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: c3-kind-admin\n \ - labels:\n amber.io/component-id: c3\n", - ), - ( - deployments_dir.join("c7-kind-helper.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: c7-kind-helper\n \ - labels:\n amber.io/component-id: c7\nspec:\n template:\n spec:\n \ - initContainers:\n - name: wait-mesh-config\n command:\n - \ - /amber-helper\n - wait-mesh-config\n - \ - /amber/mesh/mesh-config.json\n - scope\n - '30'\n \ - containers:\n - name: sidecar\n env:\n - name: \ - AMBER_SCENARIO_SCOPE\n value: scope\n", - ), - ( - services_dir.join("amber-router.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: amber-router\n labels:\n \ - amber.io/component-id: router\n", - ), - ( - services_dir.join("c3-kind-admin.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: c3-kind-admin\n labels:\n \ - amber.io/component-id: c3\n", - ), - ( - services_dir.join("c7-kind-helper.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: c7-kind-helper\n labels:\n \ - amber.io/component-id: c7\n", - ), - ( - netpol_dir.join("amber-router-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - amber-router-netpol\n labels:\n amber.io/component-id: router\n", - ), - ( - netpol_dir.join("c3-kind-admin-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - c3-kind-admin-netpol\n labels:\n amber.io/component-id: c3\n", - ), - ( - netpol_dir.join("c7-kind-helper-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - c7-kind-helper-netpol\n labels:\n amber.io/component-id: c7\nspec:\n \ - podSelector:\n matchLabels:\n amber.io/component: c7-kind-helper\n \ - policyTypes:\n - Ingress\n ingress:\n - from:\n - podSelector:\n \ - matchLabels:\n amber.io/component: c3-kind-admin\n ports:\n - \ - protocol: TCP\n port: 23007\n", - ), - ] - { - fs::write(path, body).expect("kubernetes artifact file should be written"); - } - write_json( - &site_state_root.join("manager-state.json"), - &SiteManagerState { - schema: SITE_STATE_SCHEMA.to_string(), - version: SITE_STATE_VERSION, - run_id: "run-123".to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - status: SiteLifecycleStatus::Running, - artifact_dir: temp.path().display().to_string(), - supervisor_pid: 1, - process_pid: None, - compose_project: None, - kubernetes_namespace: Some("amber-run-123-kind-local".to_string()), - port_forward_pid: None, - context: Some("kind-test".to_string()), - router_control: Some("127.0.0.1:24100".to_string()), - router_mesh_addr: Some("127.0.0.1:24000".to_string()), - router_identity_id: Some("/site/kind_local/router".to_string()), - router_public_key_b64: Some( - base64::engine::general_purpose::STANDARD.encode([7u8; 32]), - ), - last_error: None, - }, - ) - .expect("site manager state should be written"); - - prepare_dynamic_kubernetes_child_artifact( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - artifact_dir: temp.path().display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: Some("amber-run-123-kind-local".to_string()), - context: Some("kind-test".to_string()), - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - component_ids: vec![7], - assigned_components: vec!["/job/root".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(7, "/job/root", "http", "http", "http", 8080), - )]), - routed_inputs: Vec::new(), - }, - temp.path(), - &BTreeMap::from([( - "/site/kind_local/router".to_string(), - MeshIdentityPublic { - id: "/site/kind_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh-scope-test".to_string()), - }, - )]), - ) - .expect("kubernetes dynamic child artifact should be prepared"); - - let overlay_plan: MeshProvisionPlan = read_json( - &temp.path().join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("overlay mesh plan should be readable"); - assert_eq!(overlay_plan.targets.len(), 2); - let overlay_scopes = overlay_plan - .targets - .iter() - .map(|target| target.config.identity.mesh_scope.clone()) - .collect::>(); - assert!( - overlay_plan - .targets - .iter() - .all(|target| target.config.identity.mesh_scope.as_deref() == Some("mesh-scope-test")), - "dynamic kubernetes overlay plan should be projected onto the live run mesh scope: \ - {overlay_scopes:?}", - ); - let overlay_router = overlay_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .expect("overlay router target should exist"); - assert_eq!( - overlay_router - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job/root"] - ); - assert_eq!( - overlay_router - .config - .inbound - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("amber_export_5062ceb53a8ac5c2", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); - - let embedded_plan = read_embedded_kubernetes_mesh_provision_plan(temp.path()) - .expect("embedded kubernetes plan should be readable"); - assert_eq!(embedded_plan.targets.len(), 1); - assert!( - embedded_plan - .targets - .iter() - .all(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - ); - assert_eq!( - embedded_plan.targets[0].config.identity.id, "/job/root", - "embedded provision plan should keep only child component targets" - ); - assert!( - embedded_plan - .targets - .iter() - .all(|target| target.config.identity.mesh_scope.as_deref() == Some("mesh-scope-test")), - "embedded provision plan should inherit the live run mesh scope", - ); - assert_eq!(embedded_plan.existing_peer_identities.len(), 1); - assert_eq!( - embedded_plan.existing_peer_identities[0].id, - "/site/kind_local/router" - ); - assert_eq!( - embedded_plan.existing_peer_identities[0] - .mesh_scope - .as_deref(), - Some("mesh-scope-test"), - "existing router identity should preserve the live run mesh scope", - ); - let helper_deployment = fs::read_to_string(deployments_dir.join("c7-kind-helper.yaml")) - .expect("kubernetes child deployment should be readable"); - assert!( - helper_deployment.contains("- mesh-scope-test"), - "dynamic kubernetes deployment should rewrite the wait-mesh-config expected scope", - ); - assert!( - helper_deployment.contains("value: mesh-scope-test"), - "dynamic kubernetes deployment should rewrite AMBER_SCENARIO_SCOPE onto the live run scope", - ); - assert!( - !helper_deployment.contains("value: scope"), - "dynamic kubernetes deployment should not retain the compiled child scope", - ); - let helper_netpol = fs::read_to_string(netpol_dir.join("c7-kind-helper-netpol.yaml")) - .expect("kubernetes child network policy should be readable"); - assert!( - helper_netpol.contains("amber.io/component: amber-router"), - "dynamic kubernetes child network policy should admit router ingress for proxy exports", - ); - assert!( - helper_netpol.contains("port: 23007"), - "dynamic kubernetes child network policy should retain the child mesh port for router \ - ingress", - ); - - let kustomization = fs::read_to_string(temp.path().join("kustomization.yaml")) - .expect("kustomization should be readable"); - assert!(kustomization.contains("03-deployments/c7-kind-helper.yaml")); - assert!(kustomization.contains("04-services/c7-kind-helper.yaml")); - assert!(kustomization.contains("05-networkpolicies/c7-kind-helper-netpol.yaml")); - assert!(kustomization.contains("02-rbac/amber-provisioner-job.yaml")); - assert!(!kustomization.contains("03-deployments/amber-router.yaml")); - assert!(!kustomization.contains("03-deployments/c3-kind-admin.yaml")); - assert!(!kustomization.contains("amber-router-external")); - assert!( - !deployments_dir.join("amber-router.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated router resources", - ); - assert!( - !deployments_dir.join("c3-kind-admin.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated creator resources", - ); - assert!( - !services_dir.join("amber-router.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated router services", - ); - assert!( - !services_dir.join("c3-kind-admin.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated creator services", - ); - - let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files( - &read_artifact_snapshot(temp.path()).expect("prepared child artifact should be readable"), - ) - .expect("destroy bundle should project"); - let destroy_kustomization = destroy_bundle - .get("kustomization.yaml") - .expect("destroy bundle should include kustomization"); - assert!(destroy_kustomization.contains("03-deployments/c7-kind-helper.yaml")); - assert!(destroy_kustomization.contains("02-rbac/amber-provisioner-job.yaml")); - assert!(!destroy_kustomization.contains("01-configmaps/amber-mesh-provision.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-role.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-rolebinding.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); -} - -#[test] -fn kubernetes_peer_addrs_are_derived_from_mesh_secret_names() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/kind_helper".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }], - }; - let plan_json = serde_json::to_string_pretty(&mesh_plan).expect("mesh plan should serialize"); - let indented_json = plan_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - - let peer_addrs = - kubernetes_peer_addrs_for_artifact(temp.path()).expect("peer addrs should resolve"); - assert_eq!( - peer_addrs - .get("/job-compose/kind_helper") - .map(String::as_str), - Some("c7-kind-helper:23000"), - ); -} - -#[test] -fn kubernetes_dynamic_route_overlay_uses_runtime_plan_with_router_target() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - - let component_target = amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/kind_helper".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }; - let embedded_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![component_target.clone()], - }; - let runtime_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - component_target, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/kind_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23001)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "amber-router-mesh".to_string(), - namespace: None, - }, - }, - ], - }; - let embedded_json = - serde_json::to_string_pretty(&embedded_plan).expect("embedded plan should serialize"); - let indented_json = embedded_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - write_json(&temp.path().join("mesh-provision-plan.json"), &runtime_plan) - .expect("runtime plan should be written"); - - build_kubernetes_route_overlay_base( - temp.path(), - &["/job-compose/kind_helper".to_string()], - &BTreeMap::new(), - &BTreeMap::new(), - ) - .expect("runtime plan should supply the router target"); -} - -#[test] -fn kubernetes_expected_workloads_reads_current_artifact_names() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - fs::create_dir_all(artifact_root.join("03-deployments")) - .expect("deployment dir should be created"); - fs::create_dir_all(artifact_root.join("02-rbac")).expect("rbac dir should be created"); - fs::write( - artifact_root.join("03-deployments").join("app.yaml"), - r#" -apiVersion: apps/v1 -kind: Deployment -metadata: - name: child-helper ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: amber-router -"#, - ) - .expect("deployment yaml should be written"); - fs::write( - artifact_root.join("02-rbac").join("job.yaml"), - r#" -apiVersion: batch/v1 -kind: Job -metadata: - name: amber-provisioner-1234 -"#, - ) - .expect("job yaml should be written"); - - let workloads = supervisor::kubernetes_expected_workloads(artifact_root) - .expect("artifact workloads should parse"); - assert_eq!(workloads.jobs, vec!["amber-provisioner-1234"]); - assert_eq!(workloads.deployments, vec!["amber-router", "child-helper"]); -} - -#[test] -fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { - let temp = tempdir().expect("tempdir should be created"); - let site_state_root = temp.path().join("state").join("direct_local"); - let child_root = site_actuator_child_root_for_site(&site_state_root, 7); - fs::create_dir_all(child_root.join("artifact")).expect("child artifact dir should exist"); - fs::write(child_root.join("artifact").join("marker.txt"), "marker") - .expect("child marker should be written"); - write_json( - &site_actuator_state_path(&site_state_root), - &SiteActuatorState { - schema: "amber.run.site_actuator_state".to_string(), - version: 1, - run_id: "run-123".to_string(), - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - children: BTreeMap::from([( - 7, - SiteActuatorChildRecord { - child_id: 7, - artifact_root: child_root.join("artifact").display().to_string(), - assigned_components: Vec::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - process_pid: None, - published: true, - }, - )]), - }, - ) - .expect("site actuator state should be written"); - - cleanup_dynamic_site_children(&site_state_root, SiteKind::Direct) - .expect("dynamic site children should be cleaned"); - - let state: SiteActuatorState = read_json( - &site_actuator_state_path(&site_state_root), - "site actuator state", - ) - .expect("site actuator state should be readable"); - assert!(state.children.is_empty()); - assert!(!child_root.exists()); -} - -#[test] -fn parse_process_table_reads_ps_output() { - assert_eq!( - parse_process_table(" 42 1\n 84 42\n").expect("process table should parse"), - HashMap::from([(42, 1), (84, 42)]) - ); -} - -#[cfg(unix)] -#[test] -fn parse_process_status_code_reads_ps_state() { - assert_eq!(parse_process_status_code("S+\n"), Some('S')); - assert_eq!(parse_process_status_code("z\n"), Some('Z')); - assert_eq!(parse_process_status_code("\n"), None); -} - -#[cfg(unix)] -#[test] -fn collect_process_tree_postorder_visits_descendants_before_parent() { - let children_by_parent = HashMap::from([(1, vec![2, 3]), (2, vec![4]), (3, vec![5, 6])]); - let mut ordered = Vec::new(); - collect_process_tree_postorder(1, &children_by_parent, &mut ordered); - assert_eq!(ordered, vec![4, 2, 5, 6, 3, 1]); -} + +use tempfile::TempDir; + +use super::*; #[test] -fn container_host_from_resolved_ip_matches_provider_and_consumer_kind() { - assert_eq!( - container_host_from_resolved_ip(SiteKind::Compose, SiteKind::Direct, Some("172.17.0.1"),), - "127.0.0.1" - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Compose, SiteKind::Vm, Some("172.17.0.1"),), - "127.0.0.1" - ); +fn site_state_paths_are_site_scoped() { + let root = Path::new("/tmp/amber-run/state"); assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Compose, Some("172.17.0.1"),), - CONTAINER_HOST_ALIAS + site_state_path(root, "site-a"), + Path::new("/tmp/amber-run/state/site-a/manager-state.json") ); assert_eq!( - container_host_from_resolved_ip(SiteKind::Direct, SiteKind::Compose, Some("172.17.0.1"),), - CONTAINER_HOST_ALIAS + desired_links_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/desired-links.json") ); assert_eq!( - container_host_from_resolved_ip( - SiteKind::Kubernetes, - SiteKind::Compose, - Some("172.17.0.1"), - ), - "172.17.0.1" + site_controller_plan_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/site-controller-plan.json") ); assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Kubernetes, Some("172.17.0.1"),), - "172.17.0.1" + site_existing_peer_ports_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/existing-peer-ports.json") ); assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Kubernetes, None), - CONTAINER_HOST_ALIAS + site_existing_peer_identities_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/existing-peer-identities.json") ); } +#[cfg(unix)] #[test] -fn containerized_consumers_bridge_runtime_links() { - assert!(link_needs_bridge_proxy(SiteKind::Direct, SiteKind::Compose)); - assert!(link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Compose)); - assert!(!link_needs_bridge_proxy( - SiteKind::Kubernetes, - SiteKind::Compose - )); - assert!(link_needs_bridge_proxy( - SiteKind::Direct, - SiteKind::Kubernetes - )); - assert!(link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Kubernetes)); - assert!(!link_needs_bridge_proxy( - SiteKind::Kubernetes, - SiteKind::Kubernetes - )); - assert!(link_needs_bridge_proxy( - SiteKind::Compose, - SiteKind::Compose - )); - assert!(!link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Direct)); -} +fn compose_site_state_root_is_private_to_the_launching_user() { + let temp = TempDir::new().expect("temp dir should create"); + let site_state_root = temp.path().join("state").join("compose-local"); -#[test] -fn bridge_proxy_bind_and_probe_addresses_match_consumer_kind() { - let compose_listen = bridge_proxy_bind_addr(SiteKind::Compose, 41000); - assert_eq!(compose_listen, SocketAddr::from(([0, 0, 0, 0], 41000))); + launch_bundle::prepare_site_state_root(&site_state_root, SiteKind::Compose) + .expect("compose state root should prepare"); + + let mode = fs::metadata(&site_state_root) + .expect("state root metadata should read") + .permissions() + .mode() + & 0o777; assert_eq!( - bridge_proxy_probe_addr(compose_listen), - SocketAddr::from(([127, 0, 0, 1], 41000)) + mode, 0o700, + "compose site controller state contains control-plane data and must not be writable by \ + unrelated local users" ); +} + +#[test] +fn site_controller_image_override_uses_dev_tag() { + let overrides = BTreeMap::from([( + "site_controller".to_string(), + "dev-site-controller".to_string(), + )]); - let kind_listen = bridge_proxy_bind_addr(SiteKind::Kubernetes, 42000); - assert_eq!(kind_listen, SocketAddr::from(([0, 0, 0, 0], 42000))); assert_eq!( - bridge_proxy_probe_addr(kind_listen), - SocketAddr::from(([127, 0, 0, 1], 42000)) + launch_bundle::site_controller_image_reference_from_overrides(&overrides), + format!( + "{}/{}:{}", + amber_images::AMBER_SITE_CONTROLLER.registry, + amber_images::AMBER_SITE_CONTROLLER.name, + "dev-site-controller", + ) ); +} + +#[test] +fn recorded_process_roots_include_site_supervisor() { + let site = SiteReceipt { + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 7, + process_pid: Some(11), + compose_project: None, + kubernetes_namespace: None, + port_forward_pid: Some(13), + context: None, + router_control: None, + router_mesh_addr: None, + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_url: None, + }; - let direct_listen = bridge_proxy_bind_addr(SiteKind::Direct, 43000); - assert_eq!(direct_listen, SocketAddr::from(([127, 0, 0, 1], 43000))); - assert_eq!(bridge_proxy_probe_addr(direct_listen), direct_listen); + assert_eq!(recorded_process_roots(&site), vec![7, 11, 13]); } #[test] -fn host_service_addressing_matches_consumer_kind() { +fn site_controller_local_router_control_uses_backend_local_control_targets() { + let artifact_dir = Path::new("/tmp/site-artifact"); + assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Compose, 41000), - SocketAddr::from(([0, 0, 0, 0], 41000)) + launch_bundle::site_controller_local_router_control(SiteKind::Compose, artifact_dir), + "unix:///amber/control/router-control.sock" ); assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 42000), - SocketAddr::from(([0, 0, 0, 0], 42000)) + launch_bundle::site_controller_local_router_control(SiteKind::Kubernetes, artifact_dir), + "amber-router:24100" ); - assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Direct, 43000), - SocketAddr::from(([127, 0, 0, 1], 43000)) + assert!( + launch_bundle::site_controller_local_router_control(SiteKind::Direct, artifact_dir) + .starts_with("unix://"), + "direct site controllers should know their local router control socket up front" ); +} + +#[test] +fn site_controller_peer_router_urls_are_local_to_the_controller_site() { assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Vm, 44000), - SocketAddr::from(([127, 0, 0, 1], 44000)) + amber_site_controller::site_controller_peer_router_url(SiteKind::Direct, 37046), + "http://127.0.0.1:37046" ); assert_eq!( - host_service_host_for_consumer(SiteKind::Compose), - "host.docker.internal" + amber_site_controller::site_controller_peer_router_url(SiteKind::Vm, 37046), + "http://127.0.0.1:37046" ); assert_eq!( - host_service_host_for_consumer(SiteKind::Direct), - "127.0.0.1" + amber_site_controller::site_controller_peer_router_url(SiteKind::Compose, 37046), + "http://amber-router:37046" ); - assert_eq!(host_service_host_for_consumer(SiteKind::Vm), "127.0.0.1"); assert_eq!( - host_service_host_for_consumer(SiteKind::Kubernetes), - container_host_for_consumer(SiteKind::Direct, SiteKind::Kubernetes) + amber_site_controller::site_controller_peer_router_url(SiteKind::Kubernetes, 37046), + "http://amber-router:37046" ); } #[test] -fn bridge_proxy_external_url_uses_consumer_aware_host() { +fn compose_consumers_use_host_alias_for_kubernetes_router_mesh() { assert_eq!( - bridge_proxy_external_url(44000, NetworkProtocol::Http, SiteKind::Compose) - .expect("http bridge proxy url should be valid"), - "http://host.docker.internal:44000" + supervisor::container_host_from_resolved_ip( + SiteKind::Kubernetes, + SiteKind::Compose, + Some("192.168.65.254"), + ), + "host.docker.internal" ); assert_eq!( - bridge_proxy_external_url(45000, NetworkProtocol::Http, SiteKind::Kubernetes) - .expect("http bridge proxy url should be valid"), - format!( - "http://{}:45000", - host_service_host_for_consumer(SiteKind::Kubernetes) + amber_site_controller::router_mesh_addr_for_consumer( + SiteKind::Kubernetes, + SiteKind::Compose, + "127.0.0.1:24077", ) + .expect("compose consumers should be able to route to kubernetes peers"), + "host.docker.internal:24077" ); } #[test] -fn router_mesh_addr_for_container_consumers_uses_projected_host() { - assert_eq!( - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Compose, "127.0.0.1:24000") - .expect("compose consumer router addr should be valid"), - "host.docker.internal:24000" +fn site_controller_image_includes_the_amber_cli_binary() { + let dockerfile = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../docker/amber-site-controller/Dockerfile"), + ) + .expect("site-controller Dockerfile should read"); + assert!( + dockerfile.contains("cargo build --locked --release -p amber-cli -p amber-site-controller") + || dockerfile.contains("cargo build --locked -p amber-cli -p amber-site-controller"), + "site-controller image must build the amber CLI so containerized controllers can spawn \ + amber subcommands:\n{dockerfile}" ); - assert_eq!( - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Kubernetes, "127.0.0.1:25000") - .expect("kubernetes consumer router addr should be valid"), - format!( - "{}:25000", - container_host_for_consumer(SiteKind::Direct, SiteKind::Kubernetes) - ) + assert!( + dockerfile.contains("COPY --from=builder /out/amber /usr/local/bin/amber"), + "site-controller image must ship the amber CLI binary alongside \ + amber-site-controller:\n{dockerfile}" ); -} - -#[test] -fn router_mesh_addr_for_local_consumers_preserves_existing_port() { - let rewritten = - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Direct, "127.0.0.1:26000") - .expect("local consumer router addr should be valid"); - #[cfg(target_os = "linux")] - assert_eq!(rewritten, "10.0.2.2:26000"); - #[cfg(not(target_os = "linux"))] - assert_eq!(rewritten, "127.0.0.1:26000"); -} - -#[test] -fn outside_proxy_mesh_listener_stays_loopback_for_local_consumers() { - let context = RunOutsideProxyContext { - mesh_scope: "scope".to_string(), - sites: BTreeMap::from([( - "direct".to_string(), - test_launched_site_with_kind(SiteKind::Direct), - )]), - exports: BTreeMap::new(), - slots: BTreeMap::from([( - "api".to_string(), - RunOutsideSlot { - required: true, - kind: CapabilityKind::Http, - url_env: "AMBER_EXTERNAL_SLOT_API_URL".to_string(), - consumer_sites: vec!["direct".to_string()], - }, - )]), - }; - - assert_eq!( - outside_proxy_mesh_listen_addr( - &context, - &[("api".to_string(), "http://127.0.0.1:9000".to_string())], - 48000, - ) - .expect("outside proxy bind addr"), - SocketAddr::from(([127, 0, 0, 1], 48000)) + assert!( + dockerfile.contains("COPY examples ./examples"), + "site-controller image must include the examples tree so amber-cli can satisfy \ + cli/build.rs:\n{dockerfile}" ); -} - -#[test] -fn outside_proxy_mesh_listener_expands_for_container_consumers() { - let context = RunOutsideProxyContext { - mesh_scope: "scope".to_string(), - sites: BTreeMap::from([ - ( - "direct".to_string(), - test_launched_site_with_kind(SiteKind::Direct), - ), - ( - "compose".to_string(), - test_launched_site_with_kind(SiteKind::Compose), - ), - ]), - exports: BTreeMap::new(), - slots: BTreeMap::from([( - "api".to_string(), - RunOutsideSlot { - required: true, - kind: CapabilityKind::Http, - url_env: "AMBER_EXTERNAL_SLOT_API_URL".to_string(), - consumer_sites: vec!["direct".to_string(), "compose".to_string()], - }, - )]), - }; - - assert_eq!( - outside_proxy_mesh_listen_addr( - &context, - &[("api".to_string(), "http://127.0.0.1:9000".to_string())], - 49000, - ) - .expect("outside proxy bind addr"), - SocketAddr::from(([0, 0, 0, 0], 49000)) + assert!( + dockerfile.contains("COPY README.md ./"), + "site-controller image must include the workspace README because amber-cli embeds \ + it:\n{dockerfile}" ); } #[test] -fn bridge_proxy_export_binding_uses_selected_listen_addr() { - assert_eq!( - bridge_proxy_export_binding("api", SocketAddr::from(([127, 0, 0, 1], 46000))), - "api=127.0.0.1:46000" +fn provisioner_image_supports_debug_builds() { + let dockerfile = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../docker/amber-provisioner/Dockerfile"), + ) + .expect("provisioner Dockerfile should read"); + assert!( + dockerfile.contains("ARG BUILD_MODE=release"), + "provisioner image should accept the shared BUILD_MODE argument so mixed-run tests can \ + use debug builds:\n{dockerfile}" ); - assert_eq!( - bridge_proxy_export_binding("api", SocketAddr::from(([0, 0, 0, 0], 47000))), - "api=0.0.0.0:47000" + assert!( + dockerfile.contains("if [ \"$BUILD_MODE\" = \"release\" ]; then") + && dockerfile.contains("cargo build -p amber-provisioner --release --locked") + && dockerfile.contains("cargo build -p amber-provisioner --locked") + && dockerfile.contains("\"${build_dir}\"/amber-provisioner"), + "provisioner image should support both release and debug output paths:\n{dockerfile}" ); } -fn test_launched_site_with_kind(kind: SiteKind) -> LaunchedSite { - LaunchedSite { - receipt: SiteReceipt { - kind, - artifact_dir: "/tmp/artifact".to_string(), - supervisor_pid: 1, - process_pid: None, - compose_project: None, - context: None, - kubernetes_namespace: None, - port_forward_pid: None, - router_mesh_addr: None, - router_control: None, - router_identity_id: None, - router_public_key_b64: None, - }, - router_identity: MeshIdentityPublic { - id: format!("/site/{kind:?}"), - public_key: [0; 32], - mesh_scope: None, - }, - router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), - router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), - } -} - #[test] -fn kubernetes_sites_get_startup_budget_after_workloads_are_ready() { +fn site_controller_command_prefers_current_amber_binary_over_stale_sibling() { + let temp = TempDir::new().expect("temp dir"); + let debug_dir = temp.path().join("target").join("debug"); + fs::create_dir_all(&debug_dir).expect("debug dir should exist"); + let amber = debug_dir.join(format!("amber{}", std::env::consts::EXE_SUFFIX)); + let site_controller = debug_dir.join(format!( + "amber-site-controller{}", + std::env::consts::EXE_SUFFIX + )); + fs::write(&amber, "").expect("amber binary should exist"); + fs::write(&site_controller, "").expect("stale site controller binary should exist"); + + let command = site_controller_command_from(&amber).expect("site controller command"); assert_eq!( - site_ready_timeout_for_kind(SiteKind::Kubernetes), - KUBERNETES_WORKLOAD_READY_TIMEOUT + KUBERNETES_SITE_READY_BUFFER + command.executable, amber, + "the running amber binary should launch its own run-site-controller subcommand instead of \ + a stale sibling amber-site-controller executable" ); + assert_eq!(command.prefix_args, vec!["run-site-controller"]); } #[test] -fn kubernetes_namespace_name_is_run_scoped() { +fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { + let temp = TempDir::new().expect("temp dir"); + let debug_dir = temp.path().join("target").join("debug"); + let deps_dir = debug_dir.join("deps"); + fs::create_dir_all(&deps_dir).expect("deps dir should exist"); + let current = deps_dir.join(format!("mixed_run-test{}", std::env::consts::EXE_SUFFIX)); + let amber = debug_dir.join(format!("amber{}", std::env::consts::EXE_SUFFIX)); + let site_controller = debug_dir.join(format!( + "amber-site-controller{}", + std::env::consts::EXE_SUFFIX + )); + fs::write(¤t, "").expect("current test binary should exist"); + fs::write(&amber, "").expect("amber binary should exist"); + fs::write(&site_controller, "").expect("stale site controller binary should exist"); + + let command = site_controller_command_from(¤t).expect("site controller command"); assert_eq!( - kubernetes_namespace_name("run-1234abcd", "kind_c"), - "amber-run-1234abcd-kind-c" - ); - assert_ne!( - kubernetes_namespace_name("run-1234abcd", "kind_c"), - kubernetes_namespace_name("run-5678efgh", "kind_c") + command.executable, amber, + "cargo test binaries should launch the freshly built amber binary instead of a stale \ + sibling amber-site-controller executable" ); + assert_eq!(command.prefix_args, vec!["run-site-controller"]); } #[test] -fn prepare_kubernetes_artifact_namespace_rewrites_kustomization_namespace() { - let temp = tempdir().expect("tempdir should be created"); - let kustomization = temp.path().join("kustomization.yaml"); - fs::write( - &kustomization, - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - scenario-old\n", - ) - .expect("kustomization should be written"); - - let namespace = prepare_kubernetes_artifact_namespace("run-1234abcd", "kind_c", temp.path()) - .expect("artifact namespace should be prepared"); +fn local_site_controller_ready_treats_absent_controller_as_optional() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "vm-local".to_string(), + kind: SiteKind::Vm, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + }; - assert_eq!(namespace, "amber-run-1234abcd-kind-c"); - assert_eq!( - fs::read_to_string(&kustomization).expect("kustomization should be readable"), - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - amber-run-1234abcd-kind-c\n" + assert!( + supervisor::local_site_controller_ready(&plan, Duration::from_millis(10)) + .expect("sites without a local controller should still be considered ready"), + "managed vm sites should not block startup on a non-existent local controller" ); } #[test] -fn prepare_kubernetes_site_artifact_for_apply_rewrites_runtime_artifact_state() { - let temp = tempdir().expect("tempdir should be created"); - let kustomization = temp.path().join("kustomization.yaml"); - fs::write( - &kustomization, - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - scenario-old\n", - ) - .expect("kustomization should be written"); - let env_file = temp.path().join("root-config.env"); - fs::write(&env_file, "AMBER_TEST_VALUE=stale\n").expect("env file should be written"); - - let mut launch_env = BTreeMap::new(); - launch_env.insert("AMBER_TEST_VALUE".to_string(), "fresh".to_string()); - let plan = SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-1234abcd".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_c".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_c/router".to_string(), - artifact_dir: temp.path().display().to_string(), - site_state_root: temp.path().join("state").display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), +fn local_site_controller_addr_requires_loopback_http() { + let mut plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "site-a".to_string(), + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), storage_root: None, runtime_root: None, router_mesh_port: None, compose_project: None, kubernetes_namespace: None, - context: Some("kind-test".to_string()), + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, observability_endpoint: None, - launch_env, + site_controller_url: Some("http://127.0.0.1:24200".to_string()), + launch_env: BTreeMap::new(), }; - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&plan, temp.path()) - .expect("artifact should be prepared"); - - assert_eq!( - supervisor_plan.kubernetes_namespace.as_deref(), - Some("amber-run-1234abcd-kind-c") - ); assert_eq!( - fs::read_to_string(&kustomization).expect("kustomization should be readable"), - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - amber-run-1234abcd-kind-c\n" + supervisor::local_site_controller_addr(&plan).expect("loopback controller address"), + Some(SocketAddr::from(([127, 0, 0, 1], 24200))) ); - assert_eq!( - fs::read_to_string(&env_file).expect("env file should be readable"), - "AMBER_TEST_VALUE=fresh\n" - ); -} -#[test] -fn external_slot_name_from_env_var_restores_slot_name() { + plan.site_controller_url = Some("http://192.168.1.10:24200".to_string()); assert_eq!( - external_slot_name_from_env_var("AMBER_EXTERNAL_SLOT_API_URL"), - "api" + supervisor::local_site_controller_addr(&plan) + .expect("non-loopback controller address should be ignored"), + None ); } #[test] -fn external_slot_env_for_site_skips_missing_weak_provider() { - let env = external_slot_env_for_site( - "consumer_site", - SiteKind::Direct, - &[RunLink { - provider_site: "provider_site".to_string(), - consumer_site: "consumer_site".to_string(), - provider_component: "/provider".to_string(), - provide: "api".to_string(), - consumer_component: "/consumer".to_string(), - slot: "upstream".to_string(), - weak: true, - protocol: NetworkProtocol::Http, - export_name: "amber_export_provider_api_http".to_string(), - external_slot_name: "amber_link_consumer_provider_api_http".to_string(), - }], - &BTreeMap::new(), - ) - .expect("weak links should not require a launched provider"); - assert!(env.is_empty()); -} - -#[test] -fn maybe_resolve_proxy_run_target_resolves_run_id_and_prefers_live_state() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-123"; - let run_root = storage_root.join("runs").join(run_id); - let artifact_dir = run_root.join("sites").join("direct_local").join("artifact"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - let state_root = run_root.join("state"); - fs::create_dir_all(state_root.join("direct_local")).expect("state dir should exist"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - framework_control_state: None, - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([( - "direct_local".to_string(), - test_site_receipt( - SiteKind::Direct, - &artifact_dir, - Some("unix:///receipt.sock"), - Some("127.0.0.1:18080"), - ), - )]), - }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - write_json( - &site_state_path(&state_root, "direct_local"), - &test_site_state( - run_id, - "direct_local", - SiteKind::Direct, - &artifact_dir, - Some("unix:///live.sock"), - Some("127.0.0.1:18081"), - ), - ) - .expect("state should serialize"); - - let resolved = maybe_resolve_proxy_run_target(run_id, Some("direct_local"), Some(storage_root)) - .expect("run target resolution should succeed") - .expect("run target should resolve"); - +fn host_service_bind_addr_matches_component_reachability() { assert_eq!( - resolved.artifact_dir, - artifact_dir - .canonicalize() - .expect("artifact dir should canonicalize") + supervisor::host_service_bind_addr_for_consumer(SiteKind::Compose, 24200), + SocketAddr::from(([0, 0, 0, 0], 24200)) ); assert_eq!( - resolved.router_control_addr.as_deref(), - Some("unix:///live.sock") + supervisor::host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 24201), + SocketAddr::from(([0, 0, 0, 0], 24201)) ); assert_eq!( - resolved.router_addr, - Some( - "127.0.0.1:18081" - .parse::() - .expect("socket addr should parse") - ) + supervisor::host_service_bind_addr_for_consumer(SiteKind::Vm, 24202), + SocketAddr::from(([0, 0, 0, 0], 24202)) ); + + let direct = supervisor::host_service_bind_addr_for_consumer(SiteKind::Direct, 24203); + if cfg!(target_os = "linux") { + assert_eq!(direct, SocketAddr::from(([0, 0, 0, 0], 24203))); + } else { + assert_eq!(direct, SocketAddr::from(([127, 0, 0, 1], 24203))); + } } #[test] -fn maybe_resolve_proxy_run_target_requires_site_for_multi_site_run() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-456"; - let run_root = storage_root.join("runs").join(run_id); - let direct_artifact = run_root.join("sites").join("direct_local").join("artifact"); - let compose_artifact = run_root - .join("sites") - .join("compose_local") - .join("artifact"); - fs::create_dir_all(&direct_artifact).expect("direct artifact dir should exist"); - fs::create_dir_all(&compose_artifact).expect("compose artifact dir should exist"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - framework_control_state: None, - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - test_site_receipt( - SiteKind::Compose, - &compose_artifact, - Some("unix:///compose.sock"), - Some("127.0.0.1:19090"), - ), - ), - ( - "direct_local".to_string(), - test_site_receipt( - SiteKind::Direct, - &direct_artifact, - Some("unix:///direct.sock"), - Some("127.0.0.1:19091"), - ), - ), - ]), +fn local_site_controller_ready_waits_for_http_listener() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let handle = thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut request = [0u8; 256]; + let _ = stream.read(&mut request); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\nConnection: close\r\n\r\n{\"ok\":true}", + ); + } + }); + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "site-a".to_string(), + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: Some(format!("http://127.0.0.1:{}", addr.port())), + launch_env: BTreeMap::new(), }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - let err = maybe_resolve_proxy_run_target(run_id, None, Some(storage_root)) - .expect_err("multi-site run ids should require --site"); - let message = err.to_string(); - assert!( - message.contains("contains multiple sites"), - "expected multi-site guidance, got: {message}" - ); assert!( - message.contains("--site "), - "expected --site guidance, got: {message}" + supervisor::local_site_controller_ready(&plan, Duration::from_secs(1)) + .expect("controller readiness should succeed"), + "local site readiness should wait for the controller HTTP listener" ); + handle.join().expect("listener thread should exit"); } -#[cfg(unix)] -#[tokio::test] -async fn stop_run_forces_supervisor_shutdown_and_cleans_up() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-stuck"; - let run_root = storage_root.join("runs").join(run_id); - let artifact_dir = run_root.join("sites").join("direct_local").join("artifact"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - - let mut stubborn_supervisor = Command::new("sh") - .arg("-c") - .arg("trap '' TERM; while :; do sleep 1 & wait $!; done") - .spawn() - .expect("stubborn supervisor should spawn"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - framework_control_state: None, - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([( - "direct_local".to_string(), - SiteReceipt { - supervisor_pid: stubborn_supervisor.id(), - ..test_site_receipt( - SiteKind::Direct, - &artifact_dir, - Some("unix:///receipt.sock"), - Some("127.0.0.1:18080"), - ) - }, - )]), - }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - - let state_root = run_root.join("state"); - let mut state = test_site_state( - run_id, - "direct_local", - SiteKind::Direct, - &artifact_dir, - Some("unix:///live.sock"), - Some("127.0.0.1:18081"), - ); - state.status = SiteLifecycleStatus::Stopped; - state.supervisor_pid = stubborn_supervisor.id(); - write_json(&site_state_path(&state_root, "direct_local"), &state) - .expect("state should serialize"); - - let result = stop_run(run_id, Some(storage_root)).await; - - let _ = stubborn_supervisor.kill(); - let _ = stubborn_supervisor.wait(); - - result.expect("stop_run should force the supervisor down and succeed"); - assert!( - !receipt_path(&run_root).is_file(), - "receipt should be removed after forced shutdown cleanup" - ); - assert!( - stop_marker_path(&run_root).is_file(), - "stop marker should be written for supervisors" +#[test] +fn compose_site_controller_status_parsing_handles_health_and_plain_running() { + assert_eq!( + supervisor::parse_container_runtime_status("running healthy\n"), + Some(("running", Some("healthy"))) ); - - let updated_state: SiteManagerState = read_json( - &site_state_path(&state_root, "direct_local"), - "site manager state", - ) - .expect("updated state should deserialize"); - assert_eq!(updated_state.status, SiteLifecycleStatus::Stopped); - assert!( - updated_state.last_error.as_deref().is_some_and(|value| { - value.contains("forcing shutdown") || value.contains("exited before confirming stop") - }), - "expected escalated shutdown cleanup to be recorded, got: {:?}", - updated_state.last_error + assert_eq!( + supervisor::parse_container_runtime_status("running\n"), + Some(("running", None)) ); + assert_eq!(supervisor::parse_container_runtime_status(""), None); } -#[cfg(unix)] -#[tokio::test] -async fn stop_bridge_proxies_terminates_dynamic_children() { - let temp = tempdir().expect("tempdir should exist"); - let child = Command::new("sh") - .current_dir(temp.path()) - .arg("-c") - .arg("sleep 30") - .spawn() - .expect("bridge proxy child should spawn"); - let pid = child.id(); - - let mut bridge_proxies = BTreeMap::from([( - BridgeProxyKey { - provider_output_dir: temp.path().display().to_string(), - export_name: "amber_export_test".to_string(), - consumer_kind: SiteKind::Compose, - }, - BridgeProxyHandle { - child, - export_name: "amber_export_test".to_string(), - listen: SocketAddr::from(([127, 0, 0, 1], 46000)), - }, - )]); - - stop_bridge_proxies(&mut bridge_proxies) - .await - .expect("bridge proxies should stop cleanly"); +#[test] +fn compose_site_controller_container_name_uses_compose_project() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "compose-site".to_string(), + kind: SiteKind::Compose, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: Some("amber_run_compose-site".to_string()), + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: Some("http://amber-site-controller:4100".to_string()), + launch_env: BTreeMap::new(), + }; - assert!( - bridge_proxies.is_empty(), - "bridge proxy cleanup should remove all tracked proxies" - ); - assert!( - !pid_is_alive(pid), - "bridge proxy cleanup should terminate the owned child process" + assert_eq!( + supervisor::compose_site_controller_container_name(&plan).as_deref(), + Some("amber_run_compose-site-amber-site-controller-1") ); } -#[tokio::test] -async fn wait_for_kubernetes_site_router_ready_waits_for_live_discovery() { - let temp = tempdir().expect("tempdir should exist"); - let site_state_root = temp.path().join("state").join("kind_local"); - let artifact_dir = temp.path().join("artifact"); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - - let stale_control = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("stale control"); - let stale_control_addr = stale_control.local_addr().expect("stale control addr"); - drop(stale_control); - let stale_mesh = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("stale mesh"); - let stale_mesh_addr = stale_mesh.local_addr().expect("stale mesh addr"); - drop(stale_mesh); - - write_json( - &site_state_root.join("manager-state.json"), - &test_site_state( - "run-test", - "kind_local", - SiteKind::Kubernetes, - &artifact_dir, - Some(&stale_control_addr.to_string()), - Some(&stale_mesh_addr.to_string()), - ), - ) - .expect("initial site manager state should serialize"); - - let live_control = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("live control listener"); - let live_control_addr = live_control.local_addr().expect("live control addr"); - let live_mesh = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("live mesh listener"); - let live_mesh_addr = live_mesh.local_addr().expect("live mesh addr"); - - let identity_body = serde_json::to_string(&MeshIdentityPublic { - id: "/site/kind_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh.scope.test".to_string()), - }) - .expect("identity body should serialize"); - let identity_response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - identity_body.len(), - identity_body - ); - - let control_task = tokio::spawn(async move { - let (mut stream, _) = live_control.accept().await.expect("control accept"); - let mut request = [0u8; 1024]; - let _ = stream.read(&mut request).await.expect("control read"); - stream - .write_all(identity_response.as_bytes()) - .await - .expect("control response should write"); - }); - let mesh_task = tokio::spawn(async move { - let _ = live_mesh.accept().await.expect("mesh accept"); - }); - let state_update_task = tokio::spawn({ - let artifact_dir = artifact_dir.clone(); - let site_state_root = site_state_root.clone(); - async move { - sleep(Duration::from_millis(300)).await; - write_json( - &site_state_root.join("manager-state.json"), - &test_site_state( - "run-test", - "kind_local", - SiteKind::Kubernetes, - &artifact_dir, - Some(&live_control_addr.to_string()), - Some(&live_mesh_addr.to_string()), - ), - ) - .expect("updated site manager state should serialize"); - } - }); +#[test] +fn compose_site_without_controller_has_no_controller_container_name() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "compose-site".to_string(), + kind: SiteKind::Compose, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: Some("amber_run_compose-site".to_string()), + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + }; - let started = Instant::now(); - wait_for_kubernetes_site_router_ready( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, - run_id: "run-test".to_string(), - mesh_scope: "mesh.scope.test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: Some("amber-test".to_string()), - context: Some("kind-test".to_string()), - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - Duration::from_secs(5), - ) - .await - .expect("router readiness wait should succeed once discovery updates"); - assert!( - started.elapsed() >= Duration::from_millis(300), - "router readiness wait should not return while manager state still points at the stale \ - endpoint" + assert_eq!( + supervisor::compose_site_controller_container_name(&plan), + None, + "compose sites without a lowered controller should not be treated as if they had a \ + special controller container" ); - - state_update_task - .await - .expect("state update task should join"); - control_task.await.expect("control task should join"); - mesh_task.await.expect("mesh task should join"); } #[test] -fn handle_otlp_connection_persists_structured_trace_stream() { - let temp = tempdir().expect("tempdir should exist"); - let requests_log = temp.path().join("observability").join("requests.log"); - let events_ndjson = temp.path().join("observability").join("events.ndjson"); - let body = ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: vec![ - KeyValue { - key: "service.name".to_string(), - value: Some(otlp_string_value("amber.run-test.bindings")), - }, - KeyValue { - key: "amber_entity_kind".to_string(), - value: Some(otlp_string_value("binding")), - }, - KeyValue { - key: "amber_edge_ref".to_string(), - value: Some(otlp_string_value("/public.a2a -> /server.a2a")), - }, - KeyValue { - key: "amber_http_subject".to_string(), - value: Some(otlp_string_value("agent card")), - }, - ], - ..Default::default() - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "amber.binding".to_string(), - ..Default::default() - }), - log_records: vec![LogRecord { - severity_text: "INFO".to_string(), - body: Some(otlp_string_value( - "request received from public by /server [headers]", - )), - trace_id: vec![0x1a; 16], - ..Default::default() - }], - ..Default::default() - }], - ..Default::default() - }], +fn reserve_loopback_port_keeps_allocations_unique_within_one_process() { + let mut ports = BTreeSet::new(); + for _ in 0..64 { + let port = reserve_loopback_port().expect("loopback port reservation should succeed"); + assert!( + ports.insert(port), + "loopback port allocator reused {port} within one process" + ); } - .encode_to_vec(); - - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let response_task = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: {}\r\nConnection: \ - close\r\n\r\n", - body.len() - ) - .expect("request head should write"); - client.write_all(&body).expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle otlp request"); - drop(stream); - - let response = response_task.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" - ); - assert!( - fs::read_to_string(&requests_log) - .expect("requests log should exist") - .contains("/v1/logs"), - "requests log should record the OTLP path" - ); - let structured = fs::read_to_string(&events_ndjson).expect("structured events should exist"); - assert!( - structured.contains("\"message\":\"request received from public by /server [headers]\""), - "structured log should contain the event body, got:\n{structured}" - ); - assert!( - structured.contains("\"amber_http_subject\":\"agent card\""), - "structured log should preserve protocol attributes, got:\n{structured}" - ); } #[test] -fn handle_otlp_connection_ignores_internal_and_json_log_payloads() { - let temp = tempdir().expect("tempdir should exist"); - let requests_log = temp.path().join("observability").join("requests.log"); - let events_ndjson = temp.path().join("observability").join("events.ndjson"); +fn reserve_loopback_port_shares_allocator_with_site_controller_runtime() { + let mixed_run_port = + reserve_loopback_port().expect("mixed-run loopback port reservation should succeed"); + for _ in 0..32 { + let runtime_port = amber_site_controller::reserve_loopback_port() + .expect("site-controller loopback port reservation should succeed"); + assert_ne!( + runtime_port, mixed_run_port, + "mixed-run and site-controller loopback reservations must use the same shared pool", + ); + } +} - let internal_body = ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: vec![ - KeyValue { - key: "amber_entity_kind".to_string(), - value: Some(otlp_string_value("binding")), - }, - KeyValue { - key: "amber_edge_ref".to_string(), - value: Some(otlp_string_value( - "/site/compose_local/router.a2a -> /runtime.a2a", - )), - }, - ], - ..Default::default() - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "amber.binding".to_string(), - ..Default::default() - }), - log_records: vec![LogRecord { - severity_text: "INFO".to_string(), - body: Some(otlp_string_value( - "request received by /runtime from /site/compose_local/router via \ - /site/compose_local/router.a2a -> /runtime.a2a [headers]", - )), - trace_id: vec![0x2b; 16], - ..Default::default() - }], - ..Default::default() - }], - ..Default::default() - }], +#[test] +fn reserve_host_port_shares_allocator_with_loopback_reservations() { + let host_port = reserve_host_port().expect("host port reservation should succeed"); + for _ in 0..32 { + let loopback_port = + reserve_loopback_port().expect("loopback port reservation should succeed"); + assert_ne!( + loopback_port, host_port, + "host and loopback reservations must use the same shared pool", + ); } - .encode_to_vec(); +} - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let internal_request = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: {}\r\nConnection: \ - close\r\n\r\n", - internal_body.len() - ) - .expect("request head should write"); - client - .write_all(&internal_body) - .expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle internal otlp request"); - drop(stream); - let response = internal_request.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" - ); - assert!( - !events_ndjson.exists(), - "internal route chatter should not produce a structured trace stream" - ); +#[cfg(unix)] +#[test] +fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { + let temp = TempDir::new().expect("temp dir"); + let kubectl = temp.path().join("kubectl"); + let log_path = temp.path().join("kubectl.log"); + let state_path = temp.path().join("namespace-state"); + fs::write( + &kubectl, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\nstate_path='{}'\nprintf '%s\\n' \"$*\" >> \ + \"$log_path\"\nif [ \"${{1:-}}\" = \"--context\" ]; then\n shift \ + 2\nfi\nstate=alive\nif [ -f \"$state_path\" ]; then\n state=$(cat \ + \"$state_path\")\nfi\nif [ \"${{1:-}}\" = \"delete\" ] && [ \"${{2:-}}\" = \ + \"namespace\" ]; then\n exit 0\nfi\nif [ \"${{1:-}}\" = \"get\" ] && [ \"${{2:-}}\" \ + = \"namespace\" ]; then\n if [ \"$state\" = \"gone\" ]; then\n echo 'Error from \ + server (NotFound): namespaces \"'\"${{3:-}}\"'\" not found' >&2\n exit 1\n fi\n \ + printf '{{\"metadata\":{{\"name\":\"%s\",\"deletionTimestamp\":\"2026-04-16T02:33:\ + 55Z\"}}}}\\n' \"${{3:-}}\"\n exit 0\nfi\nif [ \"${{1:-}}\" = \"-n\" ] && [ \ + \"${{2:-}}\" = \"test-ns\" ] && [ \"${{3:-}}\" = \"delete\" ] && [ \"${{4:-}}\" = \ + \"pods\" ]; then\n printf 'gone' > \"$state_path\"\n exit 0\nfi\necho \"unexpected \ + kubectl invocation: $*\" >&2\nexit 1\n", + log_path.display(), + state_path.display(), + ), + ) + .expect("kubectl stub"); + let mut permissions = fs::metadata(&kubectl) + .expect("kubectl metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&kubectl, permissions).expect("kubectl chmod"); - let json_body = serde_json::to_vec(&serde_json::json!({ - "event": "amber.run.starting", - "body": "starting mixed-site run run-test", - "attributes": {"amber.run_id": "run-test"} - })) - .expect("json payload should serialize"); - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let json_request = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: \ - application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - json_body.len() - ) - .expect("request head should write"); - client - .write_all(&json_body) - .expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle manager json request"); - drop(stream); - let response = json_request.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" + stop_kubernetes_namespace_with_kubectl( + &kubectl, + Some("test-context"), + "test-ns", + Duration::from_millis(20), + ) + .expect("namespace stop should force-delete remaining pods and retry"); + + let log = fs::read_to_string(&log_path).expect("kubectl log"); + let delete_namespace_calls = log + .lines() + .filter(|line| line.contains("delete namespace test-ns --ignore-not-found --wait=false")) + .count(); + assert_eq!( + delete_namespace_calls, 2, + "namespace deletion should be retried after forced pod cleanup:\n{log}" ); assert!( - !events_ndjson.exists(), - "manager json events should not be mixed into the structured trace stream" + log.lines().any(|line| { + line.contains( + "-n test-ns delete pods --all --ignore-not-found --force --grace-period=0", + ) + }), + "forced pod cleanup should run before the retry:\n{log}" ); } diff --git a/cli/src/run_inputs.rs b/cli/src/run_inputs.rs index 43cbd9c4..04c83792 100644 --- a/cli/src/run_inputs.rs +++ b/cli/src/run_inputs.rs @@ -1725,6 +1725,7 @@ mod tests { site: SiteDefinition { kind, context: None, + controller_site: None, }, router_identity_id: "/site/test/router".to_string(), assigned_components: vec!["/app".to_string()], @@ -1805,6 +1806,7 @@ mod tests { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults::default(), diff --git a/cli/src/run_logs.rs b/cli/src/run_logs.rs index eac9fd36..4bb297a3 100644 --- a/cli/src/run_logs.rs +++ b/cli/src/run_logs.rs @@ -636,7 +636,7 @@ impl NormalizedTraceInteraction { parse_story_actors(story).or_else(|| parse_story_receivers(story))?; Some(Self { severity: severity - .filter(|severity| !severity.eq_ignore_ascii_case("info")) + .filter(|severity: &&str| !severity.eq_ignore_ascii_case("info")) .map(ToString::to_string), actor, recipient, @@ -996,7 +996,7 @@ fn event_attr_str<'a>(event: &'a PersistedTraceEvent, key: &str) -> Option<&'a s fn event_attr_bool(event: &PersistedTraceEvent, key: &str) -> Option { match event.attributes.get(key)? { - JsonValue::Bool(value) => Some(*value), + JsonValue::Bool(value) => Some(value.to_owned()), _ => None, } } diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 27604b7a..df60d2c9 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -101,6 +101,50 @@ fn top_level_help_groups_authoring_and_runtime_workflows() { ); } +#[test] +fn scenario_ir_input_rejects_framework_owned_provide_kinds() { + for kind in ["component", "docker", "kvm"] { + let temp = tempfile::tempdir().expect("tempdir should exist"); + let path = temp.path().join(format!("{kind}.json")); + let ir = serde_json::json!({ + "schema": SCENARIO_IR_SCHEMA, + "version": amber_scenario::SCENARIO_IR_VERSION, + "root": 0, + "components": [ + { + "id": 0, + "moniker": "/", + "parent": null, + "children": [], + "digest": amber_manifest::ManifestDigest::new([0u8; 32]).to_string(), + "config": null, + "program": null, + "slots": {}, + "provides": { + "api": { + "kind": kind, + "endpoint": "api" + } + } + } + ], + "bindings": [], + "exports": [] + }); + fs::write( + &path, + serde_json::to_vec_pretty(&ir).expect("IR should serialize"), + ) + .expect("IR should be written"); + + let err = load_compiled_scenario_ir(&path) + .expect_err("framework-owned provide kind should be rejected"); + let rendered = err.to_string(); + assert!(rendered.contains("framework-owned"), "{rendered}"); + assert!(rendered.contains(kind), "{rendered}"); + } +} + #[test] fn attached_run_overview_stays_focused_on_identity_and_exports() { let mut output = Vec::new(); @@ -220,6 +264,76 @@ fn proxy_telemetry_keeps_router_info_without_verbose_output() { ); } +#[cfg(target_os = "linux")] +#[test] +fn direct_runtime_only_isolates_non_controller_components() { + let controller = DirectComponentPlan { + id: 1, + moniker: amber_compiler::run_plan::FrameworkComponentControllerMoniker::for_site( + "direct_local", + ) + .into_string(), + log_name: "controller".to_string(), + source_dir: None, + depends_on: Vec::new(), + sidecar: amber_compiler::reporter::direct::DirectSidecarPlan { + log_name: "controller-sidecar".to_string(), + mesh_port: 0, + mesh_config_path: "mesh/components/controller/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/controller/mesh-identity.json".to_string(), + env_passthrough: Vec::new(), + }, + program: amber_compiler::reporter::direct::DirectProgramPlan { + log_name: "controller-program".to_string(), + work_dir: "work/components/controller".to_string(), + storage_mounts: Vec::new(), + execution: DirectProgramExecutionPlan::InternalSiteController, + }, + }; + let app = DirectComponentPlan { + id: 2, + moniker: "/app".to_string(), + log_name: "app".to_string(), + source_dir: Some("/workspace/scenarios/app".to_string()), + depends_on: Vec::new(), + sidecar: amber_compiler::reporter::direct::DirectSidecarPlan { + log_name: "app-sidecar".to_string(), + mesh_port: 0, + mesh_config_path: "mesh/components/app/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), + env_passthrough: Vec::new(), + }, + program: amber_compiler::reporter::direct::DirectProgramPlan { + log_name: "app-program".to_string(), + work_dir: "work/components/app".to_string(), + storage_mounts: Vec::new(), + execution: DirectProgramExecutionPlan::Direct { + entrypoint: vec!["/bin/true".to_string()], + env: BTreeMap::new(), + }, + }, + }; + + assert_eq!( + direct_component_sidecar_network(&controller), + ProcessNetwork::Host + ); + assert!(!direct_component_uses_isolated_network(&controller)); + assert!(!direct_component_program_joins_sidecar(&controller)); + assert_eq!(direct_program_network_override(&controller), "host"); + + assert_eq!( + direct_component_sidecar_network(&app), + ProcessNetwork::Isolated + ); + assert!(direct_component_uses_isolated_network(&app)); + assert!(direct_component_program_joins_sidecar(&app)); + assert_eq!( + direct_program_network_override(&app), + "join_component_sidecar" + ); +} + #[cfg(target_os = "linux")] #[test] fn component_program_read_only_mounts_resolve_parent_escape_paths() { @@ -310,6 +424,7 @@ fn build_runtime_template_context_uses_runtime_slot_ports() { dynamic_caps_port_by_component: BTreeMap::new(), component_mesh_port_by_id: BTreeMap::new(), router_mesh_port: None, + ready: false, }; let context = @@ -888,6 +1003,142 @@ async fn cleanup_direct_runtime_removes_partial_startup_artifacts() { ); } +#[cfg(unix)] +#[tokio::test] +async fn cleanup_direct_runtime_terminates_wrapper_descendants() { + let plan_root = tempfile::tempdir().expect("temp dir should be created"); + let runtime_dir = tempfile::Builder::new() + .prefix("amber-direct-test-") + .tempdir() + .expect("runtime dir should be created"); + let runtime_root = runtime_dir.path().to_path_buf(); + let runtime_state_path = direct_runtime_state_path(plan_root.path()); + fs::create_dir_all(runtime_state_path.parent().expect("state parent")) + .expect("state parent should be created"); + fs::write(&runtime_state_path, "{}").expect("state file should be written"); + + let child_pid_path = runtime_root.join("child.pid"); + // Use a real wrapper process so the test records the long-lived descendant PID rather than + // relying on shell job-control or `setsid` utility behavior. + let child = TokioCommand::new("python3") + .arg("-c") + .arg( + r#" +import pathlib +import subprocess +import sys + +pid_path = pathlib.Path(sys.argv[1]) +child = subprocess.Popen(["sleep", "30"], start_new_session=True) +pid_path.write_text(f"{child.pid}\n", encoding="utf-8") +child.wait() +"#, + ) + .arg(&child_pid_path) + .spawn() + .expect("wrapper should spawn"); + let mut children = vec![ManagedChild { + name: "wrapper-with-descendant".to_string(), + wrapper: Some(child), + #[cfg(target_os = "linux")] + wrapper_pid: 0, + #[cfg(target_os = "linux")] + managed_pid: 0, + }]; + #[cfg(target_os = "linux")] + { + let pid = children[0] + .wrapper + .as_ref() + .and_then(tokio::process::Child::id) + .expect("child pid should be available"); + children[0].wrapper_pid = pid; + children[0].managed_pid = pid; + } + + let deadline = Instant::now() + Duration::from_secs(5); + let child_pid = loop { + if Instant::now() >= deadline { + let contents = fs::read_to_string(&child_pid_path).unwrap_or_default(); + panic!( + "descendant pid should be recorded before cleanup starts; last contents: {:?}", + contents, + ); + } + match fs::read_to_string(&child_pid_path) { + Ok(contents) => { + let trimmed = contents.trim(); + if let Ok(pid) = trimmed.parse::() { + break pid; + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => panic!("descendant pid file should be readable: {err}"), + } + sleep(Duration::from_millis(25)).await; + }; + let wrapper_pid = children[0] + .wrapper + .as_ref() + .and_then(tokio::process::Child::id) + .expect("wrapper pid should be available"); + let tree_deadline = Instant::now() + Duration::from_secs(5); + while descendant_parent_pid(child_pid) != Some(wrapper_pid) && Instant::now() < tree_deadline { + sleep(Duration::from_millis(25)).await; + } + assert_eq!( + descendant_parent_pid(child_pid), + Some(wrapper_pid), + "descendant should appear in the process table under its wrapper before cleanup" + ); + + cleanup_direct_runtime( + &mut children, + Vec::new(), + &runtime_state_path, + None, + Some(runtime_dir), + ) + .await; + + let stopped = !crate::unix_process::pid_is_alive(child_pid); + let survivor = if stopped { + None + } else { + Some( + std::process::Command::new("ps") + .args(["-o", "pid=,ppid=,pgid=,stat=,command=", "-p"]) + .arg(child_pid.to_string()) + .output() + .ok() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()) + .filter(|line| !line.is_empty()) + .unwrap_or_else(|| "".to_string()), + ) + }; + if !stopped { + let _ = unsafe { libc::kill(child_pid as i32, libc::SIGKILL) }; + } + assert!( + stopped, + "cleanup should terminate descendants that outlive their wrapper; surviving process: {}", + survivor.unwrap_or_default() + ); +} + +#[cfg(unix)] +fn descendant_parent_pid(pid: u32) -> Option { + let output = std::process::Command::new("ps") + .args(["-o", "ppid=", "-p"]) + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout).trim().parse().ok() +} + #[test] fn write_direct_runtime_state_preserves_projected_router_mesh_port() { let plan_root = tempfile::tempdir().expect("temp dir should be created"); @@ -1064,43 +1315,6 @@ fn rewrite_peer_addr_for_slirp_gateway_rewrites_loopback_only() { ); } -#[cfg(target_os = "linux")] -#[test] -fn rewrite_loopback_url_for_slirp_gateway_rewrites_loopback_only() { - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://127.0.0.1:23000/base?x=1"), - "http://10.0.2.2:23000/base?x=1" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://localhost:24000"), - "http://10.0.2.2:24000/" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://192.168.1.10:25000/path"), - "http://192.168.1.10:25000/path" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("not-a-url"), - "not-a-url" - ); -} - -#[cfg(target_os = "linux")] -#[test] -fn rewrite_sidecar_env_passthrough_for_slirp_rewrites_only_dynamic_caps_control_url() { - assert_eq!( - rewrite_sidecar_env_passthrough_for_slirp( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, - "http://127.0.0.1:25000/v1/control-state" - ), - "http://10.0.2.2:25000/v1/control-state" - ); - assert_eq!( - rewrite_sidecar_env_passthrough_for_slirp("UNRELATED_ENV", "http://127.0.0.1:25000"), - "http://127.0.0.1:25000" - ); -} - #[cfg(target_os = "linux")] #[test] fn slirp4netns_add_hostfwd_payload_uses_guest_default_address() { diff --git a/cli/src/unix_process.rs b/cli/src/unix_process.rs new file mode 100644 index 00000000..867f2b74 --- /dev/null +++ b/cli/src/unix_process.rs @@ -0,0 +1,185 @@ +#[cfg(unix)] +use std::{ + collections::{BTreeSet, HashMap}, + process::Command, +}; + +#[cfg(unix)] +use miette::{Context as _, IntoDiagnostic as _, Result}; +#[cfg(unix)] +use tokio::time::{Duration, Instant, sleep}; + +#[cfg(unix)] +pub(crate) fn pid_is_alive(pid: u32) -> bool { + let alive = unsafe { + libc::kill(pid as i32, 0) == 0 + || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + }; + alive && process_status_code(pid) != Some('Z') +} + +#[cfg(unix)] +pub(crate) async fn terminate_process_roots(root_pids: &[u32], timeout: Duration) -> Result<()> { + let mut seen = BTreeSet::new(); + let mut ordered = Vec::new(); + for root_pid in root_pids { + for pid in process_tree_postorder(*root_pid)? { + if seen.insert(pid) { + ordered.push(pid); + } + } + } + if ordered.is_empty() { + return Ok(()); + } + + send_signal_to_pids(&ordered, libc::SIGTERM); + wait_for_pids_exit(&ordered, timeout).await; + + let survivors = ordered + .iter() + .copied() + .filter(|pid| pid_is_alive(*pid)) + .collect::>(); + if survivors.is_empty() { + return Ok(()); + } + + send_signal_to_pids(&survivors, libc::SIGKILL); + wait_for_pids_exit(&survivors, Duration::from_secs(2)).await; + Ok(()) +} + +#[cfg(unix)] +fn process_status_code(pid: u32) -> Option { + let output = Command::new("ps") + .arg("-o") + .arg("stat=") + .arg("-p") + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) +} + +#[cfg(unix)] +fn parse_process_status_code(raw: &str) -> Option { + raw.split_whitespace() + .next()? + .chars() + .next() + .map(|state| state.to_ascii_uppercase()) +} + +#[cfg(unix)] +fn send_signal_to_pids(pids: &[u32], signal: i32) { + for pid in pids { + unsafe { + libc::kill(*pid as i32, signal); + } + } +} + +#[cfg(unix)] +async fn wait_for_pids_exit(pids: &[u32], timeout: Duration) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if pids.iter().all(|pid| !pid_is_alive(*pid)) { + return; + } + sleep(Duration::from_millis(100)).await; + } +} + +#[cfg(unix)] +fn process_tree_postorder(root_pid: u32) -> Result> { + let output = Command::new("ps") + .arg("-axo") + .arg("pid=,ppid=") + .output() + .into_diagnostic() + .wrap_err("failed to enumerate process tree")?; + if !output.status.success() { + return Err(miette::miette!( + "failed to enumerate process tree: status {}", + output.status + )); + } + + let parent_by_pid = parse_process_table(&String::from_utf8_lossy(&output.stdout))?; + if !parent_by_pid.contains_key(&root_pid) && !pid_is_alive(root_pid) { + return Ok(Vec::new()); + } + + let mut children_by_parent = HashMap::>::new(); + for (pid, ppid) in parent_by_pid { + children_by_parent.entry(ppid).or_default().push(pid); + } + + let mut ordered = Vec::new(); + collect_process_tree_postorder(root_pid, &children_by_parent, &mut ordered); + Ok(ordered) +} + +#[cfg(unix)] +fn collect_process_tree_postorder( + pid: u32, + children_by_parent: &HashMap>, + ordered: &mut Vec, +) { + if let Some(children) = children_by_parent.get(&pid) { + for child in children { + collect_process_tree_postorder(*child, children_by_parent, ordered); + } + } + ordered.push(pid); +} + +#[cfg(unix)] +fn parse_process_table(raw: &str) -> Result> { + let mut parent_by_pid = HashMap::new(); + for line in raw.lines() { + let mut fields = line.split_whitespace(); + let Some(pid) = fields.next() else { + continue; + }; + let Some(ppid) = fields.next() else { + continue; + }; + let pid = pid + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid process table pid `{pid}`"))?; + let ppid = ppid + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid process table parent pid `{ppid}`"))?; + parent_by_pid.insert(pid, ppid); + } + Ok(parent_by_pid) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[test] + fn parse_process_table_reads_pid_parent_pairs() { + let table = parse_process_table("10 1\n11 10\n").expect("process table should parse"); + assert_eq!(table.get(&10), Some(&1)); + assert_eq!(table.get(&11), Some(&10)); + } + + #[test] + fn collect_process_tree_postorder_lists_descendants_before_root() { + let tree = HashMap::from([(1, vec![2, 3]), (2, vec![4])]); + let mut ordered = Vec::new(); + collect_process_tree_postorder(1, &tree, &mut ordered); + assert_eq!(ordered, vec![4, 2, 3, 1]); + } +} diff --git a/cli/src/vm_runtime.rs b/cli/src/vm_runtime.rs index c71201e7..95cce33e 100644 --- a/cli/src/vm_runtime.rs +++ b/cli/src/vm_runtime.rs @@ -5,7 +5,7 @@ use std::{ fs::{self, File}, hash::{Hash as _, Hasher as _}, io::{Seek as _, SeekFrom, Write as _}, - net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener}, + net::SocketAddr, path::{Path, PathBuf}, process::{Command as ProcessCommand, ExitStatus, Stdio}, }; @@ -54,10 +54,7 @@ mod state; use self::{artifacts::*, preview::*, state::*}; pub(crate) use self::{ preview::build_vm_site_launch_preview, - state::{ - ensure_control_socket_link, vm_current_control_socket_path, - vm_endpoint_forward_ready_timeout, write_vm_runtime_state, - }, + state::{vm_current_control_socket_path, vm_endpoint_forward_ready_timeout}, }; const VM_CHILD_POLL_INTERVAL: Duration = Duration::from_millis(150); @@ -800,13 +797,13 @@ fn resolve_vm_runtime_host_path(raw_path: &str, source_dir: Option<&str>) -> Res } pub(crate) fn vm_uses_tcg_accel() -> bool { - matches!(detect_qemu_accel(), QemuAccel::Tcg) + amber_site_controller::vm_uses_tcg_accel() } fn detect_qemu_accel() -> QemuAccel { #[cfg(target_os = "macos")] { - if env::var_os("AMBER_VM_FORCE_TCG").is_some() { + if vm_uses_tcg_accel() { QemuAccel::Tcg } else { QemuAccel::Hvf @@ -815,12 +812,10 @@ fn detect_qemu_accel() -> QemuAccel { #[cfg(target_os = "linux")] { - if env::var_os("AMBER_VM_FORCE_TCG").is_some() { + if vm_uses_tcg_accel() { QemuAccel::Tcg - } else if Path::new("/dev/kvm").exists() { - QemuAccel::Kvm } else { - QemuAccel::Tcg + QemuAccel::Kvm } } diff --git a/cli/src/vm_runtime/preview.rs b/cli/src/vm_runtime/preview.rs index 219080bf..5aa2cd97 100644 --- a/cli/src/vm_runtime/preview.rs +++ b/cli/src/vm_runtime/preview.rs @@ -138,14 +138,40 @@ pub(crate) fn build_vm_site_launch_preview( mesh_plan, } = load_vm_runtime_inputs(&plan_path)?; - let port_assignments = materialize_vm_runtime( + let mut inspectability_warnings = Vec::new(); + let port_assignments = match materialize_vm_runtime( &plan_root, runtime_root, &vm_plan, &mesh_plan, router_mesh_port, true, - )?; + ) { + Ok(port_assignments) => port_assignments, + Err(err) if crate::direct_runtime::missing_existing_peer_identity(&err) => { + // Dry-run preview can still show the local VM launch shape before peer-site routers + // exist; the missing identities only block full mesh config materialization. + inspectability_warnings.push(format!( + "preview is missing one or more peer-site router identities, so mesh configs \ + remain unresolved until those sites are running: {err}" + )); + let (preview_peer_identities, preview_peer_ports) = + crate::direct_runtime::preview_placeholder_peer_mesh_state(&mesh_plan); + materialize_vm_runtime_with_existing( + &plan_root, + runtime_root, + &vm_plan, + &mesh_plan, + router_mesh_port, + VmExistingMeshState { + reuse_existing: false, + peer_ports_by_id: &preview_peer_ports, + peer_identities_by_id: &preview_peer_identities, + }, + )? + } + Err(err) => return Err(err), + }; let arch = host_arch()?; let (qemu_system, qemu_warning) = preview_qemu_system_binary(arch); let (amber_cli, amber_warning) = preview_amber_binary(); @@ -169,7 +195,6 @@ pub(crate) fn build_vm_site_launch_preview( .transpose()?; let mut virtual_machines = Vec::new(); - let mut inspectability_warnings = Vec::new(); inspectability_warnings.extend(qemu_warning); inspectability_warnings.extend(amber_warning); for component_id in &vm_plan.startup_order { diff --git a/cli/src/vm_runtime/state.rs b/cli/src/vm_runtime/state.rs index d063e7e6..ad18e209 100644 --- a/cli/src/vm_runtime/state.rs +++ b/cli/src/vm_runtime/state.rs @@ -186,6 +186,14 @@ pub(crate) fn hashed_temp_socket_path(namespace: &str, kind: &str, path: &Path) amber_mesh::stable_temp_socket_path(namespace, kind, path) } +fn vm_component_control_socket_path(work_dir: &Path, component_id: usize) -> PathBuf { + hashed_temp_socket_path( + "amber-vm-control", + &format!("sidecar-{component_id}"), + work_dir, + ) +} + pub(crate) fn assign_vm_runtime_ports( runtime_root: &Path, vm_plan: &VmPlan, @@ -220,11 +228,19 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( } mesh_port_by_peer_id.insert(peer_id.clone(), *port); } + if let Some(port) = fixed_router_mesh_port + && !reserved.insert(port) + { + return Err(miette::miette!( + "router mesh port {} conflicts with an existing vm runtime port", + port + )); + } for component in &vm_plan.components { let path = runtime_root.join(&component.mesh_config_path); let mut config = read_mesh_config_public(&path)?; - let mesh_port = allocate_runtime_port(&mut reserved, None)?; + let mesh_port = allocate_runtime_port(&mut reserved)?; if mesh_port_by_peer_id .insert(config.identity.id.clone(), mesh_port) .is_some() @@ -239,7 +255,7 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let mut route_guest_host_pairs = BTreeMap::>::new(); for route in &mut config.outbound { let guest_port = route.listen_port; - let host_port = allocate_runtime_port(&mut reserved, None)?; + let host_port = allocate_runtime_port(&mut reserved)?; route.listen_port = host_port; route_guest_host_pairs .entry(route.slot.clone()) @@ -281,7 +297,7 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let host_port = if let Some(existing) = endpoint_forwards.get(&guest_port) { *existing } else { - let host_port = allocate_runtime_port(&mut reserved, None)?; + let host_port = allocate_runtime_port(&mut reserved)?; endpoint_forwards.insert(guest_port, host_port); host_port }; @@ -316,7 +332,10 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let mut router_config = if let Some(router) = vm_plan.router.as_ref() { let path = runtime_root.join(&router.mesh_config_path); let mut config = read_mesh_config_public(&path)?; - let mesh_port = allocate_runtime_port(&mut reserved, fixed_router_mesh_port)?; + let mesh_port = match fixed_router_mesh_port { + Some(port) => port, + None => allocate_runtime_port(&mut reserved)?, + }; if let Some(existing) = mesh_port_by_peer_id.insert(config.identity.id.clone(), mesh_port) && existing != mesh_port { @@ -357,21 +376,9 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( }) } -pub(crate) fn allocate_runtime_port( - reserved: &mut BTreeSet, - preferred: Option, -) -> Result { - if let Some(preferred) = preferred { - if reserved.insert(preferred) { - return Ok(preferred); - } - return Err(miette::miette!( - "runtime port {} was requested twice in one vm runtime", - preferred - )); - } +pub(crate) fn allocate_runtime_port(reserved: &mut BTreeSet) -> Result { for _ in 0..256 { - let port = pick_free_port()?; + let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { return Ok(port); } @@ -381,12 +388,6 @@ pub(crate) fn allocate_runtime_port( )) } -pub(crate) fn pick_free_port() -> Result { - let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)) - .into_diagnostic()?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - pub(crate) fn rewrite_mesh_peer_addrs( config: &mut MeshConfigPublic, mesh_port_by_peer_id: &HashMap, @@ -611,6 +612,33 @@ pub(crate) async fn spawn_component_sidecar( work_dir.display() ) })?; + let control_socket_path = vm_component_control_socket_path(&work_dir, component.id); + let control_socket_dir = control_socket_path + .parent() + .ok_or_else(|| miette::miette!("invalid vm sidecar control socket path"))? + .to_path_buf(); + fs::create_dir_all(&control_socket_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create vm sidecar control directory {}", + control_socket_dir.display() + ) + })?; + if control_socket_path.exists() { + fs::remove_file(&control_socket_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to remove stale vm sidecar control socket {}", + control_socket_path.display() + ) + })?; + } + env_map.insert( + "AMBER_ROUTER_CONTROL_SOCKET_PATH".to_string(), + control_socket_path.display().to_string(), + ); spawn_command( format!("{}-sidecar", component.log_name), vec![router_binary.to_string()], @@ -1004,4 +1032,37 @@ mod tests { replacement.component_mesh_port_by_id ); } + + #[test] + fn vm_component_control_socket_path_stays_short_on_long_work_dirs() { + let work_dir = Path::new( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + linux-vm-framework_component-very-long/state/runs/run-123/state/vm_local/runtime/\ + work/sidecars/c2-web", + ); + let socket = vm_component_control_socket_path(work_dir, 2); + let rendered = socket.as_os_str().to_string_lossy(); + + assert!( + rendered.len() < 104, + "vm sidecar control socket path must fit within unix socket limits: {rendered}", + ); + } + + #[test] + fn allocate_runtime_port_avoids_shared_loopback_reservations() { + let reserved_port = + amber_site_controller::reserve_loopback_port().expect("shared loopback port"); + let mut reserved = BTreeSet::new(); + + for _ in 0..32 { + let port = allocate_runtime_port(&mut reserved) + .expect("vm runtime port allocation should succeed"); + assert_ne!( + port, reserved_port, + "vm runtime should not reuse loopback ports already reserved by mixed-run \ + infrastructure", + ); + } + } } diff --git a/cli/src/vm_runtime/tests.rs b/cli/src/vm_runtime/tests.rs index 69726913..93172d47 100644 --- a/cli/src/vm_runtime/tests.rs +++ b/cli/src/vm_runtime/tests.rs @@ -1,5 +1,11 @@ -use amber_compiler::reporter::vm::{MESH_PROVISION_PLAN_FILENAME, VmStorageMount}; -use amber_mesh::{MeshProtocol, OutboundRoute, TransportConfig}; +use amber_compiler::reporter::vm::{ + MESH_PROVISION_PLAN_FILENAME, VM_PLAN_FILENAME, VmStorageMount, +}; +use amber_mesh::{ + MESH_PROVISION_PLAN_VERSION, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, + MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, + MeshProvisionTargetKind, OutboundRoute, TransportConfig, +}; use super::*; @@ -449,6 +455,104 @@ fn build_vm_launch_preview_keeps_command_when_base_image_is_unresolved() { })); } +#[test] +fn build_vm_site_launch_preview_tolerates_missing_peer_router_identity() { + let temp = tempfile::tempdir().expect("temp dir"); + let plan_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + let storage_root = temp.path().join("storage"); + fs::create_dir_all(&plan_root).expect("plan root"); + fs::create_dir_all(&runtime_root).expect("runtime root"); + fs::create_dir_all(&storage_root).expect("storage root"); + let base_image = temp.path().join("base.img"); + fs::write(&base_image, []).expect("base image"); + + let vm_plan = VmPlan { + version: VM_PLAN_VERSION.to_string(), + mesh_provision_plan: MESH_PROVISION_PLAN_FILENAME.to_string(), + startup_order: vec![7], + runtime_addresses: DirectRuntimeAddressPlan::default(), + components: vec![VmComponentPlan { + id: 7, + moniker: "/app".to_string(), + log_name: "app".to_string(), + depends_on: Vec::new(), + mesh_config_path: "mesh/components/app/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), + cpus: VmScalarPlanU32::Literal { value: 1 }, + memory_mib: VmScalarPlanU32::Literal { value: 512 }, + base_image: VmHostPathPlan::Static { + path: base_image.display().to_string(), + }, + cloud_init_user_data: None, + cloud_init_vendor_data: None, + egress: VmEgressPlan::None, + storage_mounts: Vec::new(), + runtime_config: None, + mount_spec_b64: None, + }], + router: None, + }; + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: MeshIdentityTemplate { + id: "/app".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:19000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeerTemplate { + id: "/site/direct_local/router".to_string(), + }], + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/components/app".to_string(), + }, + }], + }; + + fs::write( + plan_root.join(VM_PLAN_FILENAME), + serde_json::to_string_pretty(&vm_plan).expect("serialize vm plan"), + ) + .expect("write vm plan"); + fs::write( + plan_root.join(MESH_PROVISION_PLAN_FILENAME), + serde_json::to_string_pretty(&mesh_plan).expect("serialize mesh provision plan"), + ) + .expect("write mesh provision plan"); + + let preview = build_vm_site_launch_preview( + &plan_root.join(VM_PLAN_FILENAME), + &storage_root, + &runtime_root, + None, + ) + .expect("vm preview"); + + assert_eq!(preview.virtual_machines.len(), 1); + assert!( + !preview.virtual_machines[0].command.is_empty(), + "preview should still emit a local QEMU command when peer routers are unresolved" + ); + assert!( + preview + .inspectability_warnings + .iter() + .any(|warning| warning.contains("missing one or more peer-site router identities")) + ); +} + #[test] fn render_cloud_init_multipart_keeps_generated_and_user_parts() { let rendered = render_cloud_init_multipart( diff --git a/cli/tests/compile_outputs/artifact_outputs.rs b/cli/tests/compile_outputs/artifact_outputs.rs index d4ed1dc1..30f1f47e 100644 --- a/cli/tests/compile_outputs/artifact_outputs.rs +++ b/cli/tests/compile_outputs/artifact_outputs.rs @@ -387,9 +387,10 @@ fn compile_writes_direct_artifact() { "direct plan should not persist manifest_url" ); assert!( - components - .iter() - .all(|component| component.get("source_dir").is_some()), + components.iter().all(|component| { + component["program"]["execution"]["kind"].as_str() == Some("internal_site_controller") + || component.get("source_dir").is_some() + }), "manifest-backed direct plan components should retain source_dir" ); assert_eq!( @@ -410,8 +411,21 @@ fn compile_writes_direct_artifact() { assert_eq!(proxy_json["router"]["control_port"].as_u64(), Some(0)); } +fn assert_vm_export_succeeded(output: &std::process::Output) { + assert!( + output.status.success(), + "amber compile --vm failed +stdout: +{} +stderr: +{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] -fn compile_writes_vm_artifact() { +fn compile_vm_export_uses_default_placement() { let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("cli crate should live under the workspace root"); @@ -472,91 +486,21 @@ fn compile_writes_vm_artifact() { .output() .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan_path = artifact_dir.join(VM_PLAN_FILENAME); - let provision_plan_path = artifact_dir.join("mesh-provision-plan.json"); - let run_script_path = artifact_dir.join("run.sh"); - let readme_path = artifact_dir.join("README.md"); - let env_example_path = artifact_dir.join("env.example"); - let proxy_metadata_path = artifact_dir.join("amber-proxy.json"); - - assert!(vm_plan_path.is_file(), "missing vm-plan.json"); - assert!( - provision_plan_path.is_file(), - "missing mesh-provision-plan.json" - ); - assert!(run_script_path.is_file(), "missing run.sh"); - assert!(readme_path.is_file(), "missing README.md"); - assert!(env_example_path.is_file(), "missing env.example"); - assert!(proxy_metadata_path.is_file(), "missing amber-proxy.json"); - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(&vm_plan_path).expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - assert_eq!(vm_plan["version"], VM_PLAN_VERSION); - assert_eq!(vm_plan["mesh_provision_plan"], "mesh-provision-plan.json"); - let components = vm_plan["components"] - .as_array() - .expect("vm plan components should be an array"); - assert_eq!(components.len(), 1, "{vm_plan:#}"); - assert_eq!(components[0]["base_image"]["kind"], "runtime_config"); - assert_eq!(components[0]["base_image"]["query"], "base_image"); - assert_eq!(components[0]["cpus"]["kind"], "literal"); - assert_eq!(components[0]["cpus"]["value"], 1); - assert_eq!(components[0]["memory_mib"]["kind"], "literal"); - assert_eq!(components[0]["memory_mib"]["value"], 512); - let component = components[0] - .as_object() - .expect("vm component should serialize as an object"); - let allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| value.as_str().expect("allowed path should be a string")) - .collect::>(); + assert_vm_export_succeeded(&output); assert!( - component.contains_key("runtime_config"), - "vm component should carry runtime_config when the base image depends on runtime config" + artifact_dir.join("vm-plan.json").is_file(), + "expected vm-plan.json in {}", + artifact_dir.display() ); - assert_eq!(allowed_root_leaf_paths, vec!["base_image"]); assert!( - !component.contains_key("mount_spec_b64"), - "vm component should omit mount_spec_b64 when it is unused" - ); - - let readme = fs::read_to_string(&readme_path).expect("failed to read vm README"); - assert!( - readme.contains("amber run ."), - "vm README should explain how to start the output" - ); - let env_example = fs::read_to_string(&env_example_path).expect("failed to read vm env example"); - assert!( - env_example.contains("AMBER_CONFIG_BASE_IMAGE"), - "vm env example should mention runtime config for the base image" - ); - - let proxy_metadata: Value = serde_json::from_str( - &fs::read_to_string(&proxy_metadata_path).expect("failed to read vm proxy metadata"), - ) - .expect("vm proxy metadata should be valid JSON"); - assert_eq!( - proxy_metadata["router"]["control_socket"].as_str(), - Some(DIRECT_CONTROL_SOCKET_RELATIVE_PATH) + artifact_dir.join("run.sh").is_file(), + "expected run.sh in {}", + artifact_dir.display() ); - assert_eq!(proxy_metadata["router"]["control_port"].as_u64(), Some(0)); } #[test] -fn compile_vm_artifact_supports_runtime_templates() { +fn compile_vm_export_rejects_framework_component_without_controller_site() { let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("cli crate should live under the workspace root"); @@ -564,531 +508,21 @@ fn compile_vm_artifact_supports_runtime_templates() { let outputs_root = workspace_root.join("target").join("cli-test-outputs"); fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); let outputs_dir = tempfile::Builder::new() - .prefix("vm-runtime-template-") + .prefix("vm-framework-outputs-") .tempdir_in(&outputs_root) .expect("failed to create outputs directory"); - let manifest = outputs_dir.path().join("scenario.json5"); + let admin = outputs_dir.path().join("admin.json5"); fs::write( - &manifest, + &admin, r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - base_image: { type: "string" }, - message: { type: "string" } - }, - required: ["base_image", "message"], - additionalProperties: false - }, + manifest_version: "0.3.0", slots: { - api: { kind: "http", optional: true } + ctl: { kind: "component" } }, program: { - vm: { - image: "./images/${config.base_image}", - cpus: 1, - memory_mib: 512, - cloud_init: { - user_data: "#cloud-config\nwrite_files:\n - path: /tmp/value\n content: 'msg=${config.message};api=${slots.api.url}'\n" - }, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan_path = artifact_dir.join(VM_PLAN_FILENAME); - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(&vm_plan_path).expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "runtime_template"); - assert_eq!(component["base_image"]["parts"][0]["kind"], "literal"); - assert_eq!(component["base_image"]["parts"][0]["value"], "./images/"); - assert_eq!( - component["base_image"]["parts"][1]["kind"], - "runtime_config" - ); - assert_eq!(component["base_image"]["parts"][1]["query"], "base_image"); - assert_eq!( - component["base_image"]["source_dir"].as_str(), - outputs_dir.path().to_str() - ); - - assert_eq!( - component["cloud_init_user_data"]["kind"], - "runtime_template" - ); - let cloud_init_parts = component["cloud_init_user_data"]["parts"] - .as_array() - .expect("cloud-init template parts should be an array"); - assert!( - cloud_init_parts - .iter() - .any(|part| part["config"] == "message"), - "cloud-init template should retain config interpolation" - ); - assert!( - cloud_init_parts - .iter() - .any(|part| part["slot"] == "api.url"), - "cloud-init template should retain slot interpolation" - ); - let mut allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| { - value - .as_str() - .expect("allowed_root_leaf_paths should contain strings") - .to_string() - }) - .collect::>(); - allowed_root_leaf_paths.sort(); - assert_eq!( - allowed_root_leaf_paths, - vec!["base_image".to_string(), "message".to_string()] - ); -} - -#[test] -fn compile_vm_artifact_resolves_static_child_config_for_scalars_and_cloud_init() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-static-child-config-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let child_manifest = outputs_dir.path().join("child.json5"); - fs::write( - &child_manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" }, - cpu_count: { type: "integer" }, - memory: { type: "integer" }, - banner: { type: "string" } - }, - required: ["image", "cpu_count", "memory", "banner"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.cpu_count}", - memory_mib: "${config.memory}", - cloud_init: { - user_data: "#cloud-config\nwrite_files:\n - path: /tmp/banner\n content: '${config.banner}'\n" - }, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write child manifest"); - - let root_manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &root_manifest, - format!( - r##"{{ - manifest_version: "0.1.0", - components: {{ - child: {{ - manifest: "{}", - config: {{ - image: "./images/base.qcow2", - cpu_count: 3, - memory: 1536, - banner: "hello from parent" - }} - }} - }}, - exports: {{ - http: "#child.http" - }} -}} -"##, - child_manifest.display() - ), - ) - .expect("failed to write root manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&root_manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(artifact_dir.join(VM_PLAN_FILENAME)) - .expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "static"); - assert_eq!(component["cpus"]["kind"], "literal"); - assert_eq!(component["cpus"]["value"], 3); - assert_eq!(component["memory_mib"]["kind"], "literal"); - assert_eq!(component["memory_mib"]["value"], 1536); - assert_eq!(component["cloud_init_user_data"]["kind"], "static"); - assert!( - component["cloud_init_user_data"]["value"] - .as_str() - .is_some_and(|value| value.contains("hello from parent")), - "{component:#}" - ); - let component = component - .as_object() - .expect("vm component should serialize as an object"); - assert!( - !component.contains_key("runtime_config"), - "static child config should not force runtime_config" - ); -} - -#[test] -fn compile_vm_artifact_rewrites_runtime_scalar_paths_through_child_config() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-runtime-child-config-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let child_manifest = outputs_dir.path().join("child.json5"); - fs::write( - &child_manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" }, - cpu_count: { type: "integer" }, - memory: { type: "integer" } - }, - required: ["image", "cpu_count", "memory"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.cpu_count}", - memory_mib: "${config.memory}", - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write child manifest"); - - let root_manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &root_manifest, - format!( - r##"{{ - manifest_version: "0.1.0", - config_schema: {{ - type: "object", - properties: {{ - vm: {{ - type: "object", - properties: {{ - image: {{ type: "string" }}, - cpu: {{ type: "integer" }}, - memory: {{ type: "integer" }} - }}, - required: ["image", "cpu", "memory"], - additionalProperties: false - }} - }}, - required: ["vm"], - additionalProperties: false - }}, - components: {{ - child: {{ - manifest: "{}", - config: {{ - image: "${{config.vm.image}}", - cpu_count: "${{config.vm.cpu}}", - memory: "${{config.vm.memory}}" - }} - }} - }}, - exports: {{ - http: "#child.http" - }} -}} -"##, - child_manifest.display() - ), - ) - .expect("failed to write root manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&root_manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(artifact_dir.join(VM_PLAN_FILENAME)) - .expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "runtime_config"); - assert_eq!(component["base_image"]["query"], "image"); - assert_eq!(component["cpus"]["kind"], "runtime_config"); - assert_eq!(component["cpus"]["query"], "cpu_count"); - assert_eq!(component["memory_mib"]["kind"], "runtime_config"); - assert_eq!(component["memory_mib"]["query"], "memory"); - let mut allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| { - value - .as_str() - .expect("allowed_root_leaf_paths should contain strings") - .to_string() - }) - .collect::>(); - allowed_root_leaf_paths.sort(); - assert_eq!( - allowed_root_leaf_paths, - vec![ - "vm.cpu".to_string(), - "vm.image".to_string(), - "vm.memory".to_string(), - ] - ); - assert!( - component.get("runtime_config").is_some(), - "runtime child config should carry runtime_config payload" - ); - let env_example = fs::read_to_string(artifact_dir.join("env.example")) - .expect("failed to read vm env example"); - assert!( - env_example.contains("AMBER_CONFIG_VM__IMAGE"), - "vm env example should mention runtime config for the forwarded base image" - ); - assert!( - env_example.contains("AMBER_CONFIG_VM__CPU"), - "vm env example should mention runtime config for the forwarded cpu count" - ); - assert!( - env_example.contains("AMBER_CONFIG_VM__MEMORY"), - "vm env example should mention runtime config for the forwarded memory size" - ); -} - -#[test] -fn check_rejects_invalid_vm_scalar_config_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-invalid-scalar-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - }, - required: ["image"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.missing}", - memory_mib: 512, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write manifest"); - - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("check") - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber check: {err}")); - - assert!( - !output.status.success(), - "amber check unexpectedly succeeded" - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("program.vm.cpus"), - "expected program.vm.cpus in stderr, got:\n{stderr}" - ); - assert!( - stderr.contains("config.missing"), - "expected missing config path in stderr, got:\n{stderr}" - ); -} - -#[test] -fn compile_compose_rejects_whole_config_program_image_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("program-image-whole-config-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - } - }, - program: { - image: "${config}", - entrypoint: ["run"], + path: "/bin/echo", + args: ["admin", "${slots.ctl.url}"], network: { endpoints: [ { name: "http", port: 8080, protocol: "http" } @@ -1104,76 +538,26 @@ fn compile_compose_rejects_whole_config_program_image_ref() { } "##, ) - .expect("failed to write manifest"); - - let artifact_dir = outputs_dir.path().join("compose"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--docker-compose") - .arg(&artifact_dir) - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --docker-compose: {err}")); - - assert!( - !output.status.success(), - "amber compile --docker-compose unexpectedly succeeded" - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("program.image cannot") - && stderr.contains("reference the entire runtime config object"), - "expected whole-config image rejection in stderr, got:\n{stderr}" - ); -} - -#[test] -fn compile_vm_rejects_whole_config_vm_image_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-image-whole-config-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); + .expect("failed to write admin manifest"); let manifest = outputs_dir.path().join("scenario.json5"); fs::write( &manifest, r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - } - }, - program: { - vm: { - image: "${config}", - cpus: 1, - memory_mib: 512, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } + manifest_version: "0.3.0", + components: { + admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], exports: { - http: "http" + admin_http: "#admin.http" } } "##, ) - .expect("failed to write manifest"); + .expect("failed to write root manifest"); let artifact_dir = outputs_dir.path().join("vm"); let output = Command::new(env!("CARGO_BIN_EXE_amber")) @@ -1186,13 +570,16 @@ fn compile_vm_rejects_whole_config_vm_image_ref() { assert!( !output.status.success(), - "amber compile --vm unexpectedly succeeded" + "amber compile --vm unexpectedly succeeded\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("program.vm.image cannot") - && stderr.contains("reference the entire runtime config object"), - "expected whole-config vm image rejection in stderr, got:\n{stderr}" + stderr.contains("explicit") + && stderr.contains("controlling direct site") + && stderr.contains("mixed-site placement"), + "expected controller-site rejection in stderr, got:\n{stderr}" ); } diff --git a/cli/tests/compile_outputs/mod.rs b/cli/tests/compile_outputs/mod.rs index 070827e6..581af5d1 100644 --- a/cli/tests/compile_outputs/mod.rs +++ b/cli/tests/compile_outputs/mod.rs @@ -4,10 +4,7 @@ use std::{ process::Command, }; -use amber_compiler::reporter::{ - direct::{DIRECT_CONTROL_SOCKET_RELATIVE_PATH, DIRECT_PLAN_VERSION}, - vm::{VM_PLAN_FILENAME, VM_PLAN_VERSION}, -}; +use amber_compiler::reporter::direct::{DIRECT_CONTROL_SOCKET_RELATIVE_PATH, DIRECT_PLAN_VERSION}; use amber_images::AMBER_ROUTER; use amber_manifest::ManifestDigest; use amber_template::{ diff --git a/cli/tests/direct_smoke.rs b/cli/tests/direct_smoke.rs index ad3a9c1b..a9f762bf 100644 --- a/cli/tests/direct_smoke.rs +++ b/cli/tests/direct_smoke.rs @@ -1,6 +1,9 @@ +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; + use std::{ env, fs, - net::{SocketAddr, TcpListener}, + net::SocketAddr, path::Path, process::{Command, Stdio}, thread, @@ -8,11 +11,11 @@ use std::{ }; use amber_mesh::{MeshConfig, MeshIdentity, TransportConfig, encode_config_b64}; +use port_allocator_support::reserve_test_loopback_port; use serde_json::Value; fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn workspace_root() -> std::path::PathBuf { diff --git a/cli/tests/examples.rs b/cli/tests/examples.rs index 5cc15976..2ba166d9 100644 --- a/cli/tests/examples.rs +++ b/cli/tests/examples.rs @@ -3,6 +3,7 @@ use std::{ fs, path::{Path, PathBuf}, process::Command, + sync::{Mutex, OnceLock}, }; #[path = "../src/example_catalog.rs"] @@ -12,6 +13,7 @@ mod runtime_bins_support; #[test] fn examples_check_deny_warnings() { + let _guard = example_command_lock(); let examples_dir = examples_dir(); let examples = collect_examples(); for example in &examples { @@ -93,6 +95,7 @@ fn examples_check_deny_warnings() { #[test] fn examples_compile_from_ir_matches_manifest_outputs() { + let _guard = example_command_lock(); let amber = env!("CARGO_BIN_EXE_amber"); let outputs_root = workspace_root().join("target").join("cli-test-outputs"); fs::create_dir_all(&outputs_root).expect("failed to create outputs root"); @@ -147,6 +150,13 @@ fn examples_compile_from_ir_matches_manifest_outputs() { } } +fn example_command_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + .lock() + .expect("example command lock should not be poisoned") +} + fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() diff --git a/cli/tests/kvm_smoke.rs b/cli/tests/kvm_smoke.rs index 1ef95687..9f9281fd 100644 --- a/cli/tests/kvm_smoke.rs +++ b/cli/tests/kvm_smoke.rs @@ -8,13 +8,13 @@ mod target_dir_support; mod workspace_root_support; use std::{ - fs, + env, fs, os::unix::fs::MetadataExt, path::{Path, PathBuf}, process::{Command, Stdio}, }; -use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER}; +use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER}; use outputs_root_support::cli_test_outputs_root; use target_dir_support::cargo_target_dir; use workspace_root_support::workspace_root; @@ -76,24 +76,38 @@ fn build_checker_image(workspace_root: &Path) { } fn ensure_internal_images(workspace_root: &Path) { - build_internal_image_if_needed( - AMBER_HELPER.reference, - &workspace_root.join("docker/amber-helper/Dockerfile"), - ); - build_internal_image_if_needed( - AMBER_PROVISIONER.reference, - &workspace_root.join("docker/amber-provisioner/Dockerfile"), - ); - build_internal_image_if_needed( - AMBER_ROUTER.reference, - &workspace_root.join("docker/amber-router/Dockerfile"), - ); + for (tag, dockerfile) in [ + ( + AMBER_HELPER.reference, + workspace_root.join("docker/amber-helper/Dockerfile"), + ), + ( + AMBER_PROVISIONER.reference, + workspace_root.join("docker/amber-provisioner/Dockerfile"), + ), + ( + AMBER_ROUTER.reference, + workspace_root.join("docker/amber-router/Dockerfile"), + ), + ( + AMBER_SITE_CONTROLLER.reference, + workspace_root.join("docker/amber-site-controller/Dockerfile"), + ), + ] { + ensure_internal_image(tag, &dockerfile); + } } -fn build_internal_image_if_needed(tag: &str, dockerfile: &Path) { +fn ensure_internal_image(tag: &str, dockerfile: &Path) { if docker_image_exists(tag) { return; } + if env::var_os("AMBER_TEST_USE_PREBUILT_IMAGES").is_some() { + panic!( + "AMBER_TEST_USE_PREBUILT_IMAGES is set but {tag} is not available locally. Pull and \ + retag CI images before running the KVM smoke test." + ); + } progress(format!("building internal image {tag}")); let output = Command::new("docker") diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index e5fcfd02..cb1a3c8a 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -6,13 +6,15 @@ mod cloud_image_support; mod macos_vm_support; #[path = "test_support/outputs_root.rs"] mod outputs_root_support; +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "test_support/workspace_root.rs"] mod workspace_root_support; use std::{ + collections::BTreeMap, env, fs, hash::{Hash as _, Hasher as _}, - net::{SocketAddr, TcpListener}, path::{Path, PathBuf}, process::{Child, Command, Output, Stdio}, thread, @@ -22,6 +24,7 @@ use std::{ use cloud_image_support::default_host_arch_cloud_image_filename; use macos_vm_support::resolve_aarch64_firmware; use outputs_root_support::cli_test_outputs_root; +use port_allocator_support::reserve_test_loopback_port; use tempfile::TempDir; use workspace_root_support::workspace_root; @@ -34,6 +37,10 @@ const LINUX_VM_KIND_VERSION: &str = "v0.26.0"; const LINUX_VM_KIND_NODE_IMAGE: &str = "kindest/node:v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027"; const LINUX_VM_KIND_PULL_TIMEOUT: &str = "180s"; +const LINUX_VM_DIRECT_SMOKE_FILTER_ENV: &str = "AMBER_LINUX_VM_DIRECT_SMOKE_FILTER"; +const LINUX_VM_DYNAMIC_CAPABILITIES_FILTER_ENV: &str = "AMBER_LINUX_VM_DYNAMIC_CAPABILITIES_FILTER"; +const LINUX_VM_FRAMEWORK_COMPONENT_FILTER_ENV: &str = "AMBER_LINUX_VM_FRAMEWORK_COMPONENT_FILTER"; +const LINUX_VM_MIXED_RUN_FILTER_ENV: &str = "AMBER_LINUX_VM_MIXED_RUN_FILTER"; struct GuestArch { cloud_image_filename: &'static str, @@ -75,7 +82,7 @@ impl ProvisionProfile { guest_packages: "build-essential ca-certificates curl git jq pkg-config libssl-dev bubblewrap \ slirp4netns docker.io", profile_setup: format!( - "if ! docker compose version >/dev/null 2>&1; then\n\ + "if ! docker compose version >/dev/null 2>&1; then\n\ sudo apt-get install -y --no-install-recommends docker-compose-v2 || sudo apt-get install -y --no-install-recommends docker-compose-plugin\n\ fi\n\ if ! docker buildx version >/dev/null 2>&1; then\n\ @@ -256,8 +263,7 @@ fn qemu_system_binary(arch: &GuestArch) -> PathBuf { } fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn shell_escape(text: &str) -> String { @@ -427,6 +433,33 @@ fn write_ssh_keypair(private_key: &Path) -> Result { }) } +fn append_ssh_transport_args( + command: &mut Command, + port_flag: &str, + ssh_private_key: &Path, + ssh_port: u16, +) { + command + .arg("-i") + .arg(ssh_private_key) + .arg(port_flag) + .arg(ssh_port.to_string()) + .arg("-o") + .arg("BatchMode=yes") + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg("-o") + .arg("ConnectTimeout=5") + .arg("-o") + .arg("ServerAliveInterval=15") + .arg("-o") + .arg("ServerAliveCountMax=4") + .arg("-o") + .arg("LogLevel=ERROR"); +} + impl LinuxVmHarness { fn start(output_dir: &Path, base_image: &Path) -> Result { let arch = guest_arch(); @@ -567,18 +600,8 @@ impl LinuxVmHarness { fn ssh_base_command(&self) -> Command { let mut command = Command::new("/usr/bin/ssh"); - command - .arg("-i") - .arg(&self.ssh_private_key) - .arg("-p") - .arg(self.ssh_port.to_string()) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg("-o") - .arg("LogLevel=ERROR") - .arg(format!("{GUEST_USER}@127.0.0.1")); + append_ssh_transport_args(&mut command, "-p", &self.ssh_private_key, self.ssh_port); + command.arg(format!("{GUEST_USER}@127.0.0.1")); command } @@ -628,17 +651,7 @@ impl LinuxVmHarness { fn scp_to_guest(&self, sources: &[&Path], description: &str) -> Result<(), String> { let mut command = Command::new("/usr/bin/scp"); - command - .arg("-i") - .arg(&self.ssh_private_key) - .arg("-P") - .arg(self.ssh_port.to_string()) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg("-o") - .arg("LogLevel=ERROR"); + append_ssh_transport_args(&mut command, "-P", &self.ssh_private_key, self.ssh_port); for source in sources { command.arg(source); } @@ -710,7 +723,8 @@ impl LinuxVmHarness { self.run_guest_checked( "prepare linux guest snapshot", "set -euxo pipefail\n. \"$HOME/.cargo/env\" || true\nsudo cloud-init clean --logs \ - --machine-id\nsudo rm -rf \"$HOME/amber\" \"$HOME/amber-target\"\nsudo sync\n", + --machine-id\nsudo rm -rf \"$HOME/amber\" \"$HOME/amber-target\" \ + \"$HOME/linux-vm-test-logs\"\nsudo sync\n", ) } @@ -767,18 +781,48 @@ impl LinuxVmHarness { ) } - fn run_test_command(&self, label: &str, command: &str) -> Result<(), String> { + fn run_test_command( + &self, + label: &str, + command: &str, + extra_env: &BTreeMap, + ) -> Result<(), String> { let arch = guest_arch(); + let extra_exports = extra_env + .iter() + .map(|(name, value)| format!("export {name}={}", shell_escape(value))) + .collect::>() + .join("\n"); + let extra_exports = if extra_exports.is_empty() { + String::new() + } else { + format!("{extra_exports}\n") + }; self.run_guest_checked( label, &format!( "set -euxo pipefail\n. \"$HOME/.cargo/env\"\ncd {guest}\nexport \ AMBER_TEST_KEEP_OUTPUTS=1\nexport \ + AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE=debug\nexport \ AMBER_VM_SMOKE_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ AMBER_MIXED_RUN_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ - CARGO_TARGET_DIR=\"$HOME/amber-target\"\n{command}\n", + CARGO_TARGET_DIR=\"$HOME/amber-target\"\nexport CARGO_TERM_QUIET=true\nexport \ + CARGO_TERM_PROGRESS_WHEN=never\nexport CARGO_HTTP_MULTIPLEXING=false\nexport \ + CARGO_HTTP_TIMEOUT=120\nexport \ + CARGO_NET_RETRY=5\n{extra_exports}log_dir=\"$HOME/linux-vm-test-logs\"\nmkdir -p \ + \"$log_dir\"\nstdout_log=\"$log_dir/{label}.stdout.log\"\nstderr_log=\"$log_dir/\ + {label}.stderr.log\"\nif {command} >\"$stdout_log\" 2>\"$stderr_log\"; then\ncat \ + \"$stdout_log\"\nelse\nstatus=$?\necho \"===== guest test panic lines \ + =====\"\ngrep -nE \"panicked at|assertion.*failed|thread '.*' panicked\" \ + \"$stdout_log\" \"$stderr_log\" || true\necho \"===== guest test kind load lines \ + =====\"\ngrep -n \"kind load docker-image\" \"$stdout_log\" \"$stderr_log\" || \ + true\necho \"===== guest test stdout tail ($stdout_log) =====\"\ntail -n 300 \ + \"$stdout_log\" || true\necho \"===== guest test stderr tail ($stderr_log) \ + =====\"\ntail -n 500 \"$stderr_log\" || true\nexit \"$status\"\nfi\n", guest = shell_escape(&self.guest_workspace), image_filename = arch.cloud_image_filename, + extra_exports = extra_exports, + label = label, command = command, ), ) @@ -795,11 +839,25 @@ impl LinuxVmHarness { $latest\"\nfind \"$latest\" -maxdepth 3 -type f | sort\nfor state_root in \ \"$latest/state\" \"$latest/replay-state\"; do\nif [ ! -d \"$state_root\" ]; \ then\ncontinue\nfi\necho \"===== state root: $state_root =====\"\nwhile IFS= read -r \ - path; do\necho \"----- $path -----\"\nsed -n '1,220p' \"$path\"\ndone < <(find \ + path; do\necho \"----- $path -----\"\nsed -n '1,260p' \"$path\"\ndone < <(find \ \"$state_root\" -type f \\( -name 'manager-state.json' -o -name \ - 'direct-runtime-state.json' -o -name 'vm-runtime-state.json' -o -name \ + 'direct-runtime-state.json' -o -name 'direct-runtime.json' -o -name \ + 'vm-runtime-state.json' -o -name 'vm-runtime.json' -o -name \ + 'site-controller-state.json' -o -name 'site-controller-runtime-state.json' -o -name \ 'supervisor.log' -o -name 'port-forward.log' -o -name 'site.log' -o -name \ - 'control-state.log' \\) 2>/dev/null | sort)\ndone\n" + 'outside-proxy.log' -o -name 'site-controller.log' -o -name \ + 'site-controller-plan.json' -o -name 'site-controller-runtime-plan.json' -o -name \ + 'mesh-config.json' \\) 2>/dev/null | sort)\ndone\nruns_root=\"$(find \ + {guest_workspace} -path '*/.amber-runs/runs/run-*' -type d | sort | tail -n 1)\"\nif \ + [ -n \"$runs_root\" ]; then\necho \"latest guest run root: $runs_root\"\nfind \ + \"$runs_root/state\" -maxdepth 2 -type f \\( -name 'manager-state.json' -o -name \ + 'direct-runtime.json' -o -name 'vm-runtime.json' -o -name 'supervisor.log' -o -name \ + 'port-forward.log' -o -name 'site.log' -o -name 'outside-proxy.log' -o -name \ + 'site-controller.log' -o -name 'site-controller-state.json' -o -name \ + 'site-controller-runtime-state.json' -o -name 'site-controller-plan.json' -o -name \ + 'site-controller-runtime-plan.json' -o -name 'mesh-config.json' \\) | sort | while \ + IFS= read -r path; do\necho \"----- $path -----\"\nsed -n '1,260p' \ + \"$path\"\ndone\nfi\n" ); self.ssh_output(&script) .ok() @@ -850,7 +908,7 @@ fn run_linux_guest_test( let provisioned_image = ensure_provisioned_image(profile)?; let harness = LinuxVmHarness::start(output_dir.path(), &provisioned_image)?; harness.copy_workspace()?; - harness.run_test_command(prefix, command) + harness.run_test_command(prefix, command, &BTreeMap::new()) })(); if result.is_err() || env::var_os("AMBER_TEST_KEEP_OUTPUTS").is_some() { output_dir.preserve(); @@ -864,6 +922,8 @@ fn run_linux_guest_test( fn ensure_provisioned_image(profile: ProvisionProfile) -> Result { let cache_path = provisioned_cache_path(profile); + // Keep only the provisioned base image as a persistent cache. It saves the expensive OS/tool + // setup without accumulating workspace-dependent snapshots or large build trees under target/. if cache_path.is_file() && env::var_os("AMBER_LINUX_VM_REFRESH_CACHE").is_none() { return Ok(cache_path); } @@ -916,10 +976,6 @@ fn ensure_provisioned_image(profile: ProvisionProfile) -> Result Result<(), String> { - run_linux_guest_mixed_run_filter(test_name) -} - fn run_linux_guest_mixed_run_filter(test_filter: &str) -> Result<(), String> { run_linux_guest_test( format!("linux-vm-{test_filter}-"), @@ -931,6 +987,15 @@ fn run_linux_guest_mixed_run_filter(test_filter: &str) -> Result<(), String> { ) } +// Keep the Linux VM live surface to one guest boot per test family. Narrow reruns can still use +// the same aggregate test by overriding its guest-side filter env var. +fn linux_vm_guest_filter(env_var: &str, default: &str) -> String { + env::var(env_var) + .ok() + .filter(|filter| !filter.trim().is_empty()) + .unwrap_or_else(|| default.to_owned()) +} + fn run_linux_guest_direct_smoke_test(test_name: &str) -> Result<(), String> { run_linux_guest_test( format!("linux-vm-{test_name}-"), @@ -958,12 +1023,10 @@ fn linux_vm_runs_vm_smoke_test() { #[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run tests inside \ the guest"] fn linux_vm_runs_mixed_run_tests() { - run_linux_guest_test( - "linux-vm-mixed-run-", - ProvisionProfile::MixedRun, - "cargo test -p amber-cli --test mixed_run mixed_run_ -- --ignored --nocapture \ - --test-threads=1", - ) + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_MIXED_RUN_FILTER_ENV, + "mixed_run_", + )) .unwrap_or_else(|err| panic!("{err}")); } @@ -971,145 +1034,35 @@ fn linux_vm_runs_mixed_run_tests() { #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component live tests \ inside the guest"] fn linux_vm_runs_framework_component_live_tests() { - run_linux_guest_mixed_run_filter("framework_component_").unwrap_or_else(|err| panic!("{err}")); + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_FRAMEWORK_COMPONENT_FILTER_ENV, + "framework_component_", + )) + .unwrap_or_else(|err| panic!("{err}")); } #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability live tests \ inside the guest"] fn linux_vm_runs_dynamic_capability_live_tests() { - run_linux_guest_mixed_run_filter("dynamic_capabilities_").unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability manual \ - materialization live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_manual_materialization_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_manual_materialization_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability external \ - slot root share live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_external_slot_root_share_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_external_slot_root_share_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability snapshot \ - replay dynamic child live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_snapshot_replay_dynamic_child_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_snapshot_replay_dynamic_child_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run cleanup test \ - inside the guest"] -fn linux_vm_runs_mixed_run_cleanup_after_coordinator_dies_during_setup() { - run_linux_guest_mixed_run_test("mixed_run_cleanup_after_coordinator_dies_during_setup") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run direct/compose \ - proxy smoke test inside the guest"] -fn linux_vm_runs_mixed_run_direct_compose_proxy_smoke() { - run_linux_guest_mixed_run_test("mixed_run_direct_compose_proxy_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run detached stop \ - smoke test inside the guest"] -fn linux_vm_runs_mixed_run_detached_stop_smoke() { - run_linux_guest_mixed_run_test("mixed_run_detached_stop_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run five-site \ - startup test inside the guest"] -fn linux_vm_runs_mixed_run_five_site_startup_state_and_teardown() { - run_linux_guest_mixed_run_test("mixed_run_five_site_startup_state_and_teardown") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run local \ - observability smoke test inside the guest"] -fn linux_vm_runs_mixed_run_local_observability_scenario_smoke() { - run_linux_guest_mixed_run_test("mixed_run_local_observability_scenario_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run recovery test \ - inside the guest"] -fn linux_vm_runs_mixed_run_recovers_direct_component_failure_after_setup() { - run_linux_guest_mixed_run_test("mixed_run_recovers_direct_component_failure_after_setup") - .unwrap_or_else(|err| panic!("{err}")); + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_DYNAMIC_CAPABILITIES_FILTER_ENV, + "dynamic_capabilities_", + )) + .unwrap_or_else(|err| panic!("{err}")); } #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux direct smoke tests inside \ the guest"] fn linux_vm_runs_direct_smoke_tests() { - run_linux_guest_direct_smoke_test("direct_smoke_").unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke http startup test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_python_http_server_starts_and_stops() { - run_linux_guest_direct_smoke_test("direct_smoke_python_http_server_starts_and_stops") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke relative-path \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_resolves_relative_program_path_from_manifest_dir() { - run_linux_guest_direct_smoke_test( - "direct_smoke_resolves_relative_program_path_from_manifest_dir", - ) + run_linux_guest_direct_smoke_test(&linux_vm_guest_filter( + LINUX_VM_DIRECT_SMOKE_FILTER_ENV, + "direct_smoke_", + )) .unwrap_or_else(|err| panic!("{err}")); } -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke helper-mount test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_mount_under_run_with_helper() { - run_linux_guest_direct_smoke_test("direct_smoke_mount_under_run_with_helper") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke storage upgrade \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_storage_persists_across_upgrade() { - run_linux_guest_direct_smoke_test("direct_smoke_storage_persists_across_upgrade") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke env scrub test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_does_not_leak_host_env_into_component() { - run_linux_guest_direct_smoke_test("direct_smoke_does_not_leak_host_env_into_component") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke host-fs isolation \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_blocks_host_file_reads_outside_allowed_mounts() { - run_linux_guest_direct_smoke_test("direct_smoke_blocks_host_file_reads_outside_allowed_mounts") - .unwrap_or_else(|err| panic!("{err}")); -} - #[test] fn provisioned_cache_keys_track_guest_setup() { let vm_smoke_cache_key = ProvisionProfile::VmSmoke.cache_key(); @@ -1119,3 +1072,27 @@ fn provisioned_cache_keys_track_guest_setup() { assert!(mixed_run_cache_key.starts_with("mixed-run-")); assert_ne!(vm_smoke_cache_key, mixed_run_cache_key); } + +#[test] +fn ssh_transport_args_include_fail_fast_liveness_bounds() { + let mut command = Command::new("/usr/bin/ssh"); + append_ssh_transport_args(&mut command, "-p", Path::new("/tmp/test-key"), 2222); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + for expected in [ + ["-o", "BatchMode=yes"], + ["-o", "ConnectTimeout=5"], + ["-o", "ServerAliveInterval=15"], + ["-o", "ServerAliveCountMax=4"], + ] { + assert!( + args.windows(2) + .any(|window| window[0] == expected[0] && window[1] == expected[1]), + "ssh transport args should include {} {} so dead guests fail fast: {args:?}", + expected[0], + expected[1], + ); + } +} diff --git a/cli/tests/mixed_run/dry_run.rs b/cli/tests/mixed_run/dry_run.rs index 673bbc76..20abb90c 100644 --- a/cli/tests/mixed_run/dry_run.rs +++ b/cli/tests/mixed_run/dry_run.rs @@ -1,3 +1,9 @@ +use std::collections::BTreeSet; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; + +use amber_compiler::run_plan::FrameworkComponentControllerMoniker; + use super::*; #[test] @@ -173,18 +179,104 @@ fn mixed_run_dry_run_emits_launch_bundle_without_starting_sites() { .is_empty(), "dry-run should record compose launch commands" ); + #[cfg(unix)] + { + let compose_state_mode = fs::metadata(bundle_root.join("state").join("compose_local")) + .expect("compose site state root should exist") + .permissions() + .mode() + & 0o777; + assert_eq!( + compose_state_mode, 0o700, + "compose controller state should be private in the materialized launch bundle" + ); + + let compose_yaml_path = Path::new( + compose_site["artifact_dir"] + .as_str() + .expect("launch bundle should record compose artifact dir"), + ) + .join("compose.yaml"); + let compose_yaml: serde_yaml::Value = serde_yaml::from_str( + &fs::read_to_string(&compose_yaml_path).expect("compose yaml should read"), + ) + .expect("compose yaml should parse"); + let controller = compose_yaml["services"]["amber-site-controller"] + .as_mapping() + .expect("compose yaml should include the site controller service"); + let expected_user = format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }); + assert_eq!( + controller + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + Some(expected_user.as_str()), + "compose site controller should run as the launching user" + ); + } let stitching = launch_bundle["stitching"] .as_array() .expect("launch bundle stitching should serialize as an array"); - assert_eq!(stitching.len(), 1); - assert_eq!(stitching[0]["provider_site"], json!("compose_local")); - assert_eq!(stitching[0]["consumer_site"], json!("direct_local")); + let controller_stitching = stitching + .iter() + .filter(|link| { + link["provider_component"] + .as_str() + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) + || link["consumer_component"] + .as_str() + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) + }) + .collect::>(); + let user_stitching = stitching + .iter() + .filter(|link| { + !link["provider_component"] + .as_str() + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) + && !link["consumer_component"] + .as_str() + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) + }) + .collect::>(); + assert_eq!(user_stitching.len(), 1); + assert_eq!( + controller_stitching.len(), + 2, + "the two-site fixture has program components on both sites, so dynamic capabilities need \ + controller routes in both directions", + ); + let controller_pairs = controller_stitching + .iter() + .map(|link| { + ( + link["provider_site"] + .as_str() + .expect("controller stitching should record provider site") + .to_string(), + link["consumer_site"] + .as_str() + .expect("controller stitching should record consumer site") + .to_string(), + ) + }) + .collect::>(); + assert_eq!( + controller_pairs, + BTreeSet::from([ + ("compose_local".to_string(), "direct_local".to_string()), + ("direct_local".to_string(), "compose_local".to_string()), + ]), + ); + assert_eq!(user_stitching[0]["provider_site"], json!("compose_local")); + assert_eq!(user_stitching[0]["consumer_site"], json!("direct_local")); assert_eq!( - stitching[0]["resolution"], + user_stitching[0]["resolution"], json!("requires_runtime_discovery") ); assert!( - stitching[0]["unresolved_reason"] + user_stitching[0]["unresolved_reason"] .as_str() .is_some_and(|reason| reason.contains("Docker")), "compose-backed stitching should explain why the exact external URL is not known" diff --git a/cli/tests/mixed_run/lifecycle.rs b/cli/tests/mixed_run/lifecycle.rs index 6f3f7681..3ec31f7b 100644 --- a/cli/tests/mixed_run/lifecycle.rs +++ b/cli/tests/mixed_run/lifecycle.rs @@ -1,3 +1,7 @@ +use std::collections::BTreeSet; + +use amber_compiler::run_plan::FrameworkComponentControllerMoniker; + use super::*; #[test] @@ -31,15 +35,46 @@ fn mixed_run_five_site_startup_state_and_teardown() { ["direct_a"] ]) ); + let assignments = run_plan["assignments"] + .as_object() + .expect("run plan assignments should be an object"); + let user_assignments = assignments + .iter() + .filter(|(component, _)| { + !FrameworkComponentControllerMoniker::is_synthetic_component(component) + }) + .map(|(component, site)| (component.clone(), site.clone())) + .collect::>(); assert_eq!( - run_plan["assignments"], - json!({ - "/a": "direct_a", - "/b": "compose_b", - "/c": "kind_c", - "/d": "vm_d", - "/e": "compose_e" + user_assignments, + serde_json::Map::from_iter([ + ("/a".to_string(), json!("direct_a")), + ("/b".to_string(), json!("compose_b")), + ("/c".to_string(), json!("kind_c")), + ("/d".to_string(), json!("vm_d")), + ("/e".to_string(), json!("compose_e")), + ]) + ); + let controller_assignment_sites = assignments + .iter() + .filter(|(component, _)| { + FrameworkComponentControllerMoniker::is_synthetic_component(component) + }) + .map(|(_, site)| { + site.as_str() + .expect("assignment site should be a string") + .to_string() }) + .collect::>(); + assert_eq!( + controller_assignment_sites, + BTreeSet::from([ + "compose_b".to_string(), + "compose_e".to_string(), + "direct_a".to_string(), + "kind_c".to_string(), + ]), + "live startup should not depend on the private spelling of synthetic controller monikers", ); assert_eq!( run.receipt["sites"] @@ -109,7 +144,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("direct external slots") .len(), - 2 + 5 ); assert_eq!( read_json( @@ -121,7 +156,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("compose_b external slots") .len(), - 2 + 5 ); assert_eq!( read_json( @@ -133,7 +168,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("kind external slots") .len(), - 1 + 4 ); assert_eq!( read_json( @@ -630,3 +665,146 @@ fn mixed_run_recovers_when_kubernetes_site_is_temporarily_unreachable() { stop_proxy(&mut proxy); run.stop(); } + +#[test] +#[ignore = "requires docker + kind + kubectl; run manually or in CI"] +fn mixed_run_kubernetes_site_controller_state_survives_pod_recreation() { + let temp = temp_output_dir("mixed-run-kind-controller-pvc-"); + let kubeconfig = temp.path().join("kubeconfig"); + let kind_cluster = KindCluster::from_env_or_create(&kubeconfig); + ensure_kind_internal_images(&kind_cluster); + let kubeconfig_env = kind_cluster.kubeconfig.display().to_string(); + let fixture = write_single_site_kind_fixture(temp.path(), &kind_cluster); + let storage_root = temp.path().join("state"); + let mut run = run_manifest_with_env( + &fixture.manifest, + &fixture.placement, + &storage_root, + &[("KUBECONFIG", &kubeconfig_env)], + ); + + let kind_state = wait_for_state_status( + &run.run_root, + "kind_local", + "running", + Duration::from_secs(120), + ); + let namespace = kind_state["kubernetes_namespace"] + .as_str() + .expect("kubernetes site should publish namespace"); + let pod = kubernetes_site_controller_pod_name(&kind_state, namespace); + + let pvc_phase = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("pvc") + .arg("amber-site-controller-state") + .arg("-o") + .arg("jsonpath={.status.phase}") + .output() + .expect("failed to query site-controller state PVC"); + assert!( + pvc_phase.status.success(), + "failed to query site-controller state PVC:\n{}", + String::from_utf8_lossy(&pvc_phase.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&pvc_phase.stdout).trim(), + "Bound", + "running kubernetes site-controller should have a bound state PVC" + ); + + let secret_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("secret") + .arg("amber-site-controller-seed") + .status() + .expect("failed to query site-controller seed Secret"); + assert!( + secret_status.success(), + "kubernetes site-controller seed should be stored as a Secret" + ); + + let sentinel_path = "/amber/site/state/pvc-restart-sentinel"; + let write_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("sh") + .arg("-lc") + .arg(format!("printf preserved > {sentinel_path}")) + .status() + .expect("failed to write state sentinel in site-controller pod"); + assert!(write_status.success(), "failed to write state sentinel"); + + let delete_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("delete") + .arg("pod") + .arg(&pod) + .arg("--wait=true") + .status() + .expect("failed to delete site-controller pod"); + assert!( + delete_status.success(), + "failed to delete site-controller pod" + ); + + let mut replacement_ready = false; + for _ in 0..240 { + let status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("wait") + .arg("pod") + .arg("-l") + .arg("amber.io/component=amber-site-controller") + .arg("--for=condition=Ready") + .arg("--timeout=1s") + .status() + .expect("failed to wait for replacement site-controller pod"); + if status.success() { + replacement_ready = true; + break; + } + std::thread::sleep(Duration::from_millis(500)); + } + assert!( + replacement_ready, + "replacement site-controller pod did not become ready" + ); + + let replacement_pod = kubernetes_site_controller_pod_name(&kind_state, namespace); + assert_ne!( + replacement_pod, pod, + "deployment should recreate the site-controller pod after deletion" + ); + let sentinel = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&replacement_pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("cat") + .arg(sentinel_path) + .output() + .expect("failed to read state sentinel after pod recreation"); + assert!( + sentinel.status.success(), + "state sentinel missing after pod recreation:\n{}", + String::from_utf8_lossy(&sentinel.stderr) + ); + assert_eq!(String::from_utf8_lossy(&sentinel.stdout), "preserved"); + + run.stop(); +} diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index e6c26d68..573e24c3 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -1,6 +1,28 @@ use super::*; -const FRAMEWORK_MUTATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(300); +const FRAMEWORK_CONTROL_RESPONSE_GRACE: Duration = Duration::from_secs(30); + +fn framework_control_timeout_for_vm_ready_timeout(vm_ready_timeout: Duration) -> Duration { + vm_ready_timeout.max(Duration::from_secs(300)) +} + +fn framework_control_timeout() -> Duration { + framework_control_timeout_for_vm_ready_timeout( + amber_site_controller::vm_endpoint_forward_ready_timeout(), + ) +} + +fn framework_mutation_request_timeout_for_control_timeout(control_timeout: Duration) -> Duration { + control_timeout + FRAMEWORK_CONTROL_RESPONSE_GRACE +} + +fn framework_mutation_request_timeout() -> Duration { + framework_mutation_request_timeout_for_control_timeout(framework_control_timeout()) +} + +fn framework_control_timeout_env_value() -> String { + format!("{:.3}", framework_control_timeout().as_secs_f64()) +} const FRAMEWORK_ADMIN_APP: &str = r#"import json import os @@ -11,7 +33,7 @@ from urllib.request import Request, urlopen NAME = os.environ["NAME"] PORT = int(os.environ["PORT"]) CTL_URL = os.environ["CTL_URL"].rstrip("/") -CONTROL_TIMEOUT = 300.0 +CONTROL_TIMEOUT = float(os.environ.get("CONTROL_TIMEOUT", "300.0")) def send(handler, status, body, content_type="text/plain; charset=utf-8"): payload = body.encode("utf-8") @@ -567,12 +589,13 @@ ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() fn write_framework_admin_component(root: &Path, file_name: &str, image: bool, port: u16) { let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./admin.py" }], "env": { "NAME": "admin", "PORT": port.to_string(), - "CTL_URL": "${slots.ctl.url}" + "CTL_URL": "${slots.ctl.url}", + "CONTROL_TIMEOUT": framework_control_timeout_env_value() }, "network": { "endpoints": [ @@ -587,7 +610,8 @@ fn write_framework_admin_component(root: &Path, file_name: &str, image: bool, po "env": { "NAME": "admin", "PORT": port.to_string(), - "CTL_URL": "${slots.ctl.url}" + "CTL_URL": "${slots.ctl.url}", + "CONTROL_TIMEOUT": framework_control_timeout_env_value() }, "network": { "endpoints": [ @@ -623,7 +647,7 @@ fn write_framework_worker_component( ) { let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./worker.py" }], "env": { "NAME": name, @@ -701,7 +725,7 @@ fn write_dynamic_caps_component( .collect::>(); let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./dynamic_caps_app.py" }], "env": env, "network": { @@ -739,47 +763,117 @@ fn write_dynamic_caps_component( } fn framework_control_state_path(run: &RunHandle) -> PathBuf { - run.run_root - .join("state") - .join("framework-component") - .join("control-state.json") + run.run_root.join("state") } -fn framework_control_state_plan_path(run: &RunHandle) -> PathBuf { - run.run_root - .join("state") - .join("framework-component") - .join("control-state-plan.json") +fn read_framework_control_state(control_state_root: &Path) -> Value { + framework_control_state_snapshot(control_state_root) } -fn framework_control_state_post(run: &RunHandle, path: &str, payload: &Value) -> (u16, String) { - let plan = read_json(&framework_control_state_plan_path(run)); - let listen_addr = plan["listen_addr"] +fn framework_site_controller_plan_path(run: &RunHandle) -> PathBuf { + let state_root = run.run_root.join("state"); + let entries = fs::read_dir(&state_root).expect("run state directory should be readable"); + let mut site_ids = entries + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|file_type| file_type.is_dir()) + .map(|_| entry.file_name().to_string_lossy().to_string()) + }) + .collect::>(); + site_ids.sort(); + site_ids + .into_iter() + .map(|site_id| state_root.join(site_id).join("site-controller-plan.json")) + .find(|path| path.is_file()) + .expect("run should materialize at least one site controller plan") +} + +fn framework_site_controller_state_path(run: &RunHandle) -> PathBuf { + let state_root = run.run_root.join("state"); + let entries = fs::read_dir(&state_root).expect("run state directory should be readable"); + let mut site_ids = entries + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|file_type| file_type.is_dir()) + .map(|_| entry.file_name().to_string_lossy().to_string()) + }) + .collect::>(); + site_ids.sort(); + site_ids + .into_iter() + .map(|site_id| state_root.join(site_id).join("site-controller-state.json")) + .find(|path| path.is_file()) + .expect("run should materialize at least one site controller state file") +} + +fn framework_controller_internal_route_id(run: &RunHandle, site_id: &str) -> String { + let state = read_json(&framework_site_controller_state_path(run)); + let controller_moniker = state["base_scenario"]["components"] + .as_array() + .expect("site controller state should include base scenario components") + .iter() + .find(|component| { + component["metadata"]["kind"] + == amber_compiler::run_plan::FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND + && component["metadata"]["execution_site"] == site_id + }) + .and_then(|component| component["moniker"].as_str()) + .expect("base scenario should include the local synthetic site controller component"); + amber_mesh::component_route_id( + controller_moniker, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) +} + +fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u16, String) { + let plan = read_json(&framework_site_controller_plan_path(run)); + let authority_url = plan["authority_url"] .as_str() - .expect("framework control-state plan should publish listen_addr"); - let auth_token = plan["auth_token"] + .expect("site controller plan should publish authority_url"); + let site_id = plan["site_id"] .as_str() - .expect("framework control-state plan should publish auth token"); + .expect("site controller plan should publish site_id"); + let route_id = framework_controller_internal_route_id(run, site_id); + let logical_component_id = payload["holder_component_id"] + .as_str() + .or_else(|| payload["caller_component_id"].as_str()) + .expect("dynamic capability controller request should identify the calling component"); + let peer_id = logical_component_id + .strip_prefix("components.") + .filter(|moniker| moniker.starts_with('/')) + .expect("dynamic capability logical component id should be a component moniker"); let body = serde_json::to_string(payload).expect("request body should serialize"); - let output = std::process::Command::new("curl") - .arg("-sS") - .arg("--max-time") - .arg("30.000") - .arg("-X") - .arg("POST") - .arg("-H") - .arg("content-type: application/json") - .arg("-H") - .arg(format!("x-amber-framework-auth: {auth_token}")) - .arg("--data") - .arg(body) - .arg("-o") - .arg("-") - .arg("-w") - .arg("\n%{http_code}") - .arg(format!("http://{listen_addr}{path}")) - .output() - .expect("framework control-state request should complete"); + let output = command_output_via_tempfiles( + std::process::Command::new("curl") + .arg("-sS") + .arg("--max-time") + .arg("30.000") + .arg("-X") + .arg("POST") + .arg("-H") + .arg("content-type: application/json") + .arg("-H") + .arg("x-amber-site-controller-local-only: 1") + .arg("-H") + .arg(format!("x-amber-route-id: {route_id}")) + .arg("-H") + .arg(format!("x-amber-peer-id: {peer_id}")) + .arg("--data") + .arg(body) + .arg("-o") + .arg("-") + .arg("-w") + .arg("\n%{http_code}") + .arg(format!("{}{path}", authority_url.trim_end_matches('/'))), + "framework controller request", + ); let stdout = String::from_utf8_lossy(&output.stdout); let (body, status) = stdout .rsplit_once('\n') @@ -797,7 +891,7 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { wait_for_condition( Duration::from_secs(60), || { - read_json(control_state_path)["live_children"] + read_framework_control_state(control_state_path)["live_children"] .as_array() .is_some_and(|children| { children @@ -807,7 +901,7 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { }, &format!("dynamic child `{name}` live in control state"), ); - let control_state = read_json(control_state_path); + let control_state = read_framework_control_state(control_state_path); control_state["live_children"] .as_array() .expect("live children should be an array") @@ -817,14 +911,97 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { .unwrap_or_else(|| panic!("dynamic child `{name}` should have an id")) } +fn rule_contains(rule: &Value, key: &str, expected: &str) -> bool { + rule.get(key) + .and_then(Value::as_array) + .is_some_and(|values| values.iter().any(|value| value.as_str() == Some(expected))) +} + +fn kubernetes_role(kind_state: &Value, name: &str) -> Value { + let namespace = kind_state["kubernetes_namespace"] + .as_str() + .expect("kubernetes site should publish namespace"); + let output = kubectl_for_manager_state(kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("role") + .arg(name) + .arg("-o") + .arg("json") + .output() + .unwrap_or_else(|err| panic!("failed to query Kubernetes Role {name}: {err}")); + assert!( + output.status.success(), + "failed to query Kubernetes Role {name}:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout) + .unwrap_or_else(|err| panic!("Kubernetes Role {name} should parse: {err}")) +} + +fn assert_kubernetes_site_controller_role_cannot_write_rbac(kind_state: &Value) { + let role = kubernetes_role(kind_state, "amber-site-controller"); + let rules = role["rules"] + .as_array() + .expect("site-controller Role should include rules"); + assert!( + rules.iter().all(|rule| { + !rule_contains(rule, "apiGroups", "rbac.authorization.k8s.io") + && !rule_contains(rule, "resources", "serviceaccounts") + && !rule_contains(rule, "resources", "roles") + && !rule_contains(rule, "resources", "rolebindings") + }), + "site-controller Role should not be able to create or update Kubernetes RBAC objects" + ); +} + +fn assert_kubernetes_dynamic_provisioner_role_is_secret_only(kind_state: &Value) { + let role = kubernetes_role(kind_state, "amber-provisioner-dynamic"); + let rules = role["rules"] + .as_array() + .expect("dynamic provisioner Role should include rules"); + assert_eq!( + rules.len(), + 1, + "dynamic provisioner Role should contain only the mesh-secret write rule" + ); + let rule = &rules[0]; + assert_string_array_members( + &rule["resources"], + &["secrets"], + "dynamic provisioner Role should be limited to Kubernetes Secrets", + ); + assert_string_array_members( + &rule["verbs"], + &["create", "get", "update"], + "dynamic provisioner Role should only cover the verbs required for generated secrets", + ); +} + +fn assert_kubernetes_dynamic_child_artifact_omits_provisioner_rbac(artifact: &Path) { + let kustomization = fs::read_to_string(artifact.join("kustomization.yaml")) + .expect("kubernetes dynamic child kustomization should read"); + assert!(kustomization.contains("01-configmaps/amber-mesh-provision.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-job.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-role.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-rolebinding.yaml")); + assert!(!artifact.join("02-rbac/amber-provisioner-sa.yaml").exists()); + assert!( + !artifact + .join("02-rbac/amber-provisioner-role.yaml") + .exists() + ); + assert!( + !artifact + .join("02-rbac/amber-provisioner-rolebinding.yaml") + .exists() + ); +} + fn framework_child_artifact(run: &RunHandle, site_id: &str, child_id: u64) -> PathBuf { - run.run_root - .join("state") - .join(site_id) - .join("framework-component") - .join("children") - .join(child_id.to_string()) - .join("artifact") + framework_child_artifact_dir(&run.run_root, site_id, child_id) } fn framework_manifest_url(path: &Path) -> String { @@ -868,18 +1045,23 @@ fn spawn_framework_proxy_for_site( } fn wait_for_framework_child_absent( + run: &RunHandle, control_state_path: &Path, child_name: &str, - child_roots: &[PathBuf], + child_sites: &[&str], + child_id: u64, timeout: Duration, ) { wait_for_condition( timeout, || { - let no_live_child = read_json(control_state_path)["live_children"] + let no_live_child = read_framework_control_state(control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != child_name)); - no_live_child && child_roots.iter().all(|root| !root.exists()) + no_live_child + && child_sites + .iter() + .all(|site_id| framework_child_is_absent(&run.run_root, site_id, child_id)) }, &format!("dynamic child `{child_name}` removed from control state and site artifacts"), ); @@ -911,6 +1093,7 @@ write_files: Environment=NAME=admin Environment=PORT={port} Environment=CTL_URL=${{slots.ctl.url}} + Environment=CONTROL_TIMEOUT={control_timeout} ExecStart=/usr/bin/python3 /usr/local/bin/framework-admin.py Restart=always @@ -921,6 +1104,7 @@ runcmd: - [systemctl, enable, --now, framework-admin.service] "#, script = indent_block(FRAMEWORK_ADMIN_APP, 6), + control_timeout = framework_control_timeout_env_value(), ) } @@ -1013,6 +1197,67 @@ fn write_framework_dynamic_child_manifest( ); } +fn write_framework_kubernetes_dynamic_child_fixture( + root: &Path, + kind_cluster: &KindCluster, +) -> ScenarioFixture { + fs::write(root.join("admin.py"), FRAMEWORK_ADMIN_APP).expect("failed to write admin.py"); + fs::write(root.join("worker.py"), FRAMEWORK_WORKER_APP).expect("failed to write worker.py"); + write_framework_admin_component(root, "admin.json5", true, 8080); + write_framework_worker_component(root, "child-kind-root.json5", true, "child-kind-root", 8080); + write_framework_dynamic_child_manifest(root, "child-kind.json5", "child-kind-root.json5", &[]); + + let manifest = root.join("root.json5"); + write_json( + &manifest, + &json!({ + "manifest_version": "0.3.0", + "slots": { + "realm": { "kind": "component", "optional": true } + }, + "components": { + "admin": "./admin.json5" + }, + "child_templates": { + "child_kind": { "manifest": "./child-kind.json5" } + }, + "bindings": [ + { "to": "#admin.ctl", "from": "framework.component" } + ], + "exports": { + "admin_http": "#admin.http" + } + }), + ); + + let placement = root.join("placement.json5"); + write_json( + &placement, + &json!({ + "schema": "amber.run.placement", + "version": 1, + "sites": { + "kind_local": { + "kind": "kubernetes", + "context": kind_cluster.context_name() + } + }, + "defaults": { + "image": "kind_local" + }, + "components": { + "/admin": "kind_local", + "/job-kind/root": "kind_local" + } + }), + ); + + ScenarioFixture { + manifest, + placement, + } +} + fn write_framework_matrix_fixture(root: &Path, kind_cluster: &KindCluster) -> ScenarioFixture { fs::write(root.join("admin.py"), FRAMEWORK_ADMIN_APP).expect("failed to write admin.py"); fs::write(root.join("app.py"), FRAMEWORK_MATRIX_APP).expect("failed to write app.py"); @@ -1381,42 +1626,78 @@ fn assert_string_array_members(value: &Value, expected: &[&str], message: &str) assert_eq!(actual, expected, "{message}"); } +fn framework_admin_request( + method: &str, + port: u16, + path: &str, + body: Option<&str>, + purpose: &str, +) -> (u16, String) { + let timeout = framework_mutation_request_timeout(); + http_request_with_timeout_result(method, port, path, body, timeout).unwrap_or_else(|err| { + panic!("{purpose} via framework admin {method} http://127.0.0.1:{port}{path} failed: {err}") + }) +} + +fn framework_admin_get(port: u16, path: &str, purpose: &str) -> (u16, String) { + let timeout = framework_mutation_request_timeout(); + http_get_with_timeout_result(port, path, timeout).unwrap_or_else(|err| { + panic!("{purpose} via framework admin GET http://127.0.0.1:{port}{path} failed: {err}") + }) +} + fn framework_create_child_with_request(port: u16, request: &Value) -> (u16, String) { let body = serde_json::to_string(request).expect("create request should serialize"); - http_request_with_timeout( - "POST", - port, - "/create", - Some(&body), - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response") + framework_admin_request("POST", port, "/create", Some(&body), "create request") } fn framework_destroy_child_via_admin(port: u16, name: &str) -> (u16, String) { - http_request_with_timeout( + framework_admin_request( "DELETE", port, &format!("/destroy/{name}"), None, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, + "destroy request", ) - .expect("destroy request should return an HTTP response") } fn framework_snapshot_via_admin(port: u16) -> Value { - let (status, body) = http_request_with_timeout( - "POST", - port, - "/snapshot", - Some("{}"), - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("snapshot request should return an HTTP response"); + let (status, body) = + framework_admin_request("POST", port, "/snapshot", Some("{}"), "snapshot request"); assert_eq!(status, 200, "snapshot request should succeed: {body}"); serde_json::from_str(&body).expect("snapshot response should be valid json") } +#[test] +fn framework_control_timeout_tracks_vm_ready_budget() { + assert_eq!( + framework_control_timeout_for_vm_ready_timeout(Duration::from_secs(120)), + Duration::from_secs(300), + "fast VM environments should keep the existing 300s framework control budget", + ); + assert_eq!( + framework_control_timeout_for_vm_ready_timeout(Duration::from_secs(720)), + Duration::from_secs(720), + "slow TCG VM environments should inherit the full VM-ready budget", + ); +} + +#[test] +fn framework_mutation_request_timeout_keeps_response_grace_after_control_budget() { + assert_eq!( + framework_mutation_request_timeout_for_control_timeout(Duration::from_secs(300)), + Duration::from_secs(330), + "framework admin curl requests should outlive the inner control timeout long enough to \ + receive the terminal response", + ); + assert_eq!( + framework_mutation_request_timeout_for_control_timeout(Duration::from_secs(720)), + Duration::from_secs(750), + "slow TCG control paths should keep the same response grace instead of truncating the \ + request at 300s", + ); +} + fn write_snapshot_run_inputs(root: &Path, snapshot: &Value) -> (PathBuf, PathBuf) { let scenario_path = root.join("snapshot-scenario.json"); let placement_path = root.join("snapshot-placement.json5"); @@ -1654,7 +1935,10 @@ fn framework_component_direct_create_destroy_live() { } }), ); - assert_eq!(create_status, 200, "create request should succeed"); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); @@ -1701,7 +1985,7 @@ fn framework_component_direct_create_destroy_live() { Duration::from_secs(60), || { !child_root.exists() - && read_json(&control_state_path)["live_children"] + && read_framework_control_state(&control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != "job-1")) }, @@ -1984,7 +2268,7 @@ fn framework_component_concurrent_create_serialization_live() { "required": ["name"] }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./worker.py" }], "env": { "NAME": "${config.name}", @@ -2278,15 +2562,16 @@ fn framework_component_create_to_unoffered_site_fails_deterministically_live() { ); let storage_root = temp.path().join("state"); - let output = amber_command() - .arg("run") - .arg(&manifest) - .arg("--placement") - .arg(&placement) - .arg("--storage-root") - .arg(&storage_root) - .output() - .expect("failed to run amber"); + let output = command_output_via_tempfiles( + amber_command() + .arg("run") + .arg(&manifest) + .arg("--placement") + .arg(&placement) + .arg("--storage-root") + .arg(&storage_root), + "amber run", + ); assert!( !output.status.success(), "run should fail deterministically when a bounded template allows a manifest that \ @@ -2343,7 +2628,7 @@ fn framework_component_root_external_binding_live() { "catalog_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./external_bind.py" }], "env": { "NAME": "external-worker", @@ -2398,7 +2683,7 @@ fn framework_component_root_external_binding_live() { "catalog_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["sleep", "3600"] }, "components": { @@ -2599,7 +2884,7 @@ fn framework_component_nonweak_publication_barrier_live() { "weak_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./barrier_probe.py" }], "env": { "NAME": "consumer", @@ -2631,7 +2916,7 @@ fn framework_component_nonweak_publication_barrier_live() { "delayed_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["sleep", "3600"] }, "components": { @@ -3004,12 +3289,11 @@ fn framework_component_delegated_realm_cross_site_live() { "delegated destroy should succeed; response: {destroy_response}" ); wait_for_framework_child_absent( + &run, &control_state_path, "sibling", - &[sibling_artifact - .parent() - .expect("sibling artifact should have a parent") - .to_path_buf()], + &["compose_local"], + sibling_id, Duration::from_secs(60), ); @@ -3120,13 +3404,15 @@ fn framework_component_compose_parent_standby_direct_live() { "admin" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( proxy_port, "/create/worker/job-1", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); - assert_eq!(create_status, 200, "create request should succeed"); + "create request should return an HTTP response", + ); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); @@ -3259,13 +3545,15 @@ fn framework_component_direct_parent_compose_child_live() { "admin" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( proxy_port, "/create/job-1", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); - assert_eq!(create_status, 200, "create request should succeed"); + "create request should return an HTTP response", + ); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); @@ -3393,12 +3681,11 @@ fn framework_component_dynamic_children_teardown_with_run_live() { ); wait_for_path(&mut admin_proxy, proxy_port, "/id", Duration::from_secs(60)); - let (create_direct_status, create_direct_response) = http_get_with_timeout( + let (create_direct_status, create_direct_response) = framework_admin_get( proxy_port, "/create/worker_direct/job-direct", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("direct create request should return an HTTP response"); + "direct create request should return an HTTP response", + ); assert_eq!( create_direct_status, 200, "direct child create request should succeed" @@ -3407,12 +3694,11 @@ fn framework_component_dynamic_children_teardown_with_run_live() { .expect("direct child create response should be valid json"); assert_eq!(create_direct_json["child"]["name"], "job-direct"); - let (create_compose_status, create_compose_response) = http_get_with_timeout( + let (create_compose_status, create_compose_response) = framework_admin_get( proxy_port, "/create/worker_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("compose create request should return an HTTP response"); + "compose create request should return an HTTP response", + ); assert_eq!( create_compose_status, 200, "compose child create request should succeed" @@ -3596,12 +3882,11 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { ); wait_for_path(&mut admin_proxy, proxy_port, "/id", Duration::from_secs(60)); - let (create_provider_status, create_provider_response) = http_get_with_timeout( + let (create_provider_status, create_provider_response) = framework_admin_get( proxy_port, "/create/producer/source", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("producer create request should return an HTTP response"); + "producer create request should return an HTTP response", + ); assert_eq!( create_provider_status, 200, "producer create request should succeed" @@ -3679,7 +3964,7 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { Duration::from_secs(60), || { !source_root.exists() - && read_json(&control_state_path)["live_children"] + && read_framework_control_state(&control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != "source")) }, @@ -3710,6 +3995,91 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { run.stop(); } +#[test] +#[ignore = "requires docker + kind + kubectl + qemu + an Ubuntu 24.04 cloud image matching the \ + host architecture; run manually or in CI"] +fn framework_component_kubernetes_dynamic_child_rbac_live() { + ensure_internal_images(); + let temp = temp_output_dir("framework-component-kubernetes-rbac-"); + let kubeconfig = temp.path().join("kubeconfig"); + let kind_cluster = KindCluster::from_env_or_create(&kubeconfig); + ensure_kind_internal_images(&kind_cluster); + let kubeconfig_env = kind_cluster.kubeconfig.display().to_string(); + + let fixture = write_framework_kubernetes_dynamic_child_fixture(temp.path(), &kind_cluster); + let storage_root = temp.path().join("state"); + let mut run = run_manifest_with_env( + &fixture.manifest, + &fixture.placement, + &storage_root, + &[("KUBECONFIG", &kubeconfig_env)], + ); + + let kind_state = wait_for_state_status( + &run.run_root, + "kind_local", + "running", + Duration::from_secs(120), + ); + assert_kubernetes_site_controller_role_cannot_write_rbac(&kind_state); + assert_kubernetes_dynamic_provisioner_role_is_secret_only(&kind_state); + + let control_state_path = framework_control_state_path(&run); + let creator_port = pick_free_port(); + let mut creator_proxy = spawn_framework_proxy_for_site( + &run.site_artifact_dir("kind_local"), + "admin_http", + creator_port, + &kind_state, + ); + wait_for_path( + &mut creator_proxy, + creator_port, + "/id", + framework_mutation_request_timeout(), + ); + assert_eq!( + wait_for_body( + &mut creator_proxy, + creator_port, + "/id", + Duration::from_secs(30), + ), + "admin" + ); + + let (create_status, create_response) = framework_admin_get( + creator_port, + "/create/child_kind/job-kind", + "create request should return an HTTP response", + ); + assert_eq!( + create_status, 200, + "create request should succeed; response: {create_response}" + ); + let child_id = wait_for_live_child(&control_state_path, "job-kind"); + + let root_artifact = framework_child_artifact(&run, "kind_local", child_id); + assert_kubernetes_dynamic_child_artifact_omits_provisioner_rbac(&root_artifact); + let root_port = pick_free_port(); + let mut root_proxy = + spawn_framework_proxy_for_site(&root_artifact, "http", root_port, &kind_state); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); + assert_eq!( + wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), + "child-kind-root" + ); + + stop_proxy(&mut root_proxy); + stop_proxy(&mut creator_proxy); + run.stop(); +} + #[test] #[ignore = "requires docker + kind + kubectl + qemu + an Ubuntu 24.04 cloud image matching the \ host architecture; run manually or in CI"] @@ -3752,7 +4122,7 @@ fn framework_component_kind_root_export_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3774,7 +4144,7 @@ fn framework_component_kind_root_export_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -3786,12 +4156,11 @@ fn framework_component_kind_root_export_live() { "admin", ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, "/create/child_kind/job-kind", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request should succeed; response: {create_response}" @@ -3802,7 +4171,12 @@ fn framework_component_kind_root_export_live() { let root_port = pick_free_port(); let mut root_proxy = spawn_framework_proxy_for_site(&root_artifact, "http", root_port, site_state("kind_local")); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), "child-kind-root" @@ -3855,7 +4229,7 @@ fn framework_component_kind_creator_compose_child_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3877,7 +4251,7 @@ fn framework_component_kind_creator_compose_child_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -3890,12 +4264,11 @@ fn framework_component_kind_creator_compose_child_live() { "kind creator should expose the framework admin app" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, "/create/child_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request should succeed; response: {create_response}" @@ -3910,7 +4283,12 @@ fn framework_component_kind_creator_compose_child_live() { root_port, site_state("compose_local"), ); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), "child-compose-root" @@ -3963,7 +4341,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3986,7 +4364,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &mut compose_creator_proxy, compose_creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -4004,27 +4382,21 @@ fn framework_component_kind_creator_after_compose_churn_live() { "/create/{}/{}", template_case.template, template_case.child_name ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( compose_creator_port, &create_path, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("compose churn create request should return an HTTP response"); + "compose churn create request should return an HTTP response", + ); assert_eq!( create_status, 200, "compose churn create request {create_path} should succeed; response: \ {create_response}" ); let child_id = wait_for_live_child(&control_state_path, template_case.child_name); - let child_roots = template_case + let child_sites = template_case .exports .iter() - .map(|(site_id, _, _)| { - framework_child_artifact(&run, site_id, child_id) - .parent() - .expect("dynamic child artifact should have a parent") - .to_path_buf() - }) + .map(|(site_id, _, _)| *site_id) .collect::>(); let (destroy_status, destroy_response) = framework_destroy_child_via_admin(compose_creator_port, template_case.child_name); @@ -4034,10 +4406,12 @@ fn framework_component_kind_creator_after_compose_churn_live() { template_case.child_name, ); wait_for_framework_child_absent( + &run, &control_state_path, template_case.child_name, - &child_roots, - Duration::from_secs(300), + &child_sites, + child_id, + framework_mutation_request_timeout(), ); } stop_proxy(&mut compose_creator_proxy); @@ -4053,7 +4427,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &mut kind_creator_proxy, kind_creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -4079,16 +4453,37 @@ fn framework_component_kind_creator_after_compose_churn_live() { "kind creator control path should be healthy after compose churn" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( kind_creator_port, "/create/child_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("kind create request should return an HTTP response"); + "kind create request should return an HTTP response", + ); assert_eq!( create_status, 200, "kind creator create after compose churn should succeed; response: {create_response}" ); + let child_id = wait_for_live_child(&control_state_path, "job-compose"); + + let root_artifact = framework_child_artifact(&run, "compose_local", child_id); + let root_port = pick_free_port(); + let mut root_proxy = spawn_framework_proxy_for_site( + &root_artifact, + "http", + root_port, + site_state("compose_local"), + ); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); + assert_eq!( + wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), + "child-compose-root", + "kind creator should expose the recreated compose child after compose churn" + ); + stop_proxy(&mut root_proxy); stop_proxy(&mut kind_creator_proxy); run.stop(); @@ -4136,7 +4531,7 @@ fn framework_component_cross_backend_matrix_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -4160,7 +4555,7 @@ fn framework_component_cross_backend_matrix_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -4178,12 +4573,11 @@ fn framework_component_cross_backend_matrix_live() { "/create/{}/{}", template_case.template, template_case.child_name ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, &create_path, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request {create_path} from {creator_site} should succeed; response: \ @@ -4194,15 +4588,10 @@ fn framework_component_cross_backend_matrix_live() { assert_eq!(create_json["child"]["name"], template_case.child_name); let child_id = wait_for_live_child(&control_state_path, template_case.child_name); - let child_roots = template_case + let child_sites = template_case .exports .iter() - .map(|(site_id, _, _)| { - framework_child_artifact(&run, site_id, child_id) - .parent() - .expect("dynamic child artifact should have a parent") - .to_path_buf() - }) + .map(|(site_id, _, _)| *site_id) .collect::>(); let root_site_id = template_case.exports[0].0; @@ -4214,7 +4603,12 @@ fn framework_component_cross_backend_matrix_live() { root_port, site_state(root_site_id), ); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), template_case.exports[0].2 @@ -4229,7 +4623,12 @@ fn framework_component_cross_backend_matrix_live() { port, site_state(site_id), ); - wait_for_path(&mut proxy, port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut proxy, + port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut proxy, port, "/id", Duration::from_secs(30)), *expected_id, @@ -4257,10 +4656,12 @@ fn framework_component_cross_backend_matrix_live() { template_case.child_name, ); wait_for_framework_child_absent( + &run, &control_state_path, template_case.child_name, - &child_roots, - Duration::from_secs(300), + &child_sites, + child_id, + framework_mutation_request_timeout(), ); } @@ -5873,7 +6274,8 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { wait_for_condition( Duration::from_secs(60), || { - read_json(&replay_control_state)["base_scenario"]["components"] + read_json(&framework_site_controller_state_path(&replay_run))["base_scenario"] + ["components"] .as_array() .is_some_and(|components| { components @@ -5885,9 +6287,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { ); wait_for_live_child(&replay_control_state, "job-replay"); let replay_held = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/held", + "/v1/controller/dynamic-caps/held", &json!({ "holder_component_id": "components./job-replay" }), @@ -5927,9 +6329,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { .expect("replay share should return a grant id"); let replay_ref_detail = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &json!({ "holder_component_id": "components./job-replay", "ref": replay_ref @@ -5949,9 +6351,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { .to_string(); let replay_held_detail = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &json!({ "holder_component_id": "components./job-replay", "held_id": replay_shared_held_id @@ -5974,9 +6376,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { "replay should either reuse the restored grant or create a new live one" ); - let (old_ref_status, old_ref_body) = framework_control_state_post( + let (old_ref_status, old_ref_body) = framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &json!({ "holder_component_id": "components./job-replay", "ref": old_ref diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index b78d78ce..5277782b 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -4,6 +4,8 @@ mod cloud_image_support; #[path = "../test_support/outputs_root.rs"] mod outputs_root_support; +#[path = "../test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "../test_support/runtime_bins.rs"] mod runtime_bins_support; #[path = "../test_support/workspace_root.rs"] @@ -11,7 +13,10 @@ mod workspace_root_support; use std::{ collections::BTreeSet, - env, fs, + env, + ffi::OsString, + fs, + hash::{DefaultHasher, Hash, Hasher}, io::{Read, Write}, net::{SocketAddr, TcpListener, TcpStream}, path::{Path, PathBuf}, @@ -24,10 +29,15 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER}; +use amber_images::{ + AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, DEV_IMAGE_TAGS_ENV, + INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, parse_dev_image_tag_overrides, +}; use cloud_image_support::default_host_arch_cloud_image_filename; use outputs_root_support::cli_test_outputs_root; +use port_allocator_support::reserve_test_loopback_port; use serde_json::{Value, json}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; pub(crate) use workspace_root_support::workspace_root; const COMMON_HTTP_APP: &str = r#"import json @@ -126,7 +136,15 @@ class Handler(BaseHTTPRequestHandler): ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() "#; -pub(crate) const TEST_APP_IMAGE: &str = "python:3.13-alpine"; +const TEST_APP_SOURCE_IMAGE: &str = "python:3.13-alpine"; +const TEST_APP_LOCAL_IMAGE_REPOSITORY: &str = "amber-mixed-run-test-app"; +pub(crate) const KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER: &str = "main"; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct DockerImageMeta { + id: String, + arch: String, +} pub(crate) fn outputs_root() -> PathBuf { cli_test_outputs_root(&workspace_root()) @@ -143,6 +161,36 @@ pub(crate) struct SpawnedProxy { output_dir: PathBuf, } +#[derive(serde::Deserialize)] +struct StaleRunReceipt { + run_id: String, + #[serde(default)] + observability: Option, + #[serde(default)] + bridge_proxies: Vec, + sites: std::collections::BTreeMap, +} + +#[derive(serde::Deserialize)] +struct StaleObservabilityReceipt { + #[serde(default)] + sink_pid: Option, +} + +#[derive(serde::Deserialize)] +struct StaleBridgeProxyReceipt { + pid: u32, +} + +#[derive(serde::Deserialize)] +struct StaleSiteReceipt { + supervisor_pid: u32, + #[serde(default)] + process_pid: Option, + #[serde(default)] + port_forward_pid: Option, +} + impl TestTempDir { pub(crate) fn path(&self) -> &Path { &self.path @@ -161,6 +209,7 @@ pub(crate) fn mixed_run_base_image() -> PathBuf { pub(crate) fn temp_output_dir(prefix: &str) -> TestTempDir { fs::create_dir_all(outputs_root()).expect("failed to create cli test outputs root"); + cleanup_abandoned_test_runs_once(); let temp = tempfile::Builder::new() .prefix(prefix) .tempdir_in(outputs_root()) @@ -177,9 +226,82 @@ pub(crate) fn temp_output_dir(prefix: &str) -> TestTempDir { } } +fn cleanup_abandoned_test_runs_once() { + static CLEANUP_ONCE: OnceLock<()> = OnceLock::new(); + CLEANUP_ONCE.get_or_init(|| { + for entry in fs::read_dir(outputs_root()) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", outputs_root().display())) + { + let Ok(entry) = entry else { + continue; + }; + let storage_root = entry.path(); + if storage_root.is_dir() { + cleanup_abandoned_test_runs_in_storage_root(&storage_root); + } + } + }); +} + +fn cleanup_abandoned_test_runs_in_storage_root(storage_root: &Path) { + let runs_dir = storage_root.join("runs"); + let Ok(entries) = fs::read_dir(&runs_dir) else { + return; + }; + for entry in entries { + let Ok(entry) = entry else { + continue; + }; + let run_root = entry.path(); + let receipt_path = run_root.join("receipt.json"); + if !receipt_path.is_file() { + continue; + } + let Ok(receipt_bytes) = fs::read(&receipt_path) else { + continue; + }; + let Ok(receipt) = serde_json::from_slice::(&receipt_bytes) else { + continue; + }; + if stale_run_has_live_processes(&receipt) { + continue; + } + let _ = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&receipt.run_id) + .arg("--storage-root") + .arg(storage_root), + "amber stop stale test run", + ); + } +} + +fn stale_run_has_live_processes(receipt: &StaleRunReceipt) -> bool { + receipt + .observability + .as_ref() + .and_then(|observability| observability.sink_pid) + .into_iter() + .chain(receipt.bridge_proxies.iter().map(|proxy| proxy.pid)) + .chain( + receipt + .sites + .values() + .flat_map(|site| { + [ + Some(site.supervisor_pid), + site.process_pid, + site.port_forward_pid, + ] + }) + .flatten(), + ) + .any(pid_is_alive) +} + pub(crate) fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } pub(crate) fn docker_host_ip() -> String { @@ -191,7 +313,7 @@ pub(crate) fn docker_host_ip() -> String { .arg("host.docker.internal:host-gateway"); } let output = cmd - .arg(TEST_APP_IMAGE) + .arg(test_app_image()) .arg("python3") .arg("-c") .arg("import socket; print(socket.gethostbyname('host.docker.internal'))") @@ -206,22 +328,137 @@ pub(crate) fn docker_host_ip() -> String { String::from_utf8_lossy(&output.stdout).trim().to_string() } -pub(crate) fn ensure_local_image(tag: &str) { - let inspect = Command::new("docker") +fn docker_platform_arch() -> &'static str { + match env::consts::ARCH { + "aarch64" => "arm64", + "x86_64" => "amd64", + other => panic!("mixed-run tests support only aarch64 and x86_64 hosts, found {other}"), + } +} + +fn docker_platform(expected_arch: &str) -> String { + format!("linux/{expected_arch}") +} + +fn docker_image_meta_with_binaries( + docker_bin: &Path, + tag: &str, +) -> Result, String> { + let output = Command::new(docker_bin) .arg("image") .arg("inspect") + .arg("-f") + .arg("{{.Id}}|{{.Architecture}}") .arg(tag) - .status() - .unwrap_or_else(|err| panic!("failed to inspect docker image {tag}: {err}")); - if inspect.success() { - return; + .output() + .map_err(|err| format!("failed to inspect docker image {tag}: {err}"))?; + if !output.status.success() { + return Ok(None); } - let status = Command::new("docker") + let meta = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if meta.is_empty() { + return Ok(None); + } + let (id, arch) = meta.split_once('|').ok_or_else(|| { + format!("docker image inspect for {tag} returned malformed metadata: {meta}") + })?; + if id.is_empty() || arch.is_empty() { + return Err(format!( + "docker image inspect for {tag} returned incomplete metadata: {meta}" + )); + } + Ok(Some(DockerImageMeta { + id: id.to_string(), + arch: arch.to_string(), + })) +} + +fn ensure_local_image_with_binaries( + docker_bin: &Path, + tag: &str, + expected_arch: &str, +) -> Result { + if let Some(meta) = docker_image_meta_with_binaries(docker_bin, tag)? + && meta.arch == expected_arch + { + return Ok(meta); + } + + let output = Command::new(docker_bin) .arg("pull") + .arg("--platform") + .arg(docker_platform(expected_arch)) .arg(tag) - .status() - .unwrap_or_else(|err| panic!("failed to pull docker image {tag}: {err}")); - assert!(status.success(), "docker pull failed for {tag}"); + .output() + .map_err(|err| format!("failed to pull docker image {tag}: {err}"))?; + if !output.status.success() { + return Err(format_command_output( + &format!("docker pull failed for {tag}"), + &output, + )); + } + + let meta = docker_image_meta_with_binaries(docker_bin, tag)? + .ok_or_else(|| format!("docker image {tag} is still unavailable after pull"))?; + if meta.arch != expected_arch { + return Err(format!( + "docker pull resolved {tag} to linux/{} instead of {}", + meta.arch, + docker_platform(expected_arch) + )); + } + Ok(meta) +} + +fn build_test_app_image_with_binaries( + docker_bin: &Path, + source_image: &str, + expected_arch: &str, +) -> Result { + let _ = ensure_local_image_with_binaries(docker_bin, source_image, expected_arch)?; + let local_tag = format!("{TEST_APP_LOCAL_IMAGE_REPOSITORY}:{expected_arch}"); + let build_root = tempfile::tempdir() + .map_err(|err| format!("failed to create temp dir for {local_tag}: {err}"))?; + fs::write( + build_root.path().join("Dockerfile"), + format!("FROM {source_image}\n"), + ) + .map_err(|err| { + format!( + "failed to write Dockerfile for test app image {local_tag} in {}: {err}", + build_root.path().display() + ) + })?; + let output = Command::new(docker_bin) + .arg("build") + .arg("--platform") + .arg(docker_platform(expected_arch)) + .arg("-t") + .arg(&local_tag) + .arg(build_root.path()) + .output() + .map_err(|err| format!("failed to build docker image {local_tag}: {err}"))?; + if !output.status.success() { + return Err(format_command_output( + &format!("docker build failed for {local_tag} from {source_image}"), + &output, + )); + } + Ok(local_tag) +} + +pub(crate) fn test_app_image() -> &'static str { + static IMAGE: OnceLock = OnceLock::new(); + IMAGE + .get_or_init(|| { + build_test_app_image_with_binaries( + Path::new("docker"), + TEST_APP_SOURCE_IMAGE, + docker_platform_arch(), + ) + .unwrap_or_else(|err| panic!("{err}")) + }) + .as_str() } pub(crate) fn docker_host_http_url(port: u16) -> String { @@ -255,6 +492,14 @@ pub(crate) fn http_get_with_timeout( http_request_with_timeout("GET", port, path, None, timeout) } +pub(crate) fn http_get_with_timeout_result( + port: u16, + path: &str, + timeout: Duration, +) -> Result<(u16, String), String> { + http_request_with_timeout_result("GET", port, path, None, timeout) +} + pub(crate) fn http_request_with_timeout( method: &str, port: u16, @@ -262,6 +507,16 @@ pub(crate) fn http_request_with_timeout( body: Option<&str>, timeout: Duration, ) -> Option<(u16, String)> { + http_request_with_timeout_result(method, port, path, body, timeout).ok() +} + +pub(crate) fn http_request_with_timeout_result( + method: &str, + port: u16, + path: &str, + body: Option<&str>, + timeout: Duration, +) -> Result<(u16, String), String> { let mut command = Command::new("curl"); command .arg("-sS") @@ -276,17 +531,46 @@ pub(crate) fn http_request_with_timeout( .arg("--data") .arg(body); } - let output = command - .arg("-o") - .arg("-") - .arg("-w") - .arg("\n%{http_code}") - .arg(format!("http://127.0.0.1:{port}{path}")) - .output() - .ok()?; + let url = format!("http://127.0.0.1:{port}{path}"); + let output = command_output_via_tempfiles( + command + .arg("-o") + .arg("-") + .arg("-w") + .arg("\n%{http_code}") + .arg(&url), + "curl http request", + ); + if !output.status.success() { + return Err(format_command_output( + &format!( + "curl {method} {url} failed after {:.3}s", + timeout.as_secs_f64() + ), + &output, + )); + } let stdout = String::from_utf8_lossy(&output.stdout); - let (body, status) = stdout.rsplit_once('\n')?; - Some((status.trim().parse().ok()?, body.trim().to_string())) + let Some((body, status)) = stdout.rsplit_once('\n') else { + return Err(format!( + "curl {method} {url} returned a malformed response after \ + {:.3}s\nstdout:\n{}\nstderr:\n{}", + timeout.as_secs_f64(), + stdout, + String::from_utf8_lossy(&output.stderr), + )); + }; + let status = status.trim().parse().map_err(|err| { + format!( + "curl {method} {url} returned a malformed HTTP status `{}` after {:.3}s: \ + {err}\nstdout:\n{}\nstderr:\n{}", + status.trim(), + timeout.as_secs_f64(), + stdout, + String::from_utf8_lossy(&output.stderr), + ) + })?; + Ok((status, body.trim().to_string())) } pub(crate) fn wait_for_body( @@ -305,12 +589,14 @@ pub(crate) fn wait_for_body( last_response = Some((status, body)); } if let Ok(Some(status)) = proxy.child.try_wait() { + let diagnostics = proxy_target_diagnostics(&proxy.output_dir); panic!( "amber proxy exited before http://127.0.0.1:{port}{path} became ready\nstatus: \ - {status}\noutput dir: {}\nlog ({}):\n{}", + {status}\noutput dir: {}\n{diagnostics}log ({}):\n{}", proxy.output_dir.display(), proxy.log_path.display(), - fs::read_to_string(&proxy.log_path).unwrap_or_default() + fs::read_to_string(&proxy.log_path).unwrap_or_default(), + diagnostics = diagnostics, ); } thread::sleep(Duration::from_millis(250)); @@ -318,15 +604,94 @@ pub(crate) fn wait_for_body( let last_response = last_response .map(|(status, body)| format!("last http response: {status}\n{body}\n")) .unwrap_or_else(|| "last http response: \n".to_string()); + let diagnostics = proxy_target_diagnostics(&proxy.output_dir); panic!( - "timed out waiting for http://127.0.0.1:{port}{path}\noutput dir: {}\n{last_response}log \ - ({}):\n{}", + "timed out waiting for http://127.0.0.1:{port}{path}\noutput dir: \ + {}\n{last_response}{diagnostics}log ({}):\n{}", proxy.output_dir.display(), proxy.log_path.display(), - fs::read_to_string(&proxy.log_path).unwrap_or_default() + fs::read_to_string(&proxy.log_path).unwrap_or_default(), + diagnostics = diagnostics, ); } +fn proxy_target_diagnostics(output_dir: &Path) -> String { + let Some(child_root) = output_dir.parent() else { + return String::new(); + }; + let mut files = Vec::new(); + collect_proxy_target_diagnostic_files(child_root, &mut files); + files.sort(); + files.dedup(); + if files.is_empty() { + return String::new(); + } + + let mut diagnostics = String::from("proxy target diagnostics:\n"); + for path in files { + diagnostics.push_str(&format!("----- {} -----\n", path.display())); + diagnostics.push_str(&read_diagnostic_file_excerpt(&path)); + diagnostics.push('\n'); + } + diagnostics +} + +fn collect_proxy_target_diagnostic_files(root: &Path, files: &mut Vec) { + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + let Ok(metadata) = fs::symlink_metadata(&path) else { + continue; + }; + if metadata.is_file() { + if is_proxy_target_diagnostic_file(&path) { + files.push(path); + } + continue; + } + if !metadata.is_dir() { + continue; + } + let Ok(entries) = fs::read_dir(&path) else { + continue; + }; + for entry in entries.flatten() { + stack.push(entry.path()); + } + } +} + +fn is_proxy_target_diagnostic_file(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + matches!( + name, + "direct-runtime.json" + | "vm-runtime.json" + | "direct-plan.json" + | "vm-plan.json" + | "mesh-provision-plan.json" + | "mesh-config.json" + | "site.log" + | "supervisor.log" + | "manager-state.json" + | "site-controller-runtime-state.json" + | "site-controller-state.json" + ) +} + +fn read_diagnostic_file_excerpt(path: &Path) -> String { + const MAX_BYTES: usize = 24 * 1024; + let Ok(mut bytes) = fs::read(path) else { + return "\n".to_string(); + }; + bytes.retain(|byte| *byte != 0); + if bytes.len() > MAX_BYTES { + bytes = bytes[bytes.len() - MAX_BYTES..].to_vec(); + } + String::from_utf8_lossy(&bytes).into_owned() +} + pub(crate) fn wait_for_path(proxy: &mut SpawnedProxy, port: u16, path: &str, timeout: Duration) { let _ = wait_for_body(proxy, port, path, timeout); } @@ -443,14 +808,93 @@ pub(crate) fn append_debug_file(out: &mut String, label: &str, path: &Path) { )); } +fn manager_state_value(state_path: &Path) -> Option { + fs::read_to_string(state_path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) +} + +fn append_docker_project_debug(out: &mut String, project: &str) { + let project_filter = format!("label=com.docker.compose.project={project}"); + let ps_output = match Command::new("docker") + .args([ + "ps", + "-a", + "--filter", + project_filter.as_str(), + "--format", + "{{.ID}}\t{{.Names}}\t{{.Status}}", + ]) + .output() + { + Ok(output) => output, + Err(err) => { + out.push_str(&format!( + "\ndocker compose project `{project}` status query failed:\n{err}\n" + )); + return; + } + }; + + if !ps_output.status.success() { + out.push_str(&format!( + "\ndocker compose project `{project}` status query failed\nstdout:\n{}\nstderr:\n{}\n", + String::from_utf8_lossy(&ps_output.stdout), + String::from_utf8_lossy(&ps_output.stderr), + )); + return; + } + + let containers = String::from_utf8_lossy(&ps_output.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.splitn(3, '\t'); + Some(( + parts.next()?.trim().to_string(), + parts.next()?.trim().to_string(), + parts.next()?.trim().to_string(), + )) + }) + .collect::>(); + + if containers.is_empty() { + out.push_str(&format!( + "\ndocker compose project `{project}` containers:\n\n" + )); + return; + } + + out.push_str(&format!( + "\ndocker compose project `{project}` containers:\n" + )); + for (_, name, status) in &containers { + out.push_str(&format!("{name}\t{status}\n")); + } + + for (id, name, status) in containers { + let logs_output = match Command::new("docker") + .args(["logs", "--tail", "80", id.as_str()]) + .output() + { + Ok(output) => output, + Err(err) => { + out.push_str(&format!("\ndocker logs {name} ({status}) failed:\n{err}\n")); + continue; + } + }; + out.push_str(&format!( + "\ndocker logs {name} ({status}):\nstdout:\n{}\nstderr:\n{}\n", + String::from_utf8_lossy(&logs_output.stdout), + String::from_utf8_lossy(&logs_output.stderr), + )); + } +} + pub(crate) fn site_debug_context(run_root: &Path, site_id: &str) -> String { let state_root = run_root.join("state").join(site_id); + let manager_state_path = state_root.join("manager-state.json"); let mut out = String::new(); - append_debug_file( - &mut out, - "manager state", - &state_root.join("manager-state.json"), - ); + append_debug_file(&mut out, "manager state", &manager_state_path); append_debug_file( &mut out, "supervisor log", @@ -462,6 +906,11 @@ pub(crate) fn site_debug_context(run_root: &Path, site_id: &str) -> String { &state_root.join("port-forward.log"), ); append_debug_file(&mut out, "site log", &state_root.join("site.log")); + if let Some(project) = manager_state_value(&manager_state_path) + .and_then(|state| state["compose_project"].as_str().map(ToOwned::to_owned)) + { + append_docker_project_debug(&mut out, &project); + } out } @@ -566,6 +1015,7 @@ pub(crate) fn stop_child(child: &mut std::process::Child) { } pub(crate) fn amber_command() -> Command { + ensure_local_dev_image_tag_overrides(); let mut cmd = Command::new(env!("CARGO_BIN_EXE_amber")); cmd.env("AMBER_RUNTIME_BIN_DIR", runtime_bin_dir()); if let Some(kubeconfig) = env::var_os("AMBER_TEST_KIND_KUBECONFIG") { @@ -578,6 +1028,22 @@ pub(crate) fn use_prebuilt_images() -> bool { env::var_os("AMBER_TEST_USE_PREBUILT_IMAGES").is_some() } +fn internal_image_build_mode() -> &'static str { + static MODE: OnceLock<&'static str> = OnceLock::new(); + MODE.get_or_init(|| match env::var("AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE") { + Ok(mode) if mode == "release" => "release", + Ok(mode) if mode == "debug" => "debug", + Ok(mode) => panic!( + "AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE must be `debug` or `release`, got `{mode}`" + ), + Err(env::VarError::NotPresent) => "debug", + Err(env::VarError::NotUnicode(value)) => panic!( + "AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE must be valid UTF-8, got {:?}", + value + ), + }) +} + pub(crate) fn image_platform_opt(tag: &str) -> Option { let output = Command::new("docker") .arg("image") @@ -594,112 +1060,759 @@ pub(crate) fn image_platform_opt(tag: &str) -> Option { if arch.is_empty() { return None; } - Some(format!("linux/{arch}")) -} - -pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { - if use_prebuilt_images() { - image_platform_opt(tag).unwrap_or_else(|| { + Some(format!("linux/{arch}")) +} + +pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { + if use_prebuilt_images() { + image_platform_opt(tag).unwrap_or_else(|| { + panic!( + "AMBER_TEST_USE_PREBUILT_IMAGES is set but {tag} is not available locally. Ensure \ + the image is pulled and retagged before running mixed-run tests." + ) + }); + return; + } + + if docker_image_is_fresh(tag, dockerfile) { + return; + } + + let mut last_status = None; + for attempt in 1..=3 { + let mut command = Command::new("docker"); + if docker_supports_buildx() { + command + .arg("buildx") + .arg("build") + .arg("--load") + .arg("--progress") + .arg("plain"); + } else { + command.env("DOCKER_BUILDKIT", "1"); + command.arg("build"); + } + let status = command + .arg("--build-arg") + .arg(format!("BUILD_MODE={}", internal_image_build_mode())) + .arg("-t") + .arg(tag) + .arg("-f") + .arg(dockerfile) + .arg(workspace_root()) + .status() + .unwrap_or_else(|err| panic!("failed to build {tag}: {err}")); + if status.success() { + return; + } + last_status = Some(status); + if attempt < 3 { + eprintln!("docker build attempt {attempt} failed for {tag}; retrying"); + thread::sleep(Duration::from_secs(attempt * 2)); + } + } + let status = last_status.expect("docker build should have produced a status"); + panic!("docker build failed for {tag} after retries with status {status}"); +} + +fn docker_image_is_fresh(tag: &str, dockerfile: &Path) -> bool { + if image_platform_opt(tag).is_none() { + return false; + } + let Some(image_created) = docker_image_created_at(tag) else { + return false; + }; + !source_is_newer_than_image( + image_created, + newest_internal_image_input_mtime(&workspace_root(), dockerfile), + ) +} + +fn docker_image_created_at(tag: &str) -> Option { + let output = Command::new("docker") + .arg("image") + .arg("inspect") + .arg("-f") + .arg("{{.Created}}") + .arg(tag) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let created = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let timestamp = OffsetDateTime::parse(&created, &Rfc3339).ok()?; + Some(UNIX_EPOCH + Duration::from_secs(timestamp.unix_timestamp() as u64)) +} + +fn newest_internal_image_input_mtime( + workspace_root: &Path, + dockerfile: &Path, +) -> Option { + [ + workspace_root.join("Cargo.toml"), + workspace_root.join("Cargo.lock"), + workspace_root.join("rust-toolchain.toml"), + workspace_root.join("cli"), + workspace_root.join("compiler"), + workspace_root.join("images"), + workspace_root.join("runtime"), + dockerfile.to_path_buf(), + ] + .into_iter() + .filter_map(|path| newest_path_mtime(&path)) + .max() +} + +fn newest_path_mtime(path: &Path) -> Option { + let metadata = fs::metadata(path).ok()?; + if metadata.is_file() { + return metadata.modified().ok(); + } + if !metadata.is_dir() { + return None; + } + + let mut newest = metadata.modified().ok(); + let entries = fs::read_dir(path).ok()?; + for entry in entries.filter_map(Result::ok) { + let child = entry.path(); + let child_name = child + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if child_name == "target" || child_name == ".git" { + continue; + } + if let Some(child_mtime) = newest_path_mtime(&child) { + newest = match newest { + Some(current) if current >= child_mtime => Some(current), + _ => Some(child_mtime), + }; + } + } + newest +} + +fn source_is_newer_than_image( + image_created: SystemTime, + newest_source: Option, +) -> bool { + newest_source.is_some_and(|source| source > image_created) +} + +pub(crate) fn docker_supports_buildx() -> bool { + static READY: OnceLock = OnceLock::new(); + *READY.get_or_init(|| { + Command::new("docker") + .arg("buildx") + .arg("version") + .status() + .is_ok_and(|status| status.success()) + }) +} + +pub(crate) fn ensure_amber_internal_images() { + static READY: OnceLock<()> = OnceLock::new(); + READY.get_or_init(|| { + ensure_local_dev_image_tag_overrides(); + let root = workspace_root(); + let images = amber_internal_image_refs(); + ensure_docker_image(&images.router, &root.join("docker/amber-router/Dockerfile")); + ensure_docker_image( + &images.provisioner, + &root.join("docker/amber-provisioner/Dockerfile"), + ); + ensure_docker_image(&images.helper, &root.join("docker/amber-helper/Dockerfile")); + ensure_docker_image( + &images.site_controller, + &root.join("docker/amber-site-controller/Dockerfile"), + ); + }); +} + +pub(crate) fn ensure_internal_images() { + static READY: OnceLock<()> = OnceLock::new(); + READY.get_or_init(|| { + let _ = test_app_image(); + ensure_amber_internal_images(); + }); +} + +#[derive(Clone)] +struct AmberInternalImageRefs { + router: String, + helper: String, + provisioner: String, + site_controller: String, +} + +fn amber_internal_image_refs() -> AmberInternalImageRefs { + let overrides = parse_dev_image_tag_overrides(INTERNAL_IMAGE_OVERRIDE_KEYS) + .unwrap_or_else(|err| panic!("invalid {DEV_IMAGE_TAGS_ENV}: {err}")); + let resolve = |image: &ImageRef, key: &str| { + overrides + .get(key) + .map(|tag| override_reference(image, tag)) + .unwrap_or_else(|| image.reference.to_string()) + }; + AmberInternalImageRefs { + router: resolve(&AMBER_ROUTER, "router"), + helper: resolve(&AMBER_HELPER, "helper"), + provisioner: resolve(&AMBER_PROVISIONER, "provisioner"), + site_controller: resolve(&AMBER_SITE_CONTROLLER, "site_controller"), + } +} + +fn ensure_local_dev_image_tag_overrides() { + static READY: OnceLock<()> = OnceLock::new(); + READY.get_or_init(|| { + if use_prebuilt_images() + || env::var_os("CI").is_some() + || env::var_os(DEV_IMAGE_TAGS_ENV).is_some() + { + return; + } + let mut hasher = DefaultHasher::new(); + workspace_root().hash(&mut hasher); + internal_image_build_mode().hash(&mut hasher); + let tag = format!( + "dev-mixed-run-{}-{:016x}", + internal_image_build_mode(), + hasher.finish() + ); + let overrides = INTERNAL_IMAGE_OVERRIDE_KEYS + .iter() + .map(|key| format!("{key}={tag}")) + .collect::>() + .join(","); + unsafe { + env::set_var(DEV_IMAGE_TAGS_ENV, overrides); + } + }); +} + +pub(crate) fn load_kind_image(cluster_name: &str, image: &str) { + load_kind_image_with_binaries(Path::new("kind"), Path::new("docker"), cluster_name, image) + .unwrap_or_else(|err| panic!("{err}")); +} + +fn load_kind_image_with_binaries( + kind_bin: &Path, + docker_bin: &Path, + cluster_name: &str, + image: &str, +) -> Result<(), String> { + let mut direct_error = None; + for attempt in 1..=3 { + match kind_load_docker_image(kind_bin, cluster_name, image) { + Ok(()) => return Ok(()), + Err(err) => { + direct_error = Some(err); + if attempt < 3 { + eprintln!( + "kind load docker-image attempt {attempt} failed for {image} in cluster \ + {cluster_name}; retrying" + ); + thread::sleep(Duration::from_secs(attempt * 2)); + } + } + } + } + + let direct_error = direct_error.expect("kind image load should record a failure"); + let archive = tempfile::Builder::new() + .prefix("amber-kind-image-") + .suffix(".tar") + .tempfile() + .map_err(|err| { + format!("failed to create temporary image archive for {image} in {cluster_name}: {err}") + })?; + kind_load_image_archive_with_binaries(kind_bin, docker_bin, cluster_name, image, archive) + .map_err(|archive_error| { + format!( + "{direct_error} + +kind load image-archive fallback failed for {image} in cluster {cluster_name}: +{archive_error}" + ) + }) +} + +fn kind_load_docker_image(kind_bin: &Path, cluster_name: &str, image: &str) -> Result<(), String> { + let output = Command::new(kind_bin) + .arg("load") + .arg("docker-image") + .arg("--name") + .arg(cluster_name) + .arg(image) + .output() + .map_err(|err| { + format!("failed to run `kind load docker-image` for {image} in {cluster_name}: {err}") + })?; + if output.status.success() { + return Ok(()); + } + Err(format_command_output( + &format!("kind load docker-image failed for {image} in cluster {cluster_name}"), + &output, + )) +} + +fn kind_load_image_archive_with_binaries( + kind_bin: &Path, + docker_bin: &Path, + cluster_name: &str, + image: &str, + archive: tempfile::NamedTempFile, +) -> Result<(), String> { + let archive_path = archive.path().to_path_buf(); + let docker_output = Command::new(docker_bin) + .arg("image") + .arg("save") + .arg("--output") + .arg(&archive_path) + .arg(image) + .output() + .map_err(|err| { + format!("failed to run `docker image save` for {image} in {cluster_name}: {err}") + })?; + if !docker_output.status.success() { + return Err(format_command_output( + &format!("docker image save failed for {image}"), + &docker_output, + )); + } + + let kind = Command::new(kind_bin) + .arg("load") + .arg("image-archive") + .arg("--name") + .arg(cluster_name) + .arg(&archive_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .map_err(|err| { + format!( + "failed to run `kind load image-archive` fallback for {image} in {cluster_name}: \ + {err}" + ) + })?; + if kind.status.success() { + return Ok(()); + } + + Err(format_command_output( + &format!("kind load image-archive failed for {image} in cluster {cluster_name}"), + &kind, + )) +} + +fn format_command_output(label: &str, output: &std::process::Output) -> String { + format!( + "{label} with status {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { + ensure_internal_images(); + static READY: OnceLock>> = OnceLock::new(); + let name = kind_cluster.name.clone(); + let images = amber_internal_image_refs(); + let ready_key = format!( + "{name}|{}|{}|{}|{}|{}", + images.router, + images.provisioner, + images.helper, + images.site_controller, + test_app_image() + ); + let loaded = READY.get_or_init(|| Mutex::new(BTreeSet::new())); + { + let loaded = loaded.lock().expect("kind image-load guard should lock"); + if loaded.contains(&ready_key) { + return; + } + } + load_kind_image(&name, &images.router); + load_kind_image(&name, &images.provisioner); + load_kind_image(&name, &images.helper); + load_kind_image(&name, &images.site_controller); + load_kind_image(&name, test_app_image()); + loaded + .lock() + .expect("kind image-load guard should lock") + .insert(ready_key); +} + +pub(crate) fn kill_pid(pid: u32) { + #[cfg(unix)] + unsafe { + let _ = libc::kill(pid as i32, libc::SIGTERM); + } +} + +pub(crate) fn framework_child_artifact_dir( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + let manager_state = site_manager_state(run_root, site_id); + if manager_state["kind"].as_str() == Some("kubernetes") { + let artifact_dir = + framework_child_kubernetes_cache_artifact_dir(run_root, site_id, child_id); + materialize_kubernetes_child_artifact(&manager_state, child_id, &artifact_dir); + return artifact_dir; + } + + framework_child_host_artifact_dir(run_root, site_id, child_id) +} + +pub(crate) fn framework_control_state_snapshot(control_state_root: &Path) -> Value { + let mut live_children = Vec::new(); + let entries = fs::read_dir(control_state_root) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", control_state_root.display())); + for entry in entries.filter_map(Result::ok) { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let Some(site_id) = entry.file_name().into_string().ok() else { + continue; + }; + let manager_state = site_manager_state_from_state_root(control_state_root, &site_id); + if !site_hosts_local_framework_controller(control_state_root, &site_id, &manager_state) { + continue; + } + let state = framework_site_control_state(control_state_root, &site_id); + if let Some(children) = state["live_children"].as_array() { + live_children.extend(children.iter().cloned()); + } + } + json!({ "live_children": live_children }) +} + +pub(crate) fn framework_child_is_absent(run_root: &Path, site_id: &str, child_id: u64) -> bool { + let manager_state = site_manager_state(run_root, site_id); + if manager_state["kind"].as_str() != Some("kubernetes") { + return !framework_child_host_child_root(run_root, site_id, child_id).exists(); + } + + if kubernetes_child_runtime_root_exists(&manager_state, child_id) { + return false; + } + + let cache_root = framework_child_kubernetes_cache_child_root(run_root, site_id, child_id); + if cache_root.exists() { + fs::remove_dir_all(&cache_root).unwrap_or_else(|err| { + panic!( + "failed to clear stale kubernetes child cache {}: {err}", + cache_root.display() + ) + }); + } + true +} + +fn framework_site_control_state(control_state_root: &Path, site_id: &str) -> Value { + let manager_state = site_manager_state_from_state_root(control_state_root, site_id); + let state_path = framework_site_control_state_path(control_state_root, site_id, &manager_state); + if manager_state["kind"].as_str() == Some("kubernetes") { + materialize_kubernetes_control_state(&manager_state, &state_path); + } + read_json(&state_path) +} + +fn site_hosts_local_framework_controller( + control_state_root: &Path, + site_id: &str, + manager_state: &Value, +) -> bool { + if manager_state["kind"].as_str() == Some("kubernetes") { + return control_state_root + .join(site_id) + .join("site-controller-plan.json") + .is_file(); + } + framework_site_control_state_path(control_state_root, site_id, manager_state).is_file() +} + +fn site_manager_state(run_root: &Path, site_id: &str) -> Value { + site_manager_state_from_state_root(&run_root.join("state"), site_id) +} + +fn site_manager_state_from_state_root(state_root: &Path, site_id: &str) -> Value { + let manager_state_path = state_root.join(site_id).join("manager-state.json"); + serde_json::from_slice(&fs::read(&manager_state_path).unwrap_or_else(|err| { + panic!( + "failed to read site manager state {}: {err}", + manager_state_path.display() + ) + })) + .unwrap_or_else(|err| { + panic!( + "failed to parse site manager state {}: {err}", + manager_state_path.display() + ) + }) +} + +fn framework_site_control_state_path( + control_state_root: &Path, + site_id: &str, + manager_state: &Value, +) -> PathBuf { + if manager_state["kind"].as_str() == Some("kubernetes") { + return framework_kubernetes_cache_control_state_path(control_state_root, site_id); + } + control_state_root + .join(site_id) + .join("site-controller-state.json") +} + +fn framework_kubernetes_cache_control_state_path( + control_state_root: &Path, + site_id: &str, +) -> PathBuf { + control_state_root + .join(site_id) + .join("framework-component-kubernetes-cache") + .join("site-controller-state.json") +} + +fn framework_child_host_child_root(run_root: &Path, site_id: &str, child_id: u64) -> PathBuf { + run_root + .join("state") + .join(site_id) + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +fn framework_child_host_artifact_dir(run_root: &Path, site_id: &str, child_id: u64) -> PathBuf { + framework_child_host_child_root(run_root, site_id, child_id).join("artifact") +} + +fn framework_child_kubernetes_cache_child_root( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + run_root + .join("state") + .join(site_id) + .join("framework-component-kubernetes-cache") + .join("children") + .join(child_id.to_string()) +} + +fn framework_child_kubernetes_cache_artifact_dir( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + framework_child_kubernetes_cache_child_root(run_root, site_id, child_id).join("artifact") +} + +fn kubernetes_child_runtime_root_exists(manager_state: &Value, child_id: u64) -> bool { + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let remote_child_root = format!("/amber/site/state/framework-component/children/{child_id}"); + let output = kubectl_for_manager_state(manager_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("sh") + .arg("-lc") + .arg(format!("test -d {remote_child_root}")) + .output() + .unwrap_or_else(|err| { + panic!( + "failed to probe kubernetes child runtime root {remote_child_root} in {pod}: {err}" + ) + }); + match output.status.code() { + Some(0) => true, + Some(1) => false, + _ => { + panic!( + "failed to probe kubernetes child runtime root {remote_child_root} in {pod}: {}", + String::from_utf8_lossy(&output.stderr) + ) + } + } +} + +fn materialize_kubernetes_child_artifact( + manager_state: &Value, + child_id: u64, + artifact_dir: &Path, +) { + if artifact_dir.is_dir() { + return; + } + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let remote_artifact_dir = + format!("/amber/site/state/framework-component/children/{child_id}/artifact"); + let parent = artifact_dir.parent().unwrap_or_else(|| { + panic!( + "artifact dir {} should have a parent", + artifact_dir.display() + ) + }); + fs::create_dir_all(parent).unwrap_or_else(|err| { + panic!( + "failed to create framework child parent {}: {err}", + parent.display() + ) + }); + let target = parent.join(format!("artifact-kubernetes-copy-{child_id}")); + if target.exists() { + fs::remove_dir_all(&target).unwrap_or_else(|err| { panic!( - "AMBER_TEST_USE_PREBUILT_IMAGES is set but {tag} is not available locally. Ensure \ - the image is pulled and retagged before running mixed-run tests." + "failed to clear previous kubernetes child artifact copy {}: {err}", + target.display() ) }); - return; - } - - let mut command = Command::new("docker"); - if docker_supports_buildx() { - command.arg("buildx").arg("build").arg("--load"); - } else { - command.env("DOCKER_BUILDKIT", "1"); - command.arg("build"); } + let mut command = kubectl_for_manager_state(manager_state); let status = command - .arg("-t") - .arg(tag) - .arg("-f") - .arg(dockerfile) - .arg(workspace_root()) + .arg("-n") + .arg(namespace) + .arg("cp") + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg(format!("{pod}:{remote_artifact_dir}")) + .arg(&target) .status() - .unwrap_or_else(|err| panic!("failed to build {tag}: {err}")); - assert!(status.success(), "docker build failed for {tag}"); -} - -pub(crate) fn docker_supports_buildx() -> bool { - static READY: OnceLock = OnceLock::new(); - *READY.get_or_init(|| { - Command::new("docker") - .arg("buildx") - .arg("version") - .status() - .is_ok_and(|status| status.success()) - }) -} - -pub(crate) fn ensure_internal_images() { - static READY: OnceLock<()> = OnceLock::new(); - READY.get_or_init(|| { - let root = workspace_root(); - ensure_local_image(TEST_APP_IMAGE); - ensure_docker_image( - AMBER_ROUTER.reference, - &root.join("docker/amber-router/Dockerfile"), - ); - ensure_docker_image( - AMBER_PROVISIONER.reference, - &root.join("docker/amber-provisioner/Dockerfile"), - ); - ensure_docker_image( - AMBER_HELPER.reference, - &root.join("docker/amber-helper/Dockerfile"), + .unwrap_or_else(|err| { + panic!( + "failed to copy kubernetes child artifact {remote_artifact_dir} from {pod}: {err}" + ) + }); + if !status.success() { + panic!( + "kubectl cp failed for kubernetes child artifact {remote_artifact_dir} from {pod} \ + with status {status}" ); + } + fs::rename(&target, artifact_dir).unwrap_or_else(|err| { + panic!( + "failed to move kubernetes child artifact copy {} into {}: {err}", + target.display(), + artifact_dir.display() + ) }); } -pub(crate) fn load_kind_image(cluster_name: &str, image: &str) { - let status = Command::new("kind") - .arg("load") - .arg("docker-image") - .arg("--name") - .arg(cluster_name) - .arg(image) - .status() +fn materialize_kubernetes_control_state(manager_state: &Value, state_path: &Path) { + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let parent = state_path.parent().unwrap_or_else(|| { + panic!( + "framework control state path {} should have a parent", + state_path.display() + ) + }); + fs::create_dir_all(parent).unwrap_or_else(|err| { + panic!( + "failed to create kubernetes control state cache parent {}: {err}", + parent.display() + ) + }); + let target = parent.join("site-controller-state-kubernetes-copy.json"); + let output = kubectl_for_manager_state(manager_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("cat") + .arg("/amber/site/state/site-controller-state.json") + .output() .unwrap_or_else(|err| { - panic!("failed to load {image} into kind cluster {cluster_name}: {err}") + panic!("failed to read kubernetes control state from {pod} in {namespace}: {err}") }); - assert!( - status.success(), - "kind load docker-image failed for {image} in cluster {cluster_name}" - ); + if !output.status.success() { + panic!( + "failed to read kubernetes control state from {pod} in {namespace}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + fs::write(&target, &output.stdout).unwrap_or_else(|err| { + panic!( + "failed to write temporary kubernetes control state cache {}: {err}", + target.display() + ) + }); + fs::rename(&target, state_path).unwrap_or_else(|err| { + panic!( + "failed to move kubernetes control state cache {} into {}: {err}", + target.display(), + state_path.display() + ) + }); } -pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { - ensure_internal_images(); - static READY: OnceLock>> = OnceLock::new(); - let name = kind_cluster.name.clone(); - let loaded = READY.get_or_init(|| Mutex::new(BTreeSet::new())); - { - let loaded = loaded.lock().expect("kind image-load guard should lock"); - if loaded.contains(&name) { - return; - } +pub(crate) fn kubernetes_site_controller_pod_name( + manager_state: &Value, + namespace: &str, +) -> String { + let mut command = kubectl_for_manager_state(manager_state); + let output = command + .arg("-n") + .arg(namespace) + .arg("get") + .arg("pods") + .arg("-l") + .arg("amber.io/component=amber-site-controller") + .arg("-o") + .arg("jsonpath={.items[0].metadata.name}") + .output() + .unwrap_or_else(|err| { + panic!("failed to query kubernetes site-controller pod in {namespace}: {err}") + }); + if !output.status.success() { + panic!( + "failed to query kubernetes site-controller pod in {namespace}: {}", + String::from_utf8_lossy(&output.stderr) + ); } - load_kind_image(&name, AMBER_ROUTER.reference); - load_kind_image(&name, AMBER_PROVISIONER.reference); - load_kind_image(&name, AMBER_HELPER.reference); - load_kind_image(&name, TEST_APP_IMAGE); - loaded - .lock() - .expect("kind image-load guard should lock") - .insert(name); + let pod = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if pod.is_empty() { + panic!("kubernetes site-controller pod is missing in namespace {namespace}"); + } + pod } -pub(crate) fn kill_pid(pid: u32) { - #[cfg(unix)] - unsafe { - let _ = libc::kill(pid as i32, libc::SIGTERM); +pub(crate) fn kubectl_for_manager_state(manager_state: &Value) -> Command { + let mut command = Command::new("kubectl"); + if let Some(kubeconfig) = env::var_os("AMBER_TEST_KIND_KUBECONFIG") { + command.env("KUBECONFIG", kubeconfig); + } + if let Some(context) = manager_state["context"].as_str() { + command.arg("--context").arg(context); } + command } pub(crate) fn pid_is_alive(pid: u32) -> bool { @@ -767,14 +1880,15 @@ impl RunHandle { if self.stopped { return; } - let output = amber_command() - .arg("stop") - .arg(&self.run_id) - .arg("--storage-root") - .arg(&self.storage_root) - .envs(self.command_env.iter().map(|(key, value)| (key, value))) - .output() - .expect("failed to run amber stop"); + let output = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&self.run_id) + .arg("--storage-root") + .arg(&self.storage_root) + .envs(self.command_env.iter().map(|(key, value)| (key, value))), + "amber stop", + ); assert!( output.status.success(), "amber stop failed\nstdout:\n{}\nstderr:\n{}", @@ -790,17 +1904,47 @@ impl Drop for RunHandle { if self.stopped { return; } - let _ = amber_command() - .arg("stop") - .arg(&self.run_id) - .arg("--storage-root") - .arg(&self.storage_root) - .envs(self.command_env.iter().map(|(key, value)| (key, value))) - .output(); + let _ = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&self.run_id) + .arg("--storage-root") + .arg(&self.storage_root) + .envs(self.command_env.iter().map(|(key, value)| (key, value))), + "amber stop", + ); self.stopped = true; } } +pub(crate) fn command_output_via_tempfiles( + command: &mut Command, + label: &str, +) -> std::process::Output { + let stdout_file = tempfile::NamedTempFile::new() + .unwrap_or_else(|err| panic!("failed to create stdout temp file for {label}: {err}")); + let stderr_file = tempfile::NamedTempFile::new() + .unwrap_or_else(|err| panic!("failed to create stderr temp file for {label}: {err}")); + let status = command + .stdout(Stdio::from(stdout_file.reopen().unwrap_or_else(|err| { + panic!("failed to reopen stdout temp file for {label}: {err}") + }))) + .stderr(Stdio::from(stderr_file.reopen().unwrap_or_else(|err| { + panic!("failed to reopen stderr temp file for {label}: {err}") + }))) + .status() + .unwrap_or_else(|err| panic!("failed to run {label}: {err}")); + let stdout = fs::read(stdout_file.path()) + .unwrap_or_else(|err| panic!("failed to read stdout temp file for {label}: {err}")); + let stderr = fs::read(stderr_file.path()) + .unwrap_or_else(|err| panic!("failed to read stderr temp file for {label}: {err}")); + std::process::Output { + status, + stdout, + stderr, + } +} + pub(crate) fn parse_run_id(stdout: &[u8]) -> String { String::from_utf8_lossy(stdout) .lines() @@ -831,17 +1975,18 @@ pub(crate) fn run_manifest_with_args_and_env( extra_args: &[&str], extra_env: &[(&str, &str)], ) -> RunHandle { - let output = amber_command() - .arg("run") - .arg(manifest) - .arg("--placement") - .arg(placement) - .arg("--storage-root") - .arg(storage_root) - .args(extra_args) - .envs(extra_env.iter().copied()) - .output() - .expect("failed to run amber run"); + let output = command_output_via_tempfiles( + amber_command() + .arg("run") + .arg(manifest) + .arg("--placement") + .arg(placement) + .arg("--storage-root") + .arg(storage_root) + .args(extra_args) + .envs(extra_env.iter().copied()), + "amber run", + ); assert!( output.status.success(), "amber run failed\nstdout:\n{}\nstderr:\n{}{}", @@ -921,7 +2066,7 @@ pub(crate) fn run_manifest_expect_failure_with_env( .arg(storage_root) .args(extra_args) .envs(extra_env.iter().copied()); - let output = cmd.output().expect("failed to run amber run"); + let output = command_output_via_tempfiles(&mut cmd, "amber run"); assert!( !output.status.success(), "amber run unexpectedly succeeded\nstdout:\n{}\nstderr:\n{}", @@ -941,21 +2086,22 @@ pub(crate) fn dry_run_manifest( bundle_root: &Path, extra_args: &[&str], ) -> std::process::Output { - amber_command() - .arg("run") - .arg("-Z") - .arg("unstable-options") - .arg(manifest) - .arg("--placement") - .arg(placement) - .arg("--storage-root") - .arg(storage_root) - .args(extra_args) - .arg("--dry-run") - .arg("--emit-launch-bundle") - .arg(bundle_root) - .output() - .expect("failed to run amber run --dry-run") + command_output_via_tempfiles( + amber_command() + .arg("run") + .arg("-Z") + .arg("unstable-options") + .arg(manifest) + .arg("--placement") + .arg(placement) + .arg("--storage-root") + .arg(storage_root) + .args(extra_args) + .arg("--dry-run") + .arg("--emit-launch-bundle") + .arg(bundle_root), + "amber run --dry-run", + ) } pub(crate) fn spawn_run_manifest_with_env( @@ -1065,7 +2211,7 @@ pub(crate) fn write_image_component( "manifest_version": "0.3.0", "slots": upstreams.iter().map(|(alias, _)| ((*alias).to_string(), json!({"kind": "http"}))).collect::>(), "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./app.py" }], "env": env, "network": { @@ -1355,6 +2501,8 @@ pub(crate) fn serve_host_http_request(stream: &mut TcpStream) { pub(crate) struct KindClusterGuard { name: String, kubeconfig: PathBuf, + previous_cluster_name: Option, + previous_kubeconfig: Option, } impl KindClusterGuard { @@ -1387,9 +2535,17 @@ impl KindClusterGuard { .status(); panic!("kind create cluster failed with status {status}"); } + let previous_cluster_name = env::var_os("AMBER_TEST_KIND_CLUSTER_NAME"); + let previous_kubeconfig = env::var_os("AMBER_TEST_KIND_KUBECONFIG"); + unsafe { + env::set_var("AMBER_TEST_KIND_CLUSTER_NAME", &name); + env::set_var("AMBER_TEST_KIND_KUBECONFIG", kubeconfig); + } Self { name, kubeconfig: kubeconfig.to_path_buf(), + previous_cluster_name, + previous_kubeconfig, } } } @@ -1404,6 +2560,16 @@ impl Drop for KindClusterGuard { .arg("--kubeconfig") .arg(&self.kubeconfig) .status(); + unsafe { + match &self.previous_cluster_name { + Some(value) => env::set_var("AMBER_TEST_KIND_CLUSTER_NAME", value), + None => env::remove_var("AMBER_TEST_KIND_CLUSTER_NAME"), + } + match &self.previous_kubeconfig { + Some(value) => env::set_var("AMBER_TEST_KIND_KUBECONFIG", value), + None => env::remove_var("AMBER_TEST_KIND_KUBECONFIG"), + } + } } } @@ -1645,9 +2811,11 @@ pub(crate) fn write_single_site_vm_fixture(root: &Path) -> ScenarioFixture { "schema": "amber.run.placement", "version": 1, "sites": { - "vm_local": { "kind": "vm" } + "direct_controller": { "kind": "direct" }, + "vm_local": { "kind": "vm", "controller_site": "direct_controller" } }, "defaults": { + "path": "direct_controller", "vm": "vm_local" } }), @@ -2009,3 +3177,567 @@ pub(crate) fn namespace_exists(namespace: &str, kubeconfig: &Path, context: &str .map(|output| output.status.success()) .unwrap_or(false) } + +#[cfg(test)] +mod tests { + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + + #[test] + fn command_output_via_tempfiles_does_not_wait_for_background_stdout_writer() { + let started = Instant::now(); + let output = command_output_via_tempfiles( + Command::new("sh").arg("-c").arg("(sleep 2) & printf done"), + "background stdout writer", + ); + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout), "done"); + assert!( + started.elapsed() < Duration::from_secs(1), + "capturing to temp files should wait for the command, not background writers" + ); + } + + #[test] + fn source_is_newer_than_image_detects_newer_sources() { + let image_created = UNIX_EPOCH + Duration::from_secs(10); + let newest_source = Some(UNIX_EPOCH + Duration::from_secs(11)); + assert!(source_is_newer_than_image(image_created, newest_source)); + } + + #[test] + fn source_is_newer_than_image_ignores_older_or_missing_sources() { + let image_created = UNIX_EPOCH + Duration::from_secs(10); + assert!(!source_is_newer_than_image( + image_created, + Some(UNIX_EPOCH + Duration::from_secs(9)) + )); + assert!(!source_is_newer_than_image(image_created, None)); + } + + #[test] + fn framework_child_artifact_dir_uses_dedicated_kubernetes_cache() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let manager_state_path = temp + .path() + .join("state") + .join("kind_local") + .join("manager-state.json"); + fs::create_dir_all(manager_state_path.parent().expect("manager state parent")) + .expect("manager state dir should create"); + fs::write( + &manager_state_path, + serde_json::to_vec(&json!({ + "kind": "kubernetes", + "kubernetes_namespace": "amber-test", + })) + .expect("manager state should serialize"), + ) + .expect("manager state should write"); + + let cache_dir = framework_child_kubernetes_cache_artifact_dir(temp.path(), "kind_local", 7); + fs::create_dir_all(&cache_dir).expect("kubernetes cache dir should exist"); + + let artifact_dir = framework_child_artifact_dir(temp.path(), "kind_local", 7); + assert_eq!(artifact_dir, cache_dir); + assert!( + !artifact_dir.starts_with( + temp.path() + .join("state") + .join("kind_local") + .join("framework-component") + .join("children") + ), + "kubernetes child artifact inspection should not reuse the authoritative host state \ + path" + ); + } + + #[test] + fn framework_control_state_path_uses_dedicated_kubernetes_cache() { + let state_root = Path::new("/tmp/amber-run/state"); + let manager_state = json!({ + "kind": "kubernetes", + "kubernetes_namespace": "amber-test", + }); + + let state_path = + framework_site_control_state_path(state_root, "kind_local", &manager_state); + assert_eq!( + state_path, + framework_kubernetes_cache_control_state_path(state_root, "kind_local") + ); + assert!( + !state_path.ends_with("site-controller-state.json") + || state_path + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "framework-component-kubernetes-cache"), + "kubernetes framework control state should use the dedicated cache path, got {}", + state_path.display() + ); + } + + #[test] + fn framework_control_state_snapshot_skips_sites_without_local_controller() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let state_root = temp.path().join("state"); + let direct_root = state_root.join("direct_local"); + let vm_root = state_root.join("vm_local"); + fs::create_dir_all(&direct_root).expect("direct state root should create"); + fs::create_dir_all(&vm_root).expect("vm state root should create"); + write_json( + &direct_root.join("manager-state.json"), + &json!({ + "kind": "direct", + "site_id": "direct_local", + }), + ); + write_json( + &direct_root.join("site-controller-state.json"), + &json!({ + "live_children": [ + { + "name": "local-child", + "state": "live", + "child_id": 7, + } + ], + }), + ); + write_json( + &vm_root.join("manager-state.json"), + &json!({ + "kind": "vm", + "site_id": "vm_local", + }), + ); + + let snapshot = framework_control_state_snapshot(&state_root); + + assert_eq!(snapshot["live_children"].as_array().unwrap().len(), 1); + assert_eq!(snapshot["live_children"][0]["name"], "local-child"); + } + + #[cfg(unix)] + #[test] + fn kind_load_image_archive_uses_real_archive_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nprintf 'docker %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = image ] \ + && [ \"$2\" = inspect ] && [ \"$3\" = -f ] && [ \"$4\" = \ + '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ \"$5\" = 'test:image' ]; then\n exit \ + 1\nfi\nif [ \"$1\" = image ] && [ \"$2\" = save ] && [ \"$3\" = --output ]; \ + then\n printf 'archive' > \"$4\"\n exit 0\nfi\necho \"unexpected docker \ + invocation: $*\" >&2\nexit 1\n", + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + "#!/bin/sh\nset -eu\nprintf 'kind %s\\n' \"$*\" >> '{}'\narchive=\"${{5:-}}\"\nif \ + [ \"$1\" = load ] && [ \"$2\" = image-archive ] && [ \"$3\" = --name ] && [ \ + \"$5\" != '-' ] && [ -f \"$archive\" ]; then\n exit 0\nfi\necho \"unexpected \ + kind invocation: $*\" >&2\nexit 1\n", + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + let archive = tempfile::Builder::new() + .prefix("kind-archive-") + .suffix(".tar") + .tempfile_in(temp.path()) + .expect("temp archive should create"); + kind_load_image_archive_with_binaries( + &kind, + &docker, + "test-cluster", + "test:image", + archive, + ) + .expect("kind image-archive fallback should use a real archive path"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines() + .any(|line| line.contains("docker image save --output ")), + "docker should save to a concrete archive path:\n{log}" + ); + assert!( + log.lines().any(|line| { + line.starts_with("kind load image-archive --name test-cluster ") + && !line.ends_with(" -") + }), + "kind should receive the archive filename, not stdin:\n{log}" + ); + } + + #[cfg(unix)] + #[test] + fn kind_load_image_archive_saves_the_requested_image_tag() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +printf 'docker %s +' "$*" >> '{}' +if [ "$1" = image ] && [ "$2" = save ] && [ "$3" = --output ] && [ "$5" = 'test:image' ]; then + printf 'archive' > "$4" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + r#"#!/bin/sh +set -eu +printf 'kind %s +' "$*" >> '{}' +archive="${{5:-}}" +if [ "$1" = load ] && [ "$2" = image-archive ] && [ "$3" = --name ] && [ -f "$archive" ]; then + exit 0 +fi +echo "unexpected kind invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + let archive = tempfile::Builder::new() + .prefix("kind-archive-") + .suffix(".tar") + .tempfile_in(temp.path()) + .expect("temp archive should create"); + kind_load_image_archive_with_binaries( + &kind, + &docker, + "test-cluster", + "test:image", + archive, + ) + .expect("kind image-archive fallback should save the requested image tag"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines().any(|line| { + line.contains("docker image save --output ") && line.ends_with(" test:image") + }), + "docker save should export the requested image tag: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn load_kind_image_retries_direct_load_before_single_archive_fallback() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +printf 'docker %s +' "$*" >> '{}' +if [ "$1" = image ] && [ "$2" = save ] && [ "$3" = --output ] && [ "$5" = 'test:image' ]; then + printf 'archive' > "$4" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + r#"#!/bin/sh +set -eu +printf 'kind %s +' "$*" >> '{}' +archive="${{5:-}}" +if [ "$1" = load ] && [ "$2" = docker-image ]; then + exit 1 +fi +if [ "$1" = load ] && [ "$2" = image-archive ] && [ "$3" = --name ] && [ -f "$archive" ]; then + exit 0 +fi +echo "unexpected kind invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + load_kind_image_with_binaries(&kind, &docker, "test-cluster", "test:image") + .expect("archive fallback should run once after direct load retries are exhausted"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + let direct_count = log + .lines() + .filter(|line| line == &"kind load docker-image --name test-cluster test:image") + .count(); + let archive_count = log + .lines() + .filter(|line| line.starts_with("kind load image-archive --name test-cluster ")) + .count(); + let save_count = log + .lines() + .filter(|line| { + line.contains("docker image save --output ") && line.ends_with(" test:image") + }) + .count(); + assert_eq!( + direct_count, 3, + "kind load docker-image should retry exactly three times before falling back: +{log}" + ); + assert_eq!( + archive_count, 1, + "kind image-archive fallback should run once after retries are exhausted: +{log}" + ); + assert_eq!( + save_count, 1, + "docker image save should run once for the single archive fallback: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn ensure_local_image_pulls_expected_platform_when_missing() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\nstate_path='{}'\nprintf 'docker %s\\n' \"$*\" \ + >> \"$log_path\"\nif [ \"$1\" = image ] && [ \"$2\" = inspect ] && [ \"$3\" = -f \ + ] && [ \"$4\" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ \"$5\" = \ + 'python:3.13-alpine' ]; then\n if [ ! -f \"$state_path\" ]; then\n exit 1\n \ + fi\n printf 'sha256:test|arm64\\n'\n exit 0\nfi\nif [ \"$1\" = pull ] && [ \ + \"$2\" = --platform ] && [ \"$3\" = 'linux/arm64' ] && [ \"$4\" = \ + 'python:3.13-alpine' ]; then\n printf pulled > \"$state_path\"\n exit \ + 0\nfi\necho \"unexpected docker invocation: $*\" >&2\nexit 1\n", + log_path.display(), + state_path.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let meta = ensure_local_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("missing image should be pulled for the expected platform"); + assert_eq!( + meta, + DockerImageMeta { + id: "sha256:test".to_string(), + arch: "arm64".to_string(), + } + ); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines() + .any(|line| line == "docker pull --platform linux/arm64 python:3.13-alpine"), + "docker pull should request the expected platform:\n{log}" + ); + } + + #[cfg(unix)] + #[test] + fn ensure_local_image_repulls_expected_platform_when_local_arch_is_wrong() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +log_path='{}' +state_path='{}' +printf 'docker %s +' "$*" >> "$log_path" +if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = -f ] && [ "$4" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ "$5" = 'python:3.13-alpine' ]; then + if [ -f "$state_path" ]; then + printf 'sha256:test|arm64 +' + else + printf 'sha256:stale|amd64 +' + fi + exit 0 +fi +if [ "$1" = pull ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = 'python:3.13-alpine' ]; then + printf pulled > "$state_path" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display(), + state_path.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let meta = ensure_local_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("wrong-arch image should be repulled for the expected platform"); + assert_eq!( + meta, + DockerImageMeta { + id: "sha256:test".to_string(), + arch: "arm64".to_string(), + } + ); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines() + .any(|line| line == "docker pull --platform linux/arm64 python:3.13-alpine"), + "wrong-arch local images should be repulled for the expected platform: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn test_app_image_builds_a_local_wrapper_image() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + let dockerfile_copy = temp.path().join("Dockerfile"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +log_path='{}' +state_path='{}' +dockerfile_copy='{}' +printf 'docker %s +' "$*" >> "$log_path" +if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = -f ] && [ "$4" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ "$5" = 'python:3.13-alpine' ]; then + if [ ! -f "$state_path" ]; then + exit 1 + fi + printf 'sha256:test|arm64 +' + exit 0 +fi +if [ "$1" = pull ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = 'python:3.13-alpine' ]; then + printf pulled > "$state_path" + exit 0 +fi +if [ "$1" = build ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = -t ] && [ "$5" = 'amber-mixed-run-test-app:arm64' ]; then + cp "$6/Dockerfile" "$dockerfile_copy" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display(), + state_path.display(), + dockerfile_copy.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let local_tag = build_test_app_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("test app image should build a local wrapper image"); + assert_eq!(local_tag, "amber-mixed-run-test-app:arm64"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines().any(|line| { + line == "docker build --platform linux/arm64 -t amber-mixed-run-test-app:arm64" + || line.starts_with( + "docker build --platform linux/arm64 -t amber-mixed-run-test-app:arm64 ", + ) + }), + "test app image should be built as a local wrapper image: +{log}" + ); + assert_eq!( + fs::read_to_string(&dockerfile_copy).expect("copied Dockerfile should read"), + "FROM python:3.13-alpine +", + "test app image wrapper should inherit from the upstream python base image", + ); + } +} diff --git a/cli/tests/test_support/port_allocator.rs b/cli/tests/test_support/port_allocator.rs new file mode 100644 index 00000000..ea6a60b4 --- /dev/null +++ b/cli/tests/test_support/port_allocator.rs @@ -0,0 +1,63 @@ +use std::{ + collections::BTreeSet, + net::{SocketAddr, TcpListener}, + sync::{Mutex, OnceLock}, +}; + +const TEST_PORT_RANGE_START: u16 = 20_000; +const TEST_PORT_RANGE_END: u16 = 30_000; +static RESERVED_TEST_PORTS: OnceLock>> = OnceLock::new(); + +pub(crate) fn reserve_test_loopback_port() -> u16 { + let reserved = RESERVED_TEST_PORTS.get_or_init(|| Mutex::new(BTreeSet::new())); + let mut reserved = reserved + .lock() + .expect("test port allocator should not be poisoned"); + let span = u32::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START); + let mut next = + TEST_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; + for _ in 0..usize::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START) { + if next >= TEST_PORT_RANGE_END { + next = TEST_PORT_RANGE_START; + } + let port = next; + next += 1; + if reserved.contains(&port) { + continue; + } + match TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))) { + Ok(listener) => { + drop(listener); + reserved.insert(port); + return port; + } + Err(_) => continue, + } + } + panic!( + "failed to allocate a unique test loopback port in {}-{}", + TEST_PORT_RANGE_START, + TEST_PORT_RANGE_END - 1 + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reserve_test_loopback_port_uses_reserved_test_range() { + let port = reserve_test_loopback_port(); + assert!( + (TEST_PORT_RANGE_START..TEST_PORT_RANGE_END).contains(&port), + "test allocator returned port outside reserved range: {port}" + ); + } + + #[test] + fn reserve_test_loopback_port_does_not_reuse_ports_in_process() { + let first = reserve_test_loopback_port(); + let second = reserve_test_loopback_port(); + assert_ne!(first, second); + } +} diff --git a/cli/tests/vm_smoke.rs b/cli/tests/vm_smoke.rs index e567272a..b46bc685 100644 --- a/cli/tests/vm_smoke.rs +++ b/cli/tests/vm_smoke.rs @@ -2,6 +2,8 @@ #[path = "test_support/cloud_image.rs"] mod cloud_image_support; +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "test_support/target_dir.rs"] mod target_dir_support; #[path = "test_support/workspace_root.rs"] @@ -10,7 +12,7 @@ mod workspace_root_support; use std::{ env, fs, io::{Read, Write}, - net::{SocketAddr, TcpListener, TcpStream}, + net::{SocketAddr, TcpStream}, path::{Path, PathBuf}, process::{Command, Stdio}, thread, @@ -18,6 +20,7 @@ use std::{ }; use cloud_image_support::default_host_arch_cloud_image_filename; +use port_allocator_support::reserve_test_loopback_port; use target_dir_support::cargo_target_dir; use workspace_root_support::workspace_root; @@ -86,8 +89,7 @@ fn emit_wait_heartbeat( } fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn ensure_runtime_binaries_built(workspace_root: &Path) -> PathBuf { diff --git a/compiler/config/Cargo.toml b/compiler/config/Cargo.toml index 27cfbdc7..fec0b76c 100644 --- a/compiler/config/Cargo.toml +++ b/compiler/config/Cargo.toml @@ -7,4 +7,3 @@ amber-template = { workspace = true } jsonptr = { workspace = true } jsonschema = { workspace = true } serde_json = { workspace = true } -serde_jcs = "0.1.0" diff --git a/compiler/config/src/schema.rs b/compiler/config/src/schema.rs index 589ff5be..97877d00 100644 --- a/compiler/config/src/schema.rs +++ b/compiler/config/src/schema.rs @@ -1484,10 +1484,24 @@ fn ensure_leaf_schema_supported(schema: &Map) -> Result<()> { } pub fn canonical_json(v: &Value) -> Value { - let canonical_bytes = - serde_jcs::to_vec(v).expect("serializing config schema to canonical JSON should succeed"); - serde_json::from_slice(&canonical_bytes) - .expect("canonical JSON bytes should deserialize back into JSON value") + match v { + Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), + Value::Object(map) => { + let mut entries: Vec<_> = map.iter().collect(); + entries.sort_by(|(left, _), (right, _)| canonical_json_key_order(left, right)); + Value::Object( + entries + .into_iter() + .map(|(key, value)| (key.clone(), canonical_json(value))) + .collect(), + ) + } + _ => v.clone(), + } +} + +fn canonical_json_key_order(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) } // Produce a minimized schema that only includes explicitly allowed leaf paths. diff --git a/compiler/manifest/README.md b/compiler/manifest/README.md index d69d4cc4..50507133 100644 --- a/compiler/manifest/README.md +++ b/compiler/manifest/README.md @@ -351,7 +351,7 @@ Supported `from` sources (current): * `slots.`: mount a storage slot as a directory. The referenced slot must exist and have `kind: "storage"`. * `framework.docker`: requires `experimental_features: ["docker"]`. In Docker Compose output, this - injects a Docker socket mount backed by the framework docker gateway. + injects a Docker socket mount backed by the router-owned framework docker gateway. * `framework.kvm`: requires `experimental_features: ["kvm"]`. In Docker Compose output, adds a `/dev/kvm` device mapping to the container for hardware-accelerated virtualization (QEMU/KVM). Also adds the `kvm` group to the container via `group_add`, using the `AMBER_KVM_GID` environment @@ -359,9 +359,11 @@ Supported `from` sources (current): `stat -c %g /dev/kvm`). The generated `env.example` includes this variable. Other targets (e.g. Kubernetes) do not yet support this capability. -Reserved (not implemented yet): +Reserved (not implemented for mounts): -* `framework.` other than `framework.docker` and `framework.kvm` +* `framework.component` exists only as a binding source; it cannot be mounted. +* `framework.` other than `framework.component`, `framework.docker`, and + `framework.kvm` Mount value formatting: @@ -567,9 +569,9 @@ config_schema: { ### Capability declaration shape -Both slots and provides share: +Slots, resources, and provides use capability declarations: -* `kind`: `"mcp" | "llm" | "http" | "docker" | "a2a" | "storage"` +* `kind`: `"mcp" | "llm" | "http" | "component" | "docker" | "a2a" | "storage" | "kvm"` * `profile` (optional): string qualifier (often used for `"mcp"`) Example: @@ -689,6 +691,9 @@ Notes: * This crate enforces that each provide declares an `endpoint` and that it refers to a declared endpoint name. * `provides` cannot declare `kind: "storage"`. Storage is routed through `slots` and `bindings`, then consumed via `program.mounts`. +* `provides` cannot declare framework-owned kinds: `"component"`, `"docker"`, or `"kvm"`. + Those capability sources are created only by `framework.component`, `framework.docker`, and + `framework.kvm`. * To forward a child capability, use `exports` pointing at `#child.`. ### `exports` @@ -804,6 +809,7 @@ Framework capabilities are a fixed compiler-known list. Current framework capabilities: +* `component` (`framework.component`) — framework-provided dynamic child component management. * `docker` (`framework.docker`) — requires `experimental_features: ["docker"]` in the same manifest. * `kvm` (`framework.kvm`) — requires `experimental_features: ["kvm"]`. Docker Compose only. Adds a `/dev/kvm` device mapping to the container. diff --git a/compiler/manifest/src/document.rs b/compiler/manifest/src/document.rs index 2bd1d70b..a9f57e8a 100644 --- a/compiler/manifest/src/document.rs +++ b/compiler/manifest/src/document.rs @@ -212,7 +212,10 @@ fn labels_for_manifest_error( ManifestError::MissingProvideEndpoint { name } => { labels_for_missing_provide_endpoint(spans, name) } - ManifestError::UnsupportedProvideKind { name, .. } => labels_for_provide_kind(spans, name), + ManifestError::UnsupportedProvideKind { name, .. } + | ManifestError::FrameworkOwnedProvideKind { name, .. } => { + labels_for_provide_kind(spans, name) + } ManifestError::UnsupportedResourceKind { name, .. } => { labels_for_resource_decl(spans, name, "unsupported resource kind here") } diff --git a/compiler/manifest/src/error.rs b/compiler/manifest/src/error.rs index 0678e509..1a3d01f6 100644 --- a/compiler/manifest/src/error.rs +++ b/compiler/manifest/src/error.rs @@ -215,6 +215,14 @@ pub enum Error { kind: crate::CapabilityKind, }, + #[error("provide `{name}` cannot declare framework-owned capability kind `{kind}`")] + #[diagnostic(code(manifest::framework_owned_provide_kind), help("{help}"))] + FrameworkOwnedProvideKind { + name: String, + kind: crate::CapabilityKind, + help: String, + }, + #[error("duplicate mount name `{name}`")] #[diagnostic(code(manifest::duplicate_mount_name))] DuplicateMountName { name: String }, diff --git a/compiler/manifest/src/manifest/mod.rs b/compiler/manifest/src/manifest/mod.rs index d8e3ba4a..98a19b89 100644 --- a/compiler/manifest/src/manifest/mod.rs +++ b/compiler/manifest/src/manifest/mod.rs @@ -309,6 +309,31 @@ fn validate_resource_decls(resources: &BTreeMap) -> Ok(()) } +fn framework_owned_provide_kind_help(kind: CapabilityKind) -> String { + match kind { + CapabilityKind::Kvm => "`kvm` access is created by `framework.kvm`. Use `program.mounts` \ + with `from: \"framework.kvm\"` instead of declaring a `provides` \ + entry." + .to_string(), + CapabilityKind::Component | CapabilityKind::Docker => { + let framework_capability = kind.as_str(); + format!( + "`{kind}` capabilities are created by `framework.{framework_capability}`. Declare \ + a `slots.: {{ kind: \"{kind}\" }}` consumer slot and bind it from \ + `framework.{framework_capability}` instead of declaring a `provides` entry." + ) + } + _ => "`provides` cannot declare this framework-owned capability kind.".to_string(), + } +} + +fn is_framework_owned_provide_kind(kind: CapabilityKind) -> bool { + matches!( + kind, + CapabilityKind::Component | CapabilityKind::Docker | CapabilityKind::Kvm + ) +} + fn validate_environment_extends( environments: &BTreeMap, ) -> Result<(), Error> { @@ -728,6 +753,14 @@ fn validate_endpoints( } for (provide_name, provide) in provides { + if is_framework_owned_provide_kind(provide.decl.kind) { + return Err(Error::FrameworkOwnedProvideKind { + name: provide_name.to_string(), + kind: provide.decl.kind, + help: framework_owned_provide_kind_help(provide.decl.kind), + }); + } + if provide.decl.kind == CapabilityKind::Storage { return Err(Error::UnsupportedProvideKind { name: provide_name.to_string(), diff --git a/compiler/manifest/src/manifest/tests/components.rs b/compiler/manifest/src/manifest/tests/components.rs index 3187dcd5..720a0b1e 100644 --- a/compiler/manifest/src/manifest/tests/components.rs +++ b/compiler/manifest/src/manifest/tests/components.rs @@ -396,31 +396,66 @@ fn endpoint_validation_still_rejects_unknown_provide_with_conditional_literal_en } #[test] -fn docker_capability_kind_parses_for_slots_and_provides() { +fn framework_owned_capability_kinds_parse_for_slots() { let m: Manifest = r#" { manifest_version: "0.1.0", - program: { - image: "x", - entrypoint: ["x"], - network: { endpoints: [ { name: "endpoint", port: 80 } ] } - }, slots: { - worker: { kind: "docker" } + component: { kind: "component" }, + docker: { kind: "docker" }, + kvm: { kind: "kvm", optional: true }, }, - provides: { - api: { kind: "docker", endpoint: "endpoint" } - }, - exports: { api: "api" }, } "# .parse() .unwrap(); - let worker = m.slots.get("worker").expect("worker slot"); - assert_eq!(worker.decl.kind, CapabilityKind::Docker); + let component = m.slots.get("component").expect("component slot"); + assert_eq!(component.decl.kind, CapabilityKind::Component); - let api = m.provides.get("api").expect("api provide"); - assert_eq!(api.decl.kind, CapabilityKind::Docker); - assert_eq!(api.endpoint.as_deref(), Some("endpoint")); + let docker = m.slots.get("docker").expect("docker slot"); + assert_eq!(docker.decl.kind, CapabilityKind::Docker); + + let kvm = m.slots.get("kvm").expect("kvm slot"); + assert_eq!(kvm.decl.kind, CapabilityKind::Kvm); +} + +#[test] +fn framework_owned_capability_kinds_cannot_be_provided_by_manifests() { + for (kind, expected) in [ + ("component", CapabilityKind::Component), + ("docker", CapabilityKind::Docker), + ("kvm", CapabilityKind::Kvm), + ] { + let raw = parse_raw(&format!( + r#" + {{ + manifest_version: "0.1.0", + program: {{ + image: "x", + entrypoint: ["x"], + network: {{ endpoints: [ {{ name: "endpoint", port: 80 }} ] }} + }}, + provides: {{ + api: {{ kind: "{kind}", endpoint: "endpoint" }} + }}, + exports: {{ api: "api" }}, + }} + "# + )); + + let err = match raw.validate() { + Ok(_) => panic!("{kind} provide should be rejected"), + Err(err) => err, + }; + match err { + Error::FrameworkOwnedProvideKind { + name, kind: actual, .. + } => { + assert_eq!(name, "api"); + assert_eq!(actual, expected); + } + other => panic!("expected FrameworkOwnedProvideKind for {kind}, got: {other}"), + } + } } diff --git a/compiler/scenario/README.md b/compiler/scenario/README.md index ba41b43e..979073e2 100644 --- a/compiler/scenario/README.md +++ b/compiler/scenario/README.md @@ -10,6 +10,11 @@ Data model for the fully linked scenario produced by the compiler. - `ScenarioIr`: serde-friendly JSON IR wrapper with schema/version headers and conversions to/from `Scenario`. - `ScenarioIrError`: validation errors when loading IR into a `Scenario`. +User-loaded Scenario IR follows the same framework capability ownership rules as manifests: +component, Docker, and KVM capability providers are framework-owned. User-authored IR should +represent those sources as framework bindings (`framework.component`, `framework.docker`, or +`framework.kvm`), not as component `provides` entries. + ## Graph utilities - `graph::topo_order`: dependency ordering by non-weak component bindings with cycle detection (framework and external bindings do not introduce component dependencies). - `graph::component_path[_for]`: stable path strings for diagnostics. diff --git a/compiler/src/reporter/mod.rs b/compiler/src/reporter/mod.rs index b406254f..8263f8d8 100644 --- a/compiler/src/reporter/mod.rs +++ b/compiler/src/reporter/mod.rs @@ -95,7 +95,32 @@ impl CompiledScenario { pub fn derive_from_ir(&self, scenario_ir: ScenarioIr) -> Result { let mut compiled = Self::from_ir(scenario_ir)?; + let required_slots = compiled + .scenario + .components + .len() + .max(compiled.scenario.root.0 + 1); compiled.resolved_urls = self.resolved_urls.clone(); + if compiled.resolved_urls.len() < required_slots { + compiled.resolved_urls.resize(required_slots, None); + } + let previous_components = self + .scenario_ir + .components + .iter() + .map(|component| (component.id, component)) + .collect::>(); + for component in &mut compiled.scenario_ir.components { + let Some(previous) = previous_components.get(&component.id) else { + continue; + }; + if component.resolved_url.is_none() { + component.resolved_url = previous.resolved_url.clone(); + } + if component.exports.is_empty() { + component.exports = previous.exports.clone(); + } + } compiled.source_context = self.source_context.clone(); Ok(compiled) } diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index 7d988ead..66b20d4f 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, + fmt, path::PathBuf, }; @@ -7,7 +8,7 @@ use amber_json5 as json5; use amber_manifest::{ BindingSource as ManifestBindingSource, BindingTarget as ManifestBindingTarget, CapabilityKind, ChildTemplateDecl, ChildTemplateManifestDecl, ComponentDecl, Manifest, ManifestRef, - MountSource, NetworkProtocol, Program as ManifestProgram, SlotDecl, + MountSource, NetworkProtocol, Program as ManifestProgram, RuntimeBackend, SlotDecl, }; use amber_mesh::dynamic_caps::DynamicCapabilitiesSnapshotIr; use amber_scenario::{ @@ -164,6 +165,8 @@ pub struct SiteDefinition { pub kind: SiteKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_site: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -175,6 +178,104 @@ pub enum SiteKind { Kubernetes, } +pub const FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND: &str = + "amber.framework_component.controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_IMAGE: &str = "__amber_internal/site-controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PATH: &str = "/__amber_internal/site-controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE: &str = + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrameworkComponentControllerMetadata { + pub kind: String, + pub execution_site: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub grants: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrameworkComponentGrantMetadata { + pub managed_site: String, + pub authority_realm_id: usize, + pub authority_realm_moniker: String, +} + +pub fn framework_component_controller_metadata( + metadata: Option<&Value>, +) -> Option { + metadata + .cloned() + .and_then(|value| { + serde_json::from_value::(value).ok() + }) + .filter(|metadata| metadata.kind == FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND) +} + +pub(crate) fn lower_framework_component_bindings_for_single_site( + compiled: &CompiledScenario, + site_id: &str, + site: SiteDefinition, +) -> Result { + let scenario = compiled.scenario(); + let assignments_by_component = scenario + .components_iter() + .map(|(component_id, _)| (component_id, site_id.to_string())) + .collect::>(); + let offered_sites = BTreeMap::from([(site_id.to_string(), site)]); + let controller_sites = if has_public_framework_component_bindings(scenario) { + BTreeSet::from([site_id.to_string()]) + } else { + BTreeSet::new() + }; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &PlacementDefaults { + path: Some(site_id.to_string()), + image: Some(site_id.to_string()), + vm: Some(site_id.to_string()), + }, + &controller_sites, + )?; + compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings for site `{site_id}`: {err}" + )) + }) +} + +pub fn lower_framework_component_bindings_for_placement( + compiled: &CompiledScenario, + placement: &PlacementFile, +) -> Result { + let scenario = compiled.scenario(); + let offered_sites = normalize_site_definitions(&placement_site_definitions(Some(placement)))?; + let assignments_by_component = resolve_assignments( + scenario, + &offered_sites, + &placement.defaults, + placement.components.clone(), + )?; + validate_storage_locality(scenario, &assignments_by_component)?; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &placement.defaults, + &BTreeSet::new(), + )?; + compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings for placement-aware scenario: {err}" + )) + }) +} + #[derive(Debug, Error)] pub enum PlacementParseError { #[error("invalid placement file: {0}")] @@ -263,30 +364,18 @@ pub fn build_run_plan_with_activation( placement: Option<&PlacementFile>, activation_override: Option<&RunPlanActivationState>, ) -> Result { - let scenario = compiled.scenario(); - let offered_sites = placement_site_definitions(placement); + let offered_sites = normalize_site_definitions(&placement_site_definitions(placement))?; let defaults = placement .map(|placement| placement.defaults.clone()) .unwrap_or_else(default_placement_defaults); - let endpoint_plan = build_endpoint_plan(scenario) - .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; - let mesh_plan = build_mesh_plan( - scenario, - &endpoint_plan, - MeshOptions { - backend_label: "run plan", - }, - ) - .map_err(|err| RunPlanError::Other(format!("failed to build mesh plan: {err}")))?; - let placement_components = placement_component_overrides(placement); - let assignments_by_component = resolve_assignments( - scenario, + let mut assignments_by_component = resolve_assignments( + compiled.scenario(), &offered_sites, &defaults, placement_components.clone(), )?; - validate_storage_locality(scenario, &assignments_by_component)?; + validate_storage_locality(compiled.scenario(), &assignments_by_component)?; let ( standby_sites, @@ -304,13 +393,16 @@ pub fn build_run_plan_with_activation( activation.active_site_capabilities.clone(), ) } else { - let standby_sites = analyze_standby_sites(scenario, &offered_sites, &defaults)? + let standby_sites = analyze_standby_sites(compiled.scenario(), &offered_sites, &defaults)? .into_iter() .collect::>(); - let static_active_sites = assignments_by_component - .values() - .cloned() - .collect::>(); + let static_active_sites = expand_sites_to_execution_sites( + &assignments_by_component + .values() + .cloned() + .collect::>(), + &offered_sites, + )?; let standby_site_set = standby_sites.iter().cloned().collect::>(); let control_only_sites = Vec::new(); let control_only_site_set = control_only_sites.iter().cloned().collect::>(); @@ -326,6 +418,12 @@ pub fn build_run_plan_with_activation( .filter(|site_id| !control_only_site_set.contains(*site_id)) .cloned() .collect::>(); + let privileged_control_sites = initial_privileged_control_sites( + compiled.scenario(), + &offered_sites, + &assignments_by_component, + &standby_sites, + )?; let active_site_capabilities: BTreeMap = initial_active_sites .iter() @@ -335,7 +433,7 @@ pub fn build_run_plan_with_activation( ActiveSiteCapabilities { cross_site_routing: true, dynamic_workloads: dynamic_enabled_sites.contains(site_id), - privileged_control: true, + privileged_control: privileged_control_sites.contains(site_id), }, ) }) @@ -348,6 +446,37 @@ pub fn build_run_plan_with_activation( active_site_capabilities, ) }; + + let controller_sites = + controller_execution_sites_for_active_sites(&offered_sites, &active_site_capabilities)?; + let lowered = inject_site_controller_components( + compiled.scenario(), + &offered_sites, + &assignments_by_component, + &defaults, + &controller_sites, + )?; + assignments_by_component.extend(lowered.synthetic_assignments.clone()); + let lowered_compiled = compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings: {err}" + )) + })?; + let compiled = &lowered_compiled; + let scenario = compiled.scenario(); + let endpoint_plan = build_endpoint_plan(scenario) + .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; + let mesh_plan = build_mesh_plan( + scenario, + &endpoint_plan, + MeshOptions { + backend_label: "run plan", + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to build mesh plan: {err}")))?; + let initial_active_site_set = initial_active_sites .iter() .map(String::as_str) @@ -504,6 +633,26 @@ fn validate_activation_override( }); } } + + let initial_active_sites = activation + .initial_active_sites + .iter() + .cloned() + .collect::>(); + for site_id in activation + .standby_sites + .iter() + .chain(&activation.initial_active_sites) + { + let execution_site = site_execution_site_id(offered_sites, site_id)?; + if initial_active_sites.contains(execution_site) { + continue; + } + return Err(RunPlanError::Other(format!( + "activation state marks site `{site_id}` active, but omits its controller execution \ + site `{execution_site}` from initial_active_sites" + ))); + } Ok(()) } @@ -511,6 +660,13 @@ pub fn build_homogeneous_export_run_plan( compiled: &CompiledScenario, requested_kind: SiteKind, ) -> Result { + if requested_kind == SiteKind::Vm { + return Err(RunPlanError::Other( + "homogeneous vm exports are unsupported; vm workloads require an explicit controlling \ + direct site via placement.sites..controller_site" + .to_string(), + )); + } let site_id = homogeneous_export_site_id(requested_kind).to_string(); let placement = PlacementFile { schema: PLACEMENT_SCHEMA.to_string(), @@ -520,6 +676,7 @@ pub fn build_homogeneous_export_run_plan( SiteDefinition { kind: requested_kind, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -615,11 +772,7 @@ pub fn plan_dynamic_fragment( .flat_map(|link| [link.provider_site.clone(), link.consumer_site.clone()]), ) .collect::>(); - let site_plans = render_site_plan_subset( - scenario, - &planning, - affected_sites.iter().map(String::as_str), - )?; + let site_plans = render_site_plan_subset(&planning, affected_sites.iter().map(String::as_str))?; let assignments = fragment_components .iter() .filter_map(|component_id| { @@ -650,7 +803,7 @@ pub fn build_site_plan_subset( ) -> Result, RunPlanError> { let planning = resolve_dynamic_planning_state(scenario, placement, activation, existing_assignments)?; - render_site_plan_subset(scenario, &planning, site_ids.iter().map(String::as_str)) + render_site_plan_subset(&planning, site_ids.iter().map(String::as_str)) } fn resolve_assignments( @@ -701,6 +854,7 @@ fn resolve_assignments( #[derive(Clone, Debug)] struct DynamicPlanningState { + scenario: Scenario, mesh_scope: String, offered_sites: BTreeMap, active_site_capabilities: BTreeMap, @@ -714,18 +868,31 @@ fn resolve_dynamic_planning_state( activation: &RunPlanActivationState, existing_assignments: &BTreeMap, ) -> Result { - let offered_sites = placement_site_definitions(Some(placement)); + let offered_sites = normalize_site_definitions(&placement_site_definitions(Some(placement)))?; validate_activation_override(&offered_sites, activation)?; let mut explicit_components = placement.components.clone(); explicit_components.extend(existing_assignments.clone()); - let assignments_by_component = resolve_assignments( + let mut assignments_by_component = resolve_assignments( scenario, &offered_sites, &placement.defaults, explicit_components, )?; validate_storage_locality(scenario, &assignments_by_component)?; + let controller_sites = controller_execution_sites_for_active_sites( + &offered_sites, + &activation.active_site_capabilities, + )?; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &placement.defaults, + &controller_sites, + )?; + assignments_by_component.extend(lowered.synthetic_assignments.clone()); + let scenario = lowered.scenario; let active_sites = activation .dynamic_enabled_sites @@ -737,25 +904,26 @@ fn resolve_dynamic_planning_state( .find(|site_id| !active_sites.contains(site_id.as_str())) { return Err(RunPlanError::Other(format!( - "component placement selected site `{site_id}`, but the frozen activation state does \ - not allow dynamic workloads there" - ))); + "component placement selected site `{site_id}`, but the frozen activation state \ + does not allow dynamic workloads there" + ))); } - let endpoint_plan = build_endpoint_plan(scenario) + let endpoint_plan = build_endpoint_plan(&scenario) .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; let mesh_plan = build_mesh_plan( - scenario, + &scenario, &endpoint_plan, MeshOptions { backend_label: "dynamic plan", }, ) .map_err(|err| RunPlanError::Other(format!("failed to build dynamic mesh plan: {err}")))?; - let links = build_cross_site_links(scenario, &mesh_plan, &assignments_by_component); + let links = build_cross_site_links(&scenario, &mesh_plan, &assignments_by_component); Ok(DynamicPlanningState { - mesh_scope: scenario_mesh_scope(&ScenarioIr::from(scenario))?, + scenario: scenario.clone(), + mesh_scope: scenario_mesh_scope(&ScenarioIr::from(&scenario))?, offered_sites, active_site_capabilities: activation.active_site_capabilities.clone(), assignments_by_component, @@ -764,10 +932,10 @@ fn resolve_dynamic_planning_state( } fn render_site_plan_subset<'a>( - scenario: &Scenario, planning: &DynamicPlanningState, site_ids: impl IntoIterator, ) -> Result, RunPlanError> { + let scenario = &planning.scenario; let links_by_provider_site = planning .links @@ -968,6 +1136,7 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -975,6 +1144,7 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -982,11 +1152,55 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Vm, context: None, + controller_site: None, }, ), ]) } +fn normalize_site_definitions( + site_definitions: &BTreeMap, +) -> Result, RunPlanError> { + let direct_sites = site_definitions + .iter() + .filter_map(|(site_id, site)| (site.kind == SiteKind::Direct).then_some(site_id.clone())) + .collect::>(); + let mut normalized = site_definitions.clone(); + + for (site_id, site) in &mut normalized { + if site.kind != SiteKind::Vm { + if site.controller_site.is_some() { + return Err(RunPlanError::Other(format!( + "site `{site_id}` sets `controller_site`, but only vm sites support explicit \ + controller managers" + ))); + } + continue; + } + + if let Some(controller_site) = site.controller_site.as_deref() { + let controller = site_definitions.get(controller_site).ok_or_else(|| { + RunPlanError::Other(format!( + "vm site `{site_id}` references unknown controller_site `{controller_site}`" + )) + })?; + if controller.kind != SiteKind::Direct { + return Err(RunPlanError::Other(format!( + "vm site `{site_id}` must use a direct controller_site, but \ + `{controller_site}` is {:?}", + controller.kind + ))); + } + continue; + } + if direct_sites.len() == 1 { + site.controller_site = Some(direct_sites[0].clone()); + } + } + + Ok(normalized) +} + fn default_placement_defaults() -> PlacementDefaults { PlacementDefaults { path: Some("direct_local".to_string()), @@ -995,9 +1209,111 @@ fn default_placement_defaults() -> PlacementDefaults { } } +fn site_execution_site_id<'a>( + site_definitions: &'a BTreeMap, + site_id: &'a str, +) -> Result<&'a str, RunPlanError> { + let site = site_definitions + .get(site_id) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: site_id.to_string(), + })?; + match site.kind { + SiteKind::Vm => Ok(site.controller_site.as_deref().unwrap_or(site_id)), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + +fn framework_controller_execution_site_id<'a>( + site_definitions: &'a BTreeMap, + site_id: &'a str, +) -> Result<&'a str, RunPlanError> { + let site = site_definitions + .get(site_id) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: site_id.to_string(), + })?; + match site.kind { + SiteKind::Vm => site.controller_site.as_deref().ok_or_else(|| { + RunPlanError::Other(format!( + "vm site `{site_id}` requires an explicit controller_site because no controlling \ + direct site is available for framework.component" + )) + }), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + +fn controller_execution_sites_for_active_sites( + site_definitions: &BTreeMap, + active_site_capabilities: &BTreeMap, +) -> Result, RunPlanError> { + active_site_capabilities + .iter() + .filter(|(_, capabilities)| capabilities.privileged_control) + .map(|(site_id, _)| framework_controller_execution_site_id(site_definitions, site_id)) + .collect::, _>>() + .map(|sites| sites.into_iter().map(str::to_string).collect()) +} + +fn initial_privileged_control_sites( + scenario: &Scenario, + _offered_sites: &BTreeMap, + assignments_by_component: &BTreeMap, + standby_sites: &[String], +) -> Result, RunPlanError> { + let moniker_to_site = assignments_by_component + .iter() + .map(|(component_id, site_id)| { + ( + graph::component_path(scenario, *component_id), + site_id.clone(), + ) + }) + .collect::>(); + let mut privileged_control_sites = standby_sites.iter().cloned().collect::>(); + privileged_control_sites.extend(assignments_by_component.values().cloned()); + for (component_id, component) in scenario.components_iter() { + if component.child_templates.is_empty() { + continue; + } + if let Some(site_id) = assignments_by_component.get(&component_id) { + privileged_control_sites.insert(site_id.clone()); + } + } + for binding in &scenario.bindings { + let BindingFrom::Framework(framework) = &binding.from else { + continue; + }; + if framework.capability.as_str() != "component" { + continue; + } + privileged_control_sites.insert(framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?); + } + Ok(privileged_control_sites) +} + +fn expand_sites_to_execution_sites( + site_ids: &BTreeSet, + site_definitions: &BTreeMap, +) -> Result, RunPlanError> { + let mut expanded = BTreeSet::new(); + for site_id in site_ids { + expanded.insert(site_id.clone()); + expanded.insert(site_execution_site_id(site_definitions, site_id)?.to_string()); + } + Ok(expanded) +} + #[derive(Clone, Debug)] struct FrozenChildTemplateSpec { manifests: Option>, + possible_backends: Vec, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -1027,20 +1343,24 @@ fn analyze_standby_sites( let mut requested_sites = BTreeSet::new(); for template in collect_frozen_child_templates(scenario)? { - let Some(keys) = template.manifests.as_ref() else { - continue; - }; - for key in keys { - requested_sites.extend(analyze_manifest_root_sites( - &scenario.manifest_catalog, + if let Some(keys) = template.manifests.as_ref() { + for key in keys { + requested_sites.extend(analyze_manifest_root_sites( + &scenario.manifest_catalog, + site_definitions, + defaults, + key, + )?); + } + } else { + requested_sites.extend(site_ids_for_possible_backends( site_definitions, - defaults, - key, - )?); + &template.possible_backends, + )); } } - Ok(requested_sites) + expand_sites_to_execution_sites(&requested_sites, site_definitions) } fn collect_frozen_child_templates( @@ -1052,6 +1372,7 @@ fn collect_frozen_child_templates( templates.extend(component.child_templates.values().map(|template| { FrozenChildTemplateSpec { manifests: template.manifests.clone(), + possible_backends: template.possible_backends.clone(), } })); } @@ -1095,7 +1416,10 @@ fn freeze_manifest_child_template( } }; - Ok(FrozenChildTemplateSpec { manifests }) + Ok(FrozenChildTemplateSpec { + manifests, + possible_backends: template.possible_backends.clone(), + }) } fn resolve_catalog_key( @@ -1152,6 +1476,80 @@ fn analyze_manifest_root_sites( Ok(analyzer.required_sites) } +fn framework_component_authority_child_execution_sites( + scenario: &Scenario, + site_definitions: &BTreeMap, + defaults: &PlacementDefaults, + authority_realm: ComponentId, +) -> Result, RunPlanError> { + let mut sites = BTreeSet::new(); + for template in scenario.component(authority_realm).child_templates.values() { + sites.extend(child_template_execution_sites( + scenario, + site_definitions, + defaults, + template, + )?); + } + Ok(sites) +} + +fn child_template_execution_sites( + scenario: &Scenario, + site_definitions: &BTreeMap, + defaults: &PlacementDefaults, + template: &amber_scenario::ChildTemplate, +) -> Result, RunPlanError> { + let mut requested_sites = BTreeSet::new(); + if let Some(keys) = template.manifests.as_ref() { + for key in keys { + requested_sites.extend(analyze_manifest_root_sites( + &scenario.manifest_catalog, + site_definitions, + defaults, + key, + )?); + } + } else { + requested_sites.extend(site_ids_for_possible_backends( + site_definitions, + &template.possible_backends, + )); + } + requested_sites + .iter() + .map(|site_id| framework_controller_execution_site_id(site_definitions, site_id)) + .collect::, _>>() + .map(|sites| sites.into_iter().map(str::to_string).collect()) +} + +fn site_ids_for_possible_backends( + site_definitions: &BTreeMap, + possible_backends: &[RuntimeBackend], +) -> BTreeSet { + if possible_backends.is_empty() { + return BTreeSet::new(); + } + let possible_backends = possible_backends.iter().copied().collect::>(); + site_definitions + .iter() + .filter_map(|(site_id, site)| { + possible_backends + .contains(&runtime_backend_for_site_kind(site.kind)) + .then_some(site_id.clone()) + }) + .collect() +} + +fn runtime_backend_for_site_kind(kind: SiteKind) -> RuntimeBackend { + match kind { + SiteKind::Direct => RuntimeBackend::Direct, + SiteKind::Vm => RuntimeBackend::Vm, + SiteKind::Compose => RuntimeBackend::Compose, + SiteKind::Kubernetes => RuntimeBackend::Kubernetes, + } +} + impl FragmentAnalyzer<'_> { fn walk( &mut self, @@ -1838,76 +2236,688 @@ fn render_site_artifact_files( mesh_scope: &str, force_router: bool, ) -> Result, RunPlanError> { - let mut files = match site_kind { + let files = match site_kind { SiteKind::Direct => { - crate::targets::direct::emit_direct_artifact(compiled, force_router) - .map_err(|err| RunPlanError::Other(format!("failed to render direct site: {err}")))? - .files + crate::targets::direct::emit_direct_artifact_with_options( + compiled, + crate::targets::direct::DirectArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render direct site: {err}")))? + .files } SiteKind::Vm => { - crate::targets::vm::emit_vm_artifact(compiled, force_router) - .map_err(|err| RunPlanError::Other(format!("failed to render vm site: {err}")))? - .files + crate::targets::vm::emit_vm_artifact_with_options( + compiled, + crate::targets::vm::VmArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render vm site: {err}")))? + .files } SiteKind::Compose => { - crate::targets::mesh::docker_compose::emit_docker_compose_artifact( + crate::targets::mesh::docker_compose::emit_docker_compose_artifact_with_options( compiled, - force_router, + crate::targets::mesh::docker_compose::DockerComposeArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, ) .map_err(|err| RunPlanError::Other(format!("failed to render compose site: {err}")))? .files } SiteKind::Kubernetes => { - crate::targets::mesh::kubernetes::emit_kubernetes_artifact(compiled, force_router) - .map_err(|err| { - RunPlanError::Other(format!("failed to render kubernetes site: {err}")) - })? - .files + crate::targets::mesh::kubernetes::emit_kubernetes_artifact_with_options( + compiled, + crate::targets::mesh::kubernetes::KubernetesArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render kubernetes site: {err}")))? + .files } }; - rewrite_router_identity(site_kind, &mut files, router_identity_id); - rewrite_mesh_scope( - &mut files, - &scenario_mesh_scope(compiled.scenario_ir())?, - mesh_scope, - ); Ok(files .into_iter() .map(|(path, contents)| (path.to_string_lossy().into_owned(), contents)) .collect()) } -fn rewrite_router_identity( - site_kind: SiteKind, - files: &mut BTreeMap, - router_identity_id: &str, -) { - let existing_router_id = match site_kind { - SiteKind::Direct => crate::targets::direct::ROUTER_IDENTITY_ID, - SiteKind::Vm => crate::targets::vm::ROUTER_IDENTITY_ID, - SiteKind::Compose | SiteKind::Kubernetes => { - crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID +#[derive(Clone, Debug)] +struct LoweredFrameworkComponentScenario { + scenario: Scenario, + synthetic_assignments: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct FrameworkComponentGrantKey { + managed_site: String, + authority_realm: ComponentId, + authority_realm_moniker: String, +} + +#[derive(Clone, Debug)] +struct ControllerComponentRewrite { + component_id: ComponentId, + provide_by_grant: BTreeMap, +} + +const FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME: &str = "framework_component"; +const FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX: &str = + "/__amber_internal_framework_component_controller"; +const FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX: &str = "__amber_internal_framework_component"; +const FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX: &str = + "__amber_internal_framework_component_controller_remote"; +const FRAMEWORK_COMPONENT_CONTROLLER_PORT_BASE: u16 = 32000; +const FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH: &str = "/usr/local/bin/amber-site-controller"; +const FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH: &str = + "/amber/site/state/site-controller-plan.json"; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FrameworkComponentSiteToken(String); + +impl FrameworkComponentSiteToken { + pub fn for_site(site_id: &str) -> Self { + const HEX: &[u8; 16] = b"0123456789abcdef"; + + // These tokens are embedded in synthetic slot/provide map keys, so lossy sanitization is + // not enough: distinct authored site ids must remain distinct after lowering. + let mut out = String::with_capacity("site_".len() + site_id.len() * 2); + out.push_str("site_"); + for byte in site_id.bytes() { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); } - }; - if existing_router_id == router_identity_id { - return; + Self(out) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn controller_grant_provide_name(&self, authority_realm: ComponentId) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX}__site_{}__authority_{}", + self.as_str(), + authority_realm.0, + ) + } + + pub fn controller_remote_slot_name(&self) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX}__{}", + self.as_str(), + ) } - for contents in files.values_mut() { - *contents = contents.replace(existing_router_id, router_identity_id); +} + +impl fmt::Display for FrameworkComponentSiteToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) } } -fn rewrite_mesh_scope( - files: &mut BTreeMap, - existing_mesh_scope: &str, - mesh_scope: &str, -) { - if existing_mesh_scope == mesh_scope { - return; +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FrameworkComponentControllerMoniker(String); + +impl FrameworkComponentControllerMoniker { + pub fn for_site(site_id: &str) -> Self { + Self(format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", + FrameworkComponentSiteToken::for_site(site_id), + )) + } + + fn from_synthetic_string(moniker: String) -> Self { + debug_assert!(Self::is_synthetic_component(moniker.as_str())); + Self(moniker) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn is_synthetic_component(moniker: &str) -> bool { + moniker + .strip_prefix(FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX) + .is_some_and(|suffix| suffix.starts_with('/') && suffix.len() > 1) + } +} + +impl fmt::Display for FrameworkComponentControllerMoniker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +fn has_public_framework_component_bindings(scenario: &Scenario) -> bool { + scenario.bindings.iter().any(|binding| { + matches!( + &binding.from, + BindingFrom::Framework(framework) if framework.capability.as_str() == "component" + ) + }) +} + +fn inject_site_controller_components( + scenario: &Scenario, + offered_sites: &BTreeMap, + assignments_by_component: &BTreeMap, + defaults: &PlacementDefaults, + controller_sites: &BTreeSet, +) -> Result { + let framework_bindings = scenario + .bindings + .iter() + .filter_map(|binding| match &binding.from { + BindingFrom::Framework(framework) if framework.capability.as_str() == "component" => { + Some((binding, framework)) + } + _ => None, + }) + .collect::>(); + if framework_bindings.is_empty() && controller_sites.is_empty() { + return Ok(LoweredFrameworkComponentScenario { + scenario: scenario.clone(), + synthetic_assignments: BTreeMap::new(), + }); + } + + let moniker_to_site = assignments_by_component + .iter() + .map(|(component_id, site_id)| { + ( + graph::component_path(scenario, *component_id), + site_id.clone(), + ) + }) + .collect::>(); + let mut grants_by_execution_site = + BTreeMap::>::new(); + let mut remote_controller_edges = BTreeSet::<(String, String)>::new(); + let dynamic_capability_execution_sites = scenario + .components_iter() + .filter(|(_, component)| component.program.is_some()) + .map(|(component_id, _)| { + assignments_by_component + .get(&component_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "program component `{}` has no site assignment", + graph::component_path(scenario, component_id) + )) + }) + .and_then(|site_id| { + framework_controller_execution_site_id(offered_sites, site_id) + .map(str::to_string) + }) + }) + .collect::, _>>()?; + for local_site_id in &dynamic_capability_execution_sites { + for remote_site_id in &dynamic_capability_execution_sites { + if local_site_id != remote_site_id { + remote_controller_edges.insert((local_site_id.clone(), remote_site_id.clone())); + } + } + } + for (binding, framework) in &framework_bindings { + let managed_site = framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?; + let execution_site = + framework_controller_execution_site_id(offered_sites, &managed_site)?.to_string(); + let recipient_moniker = graph::component_path(scenario, binding.to.component); + let recipient_site = moniker_to_site + .get(&recipient_moniker) + .ok_or_else(|| { + RunPlanError::Other(format!( + "framework.component recipient `{recipient_moniker}` has no site assignment" + )) + })? + .clone(); + let recipient_execution_site = + framework_controller_execution_site_id(offered_sites, &recipient_site)?.to_string(); + if recipient_execution_site != execution_site { + remote_controller_edges + .insert((recipient_execution_site.clone(), execution_site.clone())); + } + for target_execution_site in framework_component_authority_child_execution_sites( + scenario, + offered_sites, + defaults, + framework.authority, + )? { + if target_execution_site != recipient_execution_site { + remote_controller_edges + .insert((recipient_execution_site.clone(), target_execution_site)); + } + } + grants_by_execution_site + .entry(execution_site) + .or_default() + .insert(FrameworkComponentGrantKey { + managed_site, + authority_realm: framework.authority, + authority_realm_moniker: graph::component_path(scenario, framework.authority), + }); + } + + let mut used_ports = scenario + .components_iter() + .flat_map(|(_, component)| { + component + .program + .as_ref() + .and_then(|program| program.network()) + .into_iter() + .flat_map(|network| network.endpoints.iter().map(|endpoint| endpoint.port)) + }) + .collect::>(); + let mut rewritten = scenario.clone(); + let mut synthetic_assignments = BTreeMap::new(); + let mut controller_rewrites = BTreeMap::::new(); + for (component_id, component) in rewritten.components_iter() { + let Some(metadata) = framework_component_controller_metadata(component.metadata.as_ref()) + else { + continue; + }; + if !offered_sites.contains_key(&metadata.execution_site) { + return Err(RunPlanError::UnknownSite { + site_id: metadata.execution_site, + }); + } + let provide_by_grant = metadata + .grants + .into_iter() + .map(|(provide, grant)| { + ( + FrameworkComponentGrantKey { + managed_site: grant.managed_site, + authority_realm: ComponentId(grant.authority_realm_id), + authority_realm_moniker: grant.authority_realm_moniker, + }, + provide, + ) + }) + .collect(); + synthetic_assignments + .entry(component_id) + .or_insert_with(|| metadata.execution_site.clone()); + if controller_rewrites + .insert( + metadata.execution_site.clone(), + ControllerComponentRewrite { + component_id, + provide_by_grant, + }, + ) + .is_some() + { + return Err(RunPlanError::Other(format!( + "site `{}` already has multiple synthetic framework.component controllers", + metadata.execution_site + ))); + } + } + + let execution_sites = controller_sites + .iter() + .cloned() + .chain(grants_by_execution_site.keys().cloned()) + .collect::>(); + for execution_site in execution_sites { + let grants = grants_by_execution_site + .remove(&execution_site) + .unwrap_or_default(); + if let Some(rewrite) = controller_rewrites.get(&execution_site) { + let missing_grants = grants + .iter() + .filter(|grant| !rewrite.provide_by_grant.contains_key(*grant)) + .collect::>(); + if !missing_grants.is_empty() { + return Err(RunPlanError::Other(format!( + "site `{execution_site}` already has a synthetic framework.component \ + controller, but it is missing {} lowered framework.component grant(s)", + missing_grants.len() + ))); + } + continue; + } + let controller_id = ComponentId(rewritten.components.len()); + let port = next_framework_component_controller_port(&mut used_ports)?; + let site = offered_sites + .get(&execution_site) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: execution_site.clone(), + })?; + let mut grants_metadata = BTreeMap::new(); + let mut provides = BTreeMap::from([( + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + framework_component_controller_internal_provide_decl()?, + )]); + let mut provide_by_grant = BTreeMap::new(); + for grant in grants { + let provide_name = framework_component_controller_provide_name(&grant); + grants_metadata.insert( + provide_name.clone(), + FrameworkComponentGrantMetadata { + managed_site: grant.managed_site.clone(), + authority_realm_id: grant.authority_realm.0, + authority_realm_moniker: grant.authority_realm_moniker.clone(), + }, + ); + provides.insert( + provide_name.clone(), + framework_component_controller_provide_decl()?, + ); + provide_by_grant.insert(grant, provide_name); + } + let metadata = FrameworkComponentControllerMetadata { + kind: FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND.to_string(), + execution_site: execution_site.clone(), + grants: grants_metadata, + }; + rewritten.components.push(Some(Component { + id: controller_id, + parent: Some(rewritten.root), + moniker: unique_framework_component_controller_moniker(&rewritten, &execution_site) + .into_string() + .into(), + digest: rewritten.component(rewritten.root).digest, + config: None, + config_schema: None, + program: Some(framework_component_controller_program(site.kind, port)?), + slots: BTreeMap::new(), + provides, + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value(metadata) + .expect("framework controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + rewritten + .component_mut(rewritten.root) + .children + .push(controller_id); + synthetic_assignments.insert(controller_id, execution_site.clone()); + controller_rewrites.insert( + execution_site, + ControllerComponentRewrite { + component_id: controller_id, + provide_by_grant, + }, + ); + } + + for (local_site_id, remote_site_id) in remote_controller_edges { + let local_controller_id = controller_rewrites + .get(&local_site_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "site `{local_site_id}` needs a remote-controller route to `{remote_site_id}` \ + but has no synthetic framework.component controller" + )) + })? + .component_id; + let remote_controller_id = controller_rewrites + .get(&remote_site_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "site `{local_site_id}` needs a remote-controller route to `{remote_site_id}` \ + but the remote site has no synthetic framework.component controller" + )) + })? + .component_id; + let slot_name = framework_component_controller_remote_slot_name(&remote_site_id); + rewritten.component_mut(local_controller_id).slots.insert( + slot_name.clone(), + framework_component_controller_remote_slot_decl()?, + ); + rewritten.bindings.push(BindingEdge { + from: BindingFrom::Component(ProvideRef { + component: remote_controller_id, + name: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + }), + to: SlotRef { + component: local_controller_id, + name: slot_name, + }, + // Remote controller routes must exist without introducing a cross-site startup cycle. + weak: true, + }); + } + + for binding in &mut rewritten.bindings { + let BindingFrom::Framework(framework) = &binding.from else { + continue; + }; + if framework.capability.as_str() != "component" { + continue; + } + let grant = FrameworkComponentGrantKey { + managed_site: framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?, + authority_realm: framework.authority, + authority_realm_moniker: graph::component_path(scenario, framework.authority), + }; + let execution_site = + framework_controller_execution_site_id(offered_sites, &grant.managed_site)?; + let rewrite = controller_rewrites.get(execution_site).expect( + "synthetic controller rewrite should exist for each lowered framework.component grant", + ); + let provide = rewrite + .provide_by_grant + .get(&grant) + .expect("lowered framework.component grant should have a synthetic provide"); + binding.from = BindingFrom::Component(ProvideRef { + component: rewrite.component_id, + name: provide.clone(), + }); + } + + rewritten.normalize_order(); + rewritten.assert_invariants(); + Ok(LoweredFrameworkComponentScenario { + scenario: rewritten, + synthetic_assignments, + }) +} + +fn framework_component_managed_site_id( + scenario: &Scenario, + moniker_to_site: &BTreeMap, + authority_realm: ComponentId, + recipient_component: ComponentId, +) -> Result { + let authority_moniker = graph::component_path(scenario, authority_realm); + if let Some(site_id) = moniker_to_site.get(&authority_moniker) { + return Ok(site_id.clone()); + } + + let recipient_moniker = graph::component_path(scenario, recipient_component); + if authority_moniker != "/" + && recipient_moniker != authority_moniker + && !recipient_moniker.starts_with(&format!("{authority_moniker}/")) + { + return Err(RunPlanError::Other(format!( + "framework.component recipient `{recipient_moniker}` is not inside authority \ + realm `{authority_moniker}`" + ))); + } + + let recipient_segments = recipient_moniker + .trim_start_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + let authority_depth = authority_moniker + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .count(); + for depth in authority_depth..=recipient_segments.len() { + let moniker = if depth == 0 { + "/".to_string() + } else { + format!("/{}", recipient_segments[..depth].join("/")) + }; + if let Some(site_id) = moniker_to_site.get(&moniker) { + return Ok(site_id.clone()); + } + } + + Err(RunPlanError::Other(format!( + "framework.component authority realm `{authority_moniker}` has no assigned component \ + on the path to recipient `{recipient_moniker}`" + ))) +} + +fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKey) -> String { + FrameworkComponentSiteToken::for_site(grant.managed_site.as_str()) + .controller_grant_provide_name(grant.authority_realm) +} + +pub fn framework_component_controller_remote_slot_name(site_id: &str) -> String { + FrameworkComponentSiteToken::for_site(site_id).controller_remote_slot_name() +} + +fn unique_framework_component_controller_moniker( + scenario: &Scenario, + execution_site: &str, +) -> FrameworkComponentControllerMoniker { + let base = FrameworkComponentControllerMoniker::for_site(execution_site); + let used = scenario + .components_iter() + .map(|(_, component)| component.moniker.as_str()) + .collect::>(); + if !used.contains(base.as_str()) { + return base; } - for contents in files.values_mut() { - *contents = contents.replace(existing_mesh_scope, mesh_scope); + let mut suffix = 1usize; + loop { + let candidate = format!("{base}-{suffix}"); + if !used.contains(candidate.as_str()) { + return FrameworkComponentControllerMoniker::from_synthetic_string(candidate); + } + suffix += 1; + } +} + +fn next_framework_component_controller_port( + used_ports: &mut BTreeSet, +) -> Result { + for port in FRAMEWORK_COMPONENT_CONTROLLER_PORT_BASE..=u16::MAX { + if used_ports.insert(port) { + return Ok(port); + } } + Err(RunPlanError::Other( + "unable to allocate a synthetic framework.component controller port".to_string(), + )) +} + +fn framework_component_controller_program( + kind: SiteKind, + port: u16, +) -> Result { + let value = match kind { + SiteKind::Direct => serde_json::json!({ + "path": FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PATH, + "network": { + "endpoints": [ + { + "name": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + "port": port, + "protocol": "http" + } + ] + } + }), + SiteKind::Compose | SiteKind::Kubernetes => serde_json::json!({ + "image": FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_IMAGE, + "entrypoint": [ + FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH, + "--plan", + FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH + ], + "network": { + "endpoints": [ + { + "name": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + "port": port, + "protocol": "http" + } + ] + } + }), + SiteKind::Vm => { + return Err(RunPlanError::Other( + "synthetic framework.component controllers must execute on non-vm sites" + .to_string(), + )); + } + }; + serde_json::from_value(value).map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic framework.component controller program: {err}" + )) + }) +} + +fn framework_component_controller_provide_decl() -> Result +{ + serde_json::from_value(serde_json::json!({ + "kind": "component", + "endpoint": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + })) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic framework.component controller provide: {err}" + )) + }) +} + +fn framework_component_controller_internal_provide_decl() +-> Result { + serde_json::from_value(serde_json::json!({ + "kind": "http", + "endpoint": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + })) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic site controller internal provide: {err}" + )) + }) +} + +fn framework_component_controller_remote_slot_decl() -> Result { + serde_json::from_value(serde_json::json!({ "kind": "http" })).map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic remote site controller slot: {err}" + )) + }) } #[cfg(test)] @@ -2081,66 +3091,1379 @@ mod tests { }"# } - fn storage_consumer_manifest() -> &'static str { - r#"{ + fn storage_consumer_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { state: { kind: "storage" } }, + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] }, + mounts: [{ from: "slots.state", path: "/data" }] + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn dynamic_parent_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component", optional: true } }, + child_templates: { + worker: { + manifest: "./worker.json5", + bindings: { realm: "slots.realm" } + } + }, + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn dynamic_worker_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component" } }, + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"] + } +}"# + } + + fn framework_admin_path_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn framework_admin_image_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn framework_admin_vm_manifest() -> &'static str { + r##"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + vm: { + image: "ubuntu.img", + cpus: 1, + memory_mib: 512, + network: { + endpoints: [{ name: "http", port: 8080, protocol: "http" }], + egress: "none" + }, + cloud_init: { + user_data: "#cloud-config\nwrite_files:\n - path: /etc/ctl-url\n content: '${slots.ctl.url}'\nruncmd:\n - [sh, -lc, 'sleep infinity']\n" + } + } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"## + } + + fn framework_authority_manifest() -> &'static str { + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8081, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { + api: "api", + admin_api: "#admin.api" + } +}"## + } + + fn framework_controller_component_for_site<'a>( + scenario: &'a Scenario, + execution_site: &str, + ) -> ( + ComponentId, + &'a Component, + FrameworkComponentControllerMetadata, + ) { + if let Some(controller) = + scenario + .components_iter() + .find_map(|(component_id, component)| { + framework_component_controller_metadata(component.metadata.as_ref()) + .filter(|metadata| metadata.execution_site == execution_site) + .map(|metadata| (component_id, component, metadata)) + }) + { + return controller; + } + + let seen = scenario + .components_iter() + .map(|(_, component)| { + format!( + "{} metadata={:?}", + component.moniker.as_str(), + component.metadata.as_ref() + ) + }) + .collect::>() + .join("\n"); + let bindings = scenario + .bindings + .iter() + .map(|binding| { + format!( + "to={}.{} from={:?}", + graph::component_path(scenario, binding.to.component), + binding.to.name, + binding.from + ) + }) + .collect::>() + .join("\n"); + panic!( + "synthetic framework.component controller for site `{execution_site}` should be \ + present;\ncomponents:\n{seen}\nbindings:\n{bindings}" + ) + } + + fn framework_controller_moniker_for_site(site_id: &str) -> String { + FrameworkComponentControllerMoniker::for_site(site_id).into_string() + } + + async fn compile(root: &Path) -> CompiledScenario { + let compiler = Compiler::new(Resolver::new(), DigestStore::default()) + .with_registry(ResolverRegistry::default()); + let output = compiler + .compile( + manifest_ref(root), + CompileOptions { + resolve: Default::default(), + optimize: Default::default(), + }, + ) + .await + .expect("scenario should compile"); + CompiledScenario::from_compile_output(&output).expect("compiled scenario") + } + + fn direct_mesh_config_for_component( + site_plan: &RunSitePlan, + moniker: &str, + ) -> amber_mesh::MeshConfigTemplate { + let mesh_provision_plan: amber_mesh::MeshProvisionPlan = serde_json::from_str( + site_plan + .artifact_files + .get(crate::targets::direct::MESH_PROVISION_PLAN_FILENAME) + .expect("mesh provision plan should be present in site artifacts"), + ) + .expect("mesh provision plan should deserialize"); + mesh_provision_plan + .targets + .into_iter() + .find_map(|target| { + (matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Component) + && target.config.identity.id == moniker) + .then_some(target.config) + }) + .unwrap_or_else(|| panic!("mesh provision plan should include component {moniker}")) + } + + #[tokio::test] + async fn vm_site_defaults_controller_site_to_sole_direct_site() { + let dir = tmp_dir("run-plan-vm-controller-default-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let compiled = compile(&root).await; + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_direct: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.offered_sites["guest_vm"].controller_site.as_deref(), + Some("host_direct") + ); + assert_eq!( + plan.sites["guest_vm"].site.controller_site.as_deref(), + Some("host_direct") + ); + } + + #[tokio::test] + async fn framework_component_vm_sites_require_explicit_controller_site_when_multiple_direct_sites_exist() + { + let dir = tmp_dir("run-plan-vm-controller-ambiguous-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + + let compiled = compile(&root).await; + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_a: { kind: "direct" }, + host_b: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + path: "host_a", + image: "host_a", + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let err = build_run_plan(&compiled, Some(&placement)).expect_err("run plan should fail"); + let rendered = err.to_string(); + assert!(rendered.contains("guest_vm"), "{rendered}"); + assert!(rendered.contains("controller_site"), "{rendered}"); + assert!(rendered.contains("framework.component"), "{rendered}"); + } + + #[tokio::test] + async fn vm_site_rejects_non_direct_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-kind-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + compose_mgr: { kind: "compose" }, + guest_vm: { kind: "vm", controller_site: "compose_mgr" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let err = build_run_plan(&compiled, Some(&placement)).expect_err("run plan should fail"); + let rendered = err.to_string(); + assert!(rendered.contains("guest_vm"), "{rendered}"); + assert!(rendered.contains("compose_mgr"), "{rendered}"); + assert!(rendered.contains("direct controller_site"), "{rendered}"); + } + + #[tokio::test] + async fn vm_site_preserves_explicit_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-explicit-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_a: { kind: "direct" }, + host_b: { kind: "direct" }, + guest_vm: { kind: "vm", controller_site: "host_b" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.offered_sites["guest_vm"].controller_site.as_deref(), + Some("host_b") + ); + } + + #[tokio::test] + async fn vm_static_workloads_activate_their_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-active-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_direct: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.initial_active_sites, + vec!["guest_vm".to_string(), "host_direct".to_string()] + ); + assert_eq!(plan.dynamic_enabled_sites, plan.initial_active_sites); + assert!(plan.sites.contains_key("host_direct")); + assert!(plan.sites.contains_key("guest_vm")); + assert_eq!( + plan.sites["host_direct"].assigned_components, + vec![framework_controller_moniker_for_site("host_direct")] + ); + assert_eq!( + plan.sites["guest_vm"].assigned_components, + vec!["/vm".to_string()] + ); + } + + #[tokio::test] + async fn vm_standby_sites_activate_their_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-standby-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, vm_server_manifest()); + write(&root, dynamic_parent_manifest()); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + assert_eq!( + plan.standby_sites, + vec!["direct_local".to_string(), "vm_local".to_string()] + ); + assert_eq!( + plan.initial_active_sites, + vec![ + "compose_local".to_string(), + "direct_local".to_string(), + "vm_local".to_string(), + ] + ); + assert_eq!(plan.dynamic_enabled_sites, plan.initial_active_sites); + } + + #[tokio::test] + async fn compile_only_lowering_without_framework_bindings_does_not_inject_site_controller_components() + { + let dir = tmp_dir("compile-framework-lowering-no-controller-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { worker: "./worker.json5" }, + exports: { worker_api: "#worker.api" } +}"##, + ); + + let compiled = compile(&root).await; + let lowered = lower_framework_component_bindings_for_single_site( + &compiled, + "direct_local", + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ) + .expect("compile-time framework lowering should succeed"); + let scenario = lowered.scenario(); + + assert!( + scenario.components_iter().all(|(_, component)| { + framework_component_controller_metadata(component.metadata.as_ref()).is_none() + }), + "compile-time lowering should stay inert when no framework.component bindings exist" + ); + } + + #[tokio::test] + async fn build_run_plan_injects_site_controller_components_for_component_hosting_sites() { + let dir = tmp_dir("run-plan-controller-injection-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { worker: "./worker.json5" }, + exports: { worker_api: "#worker.api" } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let site_plan = plan + .sites + .get("direct_local") + .expect("direct site should be present"); + let scenario = Scenario::try_from(site_plan.scenario_ir.clone()) + .expect("site scenario should round-trip from IR"); + let worker_config = direct_mesh_config_for_component(site_plan, "/worker"); + + framework_controller_component_for_site(&scenario, "direct_local"); + assert!( + plan.active_site_capabilities["direct_local"].privileged_control, + "component-hosting sites should advertise privileged control because dynamic \ + capability control is site-local", + ); + assert!( + worker_config.dynamic_caps_listen.is_some(), + "components on populated sites should expose dynamic caps listeners once a local \ + controller exists", + ); + assert!( + worker_config.outbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }), + "components on populated sites should get the synthetic local controller route", + ); + } + + #[tokio::test] + async fn build_run_plan_direct_artifacts_keep_local_controller_routes_for_framework_sites() { + let dir = tmp_dir("run-plan-direct-artifact-controller-route-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let run_plan = build_run_plan(&compiled, None).expect("run plan should build"); + let site_plan = run_plan + .sites + .get("direct_local") + .expect("direct site should be present"); + let site_scenario = Scenario::try_from(site_plan.scenario_ir.clone()) + .expect("site scenario should round-trip from IR"); + let (_, controller_component, _) = + framework_controller_component_for_site(&site_scenario, "direct_local"); + let admin_config = direct_mesh_config_for_component(site_plan, "/admin"); + let controller_config = + direct_mesh_config_for_component(site_plan, controller_component.moniker.as_str()); + + assert!( + run_plan.active_site_capabilities["direct_local"].privileged_control, + "sites that provide framework.component should advertise privileged control", + ); + assert!( + admin_config.dynamic_caps_listen.is_some(), + "ordinary direct-site components should expose dynamic caps listeners when a local \ + framework controller exists", + ); + assert!( + admin_config.outbound.iter().any(|route| { + route.protocol == amber_mesh::MeshProtocol::Http + && route.capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }), + "framework-enabled direct-site components must keep their local framework.component \ + controller route", + ); + assert!( + controller_config.dynamic_caps_listen.is_none(), + "synthetic controller sidecars must not expose the dynamic caps listener", + ); + } + + #[tokio::test] + async fn framework_component_lowering_rewrites_same_site_bindings_to_synthetic_controller_components() + { + let dir = tmp_dir("run-plan-framework-component-direct-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, _, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + assert_eq!(metadata.grants.len(), 1); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/admin" + && binding.to.name == "ctl" + }) + .expect("framework binding should remain present on /admin.ctl"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "direct_local"); + assert_eq!(grant.authority_realm_moniker, "/"); + assert!(scenario.bindings.iter().all( + |binding| !matches!(&binding.from, BindingFrom::Framework(framework) if framework.capability.as_str() == "component") + )); + assert!(plan.links.is_empty()); + } + + #[tokio::test] + async fn framework_component_lowering_keeps_colliding_managed_site_grants_distinct() { + let dir = tmp_dir("run-plan-framework-component-colliding-grants-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { + dash: "./admin.json5", + underscore: "./admin.json5" + }, + bindings: [ + { to: "#dash.ctl", from: "framework.component" }, + { to: "#underscore.ctl", from: "framework.component" } + ], + exports: { + dash_api: "#dash.api", + underscore_api: "#underscore.api" + } +}"###, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "site-a".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ( + "site_a".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: None, + vm: Some("site-a".to_string()), + }, + components: BTreeMap::from([ + ("/dash".to_string(), "site-a".to_string()), + ("/underscore".to_string(), "site_a".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + let managed_sites = metadata + .grants + .values() + .map(|grant| grant.managed_site.clone()) + .collect::>(); + + assert_eq!( + managed_sites, + BTreeSet::from(["site-a".to_string(), "site_a".to_string()]), + "colliding sanitized site ids must not overwrite each other's controller grants", + ); + assert_eq!( + controller.provides.len(), + 3, + "controller should expose one internal provide plus one provide per managed site", + ); + + for (moniker, expected_site) in [("/dash", "site-a"), ("/underscore", "site_a")] { + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == moniker + && binding.to.name == "ctl" + }) + .expect("framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, expected_site); + } + } + + #[tokio::test] + async fn framework_component_lowering_reuses_existing_synthetic_controller_on_replay() { + let dir = tmp_dir("run-plan-framework-component-replay-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let replay_compiled = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should round-trip through scenario IR"); + let replay_plan = + build_run_plan(&replay_compiled, None).expect("replay run plan should build"); + let replay = CompiledScenario::from_ir(replay_plan.base_scenario) + .expect("replay scenario should deserialize"); + let controllers = replay + .scenario() + .components_iter() + .filter(|(_, component)| { + framework_component_controller_metadata(component.metadata.as_ref()).is_some() + }) + .collect::>(); + + assert_eq!( + controllers.len(), + 1, + "replaying a snapshot must reuse the existing synthetic controller instead of \ + injecting a second controller for the same site", + ); + } + + #[tokio::test] + async fn framework_component_lowering_routes_cross_site_authorities_to_their_managed_site_controller() + { + let dir = tmp_dir("run-plan-framework-component-cross-site-"); + let admin = dir.path().join("admin.json5"); + let authority = dir.path().join("authority.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&authority, framework_authority_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { authority: "./authority.json5" }, + exports: { + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" + } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "compose_local"); + let (direct_controller_id, direct_controller, _) = + framework_controller_component_for_site(scenario, "direct_local"); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/authority/admin" + && binding.to.name == "ctl" + }) + .expect("nested framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("nested framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "compose_local"); + assert_eq!(grant.authority_realm_moniker, "/authority"); + let ctl_link = plan + .links + .iter() + .find(|link| link.consumer_component == "/authority/admin" && link.slot == "ctl") + .expect("cross-site framework binding should become an ordinary cross-site link"); + assert_eq!(ctl_link.provider_site, "compose_local"); + assert_eq!(ctl_link.consumer_site, "direct_local"); + assert_eq!(ctl_link.provider_component, controller.moniker.as_str()); + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == direct_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("compose_local") + }) + .expect( + "synthetic direct-site controller should bind to the remote compose controller \ + internal route", + ); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert!( + remote_controller_binding.weak, + "remote controller bindings should not impose cross-site startup ordering", + ); + assert_eq!( + graph::component_path(scenario, remote_controller_binding.to.component), + direct_controller.moniker.as_str(), + ); + assert!( + scenario.bindings.iter().any(|binding| { + binding.to.component == controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("direct_local") + }), + "component-hosting sites keep reverse controller routes because dynamic capabilities \ + can be shared in either direction at runtime", + ); + } + + #[tokio::test] + async fn framework_component_lowering_routes_creators_to_standby_child_site_controllers() { + let dir = tmp_dir("run-plan-framework-component-standby-controller-"); + let admin = dir.path().join("admin.json5"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&worker, image_server_manifest()); + write( + &root, + r##"{ manifest_version: "0.3.0", - slots: { state: { kind: "storage" } }, - program: { - image: "busybox:1.36.1", - entrypoint: ["sh", "-c", "sleep 30"], - network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] }, - mounts: [{ from: "slots.state", path: "/data" }] + slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, + child_templates: { + worker: { manifest: "./worker.json5" } }, - provides: { api: { kind: "http", endpoint: "http" } }, - exports: { api: "api" } -}"# + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: Some("compose_local".to_string()), + vm: None, + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!(plan.standby_sites, vec!["compose_local".to_string()]); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (direct_controller_id, direct_controller, _) = + framework_controller_component_for_site(scenario, "direct_local"); + let (compose_controller_id, _compose_controller, compose_metadata) = + framework_controller_component_for_site(scenario, "compose_local"); + assert!( + compose_metadata.grants.is_empty(), + "the compose controller is needed as a standby create target, not because it owns an \ + existing framework.component grant", + ); + + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == direct_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("compose_local") + }) + .expect("direct controller should have a route to the standby compose controller"); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, compose_controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert_eq!( + graph::component_path(scenario, remote_controller_binding.to.component), + direct_controller.moniker.as_str(), + ); + assert!(remote_controller_binding.weak); + assert!( + scenario.bindings.iter().all(|binding| { + binding.to.component != compose_controller_id + || binding.to.name + != framework_component_controller_remote_slot_name("direct_local") + }), + "creating into the compose standby site should not synthesize an unused reverse route", + ); } - fn dynamic_parent_manifest() -> &'static str { - r#"{ + #[tokio::test] + async fn framework_component_lowering_keeps_colliding_remote_controller_slots_distinct() { + let dir = tmp_dir("run-plan-framework-component-colliding-remote-slots-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_image_manifest()); + write( + &root, + r##"{ manifest_version: "0.3.0", slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, child_templates: { worker: { - manifest: "./worker.json5", - bindings: { realm: "slots.realm" } + possible_backends: ["direct"] } }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "site-a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "site_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("site-a".to_string()), + image: Some("compose_local".to_string()), + vm: None, + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (compose_controller_id, _, _) = + framework_controller_component_for_site(scenario, "compose_local"); + let site_dash_slot = framework_component_controller_remote_slot_name("site-a"); + let site_underscore_slot = framework_component_controller_remote_slot_name("site_a"); + + assert_ne!( + site_dash_slot, site_underscore_slot, + "remote controller slots must preserve distinct authored site ids", + ); + let remote_bindings = scenario + .bindings + .iter() + .filter(|binding| { + binding.to.component == compose_controller_id + && binding + .to + .name + .starts_with(FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX) + }) + .collect::>(); + let remote_slots = remote_bindings + .iter() + .map(|binding| binding.to.name.clone()) + .collect::>(); + assert_eq!( + remote_bindings.len(), + 2, + "compose controller should bind to both colliding direct-site controllers", + ); + assert_eq!( + remote_slots, + BTreeSet::from([site_dash_slot, site_underscore_slot]), + "remote controller routes must not collapse onto one synthetic slot", + ); + } + + #[tokio::test] + async fn framework_component_lowering_routes_vm_template_creators_to_controller_site() { + let dir = tmp_dir("run-plan-framework-component-vm-template-controller-"); + let admin = dir.path().join("admin.json5"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write( + &admin, + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, program: { image: "busybox:1.36.1", entrypoint: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } }, provides: { api: { kind: "http", endpoint: "http" } }, exports: { api: "api" } -}"# +}"#, + ); + write(&worker, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, + child_templates: { + worker: { manifest: "./worker.json5" } + }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: Some("compose_local".to_string()), + vm: Some("vm_local".to_string()), + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.standby_sites, + vec!["direct_local".to_string(), "vm_local".to_string()] + ); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (compose_controller_id, _compose_controller, _) = + framework_controller_component_for_site(scenario, "compose_local"); + let (direct_controller_id, _direct_controller, direct_metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + assert!( + direct_metadata.grants.is_empty(), + "the direct controller is needed because it controls vm creates, not because it owns \ + an existing framework.component grant", + ); + + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == compose_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("direct_local") + }) + .expect("compose controller should route vm creates to the direct controller"); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, direct_controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert!( + scenario.bindings.iter().all(|binding| { + binding.to.component != compose_controller_id + || binding.to.name + != framework_component_controller_remote_slot_name("vm_local") + }), + "vm workload sites do not host framework.component controllers; the route target is \ + the vm controller_site", + ); } - fn dynamic_worker_manifest() -> &'static str { - r#"{ + #[tokio::test] + async fn framework_component_lowering_does_not_synthesize_legacy_controller_peer_bindings() { + let dir = tmp_dir("run-plan-framework-component-legacy-controller-peer-"); + let admin = dir.path().join("admin.json5"); + let authority = dir.path().join("authority.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&authority, framework_authority_manifest()); + write( + &root, + r##"{ manifest_version: "0.3.0", - slots: { realm: { kind: "component" } }, - program: { - path: "/usr/bin/env", - args: ["sh", "-c", "sleep 30"] + components: { authority: "./authority.json5" }, + exports: { + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" } -}"# +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + + assert!( + scenario.components_iter().all(|(_, component)| { + component.slots.keys().all(|slot| { + !slot.starts_with("__amber_internal_framework_component_controller_peer") + }) + }), + "framework controller lowering must not synthesize legacy controller-peer slots", + ); + assert!( + scenario.bindings.iter().all(|binding| { + !matches!( + &binding.from, + BindingFrom::Component(ProvideRef { name, .. }) + if name == "__amber_internal_site_controller_peer" + ) + }), + "framework controller lowering must not synthesize legacy controller-peer capability \ + bindings", + ); } - async fn compile(root: &Path) -> CompiledScenario { - let compiler = Compiler::new(Resolver::new(), DigestStore::default()) - .with_registry(ResolverRegistry::default()); - let output = compiler - .compile( - manifest_ref(root), - CompileOptions { - resolve: Default::default(), - optimize: Default::default(), - }, - ) - .await - .expect("scenario should compile"); - CompiledScenario::from_compile_output(&output).expect("compiled scenario") + #[tokio::test] + async fn framework_component_lowering_executes_vm_managers_on_their_direct_controller_site() { + let dir = tmp_dir("run-plan-framework-component-vm-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/admin" + && binding.to.name == "ctl" + }) + .expect("vm framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("vm framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("vm controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "vm_local"); + let ctl_link = plan + .links + .iter() + .find(|link| link.consumer_component == "/admin" && link.slot == "ctl") + .expect("vm framework binding should become a cross-site link to the direct manager"); + assert_eq!(ctl_link.provider_site, "direct_local"); + assert_eq!(ctl_link.consumer_site, "vm_local"); + assert_eq!(ctl_link.provider_component, controller.moniker.as_str()); + } + + #[tokio::test] + async fn compiled_scenario_derive_from_ir_extends_resolved_url_slots_for_synthetic_components() + { + let dir = tmp_dir("run-plan-derived-ir-slots-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + exports: { admin_api: "#admin.api" } +}"##, + ); + + let compiled = compile(&root).await; + let mut scenario = compiled.scenario().clone(); + let synthetic_id = ComponentId(scenario.components.len()); + scenario.components.push(Some(Component { + id: synthetic_id, + parent: Some(scenario.root), + moniker: "/__amber_internal_test".to_string().into(), + digest: scenario.component(scenario.root).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario + .component_mut(scenario.root) + .children + .push(synthetic_id); + scenario.normalize_order(); + + let derived = compiled + .derive_from_ir(ScenarioIr::from(&scenario)) + .expect("derived compiled scenario should allow injected components"); + assert!(derived.resolved_url_for_component(synthetic_id).is_none()); } #[tokio::test] @@ -2241,11 +4564,14 @@ mod tests { assert_eq!(plan.defaults.path.as_deref(), Some("direct_local")); assert_eq!( plan.sites["compose_local"].assigned_components, - vec!["/".to_string()] + vec![ + "/".to_string(), + framework_controller_moniker_for_site("compose_local"), + ] ); assert_eq!( plan.sites["direct_local"].assigned_components, - Vec::::new() + vec![framework_controller_moniker_for_site("direct_local")] ); assert!( plan.sites["direct_local"] @@ -2460,7 +4786,37 @@ mod tests { "direct_local".to_string() ] ); - assert_eq!(plan.links.len(), 6); + let user_link_count = plan + .links + .iter() + .filter(|link| { + !FrameworkComponentControllerMoniker::is_synthetic_component( + link.provider_component.as_str(), + ) && !FrameworkComponentControllerMoniker::is_synthetic_component( + link.consumer_component.as_str(), + ) + }) + .count(); + assert_eq!(user_link_count, 6); + let controller_assignment_sites = plan + .assignments + .iter() + .filter_map(|(component, site)| { + FrameworkComponentControllerMoniker::is_synthetic_component(component) + .then_some(site.clone()) + }) + .collect::>(); + assert_eq!( + controller_assignment_sites, + BTreeSet::from([ + "compose_b".to_string(), + "compose_e".to_string(), + "direct_local".to_string(), + "kind_c".to_string(), + ]), + "the five-site run plan should assign synthetic controllers for every non-VM active \ + site before any live runtime work starts", + ); } #[test] @@ -2617,6 +4973,43 @@ mod tests { } } + #[tokio::test] + async fn compose_site_artifact_keeps_router_control_socket_paths() { + let dir = tmp_dir("run-plan-compose-router-control-"); + let image_child = dir.path().join("image.json5"); + let root = dir.path().join("root.json5"); + + write(&image_child, image_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { + image: "./image.json5" + }, + exports: { + image_api: "#image.api" + } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let compose_yaml = plan + .sites + .get("compose_local") + .and_then(|site| site.artifact_files.get("compose.yaml")) + .expect("compose site should have a compose artifact"); + + assert!( + compose_yaml + .contains("AMBER_ROUTER_CONTROL_SOCKET_PATH=/amber/control/router-control.sock") + ); + assert!(compose_yaml.contains("control_socket: /router-control.sock")); + assert!(!compose_yaml.contains("/amber/control/site/compose_local/router-control.sock")); + assert!(!compose_yaml.contains("control_socket: /site/compose_local/router-control.sock")); + } + #[tokio::test] async fn build_unmanaged_export_uses_single_matching_site() { let dir = tmp_dir("run-plan-unmanaged-export-single-"); @@ -2716,4 +5109,35 @@ mod tests { .contains_key(&PathBuf::from("kustomization.yaml")) ); } + + #[tokio::test] + async fn build_homogeneous_export_run_plan_rejects_vm_without_controller_site() { + let dir = tmp_dir("run-plan-homogeneous-vm-export-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { + vm: "./vm.json5" + }, + exports: { + vm_api: "#vm.api" + } +}"##, + ); + + let compiled = compile(&root).await; + let err = build_homogeneous_export_run_plan(&compiled, SiteKind::Vm) + .expect_err("homogeneous vm export should require an explicit controlling site"); + let rendered = err.to_string(); + assert!( + rendered.contains("homogeneous vm exports are unsupported"), + "{rendered}" + ); + assert!(rendered.contains("controller_site"), "{rendered}"); + } } diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index cd4a1181..ee023e9d 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -19,6 +19,10 @@ use crate::{ GENERATED_ENV_SAMPLE_FILENAME, GENERATED_README_FILENAME, build_execution_guide, }, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, targets::{ common::{TargetError as MeshError, component_label}, mesh::{ @@ -28,7 +32,7 @@ use crate::{ }, mesh_config::{ MeshAddressing, MeshConfigBuildInput, MeshConfigBuildOptions, RouterPorts, - build_mesh_config_plan, + build_mesh_config_plan, default_mesh_config_build_options, }, plan::{MeshOptions, MeshPlan, build_mesh_plan, map_program_components}, ports::placeholder_local_route_ports, @@ -63,6 +67,13 @@ pub struct DirectArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct DirectArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DirectPlan { pub version: String, @@ -163,6 +174,7 @@ pub enum DirectProgramExecutionPlan { #[serde(default, skip_serializing_if = "Option::is_none")] mount_spec_b64: Option, }, + InternalSiteController, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -209,6 +221,31 @@ pub(crate) fn emit_direct_artifact( compiled: &CompiledScenario, force_router: bool, ) -> Result { + emit_direct_artifact_with_options( + compiled, + DirectArtifactBuildOptions { + force_router, + router_identity_id: ROUTER_IDENTITY_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_direct_artifact_with_options( + compiled: &CompiledScenario, + options: DirectArtifactBuildOptions<'_>, +) -> Result { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "direct_local", + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ) + .map_err(|err| MeshError::new(err.to_string()))?; + let compiled = &lowered; let scenario = compiled.scenario(); let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; let mesh_plan = build_mesh_plan( @@ -235,7 +272,7 @@ pub(crate) fn emit_direct_artifact( let route_ports = placeholder_local_route_ports(scenario, &endpoint_plan, &mesh_plan); let mesh_ports_by_component = placeholder_mesh_ports(program_components); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let router_ports = needs_router.then_some(RouterPorts { mesh: 0, control: 0, @@ -279,11 +316,13 @@ pub(crate) fn emit_direct_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - router_identity_id: ROUTER_IDENTITY_ID, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, component_mesh_listen_addr: "127.0.0.1", router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", - force_router, + force_router: options.force_router, + ..default_mesh_config_build_options() }, })?; @@ -325,7 +364,7 @@ pub(crate) fn emit_direct_artifact( })?; let router_plan = router_ports.map(|ports| DirectRouterPlan { - identity_id: ROUTER_IDENTITY_ID.to_string(), + identity_id: options.router_identity_id.to_string(), mesh_port: ports.mesh, control_port: 0, control_socket_path: DIRECT_CONTROL_SOCKET_RELATIVE_PATH.to_string(), @@ -451,28 +490,29 @@ fn build_component_plans( component.moniker.as_str() )) })?; - let dynamic_caps_port = mesh_config_plan - .component_configs - .get(id) - .and_then(|config| config.dynamic_caps_listen) - .map(|addr| addr.port()) - .ok_or_else(|| { - MeshError::new(format!( - "internal error: missing dynamic caps listen port for {}", - component.moniker.as_str() - )) - })?; let depends_on = mesh_plan .strong_deps() .get(id) .map(|deps| deps.iter().map(|dep| dep.0).collect::>()) .unwrap_or_default(); let source_dir = component_source_dir(compiled, *id, component.moniker.as_str())?; - let execution = resolve_direct_execution_plan( - direct_execution_plan(runtime_plan.execution), - source_dir.as_deref(), - component.moniker.as_str(), - )?; + let execution = if let Some(execution) = direct_internal_execution_plan(component) { + execution + } else { + resolve_direct_execution_plan( + direct_execution_plan(runtime_plan.execution), + source_dir.as_deref(), + component.moniker.as_str(), + )? + }; + let dynamic_caps_port = match execution { + DirectProgramExecutionPlan::InternalSiteController => None, + _ => mesh_config_plan + .component_configs + .get(id) + .and_then(|config| config.dynamic_caps_listen) + .map(|addr| addr.port()), + }; out.push(DirectComponentPlan { id: id.0, @@ -502,19 +542,28 @@ fn build_component_plans( Ok(out) } +fn direct_internal_execution_plan( + component: &amber_scenario::Component, +) -> Option { + framework_component_controller_metadata(component.metadata.as_ref()) + .map(|_| DirectProgramExecutionPlan::InternalSiteController) +} + fn inject_direct_dynamic_caps_env( execution: DirectProgramExecutionPlan, - dynamic_caps_port: u16, + dynamic_caps_port: Option, ) -> DirectProgramExecutionPlan { match execution { DirectProgramExecutionPlan::Direct { entrypoint, mut env, } => { - env.insert( - DYNAMIC_CAPS_API_URL_ENV.to_string(), - format!("http://127.0.0.1:{dynamic_caps_port}"), - ); + if let Some(dynamic_caps_port) = dynamic_caps_port { + env.insert( + DYNAMIC_CAPS_API_URL_ENV.to_string(), + format!("http://127.0.0.1:{dynamic_caps_port}"), + ); + } DirectProgramExecutionPlan::Direct { entrypoint, env } } DirectProgramExecutionPlan::HelperRunner { @@ -530,6 +579,9 @@ fn inject_direct_dynamic_caps_env( runtime_config, mount_spec_b64, }, + DirectProgramExecutionPlan::InternalSiteController => { + DirectProgramExecutionPlan::InternalSiteController + } } } @@ -683,6 +735,9 @@ fn resolve_direct_execution_plan( runtime_config, mount_spec_b64, }), + DirectProgramExecutionPlan::InternalSiteController => { + Ok(DirectProgramExecutionPlan::InternalSiteController) + } } } @@ -1087,7 +1142,6 @@ mod tests { }; use amber_manifest::{Manifest, ManifestRef}; - use amber_mesh::{FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, FRAMEWORK_COMPONENT_CCS_URL_ENV}; use amber_resolver::Resolver; use amber_scenario::{BindingEdge, Component, Moniker, Scenario}; use tempfile::TempDir; @@ -1171,7 +1225,7 @@ mod tests { } #[test] - fn direct_router_passthrough_includes_framework_ccs_auth() { + fn direct_framework_component_is_lowered_to_an_internal_site_controller_component() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let admin_path = dir.path().join("admin.json5"); @@ -1200,9 +1254,6 @@ mod tests { r##" {{ manifest_version: "0.3.0", - slots: {{ - realm: {{ kind: "component", optional: true }} - }}, components: {{ admin: "{admin}" }}, @@ -1236,23 +1287,13 @@ mod tests { .expect("direct plan should be emitted"), ) .expect("direct plan should deserialize"); - let router = direct_plan - .router - .expect("framework.component binding should force a router"); assert!( - router - .env_passthrough - .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CCS_URL_ENV), - "router must receive the framework CCS URL env passthrough", - ); - assert!( - router - .env_passthrough - .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV), - "router must receive the framework CCS auth env passthrough", + direct_plan.components.iter().any(|component| matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + )), + "framework.component should lower to an injected internal site controller component" ); } diff --git a/compiler/src/targets/mesh/addressing.rs b/compiler/src/targets/mesh/addressing.rs index 205fd5d6..c0f27810 100644 --- a/compiler/src/targets/mesh/addressing.rs +++ b/compiler/src/targets/mesh/addressing.rs @@ -184,10 +184,11 @@ impl Addressing for LocalAddressing<'_> { binding: &ResolvedFrameworkBinding, ) -> Result { match binding.capability.as_str() { - "component" => { - let local_port = self.local_framework_binding_port(binding)?; - Ok(format!("http://127.0.0.1:{local_port}")) - } + "component" => Err(MeshError::new(format!( + "framework.component binding {}.{} must be lowered before mesh address planning", + component_label(self.scenario, binding.consumer), + binding.slot, + ))), "docker" => match self.options.docker_binding { DockerFrameworkBindingPolicy::LoopbackTcp => { let local_port = self.local_framework_binding_port(binding)?; @@ -592,7 +593,7 @@ mod tests { } #[test] - fn build_address_plan_maps_framework_component_to_loopback_http() { + fn build_address_plan_rejects_unlowered_framework_component() { let mut consumer = component(0, "/consumer", "consumer"); consumer.slots.insert( "realm".to_string(), @@ -634,16 +635,12 @@ mod tests { docker_binding: DockerFrameworkBindingPolicy::LoopbackTcp, }, ); - let plan = build_address_plan(&mesh_plan, addressing).expect("address plan"); - - let slot_value = plan - .slot_values_by_component - .get(&ComponentId(0)) - .and_then(|slots| slots.get("realm")) - .expect("slot value"); - let SlotValue::One(value) = slot_value else { - panic!("expected singular slot value, got {slot_value:?}"); - }; - assert_eq!(value.url, "http://127.0.0.1:20000"); + let err = build_address_plan(&mesh_plan, addressing) + .expect_err("framework.component should be lowered before address planning"); + assert!( + err.to_string() + .contains("must be lowered before mesh address planning"), + "unexpected address-plan error: {err}", + ); } } diff --git a/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs b/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs index 73ddfe64..926ae9c8 100644 --- a/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs +++ b/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; +use amber_images::AMBER_ROUTER; use amber_manifest::{ProvideDecl, SlotDecl}; use amber_scenario::{ BindingEdge, BindingFrom, Component, ComponentId, Moniker, Program, ProgramMount, ProvideRef, @@ -12,8 +13,6 @@ use crate::targets::common::TargetError as MeshError; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_INTERNAL_SLOT: &str = "__amber_internal_framework_docker"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_ENDPOINT: &str = "docker"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_PROVIDE: &str = "__amber_internal_framework_docker"; -pub(crate) const FRAMEWORK_DOCKER_GATEWAY_IMAGE: &str = "amber-internal://docker-gateway"; -pub(crate) const FRAMEWORK_DOCKER_GATEWAY_ENTRYPOINT: &str = "/amber-docker-gateway"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_PORT: u16 = 23750; const FRAMEWORK_DOCKER_GATEWAY_MONIKER_BASE: &str = "/__amber_internal_framework_docker_gateway"; @@ -145,9 +144,11 @@ pub(crate) fn rewrite_framework_docker_as_injected_component( } fn injected_gateway_program() -> Result { + // The injected component is a mesh identity/provide marker. Docker Compose renders only its + // router sidecar, whose router process owns the gateway runtime. serde_json::from_value(json!({ - "image": FRAMEWORK_DOCKER_GATEWAY_IMAGE, - "entrypoint": [FRAMEWORK_DOCKER_GATEWAY_ENTRYPOINT], + "image": AMBER_ROUTER.reference, + "entrypoint": ["/amber-router"], "network": { "endpoints": [ { diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 8985380c..5d09fe73 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -16,9 +16,7 @@ use serde::{Deserialize, Serialize}; mod framework_docker_injection; -use framework_docker_injection::{ - FRAMEWORK_DOCKER_GATEWAY_PORT, rewrite_framework_docker_as_injected_component, -}; +use framework_docker_injection::rewrite_framework_docker_as_injected_component; use crate::{ config::analysis::ScenarioConfigAnalysis, @@ -29,6 +27,10 @@ use crate::{ build_execution_guide, }, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, runtime_interface::{RootInputDescriptor, collect_root_inputs}, targets::{ mesh::{ @@ -38,8 +40,9 @@ use crate::{ }, internal_images::resolve_internal_images, mesh_config::{ - MeshConfigBuildInput, MeshConfigBuildOptions, MeshServiceName, RouterPorts, - ServiceMeshAddressing, build_mesh_config_plan, default_mesh_config_build_options, + DockerGatewayMeshConfig, MeshConfigBuildInput, MeshConfigBuildOptions, + MeshServiceName, RouterPorts, ServiceMeshAddressing, build_mesh_config_plan, + default_mesh_config_build_options, }, plan::{MeshOptions, component_label, map_program_components}, ports::{LocalRoutePorts, allocate_local_route_ports, allocate_mesh_ports}, @@ -59,13 +62,13 @@ const BOUNDARY_NETWORK_NAME: &str = "amber_boundary"; const ROUTER_SERVICE_NAME: &str = "amber-router"; const ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; +const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; const PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; const HELPER_VOLUME_NAME: &str = "amber-helper-bin"; const HELPER_INIT_SERVICE: &str = "amber-init"; const HELPER_BIN_DIR: &str = "/amber/bin"; const HELPER_BIN_PATH: &str = "/amber/bin/amber-helper"; const DOCKER_MOUNT_PROXY_SPEC_ENV: &str = "AMBER_DOCKER_MOUNT_PROXY_SPEC_B64"; -const DOCKER_GATEWAY_CONFIG_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_JSON"; const DOCKER_GATEWAY_HOST_SOCK_ENV: &str = "AMBER_DOCKER_SOCK"; const DOCKER_GATEWAY_CONTAINER_SOCK: &str = "/var/run/docker.sock"; const MESH_CONFIG_DIR: &str = "/amber/mesh"; @@ -99,6 +102,29 @@ const LOG_LABEL_SERVICE_NAME: &str = "amber_service_name"; const LOG_LABEL_LIST: &str = "amber_component_moniker,amber_service_name"; pub const COMPOSE_FILENAME: &str = GENERATED_COMPOSE_FILENAME; +fn build_control_socket_init_service(volume_name: &str) -> Service { + let mut service = Service::new("busybox:1.36.1".to_string()); + service.user = Some("0:0".to_string()); + service.command = Some(vec![ + "sh".to_string(), + "-lc".to_string(), + format!( + "mkdir -p {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chown \ + {ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID} {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} \ + && chmod 0770 {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + ), + ]); + service.volumes.push(format!( + "{volume_name}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + )); + service.restart = Some("no".to_string()); + service +} + +fn component_control_init_service_name(service_name: &str) -> String { + format!("{service_name}-control-init") +} + #[derive(Clone, Copy, Debug, Default)] pub struct DockerComposeReporter; @@ -107,6 +133,13 @@ pub struct DockerComposeArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct DockerComposeArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + impl DockerComposeArtifact { pub fn compose_yaml(&self) -> &str { self.files @@ -284,23 +317,6 @@ impl MeshServiceName for ServiceNames { } } -#[derive(Clone, Debug, Serialize)] -struct DockerGatewayCallerConfig { - host: String, - #[serde(skip_serializing_if = "Option::is_none")] - port: Option, - component: String, - compose_service: String, -} - -#[derive(Clone, Debug, Serialize)] -struct DockerGatewayConfig { - listen: String, - docker_sock: String, - compose_project: String, - callers: Vec, -} - #[derive(Clone, Debug, Serialize)] struct DockerMountProxySpec { path: String, @@ -349,13 +365,44 @@ pub(crate) fn emit_docker_compose_artifact( compiled: &CompiledScenario, force_router: bool, ) -> Result { - render_docker_compose_inner(compiled.scenario(), force_router) - .map_err(DockerComposeError::into_reporter_error) + emit_docker_compose_artifact_with_options( + compiled, + DockerComposeArtifactBuildOptions { + force_router, + router_identity_id: crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_docker_compose_artifact_with_options( + compiled: &CompiledScenario, + options: DockerComposeArtifactBuildOptions<'_>, +) -> Result { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "compose_local", + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ) + .map_err(|err| ReporterError::new(err.to_string()))?; + render_docker_compose_inner( + lowered.scenario(), + options.force_router, + options.router_identity_id, + options.mesh_scope, + ) + .map_err(DockerComposeError::into_reporter_error) } fn render_docker_compose_inner( scenario: &Scenario, force_router: bool, + router_identity_id: &str, + mesh_scope: Option<&str>, ) -> DcResult { let transformed = rewrite_framework_docker_as_injected_component(scenario).map_err(dc_other)?; let s = &transformed.scenario; @@ -377,7 +424,13 @@ fn render_docker_compose_inner( // Precompute service names (injective & stable). let names: HashMap = map_program_components(s, program_components, |id, local_name| { - let base = service_base_name(id, local_name); + let base = if framework_component_controller_metadata(s.component(id).metadata.as_ref()) + .is_some() + { + SITE_CONTROLLER_SERVICE_NAME.to_string() + } else { + service_base_name(id, local_name) + }; let sidecar = format!("{base}-net"); ServiceNames { program: base.clone(), @@ -435,7 +488,13 @@ fn render_docker_compose_inner( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { + router_identity_id, + mesh_scope, force_router, + docker_gateway: docker_gateway_component.map(|_| DockerGatewayMeshConfig { + docker_sock: DOCKER_GATEWAY_CONTAINER_SOCK, + compose_project_env: COMPOSE_PROJECT_NAME_ENV, + }), ..default_mesh_config_build_options() }, }) @@ -629,22 +688,8 @@ fn render_docker_compose_inner( .entry(ROUTER_CONTROL_SOCKET_VOLUME_NAME.to_string()) .or_insert_with(EmptyMap::default); - let mut control_init_service = Service::new("busybox:1.36.1".to_string()); - control_init_service.user = Some("0:0".to_string()); - control_init_service.command = Some(vec![ - "sh".to_string(), - "-lc".to_string(), - format!( - "mkdir -p {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chown \ - {ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID} \ - {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chmod 0700 \ - {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" - ), - ]); - control_init_service.volumes.push(format!( - "{ROUTER_CONTROL_SOCKET_VOLUME_NAME}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" - )); - control_init_service.restart = Some("no".to_string()); + let control_init_service = + build_control_socket_init_service(ROUTER_CONTROL_SOCKET_VOLUME_NAME); compose.services.insert( ROUTER_CONTROL_INIT_SERVICE_NAME.to_string(), control_init_service, @@ -742,6 +787,8 @@ fn render_docker_compose_inner( let mut sidecar_env_entries = vec![ format!("AMBER_ROUTER_CONFIG_PATH={}", mesh_config_path()), format!("AMBER_ROUTER_IDENTITY_PATH={}", mesh_identity_path()), + format!("AMBER_ROUTER_CONTROL_SOCKET_PATH={ROUTER_CONTROL_SOCKET_PATH_IN_CONTAINER}"), + format!("{COMPOSE_PROJECT_NAME_ENV}=${{{COMPOSE_PROJECT_NAME_ENV}:-default}}"), ]; sidecar_env_entries.extend( mesh_config_plan @@ -751,6 +798,9 @@ fn render_docker_compose_inner( ); push_router_observability_env(&mut sidecar_env_entries); sidecar_service.environment = Some(Environment::List(sidecar_env_entries)); + sidecar_service + .extra_hosts + .push(HOST_GATEWAY_ENTRY.to_string()); sidecar_service .networks .insert(MESH_NETWORK_NAME.to_string(), EmptyMap::default()); @@ -765,22 +815,54 @@ fn render_docker_compose_inner( sidecar_service .volumes .push(format!("{sidecar_volume}:{MESH_CONFIG_DIR}:ro")); + let sidecar_control_volume = component_control_socket_volume_expr(&svc.sidecar); + compose + .volumes + .entry(sidecar_control_volume.clone()) + .or_insert_with(EmptyMap::default); + sidecar_service.volumes.push(format!( + "{sidecar_control_volume}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + )); + if Some(*id) == docker_gateway_component { + sidecar_service.user = Some("0:0".to_string()); + sidecar_service.volumes.push(format!( + "${{{DOCKER_GATEWAY_HOST_SOCK_ENV}:-{}}}:{DOCKER_GATEWAY_CONTAINER_SOCK}", + detect_default_host_docker_sock().display() + )); + } + let sidecar_control_init_service_name = component_control_init_service_name(&svc.sidecar); sidecar_service.depends_on = build_depends_on( false, - vec![( - PROVISIONER_SERVICE_NAME.to_string(), - "service_completed_successfully", - )], + vec![ + ( + PROVISIONER_SERVICE_NAME.to_string(), + "service_completed_successfully", + ), + ( + sidecar_control_init_service_name.clone(), + "service_completed_successfully", + ), + ], ); apply_default_service_hardening(&mut sidecar_service); apply_internal_service_rootfs_hardening(&mut sidecar_service); + compose.services.insert( + sidecar_control_init_service_name, + build_control_socket_init_service(&sidecar_control_volume), + ); compose .services .insert(svc.sidecar.clone(), sidecar_service); let mut egress_init_service = Service::new(images.helper.clone()); egress_init_service.user = Some("0:0".to_string()); - egress_init_service.command = Some(vec!["install-default-egress-guard".to_string()]); + let mesh_port = *mesh_ports_by_component + .get(id) + .expect("mesh port should be allocated before rendering services"); + egress_init_service.command = Some(vec![ + "install-network-guard".to_string(), + mesh_port.to_string(), + ]); egress_init_service.network_mode = Some(format!("service:{}", svc.sidecar)); egress_init_service.cap_add.push("NET_ADMIN".to_string()); egress_init_service.depends_on = @@ -791,10 +873,16 @@ fn render_docker_compose_inner( .services .insert(svc.egress_init.clone(), egress_init_service); + if Some(*id) == docker_gateway_component { + continue; + } + // depends_on: own sidecar + strong deps provider programs (+ amber-init for helper-backed services) let program_plan = program_plans.get(id).expect("program plan computed"); - let image = if Some(*id) == docker_gateway_component { - images.docker_gateway.clone() + let is_site_controller = + framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some(); + let image = if is_site_controller { + images.site_controller.clone() } else { let image_plan = program_plan.image().ok_or_else(|| { DockerComposeError::Other(format!( @@ -846,7 +934,12 @@ fn render_docker_compose_inner( if let Some(ds) = mesh_plan.strong_deps().get(id) { for dep in ds { if let Some(dep_names) = names.get(dep) { - deps.push((dep_names.program.clone(), "service_started")); + let service = if Some(*dep) == docker_gateway_component { + dep_names.sidecar.clone() + } else { + dep_names.program.clone() + }; + deps.push((service, "service_started")); } else { return Err(format!( "internal error: missing service name for dependency {}", @@ -940,10 +1033,6 @@ fn render_docker_compose_inner( program_service.environment = Some(Environment::List(env_entries)); } } - if Some(*id) == docker_gateway_component { - configure_injected_docker_gateway_service(&mut program_service, s, *id, svc)?; - apply_internal_service_rootfs_hardening(&mut program_service); - } for storage_mount in storage_mounts { program_service.volumes.push(format!( "{}:{}", @@ -951,6 +1040,11 @@ fn render_docker_compose_inner( storage_mount.mount_path )); } + if is_site_controller { + program_service + .volumes + .push(format!("{sidecar_volume}:{MESH_CONFIG_DIR}:ro")); + } configure_program_log_shipping( &mut program_service, &label, @@ -1294,39 +1388,6 @@ fn collect_framework_docker_mount_paths( out } -fn configure_injected_docker_gateway_service( - service: &mut Service, - scenario: &Scenario, - id: ComponentId, - names: &ServiceNames, -) -> DcResult<()> { - let config_json = - build_docker_gateway_config_json(scenario.component(id).moniker.as_str(), &names.program)?; - match service.environment.take() { - Some(Environment::Map(mut env)) => { - env.insert(DOCKER_GATEWAY_CONFIG_ENV.to_string(), config_json); - service.environment = Some(Environment::Map(env)); - } - Some(Environment::List(_)) => { - return Err(DockerComposeError::Other( - "internal error: injected docker gateway should not use list-style environment" - .to_string(), - )); - } - None => { - service.environment = Some(Environment::Map(BTreeMap::from([( - DOCKER_GATEWAY_CONFIG_ENV.to_string(), - config_json, - )]))); - } - } - service.volumes.push(format!( - "${{{DOCKER_GATEWAY_HOST_SOCK_ENV}:-{}}}:{DOCKER_GATEWAY_CONTAINER_SOCK}", - detect_default_host_docker_sock().display() - )); - Ok(()) -} - fn docker_proxy_ports_by_component( scenario: &Scenario, route_ports: &LocalRoutePorts, @@ -1369,29 +1430,6 @@ fn encode_docker_mount_proxy_spec_b64( Ok(base64::engine::general_purpose::STANDARD.encode(payload)) } -fn build_docker_gateway_config_json( - caller_component: &str, - caller_compose_service: &str, -) -> DcResult { - let callers = vec![DockerGatewayCallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: caller_component.to_string(), - compose_service: caller_compose_service.to_string(), - }]; - - let config = DockerGatewayConfig { - listen: format!("0.0.0.0:{FRAMEWORK_DOCKER_GATEWAY_PORT}"), - docker_sock: DOCKER_GATEWAY_CONTAINER_SOCK.to_string(), - compose_project: "${COMPOSE_PROJECT_NAME}".to_string(), - callers, - }; - - serde_json::to_string(&config).map_err(|err| { - DockerComposeError::Other(format!("failed to serialize docker gateway config: {err}")) - }) -} - fn detect_default_host_docker_sock() -> PathBuf { #[cfg(target_os = "macos")] { @@ -1540,6 +1578,10 @@ fn compose_control_socket_volume_expr() -> String { ) } +fn component_control_socket_volume_expr(service_name: &str) -> String { + format!("{service_name}-control") +} + fn sanitize_service_suffix(s: &str) -> String { let mut out = String::new(); for ch in s.chars() { diff --git a/compiler/src/targets/mesh/docker_compose/tests/mod.rs b/compiler/src/targets/mesh/docker_compose/tests/mod.rs index 944495cd..d96f70f6 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/mod.rs @@ -13,7 +13,9 @@ use amber_manifest::{ CapabilityKind, FrameworkCapabilityName, Manifest, ManifestDigest, ManifestRef, Program as ManifestProgram, ProvideDecl, SlotDecl, }; -use amber_mesh::{InboundTarget, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget}; +use amber_mesh::{ + InboundTarget, MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, +}; use amber_scenario::{ BindingEdge, BindingFrom, Component, ComponentId, Moniker, ProvideRef, ResourceDecl, Scenario, ScenarioExport, SlotRef, @@ -24,6 +26,7 @@ use url::Url; use super::{DockerComposeReporter, *}; use crate::{ + CompileOptions, Compiler, DigestStore, OptimizeOptions, linker::program_lowering::lower_program, targets::{mesh::internal_images::resolve_internal_images, storage::StorageIdentity}, }; @@ -51,6 +54,73 @@ fn render_compose( DockerComposeReporter.emit(&compiled_scenario(output)) } +fn compile_framework_component_output(root_path: &Path) -> crate::CompileOutput { + let compiler = Compiler::new(amber_resolver::Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let url = Url::from_file_path(root_path).expect("file url"); + rt.block_on(compiler.compile(ManifestRef::from_url(url), opts)) + .expect("compile scenario") +} + +#[test] +fn lowered_framework_component_controller_uses_real_internal_image_in_compose_artifacts() { + let dir = tempfile::tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + fs::write( + &root_path, + r##" + { + manifest_version: "0.1.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ] + } + "##, + ) + .expect("write root manifest"); + fs::write( + &admin_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-lc", "sleep 3600"] + }, + slots: { ctl: { kind: "component" } } + } + "#, + ) + .expect("write admin manifest"); + + let output = compile_framework_component_output(&root_path); + let artifact = render_compose(&output).expect("render compose artifact"); + let compose_yaml = artifact.compose_yaml(); + let site_controller_image = internal_images().site_controller; + + assert!( + compose_yaml.contains(&format!("image: {site_controller_image}")), + "synthetic controller components should use the real site-controller image: {compose_yaml}" + ); + assert!( + compose_yaml.contains("amber-site-controller:") + && compose_yaml.contains("amber-site-controller-net:"), + "the lowered controller should render as an ordinary program+sidecar workload: \ + {compose_yaml}" + ); + assert!( + !compose_yaml.contains("__amber_internal/site-controller"), + "compose artifacts must not leak the internal site-controller placeholder image: \ + {compose_yaml}" + ); +} + #[test] fn render_compose_image_uses_root_default_fallback() { let leaves = [rc::SchemaLeaf { @@ -391,16 +461,17 @@ fn assert_internal_service_rootfs_hardened(service: &super::Service, yaml: &str) assert_eq!(service.read_only, Some(true), "{yaml}"); } -fn injected_docker_gateway_service(compose: &super::DockerComposeFile) -> (&str, &super::Service) { +fn injected_docker_gateway_sidecar(compose: &super::DockerComposeFile) -> (&str, &super::Service) { compose .services .iter() .find_map(|(name, svc)| { - env_value(svc, super::DOCKER_GATEWAY_CONFIG_ENV) - .is_some() + svc.volumes + .iter() + .any(|volume| volume.ends_with(super::DOCKER_GATEWAY_CONTAINER_SOCK)) .then_some((name.as_str(), svc)) }) - .expect("injected docker gateway service missing") + .expect("injected docker gateway sidecar missing") } fn assert_depends_on(service: &super::Service, name: &str, condition: &str) { diff --git a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs index 68a576c8..6cd7f2e6 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs @@ -124,6 +124,79 @@ fn compose_storage_volume_names_include_identity_hash() { assert!(child_name.starts_with("amber-storage-root-state-")); } +#[test] +fn compose_sidecars_expose_control_socket_volume() { + let program = lower_test_program( + 0, + json!({ + "image": "alpine:3.20", + "entrypoint": ["sh", "-lc", "sleep infinity"], + "env": {} + }), + ); + let root = Component { + id: ComponentId(0), + parent: None, + moniker: moniker("/"), + digest: digest(0), + config: None, + config_schema: None, + program: Some(program), + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + }; + let artifact = render_compose(&compile_output(Scenario { + manifest_catalog: BTreeMap::new(), + root: ComponentId(0), + components: vec![Some(root)], + bindings: Vec::new(), + exports: Vec::new(), + })) + .expect("compose render ok"); + let compose = parse_compose(&artifact); + let sidecar_name = compose + .services + .keys() + .find_map(|name| { + let name = name.as_str(); + name.ends_with("-net").then_some(name) + }) + .expect("sidecar service"); + let sidecar = service(&compose, sidecar_name); + assert_eq!( + env_value(sidecar, "AMBER_ROUTER_CONTROL_SOCKET_PATH").as_deref(), + Some("/amber/control/router-control.sock") + ); + assert!( + sidecar + .volumes + .iter() + .any(|mount| mount.ends_with(":/amber/control")), + "sidecar should mount a writable control socket volume: {:?}", + sidecar.volumes + ); + let control_init_name = format!("{sidecar_name}-control-init"); + assert_depends_on( + sidecar, + &control_init_name, + "service_completed_successfully", + ); + let control_init_service = service(&compose, &control_init_name); + assert_eq!(control_init_service.user.as_deref(), Some("0:0")); + assert!( + control_init_service + .volumes + .iter() + .any(|mount| mount.ends_with(":/amber/control")), + "sidecar control init should mount the sidecar control volume: {:?}", + control_init_service.volumes + ); +} + #[test] fn compose_emits_otelcol_agent_and_wires_router_otel_env() { let program = lower_test_program( @@ -274,6 +347,13 @@ fn compose_emits_otelcol_agent_and_wires_router_otel_env() { .contains_key("amber_egress_c0-component-net"), "{yaml}" ); + assert!( + sidecar + .extra_hosts + .iter() + .any(|entry| entry == super::HOST_GATEWAY_ENTRY), + "{yaml}" + ); assert!( !sidecar.networks.contains_key(super::BOUNDARY_NETWORK_NAME), "{yaml}" @@ -283,6 +363,14 @@ fn compose_emits_otelcol_agent_and_wires_router_otel_env() { Some("service:c0-component-net"), "{yaml}" ); + assert_eq!( + egress_init.command.as_ref(), + Some(&vec![ + "install-network-guard".to_string(), + "23000".to_string() + ]), + "{yaml}" + ); assert_eq!(egress_init.user.as_deref(), Some("0:0"), "{yaml}"); assert!( egress_init.cap_add.iter().any(|cap| cap == "NET_ADMIN"), @@ -500,31 +588,17 @@ fn docker_compose_emits_gateway_for_framework_docker_binding() { "framework routing should not emit a bespoke docker gateway service" ); - let (gateway_name, gateway) = injected_docker_gateway_service(&compose); - let expected_network_mode = format!("service:{gateway_name}-net"); - assert_eq!( - gateway.network_mode.as_deref(), - Some(expected_network_mode.as_str()), - ); - assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); - - let gateway_config = env_value(gateway, super::DOCKER_GATEWAY_CONFIG_ENV) - .expect("gateway config env should be present"); - let gateway_config_json: Value = - serde_json::from_str(&gateway_config).expect("gateway config should parse"); - let callers = gateway_config_json["callers"] - .as_array() - .expect("callers should be an array"); - assert_eq!(callers.len(), 1); - assert_eq!(callers[0]["host"], "127.0.0.1"); - let gateway_component = callers[0]["component"] - .as_str() - .expect("gateway caller component should be a string"); + let (gateway_name, gateway) = injected_docker_gateway_sidecar(&compose); + assert_eq!(gateway.network_mode.as_deref(), None); + assert_eq!(gateway.user.as_deref(), Some("0:0")); assert!( - gateway_component.starts_with("/__amber_internal_framework_docker_gateway"), - "unexpected injected gateway component moniker: {gateway_component}" + gateway + .volumes + .iter() + .any(|volume| volume.ends_with(super::DOCKER_GATEWAY_CONTAINER_SOCK)), + "{yaml}" ); - assert_eq!(callers[0]["compose_service"], gateway_name); + assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); let program_service = service(&compose, "c0-component"); assert_depends_on(program_service, gateway_name, "service_started"); @@ -534,6 +608,125 @@ fn docker_compose_emits_gateway_for_framework_docker_binding() { ); } +#[test] +fn docker_compose_shared_framework_docker_gateway_preserves_consumer_identities() { + let slot_docker: SlotDecl = serde_json::from_value(json!({ "kind": "docker" })).unwrap(); + let root = Component { + id: ComponentId(0), + parent: None, + moniker: moniker("/"), + digest: digest(0), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: vec![ComponentId(1), ComponentId(2)], + }; + let child = |id: usize, name: &str| Component { + id: ComponentId(id), + parent: Some(ComponentId(0)), + moniker: moniker(&format!("/{name}")), + digest: digest(id as u8), + config: None, + config_schema: None, + program: Some(lower_test_program( + id, + json!({ + "image": "alpine:3.20", + "entrypoint": ["sh", "-lc", "sleep infinity"], + "env": { + "DOCKER_HOST": "${slots.docker.url}", + }, + }), + )), + slots: BTreeMap::from([("docker".to_string(), slot_docker.clone())]), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + }; + + let scenario = Scenario { + manifest_catalog: BTreeMap::new(), + root: ComponentId(0), + components: vec![Some(root), Some(child(1, "alpha")), Some(child(2, "beta"))], + bindings: vec![ + BindingEdge { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: ComponentId(0), + capability: FrameworkCapabilityName::try_from("docker").unwrap(), + }), + to: SlotRef { + component: ComponentId(1), + name: "docker".to_string(), + }, + weak: false, + }, + BindingEdge { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: ComponentId(0), + capability: FrameworkCapabilityName::try_from("docker").unwrap(), + }), + to: SlotRef { + component: ComponentId(2), + name: "docker".to_string(), + }, + weak: false, + }, + ], + exports: Vec::new(), + }; + + let output = compile_output_with_docker_feature(scenario); + let yaml = render_compose(&output).expect("compose render should succeed"); + let compose = parse_compose(&yaml); + let plan = provision_plan(&compose); + let (gateway_name, _gateway) = injected_docker_gateway_sidecar(&compose); + let gateway_target = target_for_service(&plan, gateway_name); + let gateway_inbound = gateway_target + .config + .inbound + .iter() + .find(|route| { + route.capability == super::framework_docker_injection::FRAMEWORK_DOCKER_GATEWAY_PROVIDE + }) + .expect("gateway inbound docker route"); + assert_eq!(gateway_inbound.protocol, MeshProtocol::Tcp); + match &gateway_inbound.target { + amber_mesh::InboundTarget::DockerGateway { + docker_sock, + compose_project_env, + } => { + assert_eq!(docker_sock, Path::new(super::DOCKER_GATEWAY_CONTAINER_SOCK)); + assert_eq!(compose_project_env, super::COMPOSE_PROJECT_NAME_ENV); + } + other => panic!("unexpected docker gateway target: {other:?}"), + } + assert_eq!(gateway_inbound.allowed_issuers.len(), 2); + + for service_name in ["c1-alpha", "c2-beta"] { + let program_service = service(&compose, service_name); + assert_depends_on(program_service, gateway_name, "service_started"); + assert_eq!( + env_value(program_service, "DOCKER_HOST").as_deref(), + Some("tcp://127.0.0.1:20000"), + ); + let target = target_for_service(&plan, &format!("{service_name}-net")); + let outbound = target + .config + .outbound + .iter() + .find(|route| route.slot == "docker") + .expect("consumer outbound docker route"); + assert_eq!(outbound.protocol, MeshProtocol::Tcp); + } +} + #[test] fn docker_compose_emits_framework_docker_mount_proxy_wiring() { let program = lower_test_program( @@ -586,12 +779,9 @@ fn docker_compose_emits_framework_docker_mount_proxy_wiring() { "framework routing should not emit a bespoke docker gateway service" ); - let (gateway_name, gateway) = injected_docker_gateway_service(&compose); - let expected_network_mode = format!("service:{gateway_name}-net"); - assert_eq!( - gateway.network_mode.as_deref(), - Some(expected_network_mode.as_str()), - ); + let (gateway_name, gateway) = injected_docker_gateway_sidecar(&compose); + assert_eq!(gateway.network_mode.as_deref(), None); + assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); let program_service = service(&compose, "c0-component"); let entrypoint = program_service @@ -1221,6 +1411,13 @@ fn compose_emits_export_metadata_and_labels() { ); let control_init_service = service(&compose, "amber-router-control-init"); assert_eq!(control_init_service.user.as_deref(), Some("0:0")); + assert!( + control_init_service + .command + .as_ref() + .is_some_and(|command| command.iter().any(|arg| arg.contains("chmod 0770"))), + "router control dir should be group-accessible to the injected site controller" + ); assert!( control_init_service .volumes diff --git a/compiler/src/targets/mesh/internal_images.rs b/compiler/src/targets/mesh/internal_images.rs index b408761b..169a0735 100644 --- a/compiler/src/targets/mesh/internal_images.rs +++ b/compiler/src/targets/mesh/internal_images.rs @@ -1,15 +1,14 @@ -use std::{collections::HashSet, env}; - -use amber_images::{AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, ImageRef}; - -const DEV_IMAGE_TAGS_ENV: &str = "AMBER_DEV_IMAGE_TAGS"; +use amber_images::{ + AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, + INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, parse_dev_image_tag_overrides, +}; #[derive(Clone, Debug)] pub(crate) struct InternalImages { pub(crate) helper: String, pub(crate) provisioner: String, pub(crate) router: String, - pub(crate) docker_gateway: String, + pub(crate) site_controller: String, } pub(crate) fn resolve_internal_images() -> Result { @@ -17,71 +16,18 @@ pub(crate) fn resolve_internal_images() -> Result { helper: default_reference(&AMBER_HELPER), provisioner: default_reference(&AMBER_PROVISIONER), router: default_reference(&AMBER_ROUTER), - docker_gateway: default_reference(&AMBER_DOCKER_GATEWAY), - }; - - let raw = match env::var(DEV_IMAGE_TAGS_ENV) { - Ok(value) => value, - Err(env::VarError::NotPresent) => return Ok(images), - Err(err) => { - return Err(format!("failed to read {DEV_IMAGE_TAGS_ENV}: {err}")); - } + site_controller: default_reference(&AMBER_SITE_CONTROLLER), }; - let raw = raw.trim(); - if raw.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} is set but empty; expected format \ - \"router=,helper=,provisioner=,docker_gateway=\"" - )); - } - - let mut seen = HashSet::new(); - for entry in raw.split(',') { - let entry = entry.trim(); - if entry.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains an empty entry; expected format \ - \"router=,helper=,provisioner=,docker_gateway=\"" - )); - } - - let mut parts = entry.splitn(2, '='); - let key = parts.next().unwrap().trim(); - let value = parts.next().ok_or_else(|| { - format!("{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing '='; expected key=value") - })?; - let value = value.trim(); - - if key.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a key; expected key=value" - )); - } - if value.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a tag; expected key=tag" - )); - } - if !seen.insert(key.to_string()) { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains duplicate key \"{key}\"" - )); - } - - match key { - "router" => images.router = override_reference(&AMBER_ROUTER, value), - "helper" => images.helper = override_reference(&AMBER_HELPER, value), - "provisioner" => images.provisioner = override_reference(&AMBER_PROVISIONER, value), - "docker_gateway" => { - images.docker_gateway = override_reference(&AMBER_DOCKER_GATEWAY, value) - } - _ => { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains unknown key \"{key}\"; expected router, \ - helper, provisioner, docker_gateway" - )); + for (key, value) in parse_dev_image_tag_overrides(INTERNAL_IMAGE_OVERRIDE_KEYS)? { + match key.as_str() { + "router" => images.router = override_reference(&AMBER_ROUTER, &value), + "helper" => images.helper = override_reference(&AMBER_HELPER, &value), + "provisioner" => images.provisioner = override_reference(&AMBER_PROVISIONER, &value), + "site_controller" => { + images.site_controller = override_reference(&AMBER_SITE_CONTROLLER, &value) } + _ => unreachable!("shared image override parser returned an unknown key"), } } @@ -91,7 +37,3 @@ pub(crate) fn resolve_internal_images() -> Result { fn default_reference(image: &ImageRef) -> String { image.reference.to_string() } - -fn override_reference(image: &ImageRef, tag: &str) -> String { - format!("{}/{}:{}", image.registry, image.name, tag) -} diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index 7a4dc1ca..f4f507b7 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -26,6 +26,10 @@ use crate::{ CompiledScenario, Reporter, ReporterError, execution_guide::{GENERATED_README_FILENAME, build_execution_guide}, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, runtime_interface::{RootInputDescriptor, build_runtime_interface}, targets::{ mesh::{ @@ -66,6 +70,7 @@ const MESH_CONFIG_DIR: &str = "/amber/mesh"; const MESH_SECRET_VOLUME_NAME: &str = "amber-mesh"; const ROUTER_NAME: &str = "amber-router"; const PROVISIONER_NAME: &str = "amber-provisioner"; +const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; const PROVISIONER_CONFIGMAP_NAME: &str = "amber-mesh-provision"; const PROVISIONER_SERVICE_ACCOUNT: &str = "amber-provisioner"; const PROVISIONER_ROLE_NAME: &str = "amber-provisioner"; @@ -111,6 +116,13 @@ pub struct KubernetesArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct KubernetesArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + impl Reporter for KubernetesReporter { type Artifact = KubernetesArtifact; @@ -172,6 +184,31 @@ pub(crate) fn emit_kubernetes_artifact( compiled: &CompiledScenario, force_router: bool, ) -> KubernetesResult { + emit_kubernetes_artifact_with_options( + compiled, + KubernetesArtifactBuildOptions { + force_router, + router_identity_id: crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_kubernetes_artifact_with_options( + compiled: &CompiledScenario, + options: KubernetesArtifactBuildOptions<'_>, +) -> KubernetesResult { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "kubernetes_local", + SiteDefinition { + kind: SiteKind::Kubernetes, + context: None, + controller_site: None, + }, + ) + .map_err(|err| ReporterError::new(err.to_string()))?; + let compiled = &lowered; let s = compiled.scenario(); let scenario_digest = scenario_ir_digest(s).map_err(|err| ReporterError::new(err.to_string()))?; @@ -216,14 +253,20 @@ pub(crate) fn emit_kubernetes_artifact( let names: HashMap = map_program_components(s, program_components, |id, local_name| { - let base = service_name(id, local_name); + let base = if framework_component_controller_metadata(s.component(id).metadata.as_ref()) + .is_some() + { + SITE_CONTROLLER_SERVICE_NAME.to_string() + } else { + service_name(id, local_name) + }; ComponentNames { service: base.clone(), netpol: format!("{base}-netpol"), } }); let provisioner_job_name = provisioner_job_name(&scenario_digest); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let route_ports = allocate_local_route_ports(s, &endpoint_plan, &mesh_plan) .map_err(|e| ReporterError::new(e.to_string()))?; @@ -274,7 +317,9 @@ pub(crate) fn emit_kubernetes_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - force_router, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, + force_router: options.force_router, ..default_mesh_config_build_options() }, }) @@ -291,16 +336,7 @@ pub(crate) fn emit_kubernetes_artifact( name: mesh_secret_name(ROUTER_NAME), namespace: None, }, - |router_config| { - if let Some(control_listen) = router_config.control_listen { - router_config.control_listen = Some( - format!("127.0.0.1:{}", control_listen.port()) - .parse() - .expect("control listen"), - ); - } - router_config.control_allow = Some(vec!["127.0.0.1".to_string(), "::1".to_string()]); - }, + |_| {}, ) .map_err(|err| ReporterError::new(err.to_string()))?; @@ -717,6 +753,8 @@ pub(crate) fn emit_kubernetes_artifact( let mesh_port = *mesh_ports_by_component.get(id).expect("mesh port missing"); let label = component_label(s, *id); let pod_annotations = component_pod_annotations(&label); + let is_site_controller = + framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some(); let image_origin = program_plan.image_origin().ok_or_else(|| { ReporterError::new(format!( "internal error: {} is missing a container image origin", @@ -729,14 +767,18 @@ pub(crate) fn emit_kubernetes_artifact( component_label(s, *id) )) })?; - let image_source = program_image_source(compiled, s, *id, image_origin); - let (program_image, image_source_env_var) = render_kubernetes_image( - image_plan, - &root_leaf_by_path, - &cnames.service, - &label, - image_source.as_ref(), - )?; + let (program_image, image_source_env_var) = if is_site_controller { + (images.site_controller.clone(), None) + } else { + let image_source = program_image_source(compiled, s, *id, image_origin); + render_kubernetes_image( + image_plan, + &root_leaf_by_path, + &cnames.service, + &label, + image_source.as_ref(), + )? + }; let mount_specs = config_plan.mount_specs.get(id).map(Vec::as_slice); let runtime_plan = build_component_runtime_plan( &label, @@ -871,6 +913,13 @@ pub(crate) fn emit_kubernetes_artifact( MESH_SECRET_VOLUME_NAME.to_string(), mesh_secret, )); + if is_site_controller { + container.volume_mounts.push(VolumeMount { + name: MESH_SECRET_VOLUME_NAME.to_string(), + mount_path: MESH_CONFIG_DIR.to_string(), + read_only: Some(true), + }); + } let mut sidecar_env = vec![ EnvVar::literal( @@ -1132,6 +1181,12 @@ pub(crate) fn emit_kubernetes_artifact( ); if !router_service_ports.is_empty() { + router_service_ports.push(ServicePort { + name: "control".to_string(), + port: ROUTER_CONTROL_PORT_BASE, + target_port: ROUTER_CONTROL_PORT_BASE, + protocol: "TCP", + }); let service = Service::new( ROUTER_NAME, &namespace, @@ -1233,8 +1288,13 @@ pub(crate) fn emit_kubernetes_artifact( } let egress_from_consumers = egress_allow.get(id); - let egress_to_router = egress_router_allow.get(id); - if egress_from_consumers.is_some() || egress_to_router.is_some() { + let mut egress_to_router = egress_router_allow.get(id).cloned().unwrap_or_default(); + if framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some() + && needs_router + { + egress_to_router.insert(ROUTER_CONTROL_PORT_BASE); + } + if egress_from_consumers.is_some() || !egress_to_router.is_empty() { netpol.add_egress_rule(NetworkPolicyEgressRule { to: vec![NetworkPolicyPeer { pod_selector: None, @@ -1296,7 +1356,7 @@ pub(crate) fn emit_kubernetes_artifact( } } - if let Some(ports) = egress_to_router { + if !egress_to_router.is_empty() { netpol.add_egress_rule(NetworkPolicyEgressRule { to: vec![NetworkPolicyPeer { pod_selector: Some(LabelSelector { @@ -1305,7 +1365,7 @@ pub(crate) fn emit_kubernetes_artifact( namespace_selector: None, ip_block: None, }], - ports: ports + ports: egress_to_router .iter() .map(|port| NetworkPolicyPort { protocol: "TCP", diff --git a/compiler/src/targets/mesh/kubernetes/tests/mod.rs b/compiler/src/targets/mesh/kubernetes/tests/mod.rs index bad07973..360626b7 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/mod.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeSet, fs, net::SocketAddr, path::{Path, PathBuf}, @@ -60,6 +61,114 @@ fn render_artifact(output: &crate::CompileOutput) -> super::KubernetesArtifact { .expect("render kubernetes output") } +fn compile_framework_component_output(root_path: &Path) -> crate::CompileOutput { + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let url = Url::from_file_path(root_path).expect("file url"); + rt.block_on(compiler.compile(ManifestRef::from_url(url), opts)) + .expect("compile scenario") +} + +#[test] +fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes_artifacts() { + let dir = tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + fs::write( + &root_path, + r##" + { + manifest_version: "0.1.0", + slots: { external: { kind: "http" } }, + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" }, + { to: "#admin.external", from: "self.external", weak: true } + ] + } + "##, + ) + .expect("write root manifest"); + fs::write( + &admin_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-lc", "sleep 3600"] + }, + slots: { + ctl: { kind: "component" }, + external: { kind: "http" } + } + } + "#, + ) + .expect("write admin manifest"); + + let output = compile_framework_component_output(&root_path); + let artifact = render_artifact(&output); + let deployment = artifact + .files + .get(&PathBuf::from("03-deployments/amber-site-controller.yaml")) + .expect("site controller deployment"); + let service = artifact + .files + .get(&PathBuf::from("04-services/amber-site-controller.yaml")) + .expect("site controller service"); + let netpol = artifact + .files + .get(&PathBuf::from( + "05-networkpolicies/amber-site-controller-netpol.yaml", + )) + .expect("site controller network policy"); + let site_controller_image = internal_images().site_controller; + + assert!( + deployment.contains(&site_controller_image), + "synthetic controller deployment should use the real site-controller image: {deployment}" + ); + assert!( + !deployment.contains("__amber_internal/site-controller"), + "kubernetes artifacts must not leak the internal site-controller placeholder image: \ + {deployment}" + ); + assert!( + service.contains("name: amber-site-controller"), + "the lowered controller should render an ordinary service for the controller workload: \ + {service}" + ); + let netpol_doc: serde_yaml::Value = + serde_yaml::from_str(netpol).expect("parse site controller network policy"); + let egress = netpol_doc["spec"]["egress"] + .as_sequence() + .expect("site controller network policy should have egress rules"); + let has_router_control_egress = egress.iter().any(|rule| { + let to_router = rule["to"].as_sequence().is_some_and(|peers| { + peers.iter().any(|peer| { + peer["podSelector"]["matchLabels"]["amber.io/component"].as_str() + == Some("amber-router") + }) + }); + let has_control_port = rule["ports"].as_sequence().is_some_and(|ports| { + ports.iter().any(|port| { + port["protocol"].as_str() == Some("TCP") && port["port"].as_u64() == Some(24100) + }) + }); + to_router && has_control_port + }); + assert!( + has_router_control_egress, + "site controller must be able to reach the local router control port without opening a \ + broader egress surface: {netpol}" + ); +} + fn parse_rendered_env(content: &str) -> std::collections::BTreeMap { content .lines() @@ -295,10 +404,22 @@ fn write_kubernetes_smoke_fixture(root: &Path) -> PathBuf { "\ mkdir content\n\ cd content\n\ - wget '${slots.server.url}/runtime_secret.txt'\n\ - wget '${slots.server.url}/runtime_config.txt'\n\ - wget '${slots.server.url}/static_secret.txt'\n\ - wget '${slots.server.url}/static_config.txt'\n\ + fetch() {\n\ + path=\"$1\"\n\ + attempts=0\n\ + until wget -O \"$path\" '${slots.server.url}/'\"$path\"; do\n\ + rm -f \"$path\"\n\ + attempts=$((attempts + 1))\n\ + if [ \"$attempts\" -ge 60 ]; then\n\ + exit 1\n\ + fi\n\ + sleep 1\n\ + done\n\ + }\n\ + fetch runtime_secret.txt\n\ + fetch runtime_config.txt\n\ + fetch static_secret.txt\n\ + fetch static_config.txt\n\ httpd -f -p 8080\n\ ", ], @@ -910,6 +1031,63 @@ fn kubectl_logs(namespace: &str, pod: &str, kubeconfig: &Path) -> String { } } +fn wait_for_pod_http( + namespace: &str, + pod: &str, + container: &str, + url: &str, + kubeconfig: &Path, + timeout: Duration, +) -> String { + let deadline = Instant::now() + timeout; + let mut last_status = String::new(); + let mut last_stdout = String::new(); + let mut last_stderr = String::new(); + + loop { + let output = kubectl_cmd(kubeconfig) + .arg("exec") + .arg("-n") + .arg(namespace) + .arg(pod) + .arg("-c") + .arg(container) + .arg("--") + .arg("wget") + .arg("-q") + .arg("-O") + .arg("-") + .arg(url) + .output(); + match output { + Ok(output) if output.status.success() => { + return String::from_utf8_lossy(&output.stdout).trim().to_string(); + } + Ok(output) => { + last_status = output.status.to_string(); + last_stdout = String::from_utf8_lossy(&output.stdout).to_string(); + last_stderr = String::from_utf8_lossy(&output.stderr).to_string(); + } + Err(err) => { + last_status = format!("failed to run kubectl exec: {err}"); + last_stdout.clear(); + last_stderr.clear(); + } + } + + if Instant::now() >= deadline { + let pod_logs = kubectl_logs(namespace, pod, kubeconfig); + panic!( + "pod HTTP endpoint {url} did not become ready in container {container}\nlast \ + status: {last_status}\nlast stdout:\n{last_stdout}\nlast \ + stderr:\n{last_stderr}\npod logs:\n{pod_logs}" + ); + } + + thread::sleep(Duration::from_secs(1)); + } +} + fn fetch( url: &str, port_forward: &mut PortForwardGuard, diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 8e746a88..de767309 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -163,6 +163,16 @@ fn kubernetes_emits_router_for_external_slots() { .get(&PathBuf::from("04-services/amber-router.yaml")) .expect("router service"); assert!(router_service.contains("port: 24000"), "{router_service}"); + assert!(router_service.contains("port: 24100"), "{router_service}"); + assert!(router_service.contains("name: control"), "{router_service}"); + let mesh_provision = artifact + .files + .get(&PathBuf::from("01-configmaps/amber-mesh-provision.yaml")) + .expect("mesh provision configmap"); + assert!( + mesh_provision.contains("\"control_listen\": \"0.0.0.0:24100\""), + "{mesh_provision}" + ); let router_env = artifact .files @@ -270,6 +280,114 @@ fn kubernetes_emits_router_for_external_slots() { ); } +#[test] +fn kubernetes_network_policy_only_exposes_component_mesh_port() { + let dir = tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let server_path = dir.path().join("server.json5"); + let client_path = dir.path().join("client.json5"); + + fs::write( + &root_path, + format!( + r##" + {{ + manifest_version: "0.1.0", + components: {{ + server: "{}", + client: "{}" + }}, + bindings: [ + {{ to: "#client.api", from: "#server.api" }} + ] + }} + "##, + file_url(&server_path), + file_url(&client_path), + ), + ) + .expect("write root manifest"); + + fs::write( + &server_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "server", + entrypoint: ["server"], + network: { + endpoints: [{ name: "api", port: 8080, protocol: "http" }] + } + }, + provides: { api: { kind: "http", endpoint: "api" } }, + exports: { api: "api" } + } + "#, + ) + .expect("write server manifest"); + fs::write( + &client_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "client", + entrypoint: ["client"], + env: { API_URL: "${slots.api.url}" } + }, + slots: { api: { kind: "http" } } + } + "#, + ) + .expect("write client manifest"); + + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let output = rt + .block_on(compiler.compile(ManifestRef::from_url(file_url(&root_path)), opts)) + .expect("compile scenario"); + let artifact = render_artifact(&output); + + let service_yaml = artifact + .files + .get(&PathBuf::from("04-services/c2-server.yaml")) + .expect("server service"); + let service_doc: serde_yaml::Value = + serde_yaml::from_str(service_yaml).expect("parse server service"); + let service_ports = service_doc["spec"]["ports"] + .as_sequence() + .expect("service ports") + .iter() + .map(|port| port["port"].as_u64().expect("service port")) + .collect::>(); + assert_eq!(service_ports, BTreeSet::from([23000])); + + let netpol_yaml = artifact + .files + .get(&PathBuf::from("05-networkpolicies/c2-server-netpol.yaml")) + .expect("server network policy"); + let netpol_doc: serde_yaml::Value = + serde_yaml::from_str(netpol_yaml).expect("parse server network policy"); + let ingress_ports = netpol_doc["spec"]["ingress"] + .as_sequence() + .expect("ingress rules") + .iter() + .flat_map(|rule| rule["ports"].as_sequence().into_iter().flatten()) + .map(|port| port["port"].as_u64().expect("network policy port")) + .collect::>(); + assert_eq!(ingress_ports, BTreeSet::from([23000])); + assert!( + !netpol_yaml.contains("8080"), + "program endpoint port must not be directly reachable through \ + NetworkPolicy:\n{netpol_yaml}" + ); +} + #[test] fn kubernetes_storage_mounts_emit_pvc_and_recreate_deployment() { let fixture_dir = tempdir().expect("fixture dir"); @@ -702,7 +820,7 @@ fn kubernetes_emits_otelcol_and_wires_otel_env() { } #[test] -fn kubernetes_templates_dynamic_caps_sidecar_control_env() { +fn kubernetes_templates_omit_dynamic_caps_sidecar_control_env_without_local_controller() { let dir = tempdir().expect("temp dir"); let root_path = dir.path().join("root.json5"); let worker_path = dir.path().join("worker.json5"); @@ -753,57 +871,38 @@ fn kubernetes_templates_dynamic_caps_sidecar_control_env() { }) .expect("worker deployment"); assert!( - component_deploy.contains("AMBER_DYNAMIC_CAPS_API_URL"), + !component_deploy.contains("AMBER_DYNAMIC_CAPS_API_URL"), "{component_deploy}" ); assert!( - component_deploy.contains("http://127.0.0.1:19000"), + !component_deploy.contains("http://127.0.0.1:19000"), "{component_deploy}" ); assert!( - component_deploy.contains("amber-component-sidecar-env"), + !component_deploy.contains("amber-component-sidecar-env"), "{component_deploy}" ); - let sidecar_env = artifact + if let Some(sidecar_env) = artifact .files .get(&PathBuf::from(super::COMPONENT_SIDECAR_ENV_FILE)) - .expect("component sidecar env template"); - assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_CONTROL_URL="), - "{sidecar_env}" - ); - assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_CONTROL_AUTH_TOKEN="), - "{sidecar_env}" - ); - assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64="), - "{sidecar_env}" - ); + { + assert!( + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), + "controller URLs must not be injected as ambient sidecar env: {sidecar_env}" + ); + assert!( + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), + "controller auth must not be injected as ambient sidecar env: {sidecar_env}" + ); + } let kustomization = artifact .files .get(&PathBuf::from("kustomization.yaml")) .expect("kustomization"); assert!( - kustomization.contains("name: amber-component-sidecar-env"), - "{kustomization}" - ); - assert!( - kustomization.contains(super::COMPONENT_SIDECAR_ENV_FILE), + !kustomization.contains("amber-component-sidecar-env"), "{kustomization}" ); - assert!( - !kustomization.contains(&format!( - "resources:\n- {}", - super::COMPONENT_SIDECAR_ENV_FILE - )), - "{kustomization}" - ); - assert!( - !kustomization.contains(&format!("\n- {}\n", super::COMPONENT_SIDECAR_ENV_FILE)), - "kustomization resources must not treat the sidecar env template as a manifest: \ - {kustomization}" - ); } diff --git a/compiler/src/targets/mesh/kubernetes/tests/storage.rs b/compiler/src/targets/mesh/kubernetes/tests/storage.rs index e1b0181c..c08b03b2 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/storage.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/storage.rs @@ -372,6 +372,16 @@ fn kubernetes_smoke_config_roundtrip() { .arg(&client_pod); checked_status(&mut cmd, "kubectl wait for client pod"); + let client_runtime_secret = wait_for_pod_http( + &namespace, + &client_pod, + "main", + "http://127.0.0.1:8080/runtime_secret.txt", + &kubeconfig, + Duration::from_secs(120), + ); + assert_eq!(client_runtime_secret, "test-secret-value"); + let port_forward_log = dir.path().join("port-forward.log"); let mut port_forward = PortForwardGuard::new(&namespace, &client_pod, &port_forward_log, &kubeconfig); diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index a19e7bfe..326cbe14 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -1,42 +1,54 @@ use std::collections::{BTreeSet, HashMap}; -use amber_manifest::NetworkProtocol; +use amber_manifest::{CapabilityKind, NetworkProtocol}; use amber_mesh::{ - DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, DYNAMIC_CAPS_CONTROL_URL_ENV, - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, - FRAMEWORK_COMPONENT_CCS_URL_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, - MeshIdentityTemplate, MeshPeerTemplate, MeshProtocol, OutboundRoute, component_route_id, - framework_cap_instance_id, http_route_plugins_for_capability_kind, router_export_route_id, - router_external_route_id, telemetry::SCENARIO_RUN_ID_ENV, + InboundRoute, InboundTarget, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, + MeshProtocol, OutboundRoute, component_route_id, http_route_plugins_for_capability_kind, + router_export_route_id, router_external_route_id, telemetry::SCENARIO_RUN_ID_ENV, }; use amber_scenario::{ComponentId, Scenario}; use base64::Engine as _; use sha2::Digest as _; use super::{ - plan::{MeshError, MeshPlan, component_label}, + plan::{EndpointInfo, MeshError, MeshPlan, component_label}, ports::LocalRoutePorts, }; -use crate::runtime_interface::collect_external_slots; +use crate::{ + run_plan::{ + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, framework_component_controller_metadata, + }, + runtime_interface::collect_external_slots, +}; pub(crate) const DEFAULT_ROUTER_ID: &str = "/router"; #[derive(Clone, Copy, Debug)] pub(crate) struct MeshConfigBuildOptions<'a> { pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, pub(crate) component_mesh_listen_addr: &'a str, pub(crate) router_mesh_listen_addr: &'a str, pub(crate) router_control_listen_addr: &'a str, pub(crate) force_router: bool, + pub(crate) docker_gateway: Option>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct DockerGatewayMeshConfig<'a> { + pub(crate) docker_sock: &'a str, + pub(crate) compose_project_env: &'a str, } pub(crate) fn default_mesh_config_build_options() -> MeshConfigBuildOptions<'static> { MeshConfigBuildOptions { router_identity_id: DEFAULT_ROUTER_ID, + mesh_scope: None, component_mesh_listen_addr: "0.0.0.0", router_mesh_listen_addr: "0.0.0.0", router_control_listen_addr: "0.0.0.0", force_router: false, + docker_gateway: None, } } @@ -143,7 +155,10 @@ pub(crate) fn build_mesh_config_plan( return Err(MeshError::new("router ports missing")); } - let mesh_scope = scenario_mesh_scope(scenario)?; + let mesh_scope = match options.mesh_scope { + Some(mesh_scope) => mesh_scope.to_string(), + None => scenario_mesh_scope(scenario)?, + }; let mut identities_by_component: HashMap = HashMap::new(); for &id in mesh_plan.program_components() { @@ -163,6 +178,10 @@ pub(crate) fn build_mesh_config_plan( None }; + let local_controller_component = mesh_plan.program_components().iter().copied().find(|id| { + framework_component_controller_metadata(scenario.component(*id).metadata.as_ref()).is_some() + }); + let mut consumers_by_provider: HashMap<(ComponentId, String), BTreeSet> = HashMap::new(); for binding in mesh_plan.component_bindings() { @@ -188,18 +207,6 @@ pub(crate) fn build_mesh_config_plan( let mut component_configs: HashMap = HashMap::new(); let mut component_sidecar_env_passthrough = Vec::new(); push_env_passthrough_once(&mut component_sidecar_env_passthrough, SCENARIO_RUN_ID_ENV); - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_CONTROL_URL_ENV, - ); - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, - ); - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - ); for &id in mesh_plan.program_components() { let identity = identities_by_component .get(&id) @@ -214,10 +221,34 @@ pub(crate) fn build_mesh_config_plan( let mut inbound = Vec::new(); for (provide_name, provide_decl) in &scenario.component(id).provides { - let endpoint = mesh_plan - .component_bindings() - .find(|binding| binding.provider == id && binding.provide == *provide_name) - .map(|b| b.endpoint.clone()) + let endpoint = (Some(id) == local_controller_component + && provide_name == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE) + .then(|| { + provide_decl.endpoint.as_deref().and_then(|endpoint_name| { + scenario + .component(id) + .program + .as_ref() + .and_then(|program| program.network()) + .and_then(|network| { + network + .endpoints + .iter() + .find(|endpoint| endpoint.name == endpoint_name) + }) + .map(|endpoint| EndpointInfo { + port: endpoint.port, + protocol: endpoint.protocol, + }) + }) + }) + .flatten() + .or_else(|| { + mesh_plan + .component_bindings() + .find(|binding| binding.provider == id && binding.provide == *provide_name) + .map(|b| b.endpoint.clone()) + }) .or_else(|| { mesh_plan .exports() @@ -228,6 +259,7 @@ pub(crate) fn build_mesh_config_plan( let Some(endpoint) = endpoint else { continue; }; + let capability_kind = provide_decl.decl.kind.as_str(); let protocol = mesh_protocol(endpoint.protocol)?; let mut issuers: BTreeSet = BTreeSet::new(); @@ -241,6 +273,24 @@ pub(crate) fn build_mesh_config_plan( issuers.insert(consumer_id); } } + if Some(id) == local_controller_component + && provide_name == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE + { + for consumer in mesh_plan.program_components().iter().copied() { + if consumer == id { + continue; + } + let consumer_id = identities_by_component + .get(&consumer) + .expect("consumer identity missing") + .id + .clone(); + issuers.insert(consumer_id); + } + if let Some(router_identity) = router_identity.as_ref() { + issuers.insert(router_identity.id.clone()); + } + } let route_backs_live_http_authority = !issuers.is_empty() && matches!(protocol, MeshProtocol::Http); if (route_backs_live_http_authority @@ -253,6 +303,24 @@ pub(crate) fn build_mesh_config_plan( continue; } + let target = if capability_kind == CapabilityKind::Docker.as_str() { + let docker_gateway = input.options.docker_gateway.ok_or_else(|| { + MeshError::new(format!( + "mesh config for {}.{} requires docker gateway runtime configuration", + component_label(scenario, id), + provide_name + )) + })?; + InboundTarget::DockerGateway { + docker_sock: docker_gateway.docker_sock.into(), + compose_project_env: docker_gateway.compose_project_env.to_string(), + } + } else { + InboundTarget::Local { + port: endpoint.port, + } + }; + inbound.push(InboundRoute { route_id: component_route_id(&identity.id, provide_name, protocol), capability: provide_name.clone(), @@ -263,9 +331,7 @@ pub(crate) fn build_mesh_config_plan( Some(provide_decl.decl.kind.as_str()), protocol, ), - target: InboundTarget::Local { - port: endpoint.port, - }, + target, allowed_issuers: issuers.into_iter().collect(), }); } @@ -288,12 +354,12 @@ pub(crate) fn build_mesh_config_plan( .expect("provider identity missing") .id .clone(); - let protocol = mesh_protocol(binding.endpoint.protocol)?; let provide_decl = scenario .component(binding.provider) .provides .get(&binding.provide) .expect("binding provide should exist"); + let protocol = mesh_protocol(binding.endpoint.protocol)?; outbound.push(OutboundRoute { route_id: component_route_id(&peer_id, &binding.provide, protocol), rewrite_route_id: None, @@ -350,59 +416,65 @@ pub(crate) fn build_mesh_config_plan( }); } - for binding in mesh_plan.framework_bindings() { - if binding.consumer != id || binding.capability.as_str() != "component" { - continue; - } - let listen_port = route_ports.framework_binding_port(binding).ok_or_else(|| { + if let Some(binding) = mesh_plan + .framework_bindings() + .find(|binding| binding.consumer == id && binding.capability.as_str() == "component") + { + return Err(MeshError::new(format!( + "framework.component binding {}.{} must be lowered before mesh config generation", + component_label(scenario, id), + binding.slot, + ))); + } + + if let Some(controller_id) = local_controller_component + && controller_id != id + { + let listen_port = route_ports.controller_internal_port(id).ok_or_else(|| { MeshError::new(format!( - "route port missing for {}.{}", - component_label(scenario, id), - binding.slot + "site controller control route port missing for {}", + component_label(scenario, id) )) })?; - let router_identity = router_identity.as_ref().ok_or_else(|| { - MeshError::new("framework.component bindings require router identity") - })?; - let router_addr = addressing.mesh_addr_for_router()?; - let slot_decl = scenario - .component(binding.consumer) - .slots - .get(binding.slot.as_str()) - .expect("framework binding target slot should exist"); - let authority_moniker = scenario.component(binding.authority_realm).moniker.as_str(); - let consumer_moniker = scenario.component(binding.consumer).moniker.as_str(); - let route_id = framework_cap_instance_id( - authority_moniker, - consumer_moniker, - &binding.consumer.0.to_string(), - &binding.slot, - binding.capability.as_str(), - ); + let peer_addr = addressing.mesh_addr_for_component(controller_id)?; + let peer_id = identities_by_component + .get(&controller_id) + .expect("controller identity missing") + .id + .clone(); outbound.push(OutboundRoute { - route_id, + route_id: component_route_id( + &peer_id, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + MeshProtocol::Http, + ), rewrite_route_id: None, - slot: binding.slot.clone(), - capability_kind: Some(slot_decl.decl.kind.to_string()), - capability_profile: slot_decl.decl.profile.clone(), + slot: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + capability_kind: None, + capability_profile: None, listen_port, listen_addr: None, protocol: MeshProtocol::Http, http_plugins: Vec::new(), - peer_addr: router_addr, - peer_id: router_identity.id.clone(), - capability: binding.capability.to_string(), + peer_addr, + peer_id, + capability: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), }); } let mesh_listen = format!("{}:{mesh_port}", options.component_mesh_listen_addr) .parse() .expect("mesh listen"); - let dynamic_caps_listen = route_ports.dynamic_caps_port(id).map(|port| { - format!("{}:{port}", options.component_mesh_listen_addr) - .parse() - .expect("dynamic caps listen") - }); + let dynamic_caps_listen = + if local_controller_component.is_some() && Some(id) != local_controller_component { + route_ports.dynamic_caps_port(id).map(|port| { + format!("{}:{port}", options.component_mesh_listen_addr) + .parse() + .expect("dynamic caps listen") + }) + } else { + None + }; let config_peers = required_peers(&identity.id, &inbound, &outbound); let config = MeshConfigTemplate { @@ -470,12 +542,12 @@ pub(crate) fn build_mesh_config_plan( .expect("export provider identity missing") .id .clone(); - let protocol = mesh_protocol(export.endpoint.protocol)?; let provide_decl = scenario .component(export.provider) .provides .get(&export.provide) .expect("export provide should exist"); + let protocol = mesh_protocol(export.endpoint.protocol)?; let provider_route_id = component_route_id(&peer_id, &export.provide, protocol); inbound.push(InboundRoute { route_id: router_export_route_id(&export.name, protocol), @@ -494,52 +566,6 @@ pub(crate) fn build_mesh_config_plan( }); } - let framework_bindings = mesh_plan - .framework_bindings() - .filter(|binding| binding.capability.as_str() == "component") - .collect::>(); - if !framework_bindings.is_empty() { - push_env_passthrough_once(&mut router_env_passthrough, FRAMEWORK_COMPONENT_CCS_URL_ENV); - push_env_passthrough_once( - &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, - ); - } - for binding in framework_bindings { - let slot_decl = scenario - .component(binding.consumer) - .slots - .get(binding.slot.as_str()) - .expect("framework binding target slot should exist"); - let consumer_id = identities_by_component - .get(&binding.consumer) - .expect("framework binding consumer identity missing") - .id - .clone(); - let authority_moniker = scenario.component(binding.authority_realm).moniker.as_str(); - let consumer_moniker = scenario.component(binding.consumer).moniker.as_str(); - let route_id = framework_cap_instance_id( - authority_moniker, - consumer_moniker, - &binding.consumer.0.to_string(), - &binding.slot, - binding.capability.as_str(), - ); - inbound.push(InboundRoute { - route_id, - capability: binding.capability.to_string(), - capability_kind: Some(slot_decl.decl.kind.to_string()), - capability_profile: slot_decl.decl.profile.clone(), - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), - optional: false, - }, - allowed_issuers: vec![consumer_id], - }); - } - let mesh_listen = format!("{}:{router_mesh_port}", options.router_mesh_listen_addr) .parse() .expect("mesh listen"); @@ -647,7 +673,7 @@ mod tests { sync::Arc, }; - use amber_manifest::{ManifestDigest, ProvideDecl, SlotDecl}; + use amber_manifest::{FrameworkCapabilityName, ManifestDigest, ProvideDecl, SlotDecl}; use amber_scenario::{Component, ComponentId, Moniker, ProvideRef, Scenario, ScenarioExport}; use serde_json::json; @@ -656,6 +682,7 @@ mod tests { mesh::{ plan::{ EndpointInfo, MeshPlan, ResolvedBinding, ResolvedComponentBinding, ResolvedExport, + ResolvedFrameworkBinding, }, ports::allocate_local_route_ports, }, @@ -843,4 +870,180 @@ mod tests { "unused local routes must stay closed to the router and should not be emitted", ); } + + #[test] + fn build_mesh_config_plan_opens_controller_internal_route_to_local_sidecars_and_router() { + let consumer = component( + 0, + "/consumer", + json!({ + "image": "consumer", + "entrypoint": ["consumer"], + }), + ); + let controller_moniker = + crate::run_plan::FrameworkComponentControllerMoniker::for_site("site-a").into_string(); + let mut controller = component( + 1, + controller_moniker.as_str(), + json!({ + "image": "controller", + "entrypoint": ["controller"], + "network": { + "endpoints": [ + { "name": "framework_component", "port": 8080, "protocol": "http" } + ] + } + }), + ); + controller.metadata = Some( + serde_json::to_value(crate::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: "site-a".to_string(), + grants: BTreeMap::new(), + }) + .expect("controller metadata"), + ); + controller.provides.insert( + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + serde_json::from_value::(json!({ + "kind": "component", + "endpoint": "framework_component", + })) + .expect("internal provide decl"), + ); + + let scenario = Scenario { + root: ComponentId(0), + components: vec![Some(consumer), Some(controller)], + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }; + let mesh_plan = MeshPlan::new( + vec![ComponentId(0), ComponentId(1)], + Vec::new(), + Vec::new(), + HashMap::new(), + ); + let endpoint_plan = build_endpoint_plan(&scenario).expect("endpoint plan"); + let route_ports = allocate_local_route_ports(&scenario, &endpoint_plan, &mesh_plan) + .expect("local route ports"); + let mesh_ports_by_component = + HashMap::from([(ComponentId(0), 23000), (ComponentId(1), 23001)]); + + let plan = build_mesh_config_plan(MeshConfigBuildInput { + scenario: &scenario, + mesh_plan: &mesh_plan, + route_ports: &route_ports, + mesh_ports_by_component: &mesh_ports_by_component, + router_ports: Some(RouterPorts { + mesh: 24000, + control: 24100, + }), + addressing: &StaticAddressing, + options: MeshConfigBuildOptions { + force_router: true, + router_identity_id: "/site/test/router", + ..default_mesh_config_build_options() + }, + }) + .expect("mesh config plan"); + + let controller_config = plan + .component_configs + .get(&ComponentId(1)) + .expect("controller config"); + let internal_route = controller_config + .inbound + .iter() + .find(|route| route.capability == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE) + .expect("controller internal inbound route"); + assert_eq!( + internal_route.allowed_issuers, + vec!["/consumer".to_string(), "/site/test/router".to_string()], + "controller internal routing must admit local component sidecars and the site router", + ); + assert!( + controller_config.dynamic_caps_listen.is_none(), + "synthetic controller sidecars should not expose the ordinary dynamic caps listener", + ); + + let consumer_config = plan + .component_configs + .get(&ComponentId(0)) + .expect("consumer config"); + assert!( + consumer_config.dynamic_caps_listen.is_some(), + "ordinary components should still expose the dynamic caps listener", + ); + } + + #[test] + fn build_mesh_config_plan_rejects_unlowered_framework_component_bindings() { + let mut consumer = component( + 0, + "/consumer", + json!({ + "image": "consumer", + "entrypoint": ["consumer"], + }), + ); + consumer.slots.insert( + "realm".to_string(), + serde_json::from_value::(json!({ + "kind": "component", + })) + .expect("slot decl"), + ); + + let scenario = Scenario { + root: ComponentId(0), + components: vec![Some(consumer)], + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }; + + let mesh_plan = MeshPlan::new( + vec![ComponentId(0)], + vec![ResolvedBinding::Framework(ResolvedFrameworkBinding { + consumer: ComponentId(0), + slot: "realm".to_string(), + authority_realm: ComponentId(0), + capability: FrameworkCapabilityName::try_from("component") + .expect("framework capability"), + })], + Vec::new(), + HashMap::new(), + ); + + let endpoint_plan = build_endpoint_plan(&scenario).expect("endpoint plan"); + let route_ports = allocate_local_route_ports(&scenario, &endpoint_plan, &mesh_plan) + .expect("local route ports"); + let mesh_ports_by_component = HashMap::from([(ComponentId(0), 23000)]); + + let err = build_mesh_config_plan(MeshConfigBuildInput { + scenario: &scenario, + mesh_plan: &mesh_plan, + route_ports: &route_ports, + mesh_ports_by_component: &mesh_ports_by_component, + router_ports: Some(RouterPorts { + mesh: 24000, + control: 24100, + }), + addressing: &StaticAddressing, + options: MeshConfigBuildOptions { + router_identity_id: "/site/test/router", + ..default_mesh_config_build_options() + }, + }) + .expect_err("framework.component routes must be lowered before mesh config generation"); + + assert!( + err.to_string() + .contains("must be lowered before mesh config generation"), + "unexpected mesh config error: {err}" + ); + } } diff --git a/compiler/src/targets/mesh/plan.rs b/compiler/src/targets/mesh/plan.rs index 8df17ee8..40df25ba 100644 --- a/compiler/src/targets/mesh/plan.rs +++ b/compiler/src/targets/mesh/plan.rs @@ -158,12 +158,10 @@ impl MeshPlan { } pub(crate) fn needs_router(&self) -> bool { - self.bindings.iter().any(|binding| { - binding.as_external().is_some() - || binding - .as_framework() - .is_some_and(|binding| binding.capability.as_str() == "component") - }) || !self.exports.is_empty() + self.bindings + .iter() + .any(|binding| binding.as_external().is_some()) + || !self.exports.is_empty() } pub(crate) fn exports(&self) -> &[ResolvedExport] { diff --git a/compiler/src/targets/mesh/ports.rs b/compiler/src/targets/mesh/ports.rs index b6c619be..4bfb6934 100644 --- a/compiler/src/targets/mesh/ports.rs +++ b/compiler/src/targets/mesh/ports.rs @@ -19,6 +19,7 @@ pub(crate) struct LocalRoutePorts { slot_ports_by_component: HashMap>, reserved_ports_by_component: HashMap>, dynamic_caps_ports_by_component: HashMap, + controller_internal_ports_by_component: HashMap, component_binding_ports: HashMap, u16>, external_binding_ports: HashMap, u16>, framework_binding_ports: HashMap, u16>, @@ -45,6 +46,12 @@ impl LocalRoutePorts { .copied() } + pub(crate) fn controller_internal_port(&self, component: ComponentId) -> Option { + self.controller_internal_ports_by_component + .get(&component) + .copied() + } + pub(crate) fn component_binding_port(&self, binding: &ResolvedComponentBinding) -> Option { self.component_binding_ports .get(&BindingIdentity::from(binding)) @@ -198,6 +205,8 @@ pub(crate) fn allocate_local_route_ports( for id in mesh_plan.program_components() { let mut allocator = ComponentRoutePortAllocator::new(endpoint_plan, *id); let dynamic_caps_port = allocator.allocate_reserved(scenario, *id, "dynamic caps api")?; + let controller_internal_port = + allocator.allocate_reserved(scenario, *id, "site controller control route")?; for binding in mesh_plan.bindings_for_consumer(*id) { let port = allocator.allocate(scenario, *id, binding.slot())?; @@ -220,6 +229,8 @@ pub(crate) fn allocate_local_route_ports( let (reserved_ports, slot_ports) = allocator.finish(); out.dynamic_caps_ports_by_component .insert(*id, dynamic_caps_port); + out.controller_internal_ports_by_component + .insert(*id, controller_internal_port); out.reserved_ports_by_component.insert(*id, reserved_ports); out.slot_ports_by_component.insert(*id, slot_ports); } @@ -239,6 +250,9 @@ pub(crate) fn placeholder_local_route_ports( let dynamic_caps_port = allocator .allocate_reserved(scenario, *id, "dynamic caps api") .expect("dynamic caps api port allocation should not overflow"); + let controller_internal_port = allocator + .allocate_reserved(scenario, *id, "site controller control route") + .expect("site controller control route port allocation should not overflow"); for binding in mesh_plan.bindings_for_consumer(*id) { let port = allocator @@ -263,6 +277,8 @@ pub(crate) fn placeholder_local_route_ports( let (reserved, slot_ports) = allocator.finish(); out.dynamic_caps_ports_by_component .insert(*id, dynamic_caps_port); + out.controller_internal_ports_by_component + .insert(*id, controller_internal_port); out.reserved_ports_by_component.insert(*id, reserved); out.slot_ports_by_component.insert(*id, slot_ports); } diff --git a/compiler/src/targets/vm/mod.rs b/compiler/src/targets/vm/mod.rs index 13db0b0f..881df328 100644 --- a/compiler/src/targets/vm/mod.rs +++ b/compiler/src/targets/vm/mod.rs @@ -31,7 +31,7 @@ use crate::{ }, mesh_config::{ MeshAddressing, MeshConfigBuildInput, MeshConfigBuildOptions, RouterPorts, - build_mesh_config_plan, + build_mesh_config_plan, default_mesh_config_build_options, }, plan::{MeshOptions, MeshPlan, build_mesh_plan, map_program_components}, ports::placeholder_local_route_ports, @@ -68,6 +68,13 @@ pub struct VmArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct VmArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct VmPlan { pub version: String, @@ -201,8 +208,34 @@ impl Reporter for VmReporter { pub(crate) fn emit_vm_artifact( compiled: &CompiledScenario, force_router: bool, +) -> Result { + emit_vm_artifact_with_options( + compiled, + VmArtifactBuildOptions { + force_router, + router_identity_id: ROUTER_IDENTITY_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_vm_artifact_with_options( + compiled: &CompiledScenario, + options: VmArtifactBuildOptions<'_>, ) -> Result { let scenario = compiled.scenario(); + if scenario.bindings.iter().any(|binding| { + matches!( + &binding.from, + amber_scenario::BindingFrom::Framework(framework) + if framework.capability.as_str() == "component" + ) + }) { + return Err(MeshError::new( + "vm reporter does not support framework.component without an explicit controlling \ + direct site; use mixed-site placement instead", + )); + } let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; let mesh_plan = build_mesh_plan( scenario, @@ -248,7 +281,7 @@ pub(crate) fn emit_vm_artifact( let route_ports = placeholder_local_route_ports(scenario, &endpoint_plan, &mesh_plan); let mesh_ports_by_component = placeholder_mesh_ports(program_components); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let router_ports = needs_router.then_some(RouterPorts { mesh: 0, control: 0, @@ -292,11 +325,13 @@ pub(crate) fn emit_vm_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - router_identity_id: ROUTER_IDENTITY_ID, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, component_mesh_listen_addr: "127.0.0.1", router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", - force_router, + force_router: options.force_router, + ..default_mesh_config_build_options() }, })?; @@ -338,7 +373,7 @@ pub(crate) fn emit_vm_artifact( })?; let router_plan = router_ports.map(|ports| VmRouterPlan { - identity_id: ROUTER_IDENTITY_ID.to_string(), + identity_id: options.router_identity_id.to_string(), mesh_port: ports.mesh, control_port: 0, control_socket_path: DIRECT_CONTROL_SOCKET_RELATIVE_PATH.to_string(), @@ -1084,3 +1119,88 @@ fn component_source_dir( fn vm_runtime_addresses_is_empty(plan: &DirectRuntimeAddressPlan) -> bool { plan.slots_by_scope.is_empty() && plan.slot_items_by_scope.is_empty() } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use amber_manifest::ManifestRef; + use amber_resolver::Resolver; + use tempfile::TempDir; + use url::Url; + + use super::*; + use crate::{CompileOptions, Compiler, DigestStore}; + + fn write_file(path: &Path, contents: &str) { + std::fs::write(path, contents).expect("fixture file should write"); + } + + fn file_url(path: &Path) -> Url { + Url::from_file_path(path).expect("fixture path should convert to file url") + } + + #[test] + fn vm_reporter_rejects_framework_component_without_a_controller_site() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + + write_file( + &admin_path, + r#" + { + manifest_version: "0.3.0", + slots: { + ctl: { kind: "component" } + }, + program: { + path: "/bin/echo", + args: ["admin", "${slots.ctl.url}"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" } + } + "#, + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ + admin: "{admin}" + }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + exports: {{ + admin_http: "#admin.http" + }} + }} + "##, + admin = file_url(&admin_path), + ), + ); + + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + let output = runtime + .block_on(compiler.compile( + ManifestRef::from_url(file_url(&root_path)), + CompileOptions::default(), + )) + .expect("fixture should compile"); + let compiled = CompiledScenario::from_compile_output(&output) + .expect("fixture should materialize compiled scenario"); + let err = emit_vm_artifact(&compiled, false) + .expect_err("vm reporter should reject framework.component"); + assert!( + err.to_string().contains("explicit controlling direct site"), + "vm reporter should explain that framework.component requires a controlling direct \ + site: {err}", + ); + } +} diff --git a/docker/amber-cli/Dockerfile b/docker/amber-cli/Dockerfile index 25037867..750cb64a 100644 --- a/docker/amber-cli/Dockerfile +++ b/docker/amber-cli/Dockerfile @@ -40,6 +40,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -48,19 +49,21 @@ COPY runtime/scenario-runner/Cargo.toml runtime/scenario-runner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src runtime/scenario-runner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs runtime/scenario-runner/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/scenario-runner/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/scenario-runner/src/lib.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY cli ./cli COPY compiler ./compiler COPY examples ./examples COPY images ./images +COPY runtime/docker-gateway ./runtime/docker-gateway COPY runtime/helper ./runtime/helper COPY runtime/mesh ./runtime/mesh COPY runtime/proxy ./runtime/proxy COPY runtime/router ./runtime/router COPY runtime/scenario-runner ./runtime/scenario-runner +COPY runtime/site-controller ./runtime/site-controller COPY docker/images.json docker/images.json ARG BUILD_MODE=release @@ -70,19 +73,21 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/app/target,sharing=locked \ target=$(cat /tmp/rust-target) && \ if [ "$BUILD_MODE" = "release" ]; then \ - AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked --release -p amber-cli --target "${target}"; \ + AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked --release -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=release; \ else \ - AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked -p amber-cli --target "${target}"; \ + AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=debug; \ fi && \ install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber /out/amber && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-site-controller /out/amber-site-controller && \ install -d -m 0755 /out/workdir FROM gcr.io/distroless/static-debian13 AS runtime COPY --from=builder --chown=65532:65532 /out/workdir /app COPY --from=builder --chown=65532:65532 /out/amber /amber +COPY --from=builder --chown=65532:65532 /out/amber-site-controller /amber-site-controller USER 65532:65532 WORKDIR /app diff --git a/docker/amber-helper/Dockerfile b/docker/amber-helper/Dockerfile index 14523a56..0a72c9d2 100644 --- a/docker/amber-helper/Dockerfile +++ b/docker/amber-helper/Dockerfile @@ -39,6 +39,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -47,8 +48,8 @@ COPY runtime/scenario-runner/Cargo.toml runtime/scenario-runner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src runtime/scenario-runner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs runtime/scenario-runner/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/scenario-runner/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/scenario-runner/src/lib.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-provisioner/Dockerfile b/docker/amber-provisioner/Dockerfile index 090e906c..8607df9f 100644 --- a/docker/amber-provisioner/Dockerfile +++ b/docker/amber-provisioner/Dockerfile @@ -37,13 +37,14 @@ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ COPY runtime/scenario-runner/Cargo.toml runtime/scenario-runner/ -RUN mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/scenario-runner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/scenario-runner/src/lib.rs +RUN mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/scenario-runner/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/scenario-runner/src/lib.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs RUN cargo fetch --locked COPY compiler ./compiler @@ -51,9 +52,16 @@ COPY runtime/mesh ./runtime/mesh COPY runtime/provisioner ./runtime/provisioner COPY runtime/scenario-runner ./runtime/scenario-runner +ARG BUILD_MODE=release RUN target=$(cat /tmp/rust-target) && \ - cargo build -p amber-provisioner --release --locked --target "${target}" && \ - install -D -m 0755 /app/target/"${target}"/release/amber-provisioner /out/amber-provisioner && \ + if [ "$BUILD_MODE" = "release" ]; then \ + cargo build -p amber-provisioner --release --locked --target "${target}"; \ + build_dir=release; \ + else \ + cargo build -p amber-provisioner --locked --target "${target}"; \ + build_dir=debug; \ + fi && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-provisioner /out/amber-provisioner && \ mkdir -p /out/amber/plan /out/amber/provision && \ touch /out/amber/plan/mesh-provision-plan.json diff --git a/docker/amber-router/Dockerfile b/docker/amber-router/Dockerfile index 54f96db8..db038b97 100644 --- a/docker/amber-router/Dockerfile +++ b/docker/amber-router/Dockerfile @@ -39,6 +39,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -47,11 +48,12 @@ COPY runtime/scenario-runner/Cargo.toml runtime/scenario-runner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src runtime/scenario-runner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs runtime/scenario-runner/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/scenario-runner/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/scenario-runner/src/lib.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler +COPY runtime/docker-gateway ./runtime/docker-gateway COPY runtime/mesh ./runtime/mesh COPY runtime/router ./runtime/router COPY runtime/scenario-runner ./runtime/scenario-runner diff --git a/docker/amber-docker-gateway/Dockerfile b/docker/amber-site-controller/Dockerfile similarity index 60% rename from docker/amber-docker-gateway/Dockerfile rename to docker/amber-site-controller/Dockerfile index 2f55f6a5..79bf4038 100644 --- a/docker/amber-docker-gateway/Dockerfile +++ b/docker/amber-site-controller/Dockerfile @@ -26,8 +26,8 @@ RUN case "${TARGETARCH}" in \ FROM base-builder AS builder -# Prefetch Rust dependencies for better Docker layer caching. COPY Cargo.toml Cargo.lock ./ +COPY README.md ./ COPY cli/Cargo.toml cli/ COPY compiler/config/Cargo.toml compiler/config/ COPY compiler/Cargo.toml compiler/ @@ -38,21 +38,32 @@ COPY compiler/json5/Cargo.toml compiler/json5/ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ -COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ +COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/scenario-runner/Cargo.toml runtime/scenario-runner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/scenario-runner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/scenario-runner/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/scenario-runner/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/scenario-runner/src/lib.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ cargo fetch --locked +COPY cli ./cli +COPY compiler ./compiler +COPY examples ./examples +COPY images ./images COPY runtime/docker-gateway ./runtime/docker-gateway +COPY runtime/helper ./runtime/helper +COPY runtime/mesh ./runtime/mesh +COPY runtime/proxy ./runtime/proxy +COPY runtime/router ./runtime/router COPY runtime/scenario-runner ./runtime/scenario-runner +COPY runtime/site-controller ./runtime/site-controller +COPY docker/images.json docker/images.json ARG BUILD_MODE=release RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ @@ -60,16 +71,20 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/app/target,sharing=locked \ target=$(cat /tmp/rust-target) && \ if [ "$BUILD_MODE" = "release" ]; then \ - cargo build --locked --release -p amber-docker-gateway --target "${target}"; \ + cargo build --locked --release -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=release; \ else \ - cargo build --locked -p amber-docker-gateway --target "${target}"; \ + cargo build --locked -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=debug; \ fi && \ - install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-docker-gateway /out/amber-docker-gateway + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber /out/amber && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-site-controller /out/amber-site-controller + +FROM docker:28-cli AS runtime -FROM gcr.io/distroless/static-debian13 AS runtime +RUN apk add --no-cache kubectl -COPY --from=builder /out/amber-docker-gateway /amber-docker-gateway +COPY --from=builder /out/amber /usr/local/bin/amber +COPY --from=builder /out/amber-site-controller /usr/local/bin/amber-site-controller -ENTRYPOINT ["/amber-docker-gateway"] +ENTRYPOINT ["/usr/local/bin/amber-site-controller"] diff --git a/docker/images.json b/docker/images.json index 7631a4bc..2bcda4f1 100644 --- a/docker/images.json +++ b/docker/images.json @@ -17,18 +17,18 @@ "name": "amber-router", "context": ".", "dockerfile": "docker/amber-router/Dockerfile", - "version": "v0.1.x" + "version": "v0.2.x" }, { - "name": "amber-docker-gateway", + "name": "amber-provisioner", "context": ".", - "dockerfile": "docker/amber-docker-gateway/Dockerfile", + "dockerfile": "docker/amber-provisioner/Dockerfile", "version": "v0.1.x" }, { - "name": "amber-provisioner", + "name": "amber-site-controller", "context": ".", - "dockerfile": "docker/amber-provisioner/Dockerfile", + "dockerfile": "docker/amber-site-controller/Dockerfile", "version": "v0.1.x" } ] diff --git a/examples/observability-debug/README.md b/examples/observability-debug/README.md index e2c79fcd..5a692410 100644 --- a/examples/observability-debug/README.md +++ b/examples/observability-debug/README.md @@ -1,16 +1,17 @@ # Observability Debug Tutorial -This example is for reading a scenario the way a user experiences it. +This example is for debugging an Amber scenario as a user thinks about it: -The scenario has two components and one external upstream: +- components: `client`, `server` +- user-facing edge refs: `/client.server_api -> /server.api`, `/client.ext_api -> external.ext_api`, `public` -- `public -> /server` -- `/client.server_api -> /server.api` -- `/client.ext_api -> external.ext_api` +The dashboard should not force you to think about Amber's internal mesh. Instead, it gives you three resources: -The important output is Amber's interaction trace stream. `amber run`, `amber attach`, and -`amber logs` render that stream for humans, and the same data is persisted for tooling in -`observability/events.ndjson` under the run root. +- `amber..client`: the client program's stdout/stderr +- `amber..server`: the server program's stdout/stderr +- `amber..bindings`: structured request/response telemetry for the scenario's routed edges + +`` is the Compose project name. If you do not set one, it defaults to `default`. ## Prereqs @@ -18,52 +19,55 @@ The important output is Amber's interaction trace stream. `amber run`, `amber at - `amber` CLI on `PATH` - `python3` on the host -## 1) Start the local upstream +## 1) Start the dashboard ```sh -python3 examples/observability-debug/upstream-sse.py --port 38081 +docker rm -f amber-dashboard >/dev/null 2>&1 || true +amber dashboard --detach ``` -## 2) Start Amber +Dashboard UI: `http://127.0.0.1:18888` -Run the scenario in the foreground: +## 2) Compile + start the scenario ```sh -cd examples/observability-debug -amber run . +OUT=/tmp/amber-observability-debug +rm -rf "$OUT" +amber compile examples/observability-debug/scenario.json5 --docker-compose "$OUT" +COMPOSE_PROJECT_NAME=amberdemo docker compose -f "$OUT/compose.yaml" up -d ``` -When Amber prompts for the `ext_api` slot, provide: +If you are validating local framework changes, set `AMBER_DEV_IMAGE_TAGS` when you run `amber compile`. For this example, router-only changes need `router=`. If your changes affect provisioning or generated mesh config metadata, use `router=,provisioner=` so the provisioned sidecar config matches the runtime you are testing. If your changes affect the site controller image used for compose or Kubernetes sites, add `site_controller=` as well. -```text -http://127.0.0.1:38081 -``` +## 3) Start the local upstream (Terminal A) -Amber prints the run id, run root, exported localhost URLs, and then tails the interaction stream. +```sh +python3 examples/observability-debug/upstream-sse.py --port 38081 +``` -If you prefer a managed background run: +## 4) Wire the external slot and export (Terminal B) ```sh -cd examples/observability-debug -amber run . --detach -amber attach +OUT=/tmp/amber-observability-debug +amber proxy "$OUT" \ + --project-name amberdemo \ + --slot ext_api=127.0.0.1:38081 \ + --export public=127.0.0.1:38080 ``` -`amber ps` lists active runs, `amber logs ` replays the persisted interaction story, and -the run root contains `observability/events.ndjson` for machine tooling. +This enables two manifest-level edges: -If you are validating local runtime image changes, set `AMBER_DEV_IMAGE_TAGS` before `amber run`. -For router-only changes, `router=` is usually enough. If you changed provisioning or generated -mesh metadata too, also include `provisioner=`. +- `/client -> ext_api` +- `public -> /server` -## 3) Generate host traffic +Brief `502` or `503` responses are expected until the slot is registered and the upstream is reachable. -In another terminal, use the `public` URL printed by Amber: +## 5) Generate explicit host traffic (Terminal C) ```sh curl -sS \ -H 'x-amber-tutorial: host-export' \ - http://127.0.0.1:/rpc + http://127.0.0.1:38080/rpc ``` Expected response: @@ -72,41 +76,97 @@ Expected response: {"jsonrpc":"2.0","id":"server-static","method":"tools/list","result":{"source":"server","ok":true}} ``` -The `client` also generates traffic in a loop, so the interaction stream should cover all three -manifest-level stories. +The `client` also generates traffic in a loop, so after this call you have all three stories in the dashboard: + +- `public -> /server` +- `/client.server_api -> /server.api` +- `/client.ext_api -> external.ext_api`, including SSE + +## 6) Read the telemetry + +Open `http://127.0.0.1:18888`. + +### Component resources + +`amber.amberdemo.client` shows the client program's own logs, for example: -## 4) Read the interaction story +- `[client] internal rpc -> server` +- `[client] external rpc -> upstream` +- `[client] external sse -> upstream` -Look for these edges: +`amber.amberdemo.server` shows the server program's own logs, for example: + +- `[server] listening on :9000` +- `[server] received GET /rpc x-amber-tutorial=host-export` +- `[server] responded 200 /rpc id=server-static` + +### Edge resource + +`amber.amberdemo.bindings` is the important one. It contains the request/response lifecycle for user-facing edges. + +Look for these edge refs: - `public` - `/client.server_api -> /server.api` - `/client.ext_api -> external.ext_api` -You should see protocol-aware lines such as: +A novice should be able to read the logs as a story. -- `get agent card` -- `list tools` -- `call tool ...` -- `progress update` +Examples you should see: -Useful things to verify: +- `request received from public by /server [headers]` +- `response sent from /server to public: tools/list result (id=server-static) [body]` +- `request sent from /client.server_api to /server.api [headers]` +- `response received by /client.server_api from /server.api: tools/list result (id=server-static) [body]` +- `request sent from /client.ext_api to external.ext_api: tools/call (id=external-1) [body]` +- `response received by /client.ext_api from external.ext_api: notifications/progress response (id=sse-2) [stream event]` -- related events share a trace id -- the renderer uses scenario edges and components instead of mesh/router internals -- request, response, and stream phases are still visible, but secondary to the semantic action +Important details: -If you started the run detached, `amber logs ` shows the same interaction stream from the -persisted `events.ndjson` file. +- the same request/response chain shares a `traceId` +- request and response logs use slot/capability edge refs instead of mesh/router route ids +- protocol-aware fields are extracted when Amber can understand them: JSON-RPC method/id, MCP tool/progress fields, SSE event ids, and so on -## Cleanup +## API checks + +List the resources the tutorial should create: + +```sh +curl -sS http://127.0.0.1:18888/api/telemetry/resources | jq . +``` + +Show the `public` edge story: -For a foreground run, press `Ctrl-C` in the `amber run` terminal. +```sh +curl -sS 'http://127.0.0.1:18888/api/telemetry/logs?resource=amber.amberdemo.bindings&limit=500' \ + | jq -r ' + .data.resourceLogs[]?.scopeLogs[]?.logRecords[]? + | [ + ([.attributes[]? | select(.key == "event") | .value.stringValue][0] // ""), + (.body.stringValue // ""), + (.traceId // "") + ] + | @tsv' \ + | rg 'public|host-export' +``` -For a detached run: +Show the internal client/server edge story: ```sh -amber stop +curl -sS 'http://127.0.0.1:18888/api/telemetry/traces?resource=amber.amberdemo.bindings&limit=200' \ + | jq -r ' + .data.resourceSpans[]?.scopeSpans[]?.spans[]? + | select((.attributes[]? | select(.key == "amber_edge_ref") | .value.stringValue) == "/client.server_api -> /server.api") + | [.traceId, .name, ([.events[]?.name] | join(" | "))] + | @tsv' ``` -Then stop the upstream process if it is still running. +## Cleanup + +```sh +OUT=/tmp/amber-observability-debug +COMPOSE_PROJECT_NAME=amberdemo docker compose -f "$OUT/compose.yaml" down -v +docker rm -f amber-dashboard >/dev/null 2>&1 || true +pkill -f 'upstream-sse.py --port 38081' >/dev/null 2>&1 || true +pkill -f 'amber proxy /tmp/amber-observability-debug' >/dev/null 2>&1 || true +``` diff --git a/images/src/lib.rs b/images/src/lib.rs index f531fea8..9db30abe 100644 --- a/images/src/lib.rs +++ b/images/src/lib.rs @@ -1,5 +1,14 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + env, +}; + pub mod versioning; +pub const DEV_IMAGE_TAGS_ENV: &str = "AMBER_DEV_IMAGE_TAGS"; +pub const INTERNAL_IMAGE_OVERRIDE_KEYS: &[&str] = + &["router", "helper", "provisioner", "site_controller"]; + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub struct ImageRef { pub name: &'static str, @@ -8,4 +17,126 @@ pub struct ImageRef { pub reference: &'static str, } +pub fn override_reference(image: &ImageRef, tag: &str) -> String { + format!("{}/{}:{}", image.registry, image.name, tag) +} + +pub fn parse_dev_image_tag_overrides( + allowed_keys: &[&str], +) -> Result, String> { + match env::var(DEV_IMAGE_TAGS_ENV) { + Ok(raw) => parse_dev_image_tag_overrides_from_raw(Some(raw.as_str()), allowed_keys), + Err(env::VarError::NotPresent) => Ok(BTreeMap::new()), + Err(err) => Err(format!("failed to read {DEV_IMAGE_TAGS_ENV}: {err}")), + } +} + +pub fn parse_dev_image_tag_overrides_from_raw( + raw: Option<&str>, + allowed_keys: &[&str], +) -> Result, String> { + let Some(raw) = raw else { + return Ok(BTreeMap::new()); + }; + let raw = raw.trim(); + if raw.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} is set but empty; expected format {}", + expected_override_format(allowed_keys), + )); + } + + let allowed = allowed_keys + .iter() + .map(|key| (*key).to_string()) + .collect::>(); + let mut overrides = BTreeMap::new(); + for entry in raw.split(',') { + let entry = entry.trim(); + if entry.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains an empty entry; expected format {}", + expected_override_format(allowed_keys), + )); + } + + let mut parts = entry.splitn(2, '='); + let key = parts.next().unwrap_or_default().trim(); + let value = parts.next().ok_or_else(|| { + format!("{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing '='; expected key=value") + })?; + let value = value.trim(); + + if key.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a key; expected key=value" + )); + } + if value.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a tag; expected key=tag" + )); + } + if !allowed.contains(key) { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains unknown key \"{key}\"; expected {}", + allowed_keys.join(", "), + )); + } + if overrides + .insert(key.to_string(), value.to_string()) + .is_some() + { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains duplicate key \"{key}\"" + )); + } + } + Ok(overrides) +} + +fn expected_override_format(allowed_keys: &[&str]) -> String { + let entries = allowed_keys + .iter() + .map(|key| format!("{key}=")) + .collect::>() + .join(","); + format!("\"{entries}\"") +} + include!(concat!(env!("OUT_DIR"), "/images.rs")); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_dev_image_tag_overrides_accepts_site_controller_key() { + let overrides = parse_dev_image_tag_overrides_from_raw( + Some("router=dev-router,site_controller=dev-controller"), + INTERNAL_IMAGE_OVERRIDE_KEYS, + ) + .expect("dev image tag overrides should parse"); + + assert_eq!( + overrides.get("router").map(String::as_str), + Some("dev-router") + ); + assert_eq!( + overrides.get("site_controller").map(String::as_str), + Some("dev-controller") + ); + } + + #[test] + fn parse_dev_image_tag_overrides_rejects_unknown_key() { + let err = parse_dev_image_tag_overrides_from_raw( + Some("router=dev-router,unknown=dev"), + INTERNAL_IMAGE_OVERRIDE_KEYS, + ) + .expect_err("unknown override keys should be rejected"); + + assert!(err.contains("unknown key")); + assert!(err.contains("site_controller")); + } +} diff --git a/runtime/docker-gateway/Cargo.toml b/runtime/docker-gateway/Cargo.toml index ca71e1f9..42077392 100644 --- a/runtime/docker-gateway/Cargo.toml +++ b/runtime/docker-gateway/Cargo.toml @@ -3,7 +3,6 @@ name = "amber-docker-gateway" edition = "2024" [dependencies] -base64 = { workspace = true } bytes = "1.10.1" dashmap = "6.1.0" http-body-util = "0.1.3" @@ -13,10 +12,9 @@ hyperlocal = { version = "0.9.1", features = ["client"] } moka = { version = "0.12.10", features = ["sync"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } -serde_urlencoded = "0.7.1" serde_with = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "sync", "time", "signal"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "sync", "time"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } diff --git a/runtime/docker-gateway/README.md b/runtime/docker-gateway/README.md index 1021ac5b..49b3c9b2 100644 --- a/runtime/docker-gateway/README.md +++ b/runtime/docker-gateway/README.md @@ -1,82 +1,59 @@ # Amber Docker Gateway -Amber Docker Gateway is a small policy layer that sits between Amber components and the host -Docker Engine API. It accepts TCP connections (for example from `uds2tcp`), authenticates callers -based on the peer address, injects Compose and ownership labels on create, and scopes all -operations to the owning component inside the scenario. +Amber Docker Gateway is the Docker Engine policy layer used by Amber router sidecars for +`framework.docker` capabilities. It is a Rust library, not a standalone network service: the +router accepts a Noise-authenticated mesh stream, uses the authenticated mesh peer id as the caller +identity, and hands the raw TCP stream to the gateway handler. -The gateway is designed to run as an Amber component injected by the compiler. It behaves like a -normal component container, except that it receives a host `docker.sock` mount. +There is deliberately no source-IP, source-port, or caller-supplied-header authentication path. +All Amber component communication to the gateway travels over the mesh. The host Docker socket is +mounted only into the injected gateway sidecar, and the socket peer address is not part of the +authorization model. -## What it does +## What It Does -- Authenticates each TCP connection using the peer IP (and optional port) resolved from caller - host entries in config. +- Uses the mesh peer id supplied by the router as the owning component identity. - Injects ownership labels and `com.docker.compose.project` on create so Compose can clean up - with `--remove-orphans`. + resources with `--remove-orphans`. - Enforces per-component scoping: only the owning component can inspect, modify, exec, or delete its containers, networks, and volumes. -- Blocks endpoints that cannot be scoped safely (default-deny). -- Forwards all allowed requests to the upstream Docker daemon over a Unix socket using - `hyperlocal`. -- Supports HTTP upgrade/hijack (exec/attach). +- Blocks endpoints that cannot be scoped safely. +- Forwards allowed requests to the upstream Docker daemon over a Unix socket. +- Supports HTTP upgrade/hijack for Docker exec and attach flows. ## Configuration -The gateway reads configuration from one of these environment variables: - -- `AMBER_DOCKER_GATEWAY_CONFIG_B64` (base64 JSON) -- `AMBER_DOCKER_GATEWAY_CONFIG_JSON` (raw JSON) - -Example config: +The router creates `DockerGatewayRuntime` from: ```json { - "listen": "0.0.0.0:23750", "docker_sock": "/var/run/docker.sock", - "compose_project": "amber-scenario-123", - "callers": [ - { - "host": "c2-client-net", - "port": 41732, - "component": "component-a", - "compose_service": "c2-client-net" - } - ] + "compose_project": "amber-scenario-123" } ``` -Notes: - -- `compose_project` is required. It is the Docker Compose project name for the scenario. -- `callers` is required. Each entry maps a peer `host` (and optional `port`) to a component and - its Compose service name. The gateway resolves caller hosts periodically and authenticates peers - by the resolved source IP (plus optional source port). -- If your TCP proxy uses ephemeral source ports, omit `port` and match by resolved IP only. -- Ownership labels are fixed to `amber.component` and `amber.project`. +`docker_sock` is the Docker daemon Unix socket inside the sidecar. `compose_project` is the +scenario's Docker Compose project name and is used for Docker resource scoping. -## Labels and scoping +## Labels and Scoping On create, the gateway injects these labels: -- Ownership labels: `amber.component`, `amber.project` -- Compose project label: `com.docker.compose.project` (normalized to outer scenario project) +- `amber.component` +- `amber.project` +- `com.docker.compose.project` -These labels are used for two purposes: +The Amber labels enforce ownership. The Compose project label keeps resources created through the +gateway in the scenario's Compose project so normal Compose teardown removes them. -1. Compose grouping (`com.docker.compose.project`) so `docker compose --remove-orphans` - works for resources created through the gateway. -2. Ownership enforcement so only the owning component and scenario project can act on the resource. - -## Allowed API surface +## Allowed API Surface The gateway is default-deny. It only allows endpoints that can be scoped safely: -- `/containers/create`, `/networks/create`, `/volumes/create` (labels injected) -- `/images/{name}/json` (read-only image inspect; required by `docker compose up --no-build`) -- Container-scoped endpoints (start/stop/logs/inspect/remove/exec/etc) after ownership check -- Exec endpoints (`/exec/{id}/start`, `/exec/{id}/json`, `/exec/{id}/resize`) after mapping - exec ID to container ownership +- `/containers/create`, `/networks/create`, `/volumes/create` +- `/images/{name}/json` +- Container-scoped endpoints after ownership checks +- Exec endpoints after mapping exec ID to container ownership - List/prune endpoints with label filters injected: - `/containers/json`, `/events`, `/containers/prune` - `/networks`, `/networks/prune` @@ -86,16 +63,3 @@ Explicitly blocked: - `/networks/{id}/connect` and `/disconnect` - Any other endpoint that cannot be scoped to a single component - -If you need to allow a specific endpoint, update the policy in the handler to explicitly -authorize it. - -## Running locally (example) - -```sh -export AMBER_DOCKER_GATEWAY_CONFIG_JSON='{"listen":"0.0.0.0:23750","docker_sock":"/var/run/docker.sock","compose_project":"dev","callers":[{"host":"localhost","component":"dev","compose_service":"dev"}]}' -./target/debug/amber-docker-gateway -``` - -In production, the component should mount the host Docker socket at `/var/run/docker.sock` and -receive the config via environment variables. diff --git a/runtime/docker-gateway/src/lib.rs b/runtime/docker-gateway/src/lib.rs index 59ed24a1..208df83c 100644 --- a/runtime/docker-gateway/src/lib.rs +++ b/runtime/docker-gateway/src/lib.rs @@ -1,13 +1,10 @@ use std::{ collections::{HashMap, HashSet}, - env, - net::{IpAddr, SocketAddr}, path::PathBuf, sync::Arc, time::Duration, }; -use base64::Engine as _; use bytes::Bytes; use http_body_util::{BodyExt, Full, combinators::BoxBody}; use hyper::{ @@ -28,19 +25,16 @@ use serde::{Deserialize, de::DeserializeOwned}; use serde_with::{DefaultOnNull, serde_as}; use thiserror::Error; use tokio::{ - net::{TcpListener, UnixStream}, - sync::RwLock, - time::{sleep, timeout}, + io::{AsyncRead, AsyncWrite}, + net::UnixStream, + time::timeout, }; -const CONFIG_B64_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_B64"; -const CONFIG_JSON_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_JSON"; const COMPOSE_PROJECT_LABEL: &str = "com.docker.compose.project"; #[cfg(test)] const COMPOSE_SERVICE_LABEL: &str = "com.docker.compose.service"; const AMBER_COMPONENT_LABEL: &str = "amber.component"; const AMBER_PROJECT_LABEL: &str = "amber.project"; -const CALLER_REFRESH_INTERVAL: Duration = Duration::from_secs(5); const SHUTDOWN_CLEANUP_TIMEOUT: Duration = Duration::from_secs(8); const EXEC_CACHE_TTL: Duration = Duration::from_secs(5 * 60); const EXEC_CACHE_MAX_ENTRIES: u64 = 8_192; @@ -49,55 +43,28 @@ pub type BoxError = Box; pub type ProxyBody = BoxBody; type GatewayResult = Result>>; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ShutdownReason { - Interrupt, - Terminated, -} - #[derive(Debug, Error)] pub enum DockerGatewayError { - #[error("missing docker gateway config (set {CONFIG_B64_ENV} or {CONFIG_JSON_ENV})")] - MissingConfig, #[error("invalid docker gateway config: {0}")] InvalidConfig(String), - #[error("failed to bind {addr}: {source}")] - BindFailed { - addr: SocketAddr, - #[source] - source: std::io::Error, - }, } #[derive(Clone, Debug, Deserialize)] pub struct DockerGatewayConfig { - pub listen: SocketAddr, pub docker_sock: PathBuf, pub compose_project: String, - pub callers: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct CallerConfig { - pub host: String, - #[serde(default)] - pub port: Option, - pub component: String, - pub compose_service: String, } #[derive(Clone)] struct ConnState { state: Arc, - peer: SocketAddr, - identity: Option, + identity: CallerIdentity, } struct State { cfg: Arc, client: Client, exec_map: Cache, - callers_by_ip: RwLock>>, } #[derive(Clone, Debug)] @@ -111,12 +78,6 @@ struct CallerIdentity { component: String, } -#[derive(Clone, Debug)] -struct ResolvedCaller { - component: String, - port: Option, -} - #[derive(Debug, Deserialize)] struct ContainerInspectResponse { #[serde(rename = "Config")] @@ -279,63 +240,58 @@ struct NetworkingConfigRequest { } impl DockerGatewayConfig { - pub fn from_env() -> Result { - if let Ok(b64) = env::var(CONFIG_B64_ENV) { - if b64.trim().is_empty() { - return Err(DockerGatewayError::MissingConfig); - } - let decoded = base64::engine::general_purpose::STANDARD - .decode(b64.as_bytes()) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - let parsed = serde_json::from_slice(&decoded) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - return Self::validate(parsed); - } - - if let Ok(raw) = env::var(CONFIG_JSON_ENV) { - if raw.trim().is_empty() { - return Err(DockerGatewayError::MissingConfig); - } - let parsed = serde_json::from_str(&raw) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - return Self::validate(parsed); - } - - Err(DockerGatewayError::MissingConfig) - } - - fn validate(config: Self) -> Result { + pub fn validate(config: Self) -> Result { if config.compose_project.trim().is_empty() { return Err(DockerGatewayError::InvalidConfig( "compose_project must not be empty".to_string(), )); } - if config.callers.is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "callers must not be empty".to_string(), - )); - } - for caller in &config.callers { - if caller.host.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller host must not be empty".to_string(), - )); - } - if caller.component.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller component must not be empty".to_string(), - )); - } - if caller.compose_service.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller compose_service must not be empty".to_string(), - )); - } - } Ok(config) } } +#[derive(Clone)] +pub struct DockerGatewayRuntime { + state: Arc, +} + +impl DockerGatewayRuntime { + pub fn new(config: DockerGatewayConfig) -> Result { + Ok(Self { + state: Arc::new(State::new(DockerGatewayConfig::validate(config)?)), + }) + } + + pub async fn serve_connection(&self, stream: S, peer_id: String) -> Result<(), BoxError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let conn_state = Arc::new(ConnState { + state: self.state.clone(), + identity: CallerIdentity { component: peer_id }, + }); + let svc = service_fn(move |req: Request| { + let conn_state = conn_state.clone(); + async move { + let req = req.map(box_body_from_incoming); + Ok::<_, std::convert::Infallible>(handle(req, conn_state).await) + } + }); + + http1::Builder::new() + .preserve_header_case(true) + .title_case_headers(true) + .serve_connection(TokioIo::new(stream), svc) + .with_upgrades() + .await + .map_err(|err| -> BoxError { Box::new(err) }) + } + + pub async fn cleanup_created_resources(&self) { + self.state.cleanup_created_resources().await; + } +} + impl State { fn new(config: DockerGatewayConfig) -> Self { let connector = UnixConnector; @@ -347,67 +303,7 @@ impl State { .time_to_live(EXEC_CACHE_TTL) .max_capacity(EXEC_CACHE_MAX_ENTRIES) .build(), - callers_by_ip: RwLock::new(HashMap::new()), - } - } - - async fn refresh_callers(&self) { - let mut callers_by_ip: HashMap> = HashMap::new(); - - for caller in &self.cfg.callers { - match tokio::net::lookup_host((caller.host.as_str(), 0)).await { - Ok(addrs) => { - let mut saw_address = false; - for addr in addrs { - saw_address = true; - callers_by_ip - .entry(addr.ip()) - .or_default() - .push(ResolvedCaller { - component: caller.component.clone(), - port: caller.port, - }); - } - - if !saw_address { - tracing::warn!( - "docker gateway caller resolution returned no addresses for host {}", - caller.host - ); - } - } - Err(err) => { - tracing::warn!( - "docker gateway caller resolution failed for host {}: {err}", - caller.host - ); - } - } - } - - *self.callers_by_ip.write().await = callers_by_ip; - } - - async fn resolve_identity(&self, peer: SocketAddr) -> Option { - let callers = self.callers_by_ip.read().await; - let matches = callers.get(&peer.ip())?; - - let mut identity: Option<&str> = None; - for entry in matches { - if entry.port.is_some_and(|port| port != peer.port()) { - continue; - } - - match identity { - None => identity = Some(entry.component.as_str()), - Some(component) if component == entry.component.as_str() => {} - Some(_) => return None, - } } - - identity.map(|component| CallerIdentity { - component: component.to_string(), - }) } async fn cleanup_created_resources(&self) { @@ -499,108 +395,8 @@ impl State { } } -async fn resolve_connection_identity(state: &State, peer: SocketAddr) -> Option { - if let Some(identity) = state.resolve_identity(peer).await { - return Some(identity); - } - // Newly started compose networks can make caller hostnames resolvable a moment after startup. - // Refresh once on a miss so first requests do not fail with a transient unauthorized error. - state.refresh_callers().await; - state.resolve_identity(peer).await -} - -pub async fn run(config: DockerGatewayConfig) -> Result<(), DockerGatewayError> { - let state = Arc::new(State::new(config)); - state.refresh_callers().await; - - let refresh_state = state.clone(); - let refresh_task = tokio::spawn(async move { - loop { - sleep(CALLER_REFRESH_INTERVAL).await; - refresh_state.refresh_callers().await; - } - }); - - let listener = TcpListener::bind(state.cfg.listen) - .await - .map_err(|source| DockerGatewayError::BindFailed { - addr: state.cfg.listen, - source, - })?; - - let shutdown = shutdown_signal(); - tokio::pin!(shutdown); - - loop { - tokio::select! { - reason = &mut shutdown => { - refresh_task.abort(); - match reason { - ShutdownReason::Interrupt => { - state.cleanup_created_resources().await; - } - ShutdownReason::Terminated => { - // `docker compose down` sends SIGTERM and also performs its own teardown. - // Skipping gateway cleanup avoids duplicate deletes and "No such container" races. - tracing::warn!( - "docker gateway received SIGTERM; skipping shutdown cleanup to avoid \ - teardown races" - ); - } - } - return Ok(()); - } - accepted = listener.accept() => { - let (stream, peer) = match accepted { - Ok(value) => value, - Err(err) => { - tracing::warn!("docker gateway accept failed: {err}"); - continue; - } - }; - - let conn_state = Arc::new(ConnState { - state: state.clone(), - peer, - identity: resolve_connection_identity(state.as_ref(), peer).await, - }); - - let io = TokioIo::new(stream); - let svc = service_fn(move |req: Request| { - let conn_state = conn_state.clone(); - async move { - let req = req.map(box_body_from_incoming); - Ok::<_, std::convert::Infallible>(handle(req, conn_state).await) - } - }); - - tokio::spawn(async move { - if let Err(err) = http1::Builder::new() - .preserve_header_case(true) - .title_case_headers(true) - .serve_connection(io, svc) - .with_upgrades() - .await - { - tracing::warn!("docker gateway connection failed: {err}"); - } - }); - } - } - } -} - async fn handle(mut req: Request, conn: Arc) -> Response { - let id = match conn.identity.clone() { - Some(identity) => identity, - None => { - return docker_error( - StatusCode::UNAUTHORIZED, - format!("unauthorized peer {}", conn.peer), - ); - } - }; - + let id = conn.identity.clone(); let (ver_opt, segs) = split_version_and_segments(req.uri().path()); let version_prefix = ver_opt.unwrap_or_else(|| "".to_string()); @@ -629,7 +425,7 @@ async fn handle(mut req: Request, conn: Arc) -> Response

>(q).ok()) + .map(parse_query_pairs) .map(|params| { params.iter().any(|(k, _)| k == "fromImage") && !params.iter().any(|(k, _)| k == "fromSrc") @@ -916,34 +712,7 @@ fn build_label_filter_query(required_labels: &[String], include_all: bool) -> St } query_pairs.push(("filters".to_string(), encoded_filters)); - serde_urlencoded::to_string(query_pairs) - .expect("serializing static label query params should succeed") -} - -async fn shutdown_signal() -> ShutdownReason { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - - match signal(SignalKind::terminate()) { - Ok(mut sigterm) => { - tokio::select! { - _ = tokio::signal::ctrl_c() => ShutdownReason::Interrupt, - _ = sigterm.recv() => ShutdownReason::Terminated, - } - } - Err(_) => { - let _ = tokio::signal::ctrl_c().await; - ShutdownReason::Interrupt - } - } - } - - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - return ShutdownReason::Interrupt; - } + encode_query_pairs(&query_pairs) } async fn forward(mut req: Request, state: Arc) -> Response { @@ -1520,15 +1289,24 @@ fn ensure_host(req: &mut Request) { } } +fn parse_query_pairs(query: &str) -> Vec<(String, String)> { + url::form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect() +} + +fn encode_query_pairs(pairs: &[(String, String)]) -> String { + let mut serializer = url::form_urlencoded::Serializer::new(String::new()); + for (key, value) in pairs { + serializer.append_pair(key, value); + } + serializer.finish() +} + fn add_label_filters_to_uri(uri: &Uri, required: &[String]) -> GatewayResult { let path = uri.path().to_string(); let mut pairs: Vec<(String, String)> = match uri.query() { - Some(query) => serde_urlencoded::from_str(query).map_err(|err| { - boxed_response(docker_error( - StatusCode::BAD_REQUEST, - format!("invalid query parameters: {err}"), - )) - })?, + Some(query) => parse_query_pairs(query), None => Vec::new(), }; @@ -1570,12 +1348,7 @@ fn add_label_filters_to_uri(uri: &Uri, required: &[String]) -> GatewayResult config, - Err(err) => { - tracing::error!("docker gateway config error: {err}"); - std::process::exit(1); - } - }; - - if let Err(err) = run(config).await { - tracing::error!("docker gateway failed: {err}"); - std::process::exit(1); - } -} - -fn init_tracing() { - let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")); - tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(false) - .without_time() - .init(); -} diff --git a/runtime/docker-gateway/src/tests/compose.rs b/runtime/docker-gateway/src/tests/compose.rs index e5b8cf6c..437594e3 100644 --- a/runtime/docker-gateway/src/tests/compose.rs +++ b/runtime/docker-gateway/src/tests/compose.rs @@ -2,7 +2,7 @@ use super::*; #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_setup_injects_scoped_labels() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/networks/create", @@ -167,7 +167,7 @@ async fn compose_in_compose_setup_injects_scoped_labels() { #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_teardown_allows_owned_resource_removal() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/inner-workload/json", @@ -289,7 +289,7 @@ async fn compose_in_compose_teardown_allows_owned_resource_removal() { #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_remove_orphans_rewrites_project_filters() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/json", @@ -396,10 +396,8 @@ async fn compose_in_compose_remove_orphans_rewrites_project_filters() { async fn shutdown_cleanup_removes_owned_resources() { let docker = MockDocker::start().await; let state = State::new(DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), docker_sock: docker.socket_path.clone(), compose_project: TEST_PROJECT.to_string(), - callers: vec![default_caller()], }); docker.enqueue_json( @@ -503,7 +501,7 @@ async fn shutdown_cleanup_removes_owned_resources() { #[tokio::test(flavor = "multi_thread")] async fn versioned_paths_preserve_version_for_authorization_and_forwarding() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/v1.41/containers/workload/json", @@ -534,7 +532,7 @@ async fn versioned_paths_preserve_version_for_authorization_and_forwarding() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_uses_cached_exec_to_container_mapping() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/workload/json", @@ -590,7 +588,7 @@ async fn exec_start_uses_cached_exec_to_container_mapping() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_falls_back_to_exec_inspect_when_cache_is_missing() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/exec/exec-2/json", @@ -629,7 +627,7 @@ async fn exec_start_falls_back_to_exec_inspect_when_cache_is_missing() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_denies_access_when_exec_belongs_to_foreign_container() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/exec/exec-9/json", diff --git a/runtime/docker-gateway/src/tests/e2e.rs b/runtime/docker-gateway/src/tests/e2e.rs index 7200613f..5da0c2b9 100644 --- a/runtime/docker-gateway/src/tests/e2e.rs +++ b/runtime/docker-gateway/src/tests/e2e.rs @@ -36,25 +36,12 @@ async fn docker_daemon_e2e_enforces_scoping_and_policy() { let owned_volume = format!("amber-gw-owned-vol-{suffix}"); let foreign_volume = format!("amber-gw-foreign-vol-{suffix}"); - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: component.clone(), - compose_service: component.clone(), - }], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen, gateway_task) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component.clone(), + ) + .await; send_docker_request( &docker_sock, @@ -358,39 +345,21 @@ async fn docker_daemon_e2e_enforces_multicaller_container_exec_and_upgrade() { let owned_network_b = format!("amber-gw-owned-net-b-{suffix}"); let owned_container_a = format!("amber-gw-owned-container-a-{suffix}"); - let (caller_a_socket, caller_a_port) = reserve_bound_loopback_socket(); - let (caller_b_socket, caller_b_port) = reserve_bound_loopback_socket(); - - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![ - CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(caller_a_port), - component: component_a.clone(), - compose_service: component_a.clone(), - }, - CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(caller_b_port), - component: component_b.clone(), - compose_service: component_b.clone(), - }, - ], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in multicaller e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen_a, gateway_task_a) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component_a.clone(), + ) + .await; + let (listen_b, gateway_task_b) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component_b.clone(), + ) + .await; - let mut caller_a = GatewayClient::connect_from_socket(caller_a_socket, listen).await; - let mut caller_b = GatewayClient::connect_from_socket(caller_b_socket, listen).await; + let mut caller_a = GatewayClient::connect(listen_a).await; + let mut caller_b = GatewayClient::connect(listen_b).await; let create_network_a = caller_a .request( @@ -584,7 +553,8 @@ async fn docker_daemon_e2e_enforces_multicaller_container_exec_and_upgrade() { String::from_utf8_lossy(&allowed_exec_start.body) ); - gateway_task.abort(); + gateway_task_a.abort(); + gateway_task_b.abort(); docker_delete_container_best_effort(&docker_sock, &owned_container_a).await; docker_delete_network_best_effort(&docker_sock, &owned_network_a).await; docker_delete_network_best_effort(&docker_sock, &owned_network_b).await; @@ -626,25 +596,12 @@ async fn docker_daemon_e2e_container_create_without_network_mode() { let component = format!("amber-gw-e2e-netmode-comp-{suffix}"); let container_name = format!("amber-gw-e2e-netmode-ctr-{suffix}"); - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: component.clone(), - compose_service: component.clone(), - }], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in network-mode e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen, gateway_task) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component.clone(), + ) + .await; // Docker CLI v27+ sends EndpointsConfig:{"default":{}} when no // --network flag is given. The gateway should skip builtin names. diff --git a/runtime/docker-gateway/src/tests/gateway.rs b/runtime/docker-gateway/src/tests/gateway.rs index b5677e18..4164b7e1 100644 --- a/runtime/docker-gateway/src/tests/gateway.rs +++ b/runtime/docker-gateway/src/tests/gateway.rs @@ -1,145 +1,45 @@ use super::*; #[tokio::test(flavor = "multi_thread")] -async fn resolve_identity_uses_port_and_denies_ambiguous_component_matches() { - let state = State::new(DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), - docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![default_caller()], - }); - - { - let mut callers = state.callers_by_ip.write().await; - callers.insert( - IpAddr::V4(Ipv4Addr::LOCALHOST), - vec![ - ResolvedCaller { - component: "component-a".to_string(), - port: Some(30001), - }, - ResolvedCaller { - component: "component-b".to_string(), - port: Some(30002), - }, - ], - ); - } - - let a = state - .resolve_identity("127.0.0.1:30001".parse().expect("valid addr")) - .await - .expect("caller a should match"); - assert_eq!(a.component, "component-a"); - - let b = state - .resolve_identity("127.0.0.1:30002".parse().expect("valid addr")) - .await - .expect("caller b should match"); - assert_eq!(b.component, "component-b"); - - let unmatched = state - .resolve_identity("127.0.0.1:30003".parse().expect("valid addr")) - .await; - assert!( - unmatched.is_none(), - "unexpected identity for unmatched port" - ); - - { - let mut callers = state.callers_by_ip.write().await; - callers.insert( - IpAddr::V4(Ipv4Addr::LOCALHOST), - vec![ - ResolvedCaller { - component: "component-a".to_string(), - port: None, - }, - ResolvedCaller { - component: "component-b".to_string(), - port: None, - }, - ], - ); - } - - let ambiguous = state - .resolve_identity("127.0.0.1:30010".parse().expect("valid addr")) - .await; - assert!( - ambiguous.is_none(), - "ambiguous peer mapping should fail closed" - ); -} - -#[tokio::test(flavor = "multi_thread")] -async fn rejects_unauthorized_peer_before_proxying() { - let gateway = GatewayHarness::start(vec![CallerConfig { - host: "127.0.0.2".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }]) - .await; - - let response = send_gateway_request(gateway.addr, Method::GET, "/_ping", &[], &[]).await; - assert_eq!(response.status, StatusCode::UNAUTHORIZED); - assert!(response_message(&response.body).contains("unauthorized peer")); - assert!(gateway.requests().is_empty()); -} - -#[tokio::test(flavor = "multi_thread")] -async fn enforces_port_restrictions_for_authenticated_callers() { - let (allowed_socket, allowed_port) = reserve_bound_loopback_socket(); - let (denied_socket, denied_port) = reserve_bound_loopback_socket(); - - let gateway = GatewayHarness::start(vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(allowed_port), - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }]) - .await; +async fn mesh_peer_identity_labels_created_resources() { + let gateway = GatewayHarness::start("/component-b").await; gateway.enqueue_json( - Method::GET, - "/version", - StatusCode::OK, - serde_json::json!({"Version":"24.0"}), + Method::POST, + "/containers/create", + StatusCode::CREATED, + serde_json::json!({"Id":"new-container"}), ); - let denied = send_gateway_request_from_socket( - denied_socket, - denied_port, - gateway.addr, - Method::GET, - "/version", - &[], - &[], - ) - .await; - assert_eq!(denied.status, StatusCode::UNAUTHORIZED); - - let allowed = send_gateway_request_from_socket( - allowed_socket, - allowed_port, + let response = send_gateway_request( gateway.addr, - Method::GET, - "/version", - &[], - &[], + Method::POST, + "/containers/create", + &[("content-type", "application/json")], + br#"{"Image":"busybox"}"#, ) .await; - assert_eq!(allowed.status, StatusCode::OK); + assert_eq!(response.status, StatusCode::CREATED); let requests = gateway.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].method, Method::GET); - assert_eq!(requests[0].path, "/version"); + let create_req = requests + .iter() + .find(|req| req.method == Method::POST && req.path == "/containers/create") + .expect("create should be forwarded"); + let forwarded: serde_json::Value = + serde_json::from_slice(&create_req.body).expect("forwarded body json"); + let labels = forwarded + .get("Labels") + .and_then(|value| value.as_object()) + .expect("forwarded labels object"); + assert_eq!( + labels.get(AMBER_COMPONENT_LABEL).and_then(|v| v.as_str()), + Some("/component-b") + ); } #[tokio::test(flavor = "multi_thread")] async fn default_deny_blocks_unapproved_endpoints() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request(gateway.addr, Method::GET, "/images/json", &[], &[]).await; assert_eq!(response.status, StatusCode::FORBIDDEN); @@ -149,7 +49,7 @@ async fn default_deny_blocks_unapproved_endpoints() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_inspect_endpoint_for_prebuilt_workflows() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/images/busybox:latest/json", @@ -175,7 +75,7 @@ async fn allows_image_inspect_endpoint_for_prebuilt_workflows() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_inspect_endpoint_for_registry_qualified_names() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/images/ghcr.io/org/app:latest/json", @@ -201,7 +101,7 @@ async fn allows_image_inspect_endpoint_for_registry_qualified_names() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_from_image() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -231,7 +131,7 @@ async fn allows_image_pull_with_from_image() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_without_explicit_tag() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -260,7 +160,7 @@ async fn allows_image_pull_without_explicit_tag() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_registry_qualified_name() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -288,7 +188,7 @@ async fn allows_image_pull_with_registry_qualified_name() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_import_with_from_src() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request( gateway.addr, @@ -305,7 +205,7 @@ async fn blocks_image_import_with_from_src() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_create_with_no_query_params() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request(gateway.addr, Method::POST, "/images/create", &[], &[]).await; @@ -316,7 +216,7 @@ async fn blocks_image_create_with_no_query_params() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_create_with_both_from_image_and_from_src() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request( gateway.addr, @@ -332,7 +232,7 @@ async fn blocks_image_create_with_both_from_image_and_from_src() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_version_prefix() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/v1.45/images/create", @@ -357,7 +257,7 @@ async fn allows_image_pull_with_version_prefix() { #[tokio::test(flavor = "multi_thread")] async fn allows_info_endpoint() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/info", @@ -376,7 +276,7 @@ async fn allows_info_endpoint() { #[tokio::test(flavor = "multi_thread")] async fn blocks_network_connect_and_disconnect() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let connect = send_gateway_request( gateway.addr, @@ -403,7 +303,7 @@ async fn blocks_network_connect_and_disconnect() { #[tokio::test(flavor = "multi_thread")] async fn create_injects_owner_labels_and_normalizes_compose_project() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/containers/create", @@ -477,7 +377,7 @@ async fn create_allows_endpoints_config_with_builtin_default_network() { // Docker CLI v27+ sends EndpointsConfig:{"default":{}} when no // --network flag is given. Builtin names should be skipped, just // like add_network_mode_reference does for NetworkMode. - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/containers/create", @@ -518,7 +418,7 @@ async fn create_allows_endpoints_config_with_builtin_default_network() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_network_in_endpoints_config() { // User-defined networks in EndpointsConfig must still be authorized. - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/networks/custom-net", @@ -552,7 +452,7 @@ async fn create_rejects_foreign_network_in_endpoints_config() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_network_reference() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/networks/shared-net", @@ -586,7 +486,7 @@ async fn create_rejects_foreign_network_reference() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_named_volume_bind() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/volumes/shared-volume", @@ -620,7 +520,7 @@ async fn create_rejects_foreign_named_volume_bind() { #[tokio::test(flavor = "multi_thread")] async fn create_allows_owned_resource_references() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/base/json", @@ -690,7 +590,7 @@ async fn create_allows_owned_resource_references() { #[tokio::test(flavor = "multi_thread")] async fn list_and_prune_endpoints_force_required_label_filters() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/json", diff --git a/runtime/docker-gateway/src/tests/mod.rs b/runtime/docker-gateway/src/tests/mod.rs index 2d2d1527..17de1275 100644 --- a/runtime/docker-gateway/src/tests/mod.rs +++ b/runtime/docker-gateway/src/tests/mod.rs @@ -1,6 +1,7 @@ use std::{ collections::VecDeque, - net::{IpAddr, Ipv4Addr, SocketAddrV4}, + env, + net::{Ipv4Addr, SocketAddr}, path::PathBuf, sync::{Arc, Mutex}, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -11,7 +12,7 @@ use hyper::client::conn::http1 as client_http1; use tempfile::TempDir; use tokio::{ io::AsyncReadExt, - net::{TcpSocket, TcpStream, UnixListener}, + net::{TcpListener, TcpStream, UnixListener}, task::JoinHandle, time::timeout, }; @@ -234,23 +235,14 @@ struct GatewayHarness { } impl GatewayHarness { - async fn start(callers: Vec) -> Self { + async fn start(peer_id: impl Into) -> Self { let docker = MockDocker::start().await; - let addr = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen: addr, - docker_sock: docker.socket_path.clone(), - compose_project: TEST_PROJECT.to_string(), - callers, - }; - - let task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed: {err}"); - } - }); - - wait_until_gateway_listens(addr, &task).await; + let (addr, task) = start_gateway_listener( + docker.socket_path.clone(), + TEST_PROJECT.to_string(), + peer_id.into(), + ) + .await; Self { addr, docker, task } } @@ -291,11 +283,10 @@ struct GatewayClient { } impl GatewayClient { - async fn connect_from_socket(socket: TcpSocket, addr: SocketAddr) -> Self { - let stream = socket - .connect(addr) + async fn connect(addr: SocketAddr) -> Self { + let stream = TcpStream::connect(addr) .await - .unwrap_or_else(|err| panic!("connect from bound local source socket failed: {err}")); + .unwrap_or_else(|err| panic!("connect to gateway failed: {err}")); let io = TokioIo::new(stream); let (sender, conn) = client_http1::handshake(io) .await @@ -398,35 +389,33 @@ impl Drop for GatewayClient { } } -fn reserve_bound_loopback_socket() -> (TcpSocket, u16) { - let socket = TcpSocket::new_v4().expect("create v4 tcp socket"); - socket - .bind(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))) - .expect("bind loopback tcp socket"); - let local_port = socket.local_addr().expect("socket local addr").port(); - (socket, local_port) -} - -fn reserve_loopback_socket_addr() -> SocketAddr { - let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .expect("bind temporary loopback listener"); - let addr = listener.local_addr().expect("local addr"); - drop(listener); - addr -} - -async fn wait_until_gateway_listens(addr: SocketAddr, task: &JoinHandle<()>) { - for _ in 0..400 { - if TcpStream::connect(addr).await.is_ok() { - return; +async fn start_gateway_listener( + docker_sock: PathBuf, + compose_project: String, + peer_id: String, +) -> (SocketAddr, JoinHandle<()>) { + let runtime = DockerGatewayRuntime::new(DockerGatewayConfig { + docker_sock, + compose_project, + }) + .expect("valid gateway config"); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind gateway test listener"); + let addr = listener.local_addr().expect("gateway listener addr"); + let task = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept gateway test client"); + let runtime = runtime.clone(); + let peer_id = peer_id.clone(); + tokio::spawn(async move { + if let Err(err) = runtime.serve_connection(stream, peer_id).await { + eprintln!("gateway test connection failed: {err}"); + } + }); } - assert!( - !task.is_finished(), - "gateway task exited before listener was ready" - ); - tokio::task::yield_now().await; - } - panic!("gateway listener did not come up at {addr}"); + }); + (addr, task) } async fn send_gateway_request( @@ -442,22 +431,6 @@ async fn send_gateway_request( send_gateway_request_on_stream(stream, method, target, headers, body).await } -async fn send_gateway_request_from_socket( - socket: TcpSocket, - source_port: u16, - addr: SocketAddr, - method: Method, - target: &str, - headers: &[(&str, &str)], - body: &[u8], -) -> GatewayResponse { - let stream = socket - .connect(addr) - .await - .unwrap_or_else(|err| panic!("connect from local source port {source_port} failed: {err}")); - send_gateway_request_on_stream(stream, method, target, headers, body).await -} - async fn send_gateway_request_on_stream( stream: TcpStream, method: Method, @@ -501,15 +474,6 @@ async fn send_gateway_request_on_stream( GatewayResponse { status, body } } -fn default_caller() -> CallerConfig { - CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - } -} - fn container_labels(component: &str, project: &str) -> serde_json::Value { serde_json::json!({ "Config": { @@ -535,8 +499,7 @@ fn decode_filters(req: &CapturedRequest) -> serde_json::Value { .path_and_query .split_once('?') .expect("request should include query"); - let query_map: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let query_map: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = query_map.get("filters").expect("filters query param"); serde_json::from_str(filters).expect("filters should be valid json") } diff --git a/runtime/docker-gateway/src/tests/unit.rs b/runtime/docker-gateway/src/tests/unit.rs index 0274ac05..9ec4bba5 100644 --- a/runtime/docker-gateway/src/tests/unit.rs +++ b/runtime/docker-gateway/src/tests/unit.rs @@ -50,8 +50,7 @@ fn add_label_filters_to_uri_accepts_object_labels() { let required = vec!["c=d".to_string()]; let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); let labels = parsed_filters @@ -77,8 +76,7 @@ fn add_label_filters_to_uri_uses_object_labels_with_name_filter() { let required = vec!["amber.component=/green".to_string()]; let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); @@ -121,8 +119,7 @@ fn add_label_filters_to_uri_rewrites_compose_project_filter() { let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); let labels = parsed_filters @@ -196,44 +193,16 @@ fn parse_container_create_references_extracts_resources() { } #[test] -fn validate_rejects_empty_caller_host() { +fn validate_rejects_empty_compose_project() { let config = DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![CallerConfig { - host: " ".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }], + compose_project: " ".to_string(), }; - let err = DockerGatewayConfig::validate(config).expect_err("empty host should fail"); - assert!( - err.to_string().contains("caller host must not be empty"), - "{err}" - ); -} - -#[test] -fn validate_rejects_empty_caller_component() { - let config = DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), - docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![CallerConfig { - host: "localhost".to_string(), - port: None, - component: " ".to_string(), - compose_service: TEST_COMPONENT.to_string(), - }], - }; - - let err = DockerGatewayConfig::validate(config).expect_err("empty component should fail"); + let err = DockerGatewayConfig::validate(config).expect_err("empty compose project should fail"); assert!( err.to_string() - .contains("caller component must not be empty"), + .contains("compose_project must not be empty"), "{err}" ); } diff --git a/runtime/helper/src/lib.rs b/runtime/helper/src/lib.rs index 716f5b9b..60bd7c3e 100644 --- a/runtime/helper/src/lib.rs +++ b/runtime/helper/src/lib.rs @@ -35,6 +35,7 @@ const IP_BIN: &str = "/usr/sbin/ip"; const IPTABLES_BIN: &str = "/usr/sbin/iptables"; const IP6TABLES_BIN: &str = "/usr/sbin/ip6tables"; const DEFAULT_EGRESS_CHAIN: &str = "AMBER_EGRESS"; +const DEFAULT_INGRESS_CHAIN: &str = "AMBER_INGRESS"; const HOST_GATEWAY_NAME: &str = "host.docker.internal"; const BLOCKED_IPV4_CIDRS: &[&str] = &[ "10.0.0.0/8", @@ -96,6 +97,11 @@ pub enum HelperError { pub type Result = std::result::Result; +pub fn install_default_network_guard(allowed_ingress_tcp_ports: &[u16]) -> Result<()> { + install_default_ingress_guard(allowed_ingress_tcp_ports)?; + install_default_egress_guard() +} + pub fn install_default_egress_guard() -> Result<()> { let host_gateway_ips = resolve_host_gateway_ips(); install_egress_guard_family(IPTABLES_BIN, "-4", BLOCKED_IPV4_CIDRS, &host_gateway_ips)?; @@ -103,6 +109,12 @@ pub fn install_default_egress_guard() -> Result<()> { Ok(()) } +fn install_default_ingress_guard(allowed_tcp_ports: &[u16]) -> Result<()> { + install_ingress_guard_family(IPTABLES_BIN, allowed_tcp_ports)?; + install_ingress_guard_family(IP6TABLES_BIN, allowed_tcp_ports)?; + Ok(()) +} + pub fn wait_for_mesh_config_scope( config_path: &Path, expected_scope: &str, @@ -221,6 +233,71 @@ fn install_egress_guard_family( Ok(()) } +fn install_ingress_guard_family(iptables_bin: &str, allowed_tcp_ports: &[u16]) -> Result<()> { + run_command_allow_failure( + iptables_bin, + ["-w", "-D", "INPUT", "-j", DEFAULT_INGRESS_CHAIN], + )?; + run_command_allow_failure(iptables_bin, ["-w", "-F", DEFAULT_INGRESS_CHAIN])?; + run_command_allow_failure(iptables_bin, ["-w", "-X", DEFAULT_INGRESS_CHAIN])?; + + run_command(iptables_bin, ["-w", "-N", DEFAULT_INGRESS_CHAIN])?; + run_command( + iptables_bin, + ["-w", "-A", "INPUT", "-j", DEFAULT_INGRESS_CHAIN], + )?; + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "RETURN", + ], + )?; + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-i", + "lo", + "-j", + "RETURN", + ], + )?; + + for port in allowed_tcp_ports { + let port = port.to_string(); + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-p", + "tcp", + "--dport", + port.as_str(), + "-j", + "RETURN", + ], + )?; + } + + run_command( + iptables_bin, + ["-w", "-A", DEFAULT_INGRESS_CHAIN, "-j", "REJECT"], + )?; + Ok(()) +} + fn read_route_snapshot(family_flag: &str) -> Result { let output = run_command( IP_BIN, diff --git a/runtime/helper/src/main.rs b/runtime/helper/src/main.rs index 0ef1e7e8..c2f42932 100644 --- a/runtime/helper/src/main.rs +++ b/runtime/helper/src/main.rs @@ -15,7 +15,8 @@ use std::{ }; use amber_helper::{ - HelperError, RunPlan, build_run_plan, install_default_egress_guard, wait_for_mesh_config_scope, + HelperError, RunPlan, build_run_plan, install_default_egress_guard, + install_default_network_guard, wait_for_mesh_config_scope, }; use amber_mesh::telemetry::{ COMPONENT_MONIKER_ENV, OtlpIdentity, OtlpInstallMode, SCENARIO_SCOPE_ENV, SubscriberFormat, @@ -148,6 +149,20 @@ fn run_main() -> Result { install_default_egress_guard()?; Ok(ExitCode::SUCCESS) } + "install-network-guard" => { + let allowed_ingress_tcp_ports = args + .map(|value| { + value.parse::().map_err(|err| { + HelperError::Msg(format!( + "install-network-guard allowed ingress port must be a TCP port \ + number: {err}" + )) + }) + }) + .collect::, _>>()?; + install_default_network_guard(&allowed_ingress_tcp_ports)?; + Ok(ExitCode::SUCCESS) + } _ => Err(usage_error()), } } @@ -155,7 +170,7 @@ fn run_main() -> Result { fn usage_error() -> HelperError { HelperError::Msg( "usage: amber-helper " + [TIMEOUT_SECONDS]|install-default-egress-guard|install-network-guard [TCP_PORT...]>" .to_string(), ) } diff --git a/runtime/mesh/src/dynamic_caps.rs b/runtime/mesh/src/dynamic_caps.rs index 8ded1d06..c8071ef2 100644 --- a/runtime/mesh/src/dynamic_caps.rs +++ b/runtime/mesh/src/dynamic_caps.rs @@ -1,11 +1,7 @@ use std::fmt; -use base64::{ - Engine as _, - engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}, -}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _, VerifyingKey}; -use rand_core::OsRng; use serde::{Deserialize, Serialize}; use url::Url; @@ -16,9 +12,9 @@ pub const DYNAMIC_CAPS_REF_HOST: &str = "ref"; pub const DYNAMIC_CAPS_REF_VERSION: u32 = 1; pub const DYNAMIC_CAPS_API_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_API_URL"; -pub const DYNAMIC_CAPS_CONTROL_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_CONTROL_URL"; -pub const DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV: &str = "AMBER_DYNAMIC_CAPS_CONTROL_AUTH_TOKEN"; -pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = "AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = + "__amber_internal_site_controller"; +const DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN: &[u8] = b"amber.dynamic-capability-ref.v1\0"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -246,60 +242,13 @@ pub struct GrantSnapshotIr { pub struct DynamicCapabilityRefClaims { pub version: u32, pub run_id: String, + pub issuer_id: String, pub grant_id: String, pub holder_component_id: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub descriptor_hint: Option, } -pub fn generate_dynamic_capability_signing_seed() -> [u8; 32] { - SigningKey::generate(&mut OsRng).to_bytes() -} - -pub fn signing_key_from_seed(seed: [u8; 32]) -> SigningKey { - SigningKey::from_bytes(&seed) -} - -pub fn signing_key_from_seed_b64(raw: &str) -> Result { - let decoded = STANDARD.decode(raw.as_bytes()).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to decode dynamic capability signing seed: {err}" - )) - })?; - let seed: [u8; 32] = decoded.as_slice().try_into().map_err(|_| { - DynamicCapabilityRefError::InvalidToken( - "dynamic capability signing seed must be exactly 32 bytes".to_string(), - ) - })?; - Ok(signing_key_from_seed(seed)) -} - -pub fn signing_seed_b64(signing_key: &SigningKey) -> String { - STANDARD.encode(signing_key.to_bytes()) -} - -pub fn verify_key_b64(signing_key: &SigningKey) -> String { - STANDARD.encode(signing_key.verifying_key().to_bytes()) -} - -pub fn verify_key_from_b64(raw: &str) -> Result { - let decoded = STANDARD.decode(raw.as_bytes()).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to decode dynamic capability verify key: {err}" - )) - })?; - let bytes: [u8; 32] = decoded.as_slice().try_into().map_err(|_| { - DynamicCapabilityRefError::InvalidToken( - "dynamic capability verify key must be exactly 32 bytes".to_string(), - ) - })?; - VerifyingKey::from_bytes(&bytes).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "dynamic capability verify key is malformed: {err}" - )) - }) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExactRootRouteError { InvalidLogicalComponentId, @@ -405,11 +354,7 @@ pub fn encode_dynamic_capability_ref( claims: DynamicCapabilityRefClaims, signing_key: &SigningKey, ) -> Result { - let payload = serde_json::to_vec(&claims).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to encode dynamic capability ref claims: {err}" - )) - })?; + let payload = dynamic_capability_ref_signature_payload(&claims)?; let signature = signing_key.sign(&payload); let token = DynamicCapabilityRefToken { claims, @@ -511,16 +456,27 @@ pub fn verify_dynamic_capability_ref( parsed: &ParsedDynamicCapabilityRef, verify_key: &VerifyingKey, ) -> Result<(), DynamicCapabilityRefError> { - let claims_bytes = serde_json::to_vec(&parsed.claims).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to re-encode dynamic capability ref claims: {err}" - )) - })?; + let claims_bytes = dynamic_capability_ref_signature_payload(&parsed.claims)?; verify_key .verify(&claims_bytes, &parsed.signature) .map_err(|_| DynamicCapabilityRefError::InvalidSignature) } +fn dynamic_capability_ref_signature_payload( + claims: &DynamicCapabilityRefClaims, +) -> Result, DynamicCapabilityRefError> { + let claims_json = serde_json::to_vec(claims).map_err(|err| { + DynamicCapabilityRefError::InvalidToken(format!( + "failed to encode dynamic capability ref claims: {err}" + )) + })?; + let mut payload = + Vec::with_capacity(DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN.len() + claims_json.len()); + payload.extend_from_slice(DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN); + payload.extend_from_slice(&claims_json); + Ok(payload) +} + pub fn build_dynamic_capability_ref_url( claims: DynamicCapabilityRefClaims, signing_key: &SigningKey, @@ -552,6 +508,8 @@ pub fn build_dynamic_capability_ref_url( #[cfg(test)] mod tests { + use rand_core::OsRng; + use super::*; use crate::HttpRoutePlugin; @@ -581,6 +539,7 @@ mod tests { DynamicCapabilityRefClaims { version: DYNAMIC_CAPS_REF_VERSION, run_id: "run-123".to_string(), + issuer_id: "/controller".to_string(), grant_id: "g_abc".to_string(), holder_component_id: "components./worker".to_string(), descriptor_hint: Some("worker.http".to_string()), @@ -589,7 +548,7 @@ mod tests { #[test] fn dynamic_capability_ref_round_trip_preserves_claims_and_suffix() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); + let signing_key = SigningKey::generate(&mut OsRng); let raw = build_dynamic_capability_ref_url( test_claims(), &signing_key, @@ -609,7 +568,7 @@ mod tests { #[test] fn dynamic_capability_ref_rejects_signature_tampering() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); + let signing_key = SigningKey::generate(&mut OsRng); let raw = build_dynamic_capability_ref_url(test_claims(), &signing_key, "/", None, None) .expect("dynamic ref should build"); let mut parsed = Url::parse(&raw).expect("dynamic ref url should parse"); @@ -630,14 +589,6 @@ mod tests { assert!(matches!(err, DynamicCapabilityRefError::InvalidSignature)); } - #[test] - fn verify_key_b64_round_trip_preserves_key_material() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); - let encoded = verify_key_b64(&signing_key); - let decoded = verify_key_from_b64(&encoded).expect("verify key should decode"); - assert_eq!(decoded.to_bytes(), signing_key.verifying_key().to_bytes()); - } - #[test] fn held_entry_summary_serialization_keeps_empty_materializations() { let summary = HeldEntrySummary { diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index dfb54a9b..27fcb39f 100644 --- a/runtime/mesh/src/lib.rs +++ b/runtime/mesh/src/lib.rs @@ -1,6 +1,5 @@ use std::{ collections::HashMap, - env, net::SocketAddr, path::{Path, PathBuf}, }; @@ -207,14 +206,21 @@ pub struct MeshConfigTemplate { pub const MESH_PROVISION_PLAN_VERSION: &str = "2"; pub const MESH_CONFIG_FILENAME: &str = "mesh-config.json"; pub const MESH_IDENTITY_FILENAME: &str = "mesh-identity.json"; -pub const FRAMEWORK_COMPONENT_CCS_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CCS_URL"; -pub const FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = + dynamic_caps::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const DYNAMIC_CAPS_API_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_API_URL_ENV; -pub const DYNAMIC_CAPS_CONTROL_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_CONTROL_URL_ENV; -pub const DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV: &str = - dynamic_caps::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV; -pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = - dynamic_caps::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV; + +fn stable_temp_socket_root() -> PathBuf { + #[cfg(unix)] + { + PathBuf::from("/tmp") + } + + #[cfg(not(unix))] + { + std::env::temp_dir() + } +} pub fn stable_temp_socket_path(namespace: &str, kind: &str, path: &Path) -> PathBuf { let mut hasher = sha2::Sha256::new(); @@ -229,7 +235,7 @@ pub fn stable_temp_socket_path(namespace: &str, kind: &str, path: &Path) -> Path .iter() .map(|byte| format!("{byte:02x}")) .collect::(); - env::temp_dir() + stable_temp_socket_root() .join(namespace) .join(format!("{kind}-{suffix}.sock")) } @@ -322,6 +328,10 @@ pub enum InboundTarget { Local { port: u16, }, + DockerGateway { + docker_sock: PathBuf, + compose_project_env: String, + }, External { url_env: String, #[serde(default, skip_serializing_if = "is_false")] @@ -608,6 +618,8 @@ mod key_serde_64 { #[cfg(test)] mod tests { + use std::env; + use super::*; #[test] @@ -644,9 +656,51 @@ mod tests { ); assert_eq!( path, - env::temp_dir() + Path::new("/tmp") .join("amber-direct-control") .join("current-9308da51951bac96.sock",) ); } + + #[test] + fn source_does_not_contain_literal_nul_bytes() { + let source = include_bytes!("lib.rs"); + assert!( + !source.contains(&0), + "use escaped \\0 sequences instead of literal NUL bytes in Rust source", + ); + } + + #[cfg(unix)] + #[test] + fn stable_temp_socket_path_ignores_tmpdir_env() { + static LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); + let _guard = LOCK + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + .expect("env lock should not be poisoned"); + let original = env::var_os("TMPDIR"); + unsafe { + env::set_var("TMPDIR", "/var/folders/custom-process-temp"); + } + + let path = stable_temp_socket_path( + "amber-direct-control", + "current", + Path::new("/tmp/amber/example/sites/direct_local/artifact"), + ); + + match original { + Some(value) => unsafe { env::set_var("TMPDIR", value) }, + None => unsafe { env::remove_var("TMPDIR") }, + } + + assert_eq!( + path, + Path::new("/tmp") + .join("amber-direct-control") + .join("current-9308da51951bac96.sock"), + "stable temp socket paths must not vary with per-process TMPDIR values", + ); + } } diff --git a/runtime/proxy/src/lib.rs b/runtime/proxy/src/lib.rs index ec436977..82731dcd 100644 --- a/runtime/proxy/src/lib.rs +++ b/runtime/proxy/src/lib.rs @@ -1736,6 +1736,13 @@ fn control_status(code: u16, body: &str) -> Result<(), ControlUpdateError> { }) } +fn control_delete_status(code: u16, body: &str) -> Result<(), ControlUpdateError> { + if (200..300).contains(&code) || code == 404 { + return Ok(()); + } + control_status(code, body) +} + async fn send_control_put_json( endpoint: &ControlEndpoint, path: &str, @@ -1775,7 +1782,7 @@ async fn send_control_delete_json( ); let response = send_control_request(endpoint, &request).await?; let (code, body) = parse_http_response(&response).ok_or(ControlUpdateError::Retryable)?; - control_status(code, body) + control_delete_status(code, body) } fn parse_http_response(response: &str) -> Option<(u16, &str)> { @@ -1986,13 +1993,44 @@ where .map_err(|_| ControlUpdateError::Retryable)? .map_err(|_| ControlUpdateError::Retryable)?; let mut buf = Vec::new(); - tokio::time::timeout(CONTROL_REQUEST_TIMEOUT, stream.read_to_end(&mut buf)) - .await - .map_err(|_| ControlUpdateError::Retryable)? - .map_err(|_| ControlUpdateError::Retryable)?; + let mut chunk = [0u8; 8192]; + loop { + let read = tokio::time::timeout(CONTROL_REQUEST_TIMEOUT, stream.read(&mut chunk)) + .await + .map_err(|_| ControlUpdateError::Retryable)? + .map_err(|_| ControlUpdateError::Retryable)?; + if read == 0 { + break; + } + buf.extend_from_slice(&chunk[..read]); + if let Some(response_len) = http_response_length(&buf) + && buf.len() >= response_len + { + break; + } + } Ok(String::from_utf8_lossy(&buf).to_string()) } +fn http_response_length(buf: &[u8]) -> Option { + let header_end = find_header_end(buf)?; + let header = std::str::from_utf8(&buf[..header_end]).ok()?; + let content_length = header.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.trim() + .eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + })?; + Some(header_end + content_length) +} + +fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|index| index + 4) +} + fn mesh_protocol_from_metadata(protocol: &str) -> Result { Ok(match protocol { "http" | "https" => MeshProtocol::Http, diff --git a/runtime/proxy/src/tests.rs b/runtime/proxy/src/tests.rs index ce3dc6c2..68f74f9c 100644 --- a/runtime/proxy/src/tests.rs +++ b/runtime/proxy/src/tests.rs @@ -471,7 +471,7 @@ fn resolve_control_endpoint_preserves_nested_compose_volume_socket_path() { mesh_port: 24000, control_port: 24100, compose_project: None, - control_socket: Some("/site/compose_local/router-control.sock".to_string()), + control_socket: Some("/router-control.sock".to_string()), control_socket_volume: Some( "${COMPOSE_PROJECT_NAME:-default}_amber-router-control".to_string(), ), @@ -492,7 +492,7 @@ fn resolve_control_endpoint_preserves_nested_compose_volume_socket_path() { panic!("expected compose volume socket endpoint"); }; assert_eq!(volume, "mixed-stack_amber-router-control"); - assert_eq!(socket_path, "/site/compose_local/router-control.sock"); + assert_eq!(socket_path, "/router-control.sock"); } #[test] @@ -505,7 +505,7 @@ fn resolve_control_endpoint_prefers_router_metadata_compose_project() { mesh_port: 24000, control_port: 24100, compose_project: Some("dynamic-stack".to_string()), - control_socket: Some("/site/compose_local/router-control.sock".to_string()), + control_socket: Some("/router-control.sock".to_string()), control_socket_volume: Some( "${COMPOSE_PROJECT_NAME:-default}_amber-router-control".to_string(), ), @@ -525,7 +525,7 @@ fn resolve_control_endpoint_prefers_router_metadata_compose_project() { panic!("expected compose volume socket endpoint"); }; assert_eq!(volume, "dynamic-stack_amber-router-control"); - assert_eq!(socket_path, "/site/compose_local/router-control.sock"); + assert_eq!(socket_path, "/router-control.sock"); } #[test] @@ -637,3 +637,45 @@ async fn try_fetch_router_identity_times_out_stalled_control_requests() { ); handle.join().expect("listener thread should finish"); } + +#[tokio::test] +async fn try_fetch_router_identity_accepts_complete_http_response_without_eof() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let public_key = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + let body = format!( + r#"{{"id":"/site/test/router","public_key":"{}","mesh_scope":"scope"}}"#, + public_key + ); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("listener should accept"); + let mut request = [0u8; 256]; + let _ = stream + .read(&mut request) + .await + .expect("request should be readable"); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + stream + .write_all(response.as_bytes()) + .await + .expect("response should be writable"); + tokio::time::sleep(CONTROL_REQUEST_TIMEOUT + Duration::from_secs(1)).await; + }); + + let identity = match try_fetch_router_identity(&ControlEndpoint::Tcp(addr.to_string())).await { + Ok(identity) => identity, + Err(_) => panic!("complete HTTP response should not require EOF"), + }; + assert_eq!(identity.id, "/site/test/router"); + assert_eq!(identity.public_key, [7u8; 32]); + assert_eq!(identity.mesh_scope.as_deref(), Some("scope")); + server.await.expect("server should finish"); +} diff --git a/runtime/router/Cargo.toml b/runtime/router/Cargo.toml index 122ed2eb..afab6a6c 100644 --- a/runtime/router/Cargo.toml +++ b/runtime/router/Cargo.toml @@ -3,6 +3,7 @@ name = "amber-router" edition = "2024" [dependencies] +amber-docker-gateway = { workspace = true } amber-mesh = { workspace = true } aho-corasick = "1.1.3" base64 = { workspace = true } diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index 89272a24..28c0012f 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -1,3 +1,5 @@ +use std::env; + use super::*; #[derive(Clone)] @@ -22,6 +24,7 @@ pub(super) struct InboundRuntime { client: Arc, a2a_url_rewrite_table: Arc, dynamic_caps: Option>, + docker_gateways: DockerGatewayRuntimes, } #[derive(Debug, Deserialize)] @@ -156,6 +159,7 @@ pub async fn run_with_listeners( let trust = Arc::new(TrustBundle::new(&config)?); let inbound_routes = Arc::new(build_inbound_routes(&config)?); validate_outbound_routes(&config)?; + let docker_gateways = Arc::new(build_docker_gateway_runtimes(&config)?); let a2a_url_rewrite_table = Arc::new(a2a::UrlRewriteTable::from_routes( &config.inbound, &config.outbound, @@ -191,6 +195,7 @@ pub async fn run_with_listeners( client: client.clone(), a2a_url_rewrite_table: a2a_url_rewrite_table.clone(), dynamic_caps: dynamic_caps.clone(), + docker_gateways: docker_gateways.clone(), }, listeners_by_route.mesh.take(), )); @@ -377,6 +382,7 @@ pub(super) async fn handle_inbound( client, a2a_url_rewrite_table, dynamic_caps, + docker_gateways, } = state; let noise_keys = noise_keys_for_identity(&config.identity)?; let mut session = accept_noise(stream, &noise_keys, &trust).await?; @@ -423,20 +429,34 @@ pub(super) async fn handle_inbound( proxy_noise_to_plain(&mut session, target).await?; } } + InboundTarget::DockerGateway { .. } => { + if route.protocol != MeshProtocol::Tcp { + return Err(RouterError::InvalidConfig(format!( + "docker gateway route {} must use tcp protocol", + route.route_id + ))); + } + let gateway = docker_gateways + .get(&route.route_id) + .cloned() + .ok_or_else(|| { + RouterError::InvalidConfig(format!( + "missing docker gateway runtime for route {}", + route.route_id + )) + })?; + proxy_noise_to_docker_gateway(&mut session, remote_id, gateway).await?; + } InboundTarget::External { ref url_env, optional, } => match route.protocol { MeshProtocol::Http => { - let framework_route_id = (url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV) - .then(|| Arc::::from(route.route_id.as_str())); - let framework_peer_id = (url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV) - .then(|| Arc::::from(remote_id.as_str())); proxy_noise_to_external( &mut session, ExternalProxyRequest { - route_id: framework_route_id, - peer_id: framework_peer_id, + route_id: None, + peer_id: None, labels: HttpExchangeLabels::inbound_from_route( config.identity.id.clone().into(), remote_id.clone().into(), @@ -556,6 +576,7 @@ pub(super) async fn handle_inbound( &mut session, outbound, route.route_id.clone().into(), + remote_id.clone().into(), plugins, labels, ) @@ -675,6 +696,7 @@ pub(super) async fn handle_outbound( proxy_local_http_to_noise( &mut outbound, route.route_id.clone().into(), + config.identity.id.clone().into(), stream, plugins, HttpExchangeLabels::outbound_from_route(config.identity.id.clone().into(), &route), @@ -758,7 +780,7 @@ pub(super) fn bind_unix_listener(path: &str) -> Result Result { pub(super) fn build_inbound_routes(config: &MeshConfig) -> Result { let mut map = HashMap::new(); for route in &config.inbound { + if matches!(route.target, InboundTarget::DockerGateway { .. }) + && route.protocol != MeshProtocol::Tcp + { + return Err(RouterError::InvalidConfig(format!( + "inbound route {} targets the docker gateway but uses {} protocol", + route.route_id, + protocol_string(route.protocol) + ))); + } if route.protocol != MeshProtocol::Http && !route.http_plugins.is_empty() { return Err(RouterError::InvalidConfig(format!( "inbound route {} has http plugins but uses {} protocol", @@ -1490,6 +1531,38 @@ pub(super) fn build_inbound_routes(config: &MeshConfig) -> Result Result>, RouterError> { + let mut runtimes = HashMap::new(); + for route in &config.inbound { + let InboundTarget::DockerGateway { + docker_sock, + compose_project_env, + } = &route.target + else { + continue; + }; + let compose_project = env::var(compose_project_env) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + RouterError::InvalidConfig(format!( + "docker gateway route {} requires non-empty {compose_project_env}", + route.route_id + )) + })?; + let runtime = DockerGatewayRuntime::new(DockerGatewayConfig { + docker_sock: docker_sock.clone(), + compose_project, + }) + .map_err(|err| RouterError::InvalidConfig(err.to_string()))?; + runtimes.insert(route.route_id.clone(), Arc::new(runtime)); + } + Ok(runtimes) +} + pub(super) fn validate_outbound_routes(config: &MeshConfig) -> Result<(), RouterError> { for route in &config.outbound { if route.protocol != MeshProtocol::Http && !route.http_plugins.is_empty() { diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index cd112a0e..78fef51e 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use amber_mesh::{ - MeshProtocol, + MeshProtocol, OutboundRoute, component_protocol::{ProtocolErrorCode, ProtocolErrorResponse}, dynamic_caps::{ self as mesh_dynamic_caps, DescriptorIr, HeldEntryDetail, HeldEntryKind, HeldEntryState, @@ -10,7 +10,6 @@ use amber_mesh::{ MaterializedHandleSummary, RevokeRequest, RevokeResponse, RootAuthoritySelectorIr, ShareRequest, ShareResponse, ShareSource, }, - telemetry::SCENARIO_RUN_ID_ENV, }; use base64::Engine as _; use http_body_util::{BodyExt as _, Full}; @@ -20,23 +19,21 @@ use tokio::time::{Duration, MissedTickBehavior, interval}; use super::*; -const DYNAMIC_CAPS_CONTROL_HELD_LIST_PATH: &str = "/v1/control-state/dynamic-caps/held"; -const DYNAMIC_CAPS_CONTROL_HELD_DETAIL_PATH: &str = "/v1/control-state/dynamic-caps/held/detail"; -const DYNAMIC_CAPS_CONTROL_SHARE_PATH: &str = "/v1/control-state/dynamic-caps/share"; -const DYNAMIC_CAPS_CONTROL_INSPECT_REF_PATH: &str = "/v1/control-state/dynamic-caps/inspect-ref"; -const DYNAMIC_CAPS_CONTROL_REVOKE_PATH: &str = "/v1/control-state/dynamic-caps/revoke"; -const DYNAMIC_CAPS_CONTROL_RESOLVE_ORIGIN_PATH: &str = - "/v1/control-state/dynamic-caps/resolve-origin"; +const DYNAMIC_CAPS_CONTROLLER_HELD_LIST_PATH: &str = "/v1/controller/dynamic-caps/held"; +const DYNAMIC_CAPS_CONTROLLER_HELD_DETAIL_PATH: &str = "/v1/controller/dynamic-caps/held/detail"; +const DYNAMIC_CAPS_CONTROLLER_SHARE_PATH: &str = "/v1/controller/dynamic-caps/share"; +const DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH: &str = "/v1/controller/dynamic-caps/inspect-ref"; +const DYNAMIC_CAPS_CONTROLLER_REVOKE_PATH: &str = "/v1/controller/dynamic-caps/revoke"; +const DYNAMIC_CAPS_CONTROLLER_RESOLVE_REF_PATH: &str = "/v1/controller/dynamic-caps/resolve-ref"; +const DYNAMIC_CAPS_CONTROLLER_RESOLVE_ORIGIN_PATH: &str = + "/v1/controller/dynamic-caps/resolve-origin"; const DYNAMIC_CAPS_HANDLE_PREFIX: &str = "/v1/handles/"; const DYNAMIC_CAPS_HANDLE_ID_PREFIX: &str = "hdl_"; const DYNAMIC_CAPS_WATCH_POLL_INTERVAL: Duration = Duration::from_millis(250); #[derive(Debug)] -struct DynamicCapsControlEnv { +struct DynamicCapsControllerEnv { control_url: String, - control_auth_token: String, - verify_key_raw: String, - run_id: String, } #[derive(Clone, Debug, Serialize)] @@ -93,6 +90,12 @@ struct ControlDynamicResolveOriginRequest { source: DynamicCapabilityControlSourceRequest, } +#[derive(Clone, Debug, Serialize)] +struct ControlDynamicResolveRefRequest { + holder_component_id: String, + r#ref: String, +} + #[derive(Clone, Debug, Deserialize)] struct ControlDynamicResolveOriginResponse { held_id: String, @@ -105,14 +108,22 @@ struct ControlDynamicResolveOriginResponse { origin_peer_addr: String, } +#[derive(Clone, Debug, Deserialize)] +struct ControlDynamicResolveRefResponse { + #[serde(flatten)] + origin: ControlDynamicResolveOriginResponse, + relative_path: String, + #[serde(default)] + query: Option, + #[serde(default)] + fragment: Option, +} + #[derive(Clone)] pub(super) struct DynamicCapsRuntime { listen_addr: SocketAddr, component_id: Arc, - run_id: Arc, control_url: Arc, - control_auth_token: Arc, - verify_key: ed25519_dalek::VerifyingKey, config: Arc, client: Arc, a2a_url_rewrite_table: Arc, @@ -130,7 +141,6 @@ struct DynamicHandleRecord { handle_id: String, held_id: String, descriptor: DescriptorIr, - upstream: Option, } #[derive(Clone)] @@ -145,6 +155,49 @@ struct DynamicHandleUpstreamTasks { bridge_task: tokio::task::JoinHandle>, } +// Dynamic handle upstreams are established per request, so the response body +// must retain the upstream connection until streaming completes. +struct DynamicHandleResponseBody { + inner: BoxBody, + _upstream: DynamicHandleUpstream, +} + +impl hyper::body::Body for DynamicHandleResponseBody { + type Data = Bytes; + type Error = hyper::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + Pin::new(&mut this.inner).poll_frame(cx) + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + + fn size_hint(&self) -> hyper::body::SizeHint { + self.inner.size_hint() + } +} + +fn retain_dynamic_handle_upstream( + response: Response, + upstream: DynamicHandleUpstream, +) -> Response { + let (parts, body) = response.into_parts(); + Response::from_parts( + parts, + DynamicHandleResponseBody { + inner: body, + _upstream: upstream, + } + .boxed(), + ) +} + struct DynamicHandleProxyRoute { handle_id: String, forwarded_path: String, @@ -162,12 +215,6 @@ struct HeldWatchSnapshot { materializations: BTreeMap>, } -impl DynamicHandleUpstream { - fn is_alive(&self) -> bool { - !self._tasks.conn_task.is_finished() && !self._tasks.bridge_task.is_finished() - } -} - impl Drop for DynamicHandleUpstreamTasks { fn drop(&mut self) { self.conn_task.abort(); @@ -184,7 +231,7 @@ impl DynamicCapsRuntime { let Some(listen_addr) = config.dynamic_caps_listen else { return Ok(None); }; - let Some(control_env) = resolve_dynamic_caps_control_env()? else { + let Some(control_env) = resolve_dynamic_caps_controller_env(config.as_ref())? else { tracing::warn!( target: "amber.internal", component_id = %config.identity.id, @@ -193,15 +240,10 @@ impl DynamicCapsRuntime { ); return Ok(None); }; - let verify_key = mesh_dynamic_caps::verify_key_from_b64(&control_env.verify_key_raw) - .map_err(|err| RouterError::InvalidConfig(err.to_string()))?; Ok(Some(Arc::new(Self { listen_addr, component_id: Arc::::from(format!("components.{}", config.identity.id)), - run_id: Arc::::from(control_env.run_id), control_url: Arc::::from(control_env.control_url), - control_auth_token: Arc::::from(control_env.control_auth_token), - verify_key, config, client, a2a_url_rewrite_table, @@ -242,14 +284,11 @@ impl DynamicCapsRuntime { format!("failed to serialize dynamic capability control request: {err}"), ) })?; - let request = Request::builder() + let request_builder = Request::builder() .method(Method::POST) .uri(&url) - .header(header::CONTENT_TYPE, "application/json") - .header( - AMBER_FRAMEWORK_AUTH_HEADER, - self.control_auth_token.as_ref(), - ) + .header(header::CONTENT_TYPE, "application/json"); + let request = request_builder .body( Full::new(Bytes::from(request_body)) .map_err(|never| match never {}) @@ -294,7 +333,7 @@ impl DynamicCapsRuntime { async fn control_held_list(&self) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_HELD_LIST_PATH, + DYNAMIC_CAPS_CONTROLLER_HELD_LIST_PATH, &ControlDynamicHeldListRequest { holder_component_id: self.component_id.to_string(), }, @@ -307,7 +346,7 @@ impl DynamicCapsRuntime { held_id: &str, ) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_HELD_DETAIL_PATH, + DYNAMIC_CAPS_CONTROLLER_HELD_DETAIL_PATH, &ControlDynamicHeldDetailRequest { holder_component_id: self.component_id.to_string(), held_id: held_id.to_string(), @@ -321,7 +360,7 @@ impl DynamicCapsRuntime { source: DynamicCapabilityControlSourceRequest, ) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_RESOLVE_ORIGIN_PATH, + DYNAMIC_CAPS_CONTROLLER_RESOLVE_ORIGIN_PATH, &ControlDynamicResolveOriginRequest { holder_component_id: self.component_id.to_string(), source, @@ -330,6 +369,34 @@ impl DynamicCapsRuntime { .await } + async fn control_inspect_ref( + &self, + raw_ref: &str, + ) -> Result { + self.control_post_json( + DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH, + &ControlDynamicInspectRefRequest { + holder_component_id: self.component_id.to_string(), + r#ref: raw_ref.to_string(), + }, + ) + .await + } + + async fn control_resolve_ref( + &self, + raw_ref: &str, + ) -> Result { + self.control_post_json( + DYNAMIC_CAPS_CONTROLLER_RESOLVE_REF_PATH, + &ControlDynamicResolveRefRequest { + holder_component_id: self.component_id.to_string(), + r#ref: raw_ref.to_string(), + }, + ) + .await + } + fn next_handle_id(state: &mut DynamicCapsLocalState) -> String { let handle_id = format!( "{DYNAMIC_CAPS_HANDLE_ID_PREFIX}{:016x}", @@ -356,38 +423,6 @@ impl DynamicCapsRuntime { Ok(url.to_string()) } - fn parse_ref_for_local_holder( - &self, - raw_ref: &str, - ) -> Result { - let parsed = mesh_dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) - .map_err(|err| self.protocol_error(ProtocolErrorCode::MalformedRef, err.to_string()))?; - if parsed.claims.version != mesh_dynamic_caps::DYNAMIC_CAPS_REF_VERSION { - return Err(self.protocol_error( - ProtocolErrorCode::MalformedRef, - format!( - "dynamic capability ref version {} is unsupported", - parsed.claims.version - ), - )); - } - if parsed.claims.run_id != self.run_id.as_ref() { - return Err(self.protocol_error( - ProtocolErrorCode::MalformedRef, - "dynamic capability ref belongs to a different run", - )); - } - if parsed.claims.holder_component_id != self.component_id.as_ref() { - return Err(self.protocol_error( - ProtocolErrorCode::RecipientMismatch, - "dynamic capability ref is bound to a different holder", - )); - } - mesh_dynamic_caps::verify_dynamic_capability_ref(&parsed, &self.verify_key) - .map_err(|err| self.protocol_error(ProtocolErrorCode::MalformedRef, err.to_string()))?; - Ok(parsed) - } - fn source_from_held_detail( &self, detail: &HeldEntryDetail, @@ -623,7 +658,16 @@ impl DynamicCapsRuntime { source: DynamicCapabilityControlSourceRequest, descriptor_hint: Option, ) -> Result { - let resolved = self.control_resolve_origin(source.clone()).await?; + let resolved = self.control_resolve_origin(source).await?; + self.ensure_resolved_dynamic_handle_materialized(resolved, descriptor_hint) + .await + } + + async fn ensure_resolved_dynamic_handle_materialized( + &self, + resolved: ControlDynamicResolveOriginResponse, + descriptor_hint: Option, + ) -> Result { if resolved.origin_protocol != "http" { return Err(self.protocol_error( ProtocolErrorCode::PathEstablishmentFailed, @@ -661,13 +705,10 @@ impl DynamicCapsRuntime { handle_id: handle_id.clone(), held_id: resolved.held_id.clone(), descriptor: descriptor_hint.unwrap_or_else(|| resolved.descriptor.clone()), - upstream: None, }, ); handle_id }; - self.refresh_dynamic_handle_upstream(&handle_id, &resolved) - .await?; Ok(MaterializeResponse { held_id: resolved.held_id, handle_id: handle_id.clone(), @@ -675,11 +716,10 @@ impl DynamicCapsRuntime { }) } - async fn refresh_dynamic_handle_upstream( + async fn establish_dynamic_handle_upstream( &self, - handle_id: &str, resolved: &ControlDynamicResolveOriginResponse, - ) -> Result<(), ProtocolErrorResponse> { + ) -> Result { let peer_key = base64::engine::general_purpose::STANDARD .decode(resolved.origin_peer_key_b64.as_bytes()) .map_err(|err| { @@ -742,30 +782,21 @@ impl DynamicCapsRuntime { ); } }); - let upstream = DynamicHandleUpstream { + Ok(DynamicHandleUpstream { route_id: Arc::::from(resolved.origin_route_id.as_str()), sender: Arc::new(Mutex::new(sender)), _tasks: Arc::new(DynamicHandleUpstreamTasks { conn_task, bridge_task, }), - }; - let mut state = self.handles.lock().await; - let record = state.handles_by_id.get_mut(handle_id).ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::UnknownHandle, - "dynamic capability handle disappeared during materialization", - ) - })?; - record.upstream = Some(upstream); - Ok(()) + }) } - async fn ensure_dynamic_handle_upstream( + async fn resolve_dynamic_handle_request( &self, handle_id: &str, ) -> Result<(DynamicHandleRecordView, DynamicHandleUpstream), ProtocolErrorResponse> { - let (held_id, descriptor, current_upstream) = { + let (held_id, descriptor) = { let state = self.handles.lock().await; let record = state.handles_by_id.get(handle_id).ok_or_else(|| { self.protocol_error( @@ -773,11 +804,7 @@ impl DynamicCapsRuntime { "dynamic capability handle does not exist", ) })?; - ( - record.held_id.clone(), - record.descriptor.clone(), - record.upstream.clone(), - ) + (record.held_id.clone(), record.descriptor.clone()) }; let detail = self.control_held_detail(&held_id).await?; if detail.summary.state != HeldEntryState::Live { @@ -786,39 +813,15 @@ impl DynamicCapsRuntime { "dynamic capability handle has been revoked", )); } - if let Some(upstream) = current_upstream - && upstream.is_alive() - { - return Ok(( - DynamicHandleRecordView { - held_id, - descriptor, - }, - upstream, - )); - } let source = self.source_from_held_detail(&detail)?; let resolved = self.control_resolve_origin(source).await?; - self.refresh_dynamic_handle_upstream(handle_id, &resolved) - .await?; - let state = self.handles.lock().await; - let record = state.handles_by_id.get(handle_id).ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::UnknownHandle, - "dynamic capability handle disappeared during refresh", - ) - })?; + let upstream = self.establish_dynamic_handle_upstream(&resolved).await?; Ok(( DynamicHandleRecordView { - held_id: record.held_id.clone(), - descriptor: record.descriptor.clone(), + held_id, + descriptor, }, - record.upstream.clone().ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::PathEstablishmentFailed, - "dynamic capability upstream was not established", - ) - })?, + upstream, )) } @@ -827,23 +830,18 @@ impl DynamicCapsRuntime { request: MaterializeRequest, ) -> Result { if let Some(raw_ref) = request.r#ref.as_deref() { - let parsed = self.parse_ref_for_local_holder(raw_ref)?; + let resolved = self.control_resolve_ref(raw_ref).await?; let materialized = self - .ensure_dynamic_handle_materialized( - DynamicCapabilityControlSourceRequest::Grant { - grant_id: parsed.claims.grant_id.clone(), - }, - None, - ) + .ensure_resolved_dynamic_handle_materialized(resolved.origin, None) .await?; return Ok(MaterializeResponse { held_id: materialized.held_id, handle_id: materialized.handle_id.clone(), url: self.materialized_handle_url( &materialized.handle_id, - &parsed.relative_path, - parsed.query.as_deref(), - parsed.fragment.as_deref(), + &resolved.relative_path, + resolved.query.as_deref(), + resolved.fragment.as_deref(), )?, }); } @@ -1043,7 +1041,7 @@ impl DynamicCapsRuntime { route: DynamicHandleProxyRoute, req: Request, ) -> Response { - let Ok((record, upstream)) = self.ensure_dynamic_handle_upstream(&route.handle_id).await + let Ok((record, upstream)) = self.resolve_dynamic_handle_request(&route.handle_id).await else { return error_response( StatusCode::GONE, @@ -1068,172 +1066,147 @@ impl DynamicCapsRuntime { } } let req = Request::from_parts(parts.0, parts.1); - proxy_outbound_http_request( - OutboundHttpProxyState { - upstream: upstream.sender.clone(), - plugins: if record.descriptor.kind == "a2a" { - Arc::from([Arc::new(a2a::A2aUrlRewritePlugin::new( - self.a2a_url_rewrite_table.clone(), - )) as Arc]) - } else { - Arc::from(Vec::>::new()) - }, - route_id: upstream.route_id.clone(), - labels: HttpExchangeLabels { - kind: HttpEdgeKind::Binding, - emit_telemetry: true, - slot: None, - capability: Arc::::from(record.descriptor.label.as_str()), - capability_kind: Some(Arc::::from(record.descriptor.kind.as_str())), - capability_profile: record.descriptor.profile.as_deref().map(Arc::::from), - source_component: Some(Arc::::from(self.config.identity.id.as_str())), - source_endpoint: Arc::::from(record.held_id.as_str()), - destination_component: None, - destination_endpoint: Arc::::from(record.descriptor.label.as_str()), + retain_dynamic_handle_upstream( + proxy_outbound_http_request( + OutboundHttpProxyState { + upstream: upstream.sender.clone(), + plugins: if record.descriptor.kind == "a2a" { + Arc::from([Arc::new(a2a::A2aUrlRewritePlugin::new( + self.a2a_url_rewrite_table.clone(), + )) as Arc]) + } else { + Arc::from(Vec::>::new()) + }, + route_id: upstream.route_id.clone(), + peer_id: Arc::::from(self.config.identity.id.as_str()), + labels: HttpExchangeLabels { + kind: HttpEdgeKind::Binding, + emit_telemetry: true, + slot: None, + capability: Arc::::from(record.descriptor.label.as_str()), + capability_kind: Some(Arc::::from(record.descriptor.kind.as_str())), + capability_profile: record + .descriptor + .profile + .as_deref() + .map(Arc::::from), + source_component: Some(Arc::::from(self.config.identity.id.as_str())), + source_endpoint: Arc::::from(record.held_id.as_str()), + destination_component: None, + destination_endpoint: Arc::::from(record.descriptor.label.as_str()), + }, + dynamic_caps: Some(Arc::new(self.clone())), }, - dynamic_caps: Some(Arc::new(self.clone())), - }, - req, + req, + ) + .await, + upstream, ) - .await } } -fn nonempty_env_var(name: &str) -> Option { - env::var(name) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - -fn required_dynamic_caps_env_var( - name: &'static str, - value: Option, -) -> Result { - value.ok_or_else(|| { - RouterError::InvalidConfig(format!( - "{name} must be set when dynamic_caps_listen is configured" - )) +fn local_dynamic_caps_controller_route(config: &MeshConfig) -> Option<&OutboundRoute> { + config.outbound.iter().find(|route| { + route.protocol == MeshProtocol::Http + && route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME }) } -fn resolve_dynamic_caps_control_env() -> Result, RouterError> { - let control_url = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV); - let control_auth_token = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV); - let verify_key_raw = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV); - if control_url.is_none() && control_auth_token.is_none() && verify_key_raw.is_none() { +fn resolve_dynamic_caps_controller_env( + config: &MeshConfig, +) -> Result, RouterError> { + if config.dynamic_caps_listen.is_none() { return Ok(None); } - let run_id = - required_dynamic_caps_env_var(SCENARIO_RUN_ID_ENV, nonempty_env_var(SCENARIO_RUN_ID_ENV))?; - Ok(Some(DynamicCapsControlEnv { - control_url: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, - control_url, - )?, - control_auth_token: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, - control_auth_token, - )?, - verify_key_raw: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - verify_key_raw, - )?, - run_id, + + let controller_route = local_dynamic_caps_controller_route(config); + let controller_route = controller_route.ok_or_else(|| { + RouterError::InvalidConfig( + "dynamic_caps_listen requires a local framework.component controller route".to_string(), + ) + })?; + Ok(Some(DynamicCapsControllerEnv { + control_url: format!("http://127.0.0.1:{}", controller_route.listen_port), })) } #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - static ENV_LOCK: OnceLock> = OnceLock::new(); - - struct EnvGuard { - saved: Vec<(&'static str, Option)>, + fn mesh_config_with_internal_controller_route() -> MeshConfig { + MeshConfig { + identity: amber_mesh::MeshIdentity::generate("/component/test", None), + mesh_listen: "127.0.0.1:23000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: Some("127.0.0.1:19000".parse().expect("dynamic caps listen")), + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: vec![OutboundRoute { + route_id: "component:/controller:__amber_internal_site_controller:http".to_string(), + rewrite_route_id: None, + slot: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + capability_kind: None, + capability_profile: None, + listen_port: 19001, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "controller:23000".to_string(), + peer_id: "/controller".to_string(), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + } } - impl EnvGuard { - fn replace(pairs: [(&'static str, Option<&str>); 4]) -> Self { - let saved = pairs - .iter() - .map(|(name, value)| { - let previous = env::var(name).ok(); - unsafe { - match value { - Some(value) => env::set_var(name, value), - None => env::remove_var(name), - } - } - (*name, previous) - }) - .collect(); - Self { saved } - } + fn mesh_config_without_internal_controller_route() -> MeshConfig { + let mut config = mesh_config_with_internal_controller_route(); + config.outbound.clear(); + config } - impl Drop for EnvGuard { - fn drop(&mut self) { - for (name, value) in self.saved.drain(..) { - unsafe { - match value { - Some(value) => env::set_var(name, value), - None => env::remove_var(name), - } - } - } - } + fn mesh_config_without_dynamic_caps_listener() -> MeshConfig { + let mut config = mesh_config_with_internal_controller_route(); + config.dynamic_caps_listen = None; + config } #[test] - fn resolve_dynamic_caps_control_env_disables_listener_when_control_env_is_absent() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - (amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, None), - (amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, None), - (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - + fn resolve_dynamic_caps_controller_env_ignores_control_env_when_listener_is_disabled() { assert!( - resolve_dynamic_caps_control_env() + resolve_dynamic_caps_controller_env(&mesh_config_without_dynamic_caps_listener()) .expect("dynamic caps env should resolve") .is_none(), - "sidecars without dynamic caps control env should leave the listener disabled", + "sidecars without a dynamic caps listener must ignore controller env", ); } #[test] - fn resolve_dynamic_caps_control_env_rejects_partial_configuration() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - ( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, - Some("http://127.0.0.1:24000"), - ), - (amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, None), - (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - - let err = resolve_dynamic_caps_control_env().expect_err("partial env must fail"); + fn resolve_dynamic_caps_controller_env_rejects_missing_local_controller_route() { + let err = + resolve_dynamic_caps_controller_env(&mesh_config_without_internal_controller_route()) + .expect_err("dynamic caps without a local controller route must fail"); assert!( matches!( &err, RouterError::InvalidConfig(message) - if message.contains(amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV) + if message.contains("local framework.component controller route") ), - "partial dynamic caps control env should fail with the missing variable name: {err}", + "missing local controller route should produce a targeted error: {err}", ); } + + #[test] + fn resolve_dynamic_caps_controller_env_prefers_local_controller_route_without_auth_token() { + let config = mesh_config_with_internal_controller_route(); + let env = resolve_dynamic_caps_controller_env(&config) + .expect("dynamic caps env should resolve") + .expect("local controller route should enable dynamic caps"); + assert_eq!(env.control_url, "http://127.0.0.1:19001"); + } } #[derive(Clone)] @@ -1367,7 +1340,7 @@ async fn dynamic_caps_service( }; let response: ShareResponse = match state .control_post_json( - DYNAMIC_CAPS_CONTROL_SHARE_PATH, + DYNAMIC_CAPS_CONTROLLER_SHARE_PATH, &ControlDynamicShareRequest { caller_component_id: state.component_id.to_string(), source, @@ -1409,12 +1382,12 @@ async fn dynamic_caps_service( Err(err) => return Ok(protocol_response(&err)), } } else if let Some(raw_ref) = request.r#ref.as_deref() { - let parsed = match state.parse_ref_for_local_holder(raw_ref) { - Ok(parsed) => parsed, + let inspected = match state.control_inspect_ref(raw_ref).await { + Ok(inspected) => inspected, Err(err) => return Ok(protocol_response(&err)), }; DynamicCapabilityControlSourceRequest::Grant { - grant_id: parsed.claims.grant_id, + grant_id: inspected.grant_id, } } else { return Ok(protocol_response(&ProtocolErrorResponse { @@ -1425,7 +1398,7 @@ async fn dynamic_caps_service( }; match state .control_post_json::<_, RevokeResponse>( - DYNAMIC_CAPS_CONTROL_REVOKE_PATH, + DYNAMIC_CAPS_CONTROLLER_REVOKE_PATH, &ControlDynamicRevokeRequest { caller_component_id: state.component_id.to_string(), target: source, @@ -1448,12 +1421,9 @@ async fn dynamic_caps_service( Ok(request) => request, Err(response) => return Ok(response), }; - if let Err(err) = state.parse_ref_for_local_holder(&request.r#ref) { - return Ok(protocol_response(&err)); - } let response: InspectRefResponse = match state .control_post_json( - DYNAMIC_CAPS_CONTROL_INSPECT_REF_PATH, + DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH, &ControlDynamicInspectRefRequest { holder_component_id: state.component_id.to_string(), r#ref: request.r#ref, diff --git a/runtime/router/src/external.rs b/runtime/router/src/external.rs index 122b342e..abf61a68 100644 --- a/runtime/router/src/external.rs +++ b/runtime/router/src/external.rs @@ -1,3 +1,5 @@ +use std::env; + use super::*; #[allow(clippy::result_large_err)] @@ -213,8 +215,7 @@ fn is_disallowed_external_ip(ip: IpAddr) -> bool { } fn allows_loopback_external_target(target: &ExternalTarget) -> bool { - target.url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV - || (target.url_env.starts_with("AMBER_EXTERNAL_SLOT_") && target.url_env.ends_with("_URL")) + target.url_env.starts_with("AMBER_EXTERNAL_SLOT_") && target.url_env.ends_with("_URL") } pub(super) fn join_url(base: &Url, uri: &Uri) -> Url { diff --git a/runtime/router/src/lib.rs b/runtime/router/src/lib.rs index 82386760..53e0b181 100644 --- a/runtime/router/src/lib.rs +++ b/runtime/router/src/lib.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashMap, HashSet, VecDeque}, - env, future::Future, io, net::{IpAddr, SocketAddr}, @@ -10,6 +9,7 @@ use std::{ task::{Context, Poll}, }; +use amber_docker_gateway::{DockerGatewayConfig, DockerGatewayRuntime}; use amber_mesh::{ HttpRoutePlugin, InboundRoute, InboundTarget, MeshConfig, MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, MeshProtocol, OutboundRoute, @@ -209,6 +209,29 @@ impl HttpExchangeLabels { ) } } + InboundTarget::DockerGateway { .. } => { + if router_forwarded_export { + ( + HttpEdgeKind::Export, + false, + None, + source_from_open + .clone() + .unwrap_or_else(|| capability.clone()), + Some(local_id.clone()), + capability.clone(), + ) + } else { + ( + HttpEdgeKind::Binding, + true, + Some(remote_id.clone()), + source_from_open.unwrap_or_else(|| capability.clone()), + Some(local_id.clone()), + capability.clone(), + ) + } + } InboundTarget::External { .. } => ( HttpEdgeKind::ExternalSlot, true, @@ -304,26 +327,18 @@ struct LocalHttpProxyState { const AMBER_ROUTE_ID_HEADER: &str = "x-amber-route-id"; const AMBER_PEER_ID_HEADER: &str = "x-amber-peer-id"; -const AMBER_FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; - -fn framework_component_auth_header_value() -> Option { - env::var(amber_mesh::FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .and_then(|value| HeaderValue::from_str(&value).ok()) -} - #[derive(Clone)] struct OutboundHttpProxyState { upstream: Arc>>, plugins: Arc<[Arc]>, route_id: Arc, + peer_id: Arc, labels: HttpExchangeLabels, dynamic_caps: Option>, } type HttpClient = Client>, BoxBody>; +type DockerGatewayRuntimes = Arc>>; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum BodyMode { @@ -1009,12 +1024,6 @@ async fn proxy_local_http_request( HeaderValue::from_str(state.peer_id.as_ref()) .expect("peer id header value should be valid"), ); - if let Some(auth_token) = framework_component_auth_header_value() { - request_parts.headers.insert( - header::HeaderName::from_static(AMBER_FRAMEWORK_AUTH_HEADER), - auth_token, - ); - } if emit_telemetry { inject_trace_context(&span, &mut request_parts.headers); } @@ -1233,6 +1242,7 @@ async fn proxy_local_http_request( async fn proxy_local_http_to_noise( session: &mut NoiseSession, route_id: Arc, + peer_id: Arc, stream: tokio::net::TcpStream, plugins: Arc<[Arc]>, labels: HttpExchangeLabels, @@ -1259,6 +1269,7 @@ async fn proxy_local_http_to_noise( upstream: Arc::new(Mutex::new(sender)), plugins, route_id, + peer_id, labels, dynamic_caps, }; @@ -1470,6 +1481,16 @@ async fn proxy_http_request_to_noise( } let host_header = outgoing_host_header(&request_parts.uri, &request_parts.headers); sanitize_request_headers(&mut request_parts.headers, host_header.as_str()); + request_parts.headers.insert( + header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER), + HeaderValue::from_str(state.route_id.as_ref()) + .expect("route id header value should be valid"), + ); + request_parts.headers.insert( + header::HeaderName::from_static(AMBER_PEER_ID_HEADER), + HeaderValue::from_str(state.peer_id.as_ref()) + .expect("peer id header value should be valid"), + ); inject_trace_context(&span, &mut request_parts.headers); let upstream_uri = request_parts.uri.to_string(); @@ -1726,26 +1747,30 @@ async fn proxy_http_request(state: HttpProxyState, req: Request) -> Re }; sanitize_request_headers(&mut parts.0.headers, &host_header); - if let Some(route_id) = state.route_id.as_ref() { + if let Some(route_id) = state.route_id.as_ref() + && !parts + .0 + .headers + .contains_key(header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER)) + { parts.0.headers.insert( header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER), HeaderValue::from_str(route_id.as_ref()) .expect("route id header value should be valid"), ); } - if let Some(peer_id) = state.peer_id.as_ref() { + if let Some(peer_id) = state.peer_id.as_ref() + && !parts + .0 + .headers + .contains_key(header::HeaderName::from_static(AMBER_PEER_ID_HEADER)) + { parts.0.headers.insert( header::HeaderName::from_static(AMBER_PEER_ID_HEADER), HeaderValue::from_str(peer_id.as_ref()) .expect("peer id header value should be valid"), ); } - if let Some(auth_token) = framework_component_auth_header_value() { - parts.0.headers.insert( - header::HeaderName::from_static(AMBER_FRAMEWORK_AUTH_HEADER), - auth_token, - ); - } inject_trace_context(&span, &mut parts.0.headers); let request_body = capture_box_body( diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index 7437c2eb..5d0f6ab4 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -66,6 +66,8 @@ fn empty_box_body() -> BoxBody { .boxed() } +const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; + async fn response_text(response: Response) -> String { String::from_utf8( response @@ -232,6 +234,35 @@ fn build_inbound_routes_rejects_http_plugins_on_non_local_target() { } } +#[test] +fn build_inbound_routes_rejects_non_tcp_docker_gateway_route() { + let config = MeshConfig { + inbound: vec![InboundRoute { + route_id: "docker-route".to_string(), + capability: "docker".to_string(), + capability_kind: Some("docker".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::DockerGateway { + docker_sock: "/var/run/docker.sock".into(), + compose_project_env: "COMPOSE_PROJECT_NAME".to_string(), + }, + allowed_issuers: vec!["peer-a".to_string()], + }], + ..test_mesh_config() + }; + + let err = build_inbound_routes(&config) + .expect_err("docker gateway routes must stay on the tcp mesh path"); + match err { + RouterError::InvalidConfig(message) => { + assert!(message.contains("targets the docker gateway but uses http protocol")); + } + other => panic!("unexpected error: {other}"), + } +} + #[test] fn validate_outbound_routes_rejects_http_plugins_on_non_http_route() { let config = MeshConfig { @@ -1176,6 +1207,91 @@ async fn revoke_route_overlay_allows_same_peer_id_with_new_key_after_cleanup() { .expect("same peer id should be reusable after overlay revoke cleanup"); } +#[tokio::test] +async fn revoke_route_overlay_cleans_up_stale_dynamic_peer_registration_by_id() { + let config = test_mesh_config(); + let inbound_routes = build_inbound_routes(&config).expect("build inbound routes"); + let trust = TrustBundle::new(&config).expect("trust"); + let dynamic_route_overlays: DynamicRouteOverlays = Arc::new(RwLock::new(HashMap::new())); + let dynamic_issuers: DynamicIssuers = Arc::new(RwLock::new(HashMap::new())); + let original_peer = test_peer("dynamic-peer"); + let stale_peer = test_peer("dynamic-peer"); + let replacement_peer = test_peer("dynamic-peer"); + assert_ne!( + original_peer.public_key, stale_peer.public_key, + "test peers should use distinct keys for the same peer id" + ); + assert_ne!( + stale_peer.public_key, replacement_peer.public_key, + "replacement test peer should also use a distinct key" + ); + + apply_route_overlay( + "overlay-a", + ControlRouteOverlay { + peers: vec![ControlRouteOverlayPeer { + peer_id: original_peer.id.clone(), + peer_key: base64::engine::general_purpose::STANDARD + .encode(original_peer.public_key), + }], + inbound_routes: vec![inbound_route( + "dynamic-route", + "dynamic", + MeshProtocol::Http, + InboundTarget::Local { port: 8080 }, + &["dynamic-peer"], + )], + }, + &trust, + &inbound_routes, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await + .expect("first overlay should apply"); + + trust + .remove_dynamic_peer_by_id("dynamic-peer") + .await + .expect("simulated drift cleanup should remove the original registration"); + trust + .insert_peer(&stale_peer) + .await + .expect("simulated stale dynamic registration should install"); + + revoke_route_overlay( + "overlay-a", + &trust, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await; + + apply_route_overlay( + "overlay-b", + ControlRouteOverlay { + peers: vec![ControlRouteOverlayPeer { + peer_id: replacement_peer.id.clone(), + peer_key: base64::engine::general_purpose::STANDARD + .encode(replacement_peer.public_key), + }], + inbound_routes: vec![inbound_route( + "dynamic-route", + "dynamic", + MeshProtocol::Http, + InboundTarget::Local { port: 8080 }, + &["dynamic-peer"], + )], + }, + &trust, + &inbound_routes, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await + .expect("overlay revoke should clear a stale dynamic peer registration by id"); +} + #[tokio::test] async fn apply_route_overlay_rejects_header_invalid_peer_id() { let config = test_mesh_config(); @@ -1710,28 +1826,6 @@ fn resolve_http_external_target_with_override_rejects_loopback_ip_literals() { assert_eq!(response.status(), StatusCode::BAD_GATEWAY); } -#[test] -fn resolve_http_external_target_with_override_accepts_framework_loopback_ip_literals() { - let target = ExternalTarget { - name: "component".to_string(), - url_env: amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), - optional: false, - url_override: None, - }; - - let resolved = resolve_http_external_target_with_override( - &target, - Some("http://127.0.0.1:6167/base"), - &Uri::from_static("/v1/children"), - ) - .expect("framework target should resolve"); - - let ResolvedHttpExternalTarget::Http(url) = resolved else { - panic!("expected direct http target"); - }; - assert_eq!(url.as_str(), "http://127.0.0.1:6167/base/v1/children"); -} - #[test] fn resolve_http_external_target_with_override_accepts_external_slot_loopback_ip_literals() { let target = ExternalTarget { @@ -2088,3 +2182,625 @@ async fn late_mesh_slot_registration_succeeds_on_same_http_connection() { let _ = proxy_task.await; let _ = mesh_server_task.await; } + +#[tokio::test] +async fn outbound_http_proxy_overwrites_internal_framework_auth_headers() { + let captured = Arc::new(StdMutex::new(None::<(String, String)>)); + let (upstream_client, upstream_server) = duplex(64 * 1024); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + let route_id = req + .headers() + .get(AMBER_ROUTE_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let peer_id = req + .headers() + .get(AMBER_PEER_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + *captured.lock().expect("capture lock") = Some((route_id, peer_id)); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state = OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("framework-route"), + peer_id: Arc::::from("/component/delegate"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/framework/create") + .header(AMBER_ROUTE_ID_HEADER, "spoofed-route") + .header(AMBER_PEER_ID_HEADER, "/spoofed/peer") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + Some(( + "framework-route".to_string(), + "/component/delegate".to_string(), + )), + "router-generated internal auth headers must overwrite caller-supplied values", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + +#[tokio::test] +async fn outbound_http_proxy_does_not_attach_framework_auth_header_to_noise_hops() { + let captured = Arc::new(StdMutex::new(None::)); + let (upstream_client, upstream_server) = duplex(64 * 1024); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + *captured.lock().expect("capture lock") = req + .headers() + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state = OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("binding-route"), + peer_id: Arc::::from("/component/provider"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/binding") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + None, + "ordinary noise-to-noise binding hops must not carry the framework controller auth header", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + +#[tokio::test] +async fn outbound_http_proxy_keeps_ephemeral_upstream_alive_for_response_body() { + let (upstream_client, upstream_server) = duplex(64 * 1024); + let upstream_task = tokio::spawn(async move { + let service = service_fn(|_req: Request| async move { + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body( + Full::new(Bytes::from_static(b"a")) + .map_err(|never| match never {}) + .boxed(), + ) + .expect("upstream response should build"), + ) + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state_slot = Arc::new(Mutex::new(Some(OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("dynamic-handle-route"), + peer_id: Arc::::from("/component/dynamic"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }))); + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state_slot = state_slot.clone(); + async move { + let state = state_slot + .lock() + .await + .take() + .expect("proxy should only serve one request"); + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/id") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response body should collect") + .to_bytes(); + assert_eq!(body.as_ref(), b"a"); + drop(client); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + +#[tokio::test] +async fn proxy_noise_to_plain_preserves_connection_close_response_body() { + let client_config = test_mesh_config(); + let server_identity = MeshIdentity::generate("provider-router", Some("test-scope".to_string())); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("mesh listener should bind"); + let addr = listener.local_addr().expect("mesh listener addr"); + let server_config = MeshConfig { + identity: server_identity.clone(), + peers: vec![MeshPeer { + id: client_config.identity.id.clone(), + public_key: client_config.identity.public_key, + }], + ..test_mesh_config() + }; + let trust = Arc::new(TrustBundle::new(&server_config).expect("mesh server trust")); + let noise_keys = noise_keys_for_identity(&server_config.identity).expect("noise keys"); + + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("mesh peer should accept"); + let mut session = accept_noise(stream, &noise_keys, trust.as_ref()) + .await + .expect("mesh peer should accept noise"); + let open = session.recv_open().await.expect("open frame"); + assert_eq!(open.capability, "provider.api"); + + let (local, remote) = duplex(64 * 1024); + let bridge = tokio::spawn(async move { proxy_noise_to_plain(&mut session, local).await }); + let service = service_fn(|req: Request| async move { + assert_eq!(req.uri().path(), "/id"); + assert_eq!( + req.headers() + .get(header::CONNECTION) + .and_then(|value| value.to_str().ok()), + Some("close"), + ); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body( + Full::new(Bytes::from_static(b"provider")) + .map_err(|never| match never {}) + .boxed(), + ) + .expect("mesh response should build"), + ) + }); + http1::Builder::new() + .serve_connection(TokioIo::new(remote), service) + .await + .expect("mesh http server should complete"); + bridge + .await + .expect("mesh bridge task should complete") + .expect("mesh bridge should succeed"); + }); + + let mut outbound = connect_noise_with_key( + &addr.to_string(), + &server_identity.id, + server_identity.public_key, + &client_config, + ) + .await + .expect("mesh client should connect"); + outbound + .send_open(&OpenFrame { + route_id: "dynamic-handle-route".to_string(), + capability: "provider.api".to_string(), + protocol: MeshProtocol::Http, + slot: None, + capability_kind: None, + capability_profile: None, + }) + .await + .expect("open frame should send"); + + let (local, remote) = duplex(64 * 1024); + let client_bridge = + tokio::spawn(async move { proxy_noise_to_plain(&mut outbound, remote).await }); + let (mut sender, conn) = client_http1::handshake(TokioIo::new(local)) + .await + .expect("client handshake should succeed"); + let conn_task = tokio::spawn(async move { + conn.await.expect("client connection should complete"); + }); + + let response = sender + .send_request( + Request::builder() + .uri("/id") + .header(header::CONNECTION, "close") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("response should arrive"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response body should collect") + .to_bytes(); + assert_eq!(body.as_ref(), b"provider"); + drop(sender); + + conn_task.await.expect("client connection task should join"); + client_bridge + .await + .expect("client bridge task should complete") + .expect("client bridge should succeed"); + server_task.await.expect("server task should join"); +} + +#[tokio::test] +async fn external_proxies_do_not_attach_framework_auth_headers() { + let captured = Arc::new(StdMutex::new(None::)); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("upstream should accept"); + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + *captured.lock().expect("capture lock") = req + .headers() + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("upstream server should complete"); + }); + + let (client, vetted_external_addrs) = build_client(); + let external_overrides: ExternalOverrides = Arc::new(RwLock::new(HashMap::from([( + "catalog".to_string(), + format!("http://127.0.0.1:{}/", addr.port()), + )]))); + let state = HttpProxyState { + client, + target: ExternalTarget { + name: "catalog".to_string(), + url_env: "AMBER_EXTERNAL_SLOT_CATALOG_URL".to_string(), + optional: false, + url_override: None, + }, + labels: test_http_exchange_labels(), + config: Arc::new(test_mesh_config()), + external_overrides, + vetted_external_addrs, + mesh_upstream: Arc::new(Mutex::new(None)), + route_id: None, + peer_id: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(proxy_http_request(state, req).await) } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/catalog") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + None, + "ordinary external slots must not receive the framework auth header", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); +} + +#[tokio::test] +async fn external_proxy_preserves_forwarded_internal_auth_headers() { + let captured = Arc::new(StdMutex::new(None::<(String, String)>)); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("controller listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("controller should accept"); + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + let route_id = req + .headers() + .get(AMBER_ROUTE_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let peer_id = req + .headers() + .get(AMBER_PEER_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + *captured.lock().expect("capture lock") = Some((route_id, peer_id)); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("controller response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("controller server should complete"); + }); + + let (client, vetted_external_addrs) = build_client(); + let external_overrides: ExternalOverrides = Arc::new(RwLock::new(HashMap::from([( + "gateway".to_string(), + format!("http://127.0.0.1:{}/", addr.port()), + )]))); + let state = HttpProxyState { + client, + target: ExternalTarget { + name: "gateway".to_string(), + url_env: "AMBER_EXTERNAL_SLOT_GATEWAY_URL".to_string(), + optional: false, + url_override: None, + }, + labels: test_http_exchange_labels(), + config: Arc::new(test_mesh_config()), + external_overrides, + vetted_external_addrs, + mesh_upstream: Arc::new(Mutex::new(None)), + route_id: Some(Arc::::from("site-router-route")), + peer_id: Some(Arc::::from("/site/source/router")), + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(proxy_http_request(state, req).await) } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/v1/children") + .header(AMBER_ROUTE_ID_HEADER, "forwarded-framework-route") + .header(AMBER_PEER_ID_HEADER, "/component/delegate") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + Some(( + "forwarded-framework-route".to_string(), + "/component/delegate".to_string(), + )), + "destination router must preserve forwarded framework auth headers instead of replacing \ + them with the intermediate site-router identity", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); +} diff --git a/runtime/router/src/transport.rs b/runtime/router/src/transport.rs index 21e46095..1fcd8e6d 100644 --- a/runtime/router/src/transport.rs +++ b/runtime/router/src/transport.rs @@ -367,6 +367,24 @@ pub(super) async fn proxy_noise_to_external_tcp( proxy_noise_to_plain(session, upstream).await } +pub(super) async fn proxy_noise_to_docker_gateway( + session: &mut NoiseSession, + peer_id: String, + gateway: Arc, +) -> Result<(), RouterError> { + let (local, remote) = duplex(64 * 1024); + let mut noise_session = session.clone(); + let bridge = tokio::spawn(async move { proxy_noise_to_plain(&mut noise_session, local).await }); + + gateway + .serve_connection(remote, peer_id) + .await + .map_err(|err| RouterError::Transport(err.to_string()))?; + + let _ = bridge.await; + Ok(()) +} + #[allow(clippy::too_many_arguments)] pub(super) async fn proxy_noise_to_local_http( session: &mut NoiseSession, @@ -421,6 +439,7 @@ pub(super) async fn proxy_noise_to_noise_http( session: &mut NoiseSession, outbound: NoiseSession, route_id: Arc, + peer_id: Arc, plugins: Arc<[Arc]>, labels: HttpExchangeLabels, ) -> Result<(), RouterError> { @@ -453,6 +472,7 @@ pub(super) async fn proxy_noise_to_noise_http( upstream: Arc::new(Mutex::new(sender)), plugins, route_id, + peer_id, labels, dynamic_caps: None, }; @@ -609,6 +629,21 @@ impl TrustBundle { } Ok(()) } + + pub(super) async fn remove_dynamic_peer_by_id(&self, peer_id: &str) -> Result<(), RouterError> { + let mut inner = self.inner.write().await; + let Some(existing) = inner.noise_by_id.remove(peer_id) else { + return Ok(()); + }; + if existing.static_peer { + inner.noise_by_id.insert(peer_id.to_string(), existing); + return Err(RouterError::Auth(format!( + "peer {peer_id} is statically registered" + ))); + } + inner.id_by_noise.remove(&existing.noise); + Ok(()) + } } fn insert_static_peer( diff --git a/runtime/site-controller/Cargo.toml b/runtime/site-controller/Cargo.toml new file mode 100644 index 00000000..5b92951b --- /dev/null +++ b/runtime/site-controller/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "amber-site-controller" +edition = "2024" + +[[bin]] +name = "amber-site-controller" +path = "src/bin/amber-site-controller.rs" + +[dependencies] +amber-compiler = { workspace = true } +amber-manifest = { workspace = true } +amber-mesh = { workspace = true } +amber-proxy = { workspace = true } +amber-resolver = { workspace = true } +amber-scenario = { workspace = true } +axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } +base64 = { workspace = true } +libc = "0.2.177" +miette = { workspace = true, features = ["fancy"] } +reqwest = { workspace = true } +regex = "1.11.1" +rmcp = { workspace = true, features = ["server", "macros", "transport-streamable-http-server", "schemars"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +sha2 = "0.10.9" +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "time"] } +url = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/cli/src/framework_component/api.rs b/runtime/site-controller/src/api.rs similarity index 93% rename from cli/src/framework_component/api.rs rename to runtime/site-controller/src/api.rs index d30a031b..3e1a0c4c 100644 --- a/cli/src/framework_component/api.rs +++ b/runtime/site-controller/src/api.rs @@ -1,5 +1,6 @@ use super::{planner::*, state::*, *}; +#[cfg(test)] pub(crate) fn authorize_capability_instance<'a>( state: &'a FrameworkControlState, cap_instance_id: &str, @@ -15,6 +16,35 @@ pub(crate) fn authorize_capability_instance<'a>( Ok(record) } +pub(crate) fn authorize_capability_route<'a>( + state: &'a FrameworkControlState, + route_id: &str, + peer_id: &str, +) -> std::result::Result<&'a CapabilityInstanceRecord, ProtocolErrorResponse> { + let mut route_records = state + .capability_instances + .values() + .filter(|record| record.route_id == route_id); + let Some(first_record) = route_records.next() else { + return Err(protocol_error( + ProtocolErrorCode::Unauthorized, + "unknown framework capability instance", + )); + }; + if first_record.recipient_peer_id == peer_id { + return Ok(first_record); + } + route_records + .find(|record| record.recipient_peer_id == peer_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::Unauthorized, + "framework capability instance is not bound to the authenticated mesh peer", + ) + }) +} + +#[cfg(test)] pub(super) fn capability_instance_record<'a>( state: &'a FrameworkControlState, cap_instance_id: &str, @@ -404,7 +434,6 @@ pub(super) struct ResolvedTemplateBinding { #[derive(Clone)] pub(super) struct SyntheticSourceRecord { - pub(super) slot_name: String, pub(super) actual_source: BindingFrom, pub(super) source_child_id: Option, pub(super) weak: bool, diff --git a/runtime/site-controller/src/bin/amber-site-controller.rs b/runtime/site-controller/src/bin/amber-site-controller.rs new file mode 100644 index 00000000..fbe0d00d --- /dev/null +++ b/runtime/site-controller/src/bin/amber-site-controller.rs @@ -0,0 +1,30 @@ +use std::{env, path::PathBuf}; + +use miette::Result; + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + amber_site_controller::run_site_controller_default(parse_args()?).await +} + +fn parse_args() -> Result { + let mut args = env::args_os(); + let _exe = args.next(); + let Some(flag) = args.next() else { + return usage_error(); + }; + if flag != "--plan" { + return usage_error(); + } + let Some(plan) = args.next() else { + return usage_error(); + }; + if args.next().is_some() { + return usage_error(); + } + Ok(PathBuf::from(plan)) +} + +fn usage_error() -> Result { + Err(miette::miette!("usage: amber-site-controller --plan FILE")) +} diff --git a/cli/src/framework_component/ccs_api.rs b/runtime/site-controller/src/ccs_api.rs similarity index 73% rename from cli/src/framework_component/ccs_api.rs rename to runtime/site-controller/src/ccs_api.rs index 76be068d..9197cf83 100644 --- a/cli/src/framework_component/ccs_api.rs +++ b/runtime/site-controller/src/ccs_api.rs @@ -1,11 +1,4 @@ -use super::{ - api::*, - http::{forward_create_child, forward_destroy_child}, - orchestration::ProtocolApiError, - planner::CcsApp, - state::FrameworkControlState, - *, -}; +use super::{api::*, state::FrameworkControlState, *}; pub(crate) enum FrameworkComponentInspectRequest { ListTemplates, @@ -90,24 +83,3 @@ pub(crate) async fn execute_framework_component_inspect( } }) } - -pub(crate) async fn execute_framework_component_mutate( - app: &CcsApp, - cap_instance_id: &str, - request: FrameworkComponentMutateRequest, -) -> std::result::Result { - Ok(match request { - FrameworkComponentMutateRequest::CreateChild(request) => { - FrameworkComponentMutateResponse::CreateChild( - forward_create_child(app, cap_instance_id, request).await?, - ) - } - FrameworkComponentMutateRequest::DestroyChild { child } => { - forward_destroy_child(app, cap_instance_id, &child).await?; - FrameworkComponentMutateResponse::DestroyChild(DestroyChildResponse { - child, - destroyed: true, - }) - } - }) -} diff --git a/cli/src/framework_component/control_state_api.rs b/runtime/site-controller/src/control_state_api.rs similarity index 61% rename from cli/src/framework_component/control_state_api.rs rename to runtime/site-controller/src/control_state_api.rs index 3250a4b6..8992e38c 100644 --- a/cli/src/framework_component/control_state_api.rs +++ b/runtime/site-controller/src/control_state_api.rs @@ -1,14 +1,13 @@ use base64::Engine as _; -use serde_json::json; use super::{ dynamic_caps, orchestration::{ - ProtocolApiError, dynamic_capability_component_runtime_endpoint, load_site_actuator_plan, - load_site_manager_state, publish_dynamic_capability_origin, + ProtocolApiError, load_site_manager_state, publish_dynamic_capability_origin, + router_identity_for_overlay, }, - planner::{ControlStateApp, protocol_error}, - state::persist_control_state_update, + planner::{ControlStateApp, live_assignment_map, protocol_error}, + state::{FrameworkControlState, persist_control_state_update}, *, }; @@ -158,9 +157,56 @@ pub(crate) async fn execute_dynamic_caps_mutate( }) } -async fn resolve_dynamic_capability_origin( +pub(crate) fn local_component_runtime( app: &ControlStateApp, - request: dynamic_caps::ControlDynamicResolveOriginRequest, + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; + let assignments = live_assignment_map(state); + let site_id = assignments.get(moniker).ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!("live component `{logical_component_id}` is not assigned to a live site"), + ) + })?; + if site_id != &app.controller_plan.site_id { + return Err(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "live component `{logical_component_id}` is assigned to site `{site_id}`, not \ + local site `{}`", + app.controller_plan.site_id + ), + )); + } + let site_plan = site_controller_runtime_plan_from_controller_plan(&app.controller_plan); + app.runtime + .collect_live_component_runtime_metadata(&site_plan) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "failed to resolve live runtime metadata for component \ + `{logical_component_id}` on site `{site_id}`: {err}" + ), + ) + })? + .remove(moniker) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "live runtime metadata for component `{logical_component_id}` is unavailable \ + on site `{site_id}`" + ), + ) + }) +} + +pub(crate) async fn resolve_dynamic_capability_origin_internal( + app: &ControlStateApp, + request: dynamic_caps::InternalDynamicResolveOriginRequest, ) -> std::result::Result { let state = app.control_state.lock().await.clone(); let source_key = dynamic_caps::source_key_from_control_request(&request.source); @@ -180,82 +226,67 @@ async fn resolve_dynamic_capability_origin( "dynamic capability root authority is no longer live", )) })?; - let holder_runtime = - dynamic_capability_component_runtime_endpoint(app, &state, &request.holder_component_id)?; - let holder_plan = load_site_actuator_plan(app, &holder_runtime.site_id)?; - let origin_runtime = - dynamic_capability_component_runtime_endpoint(app, &state, &root.holder_component_id)?; - let origin_site_id = origin_runtime.site_id.clone(); - let origin_plan = load_site_actuator_plan(app, &origin_site_id)?; - let origin_manager_state = load_site_manager_state(app, &origin_site_id)?; - let origin_peer_id = origin_manager_state - .router_identity_id - .clone() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` does not expose a live router identity for dynamic \ - capability publication" - ), - )) - })?; - let origin_peer_key_b64 = origin_manager_state - .router_public_key_b64 - .clone() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` does not expose a live router public key for dynamic \ - capability publication" - ), - )) - })?; - let origin_peer_addr = origin_manager_state - .router_mesh_addr - .as_deref() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` does not expose a live router mesh address for \ - dynamic capability publication" - ), - )) - }) - .and_then(|router_mesh_addr| { - router_mesh_addr_for_consumer(origin_plan.kind, holder_plan.kind, router_mesh_addr) - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` exposes an invalid live router mesh address \ - for dynamic capability publication: {err}" - ), - )) - }) - })?; - let overlay_suffix = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( - serde_json::to_vec(&json!({ - "holder_component_id": request.holder_component_id, - "root_authority_selector": resolved_source.root_authority_selector.clone(), - })) - .expect("dynamic capability origin overlay key should serialize"), + let _origin_runtime = local_component_runtime(app, &state, &root.holder_component_id)?; + let origin_peer = router_identity_for_overlay(app, &app.controller_plan.site_id).await?; + let origin_state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let origin_receipt = super::orchestration::site_receipt_from_manager_state(&origin_state); + let origin_peer_addr = if let Some(router_mesh_addr) = + crate::runtime_api::published_router_mesh_addr_for_consumer_kind( + &origin_receipt, + request.holder_site_kind, + ) { + Ok(router_mesh_addr.to_string()) + } else { + origin_state + .router_mesh_addr + .as_deref() + .ok_or_else(|| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site `{}` does not expose a live router mesh address for dynamic \ + capability publication", + app.controller_plan.site_id + ), + )) + }) + .and_then(|router_mesh_addr| { + app.runtime + .router_mesh_addr_for_component_consumer( + app.controller_plan.kind, + request.holder_site_kind, + router_mesh_addr, + ) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site `{}` exposes an invalid live router mesh address for \ + dynamic capability publication: {err}", + app.controller_plan.site_id + ), + )) + }) + }) + }?; + let overlay_id = dynamic_caps::origin_overlay_id( + &request.holder_component_id, + &resolved_source.root_authority_selector, + ); + let route_id = dynamic_caps::origin_route_id( + &request.holder_component_id, + &resolved_source.root_authority_selector, ); - let overlay_id = format!("dynamic-cap-origin-{overlay_suffix}"); - let route_id = format!("dynamic-cap-origin-route-{overlay_suffix}"); let publish = publish_dynamic_capability_origin( app, - &origin_site_id, + &app.controller_plan.site_id, &dynamic_caps::PublishDynamicCapabilityOriginRequest { overlay_id, route_id: route_id.clone(), root_authority_selector: resolved_source.root_authority_selector.clone(), allowed_peers: vec![dynamic_caps::DynamicCapabilityAllowedPeer { - peer_id: holder_runtime.runtime.mesh_config.identity.id.clone(), - peer_key_b64: base64::engine::general_purpose::STANDARD - .encode(holder_runtime.runtime.mesh_config.identity.public_key), + peer_id: request.holder_peer_id.clone(), + peer_key_b64: request.holder_peer_key_b64.clone(), }], }, ) @@ -274,8 +305,31 @@ async fn resolve_dynamic_capability_origin( origin_route_id: publish.route_id, origin_capability: publish.capability, origin_protocol: publish.protocol, - origin_peer_id, - origin_peer_key_b64, + origin_peer_id: origin_peer.id, + origin_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(origin_peer.public_key), origin_peer_addr, }) } + +async fn resolve_dynamic_capability_origin( + app: &ControlStateApp, + request: dynamic_caps::ControlDynamicResolveOriginRequest, +) -> std::result::Result { + let state = app.control_state.lock().await.clone(); + let holder_peer = local_component_runtime(app, &state, &request.holder_component_id)? + .mesh_config + .identity; + resolve_dynamic_capability_origin_internal( + app, + dynamic_caps::InternalDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: request.source, + holder_peer_id: holder_peer.id, + holder_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(holder_peer.public_key), + holder_site_kind: app.controller_plan.kind, + }, + ) + .await +} diff --git a/cli/src/framework_component/control_state_mcp.rs b/runtime/site-controller/src/control_state_mcp.rs similarity index 59% rename from cli/src/framework_component/control_state_mcp.rs rename to runtime/site-controller/src/control_state_mcp.rs index 258f44b3..e23d69e2 100644 --- a/cli/src/framework_component/control_state_mcp.rs +++ b/runtime/site-controller/src/control_state_mcp.rs @@ -20,21 +20,24 @@ use serde_json::Value; use super::{ control_state_api::{ - self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, + DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, DynamicCapsMutateResponse, }, dynamic_caps, - http::authorize_framework_auth_header, - mcp_common::{json_response, map_protocol_api_error}, - planner::ControlStateApp, - *, + mcp_common::{McpOperationResponse, json_response, map_protocol_api_error}, + planner::SiteControllerApp, + site_controller::{ + DynamicCapsRequestAuth, authorize_dynamic_caps_mcp_session_request, + authorize_dynamic_caps_request, execute_site_controller_dynamic_caps_inspect, + execute_site_controller_dynamic_caps_mutate, + }, }; const HELP_RESOURCE_URI: &str = "amber://framework-dynamic-caps"; const OPERATION_RESOURCE_PREFIX: &str = "amber://framework-dynamic-caps/op/"; pub(crate) fn service( - app: ControlStateApp, + app: SiteControllerApp, ) -> StreamableHttpService { StreamableHttpService::new( move || Ok(FrameworkDynamicCapsMcp::new(app.clone())), @@ -45,83 +48,87 @@ pub(crate) fn service( #[derive(Clone)] pub(crate) struct FrameworkDynamicCapsMcp { - app: ControlStateApp, + app: SiteControllerApp, tool_router: ToolRouter, } impl FrameworkDynamicCapsMcp { - fn new(app: ControlStateApp) -> Self { + fn new(app: SiteControllerApp) -> Self { Self { app, tool_router: Self::tool_router(), } } - fn authorize(&self, context: &RequestContext) -> Result<(), McpError> { - let parts = context + fn http_parts<'a>( + &self, + context: &'a RequestContext, + ) -> Result<&'a Parts, McpError> { + context .extensions .get::() - .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - authorize_framework_auth_header(&parts.headers, self.app.control_state_auth_token.as_ref()) + .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None)) + } + + async fn authorize_session( + &self, + context: &RequestContext, + ) -> Result<(), McpError> { + let parts = self.http_parts(context)?; + authorize_dynamic_caps_mcp_session_request(&self.app, &parts.headers) + .await .map_err(map_protocol_api_error) } - fn render_help_resource(&self) -> String { - let mut out = String::from("# framework dynamic caps MCP\n\n"); - out.push_str("Tools:\n"); - out.push_str("- `amber.v1.framework_dynamic_caps.inspect`\n"); - out.push_str("- `amber.v1.framework_dynamic_caps.mutate`\n\n"); - out.push_str("Transport:\n"); - out.push_str("- MCP endpoint: `/mcp`\n"); - out.push_str("- HTTP endpoints remain under `/v1/control-state/dynamic-caps/...`\n\n"); - out.push_str("Inspect operations:\n"); - for op in ["held_list", "held_detail", "inspect_ref", "resolve_origin"] { - out.push_str(&format!( - "- `{op}`: read `{}{op}` for argument details\n", - OPERATION_RESOURCE_PREFIX - )); - } - out.push_str("\nMutate operations:\n"); - for op in ["share", "revoke"] { - out.push_str(&format!( - "- `{op}`: read `{}{op}` for argument details\n", - OPERATION_RESOURCE_PREFIX - )); - } - out + async fn authorize_operation( + &self, + context: &RequestContext, + expected_component_id: &str, + ) -> Result { + let parts = self.http_parts(context)?; + authorize_dynamic_caps_request(&self.app, &parts.headers, expected_component_id) + .await + .map_err(map_protocol_api_error) } - fn render_operation_resource(&self, name: &str) -> Result { + fn help_resource(&self) -> String { + "# framework dynamic caps MCP\n\n- `amber.v1.framework_dynamic_caps.inspect`\n- \ + `amber.v1.framework_dynamic_caps.mutate`\n" + .to_string() + } + + fn operation_resource(&self, name: &str) -> Result { let doc = match name { "held_list" => { - "# `held_list`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"held_list\", \"holder_component_id\": \"components./alice\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/held`." + "# `held_list`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"held_list\", \"holder_component_id\": \"components./alice\" }\n```" } "held_detail" => { - "# `held_detail`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"held_detail\", \"holder_component_id\": \"components./alice\", \"held_id\": \"held_root_...\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/held/detail`." + "# `held_detail`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"held_detail\", \"holder_component_id\": \"components./alice\", \ + \"held_id\": \"held_root_...\" }\n```" } "inspect_ref" => { - "# `inspect_ref`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"inspect_ref\", \"holder_component_id\": \"components./carol\", \"ref\": \"amber://ref/...\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/inspect-ref`." + "# `inspect_ref`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"inspect_ref\", \"holder_component_id\": \"components./carol\", \"ref\": \ + \"amber://ref/...\" }\n```" } "resolve_origin" => { - "# `resolve_origin`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{\n \"op\": \"resolve_origin\",\n \"holder_component_id\": \"components./alice\",\n \"source\": {\n \"kind\": \"root_authority\",\n \"root_authority_selector\": {\n \"kind\": \"binding\",\n \"consumer_component_id\": \"components./alice\",\n \"slot_name\": \"upstream\",\n \"provider_component_id\": \"components./provider\",\n \"provider_capability_name\": \"http\"\n }\n }\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/resolve-origin`." + "# `resolve_origin`\n\nTool: \ + `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{\n \"op\": \ + \"resolve_origin\",\n \"holder_component_id\": \"components./alice\",\n \ + \"source\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```" } "share" => { - "# `share`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\nArguments:\n\ - ```json\n{\n \"op\": \"share\",\n \"caller_component_id\": \"components./alice\",\n \"source\": {\n \"kind\": \"root_authority\",\n \"root_authority_selector\": {\n \"kind\": \"binding\",\n \"consumer_component_id\": \"components./alice\",\n \"slot_name\": \"upstream\",\n \"provider_component_id\": \"components./provider\",\n \"provider_capability_name\": \"http\"\n }\n },\n \"recipient_component_id\": \"components./carol\"\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/share`." + "# `share`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\n```json\n{\n \ + \"op\": \"share\",\n \"caller_component_id\": \"components./alice\",\n \ + \"source\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" },\n \ + \"recipient_component_id\": \"components./carol\"\n}\n```" } "revoke" => { - "# `revoke`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\nArguments:\n\ - ```json\n{\n \"op\": \"revoke\",\n \"caller_component_id\": \"components./alice\",\n \"target\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/revoke`." + "# `revoke`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\n```json\n{\n \ + \"op\": \"revoke\",\n \"caller_component_id\": \"components./alice\",\n \ + \"target\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```" } _ => { return Err(McpError::resource_not_found( @@ -155,7 +162,7 @@ enum RootAuthoritySelectorArg { }, } -impl From for RootAuthoritySelectorIr { +impl From for amber_mesh::dynamic_caps::RootAuthoritySelectorIr { fn from(value: RootAuthoritySelectorArg) -> Self { match value { RootAuthoritySelectorArg::SelfProvide { @@ -264,50 +271,60 @@ impl FrameworkDynamicCapsMcp { &self, Parameters(args): Parameters, context: RequestContext, - ) -> Result, McpError> { - self.authorize(&context)?; - let response = control_state_api::execute_dynamic_caps_inspect( - &self.app, - match args { - InspectArgs::HeldList { - holder_component_id, - } => DynamicCapsInspectRequest::HeldList( - dynamic_caps::ControlDynamicHeldListRequest { - holder_component_id, - }, - ), - InspectArgs::HeldDetail { + ) -> Result, McpError> { + let request = match args { + InspectArgs::HeldList { + holder_component_id, + } => DynamicCapsInspectRequest::HeldList(dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id, + }), + InspectArgs::HeldDetail { + holder_component_id, + held_id, + } => DynamicCapsInspectRequest::HeldDetail( + dynamic_caps::ControlDynamicHeldDetailRequest { holder_component_id, held_id, - } => DynamicCapsInspectRequest::HeldDetail( - dynamic_caps::ControlDynamicHeldDetailRequest { - holder_component_id, - held_id, - }, - ), - InspectArgs::InspectRef { + }, + ), + InspectArgs::InspectRef { + holder_component_id, + r#ref, + } => DynamicCapsInspectRequest::InspectRef( + dynamic_caps::ControlDynamicInspectRefRequest { holder_component_id, r#ref, - } => DynamicCapsInspectRequest::InspectRef( - dynamic_caps::ControlDynamicInspectRefRequest { - holder_component_id, - r#ref, - }, - ), - InspectArgs::ResolveOrigin { + }, + ), + InspectArgs::ResolveOrigin { + holder_component_id, + source, + } => DynamicCapsInspectRequest::ResolveOrigin( + dynamic_caps::ControlDynamicResolveOriginRequest { holder_component_id, - source, - } => DynamicCapsInspectRequest::ResolveOrigin( - dynamic_caps::ControlDynamicResolveOriginRequest { - holder_component_id, - source: source.into(), - }, - ), - }, + source: source.into(), + }, + ), + }; + let expected_component_id = match &request { + DynamicCapsInspectRequest::HeldList(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::HeldDetail(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::InspectRef(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::ResolveOrigin(request) => { + request.holder_component_id.as_str() + } + } + .to_string(); + let auth = self + .authorize_operation(&context, &expected_component_id) + .await?; + let response = execute_site_controller_dynamic_caps_inspect( + &self.app, + request, + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await .map_err(map_protocol_api_error)?; - match response { DynamicCapsInspectResponse::HeldList(data) => json_response("held_list", data), DynamicCapsInspectResponse::HeldDetail(data) => json_response("held_detail", data), @@ -326,36 +343,44 @@ impl FrameworkDynamicCapsMcp { &self, Parameters(args): Parameters, context: RequestContext, - ) -> Result, McpError> { - self.authorize(&context)?; - let response = control_state_api::execute_dynamic_caps_mutate( + ) -> Result, McpError> { + let request = match args { + MutateArgs::Share { + caller_component_id, + source, + recipient_component_id, + idempotency_key, + options, + } => DynamicCapsMutateRequest::Share(dynamic_caps::ControlDynamicShareRequest { + caller_component_id, + source: source.into(), + recipient_component_id, + idempotency_key, + options, + }), + MutateArgs::Revoke { + caller_component_id, + target, + } => DynamicCapsMutateRequest::Revoke(dynamic_caps::ControlDynamicRevokeRequest { + caller_component_id, + target: target.into(), + }), + }; + let expected_component_id = match &request { + DynamicCapsMutateRequest::Share(request) => request.caller_component_id.as_str(), + DynamicCapsMutateRequest::Revoke(request) => request.caller_component_id.as_str(), + } + .to_string(); + let auth = self + .authorize_operation(&context, &expected_component_id) + .await?; + let response = execute_site_controller_dynamic_caps_mutate( &self.app, - match args { - MutateArgs::Share { - caller_component_id, - source, - recipient_component_id, - idempotency_key, - options, - } => DynamicCapsMutateRequest::Share(dynamic_caps::ControlDynamicShareRequest { - caller_component_id, - source: source.into(), - recipient_component_id, - idempotency_key, - options, - }), - MutateArgs::Revoke { - caller_component_id, - target, - } => DynamicCapsMutateRequest::Revoke(dynamic_caps::ControlDynamicRevokeRequest { - caller_component_id, - target: target.into(), - }), - }, + request, + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await .map_err(map_protocol_api_error)?; - match response { DynamicCapsMutateResponse::Share(data) => json_response("share", data), DynamicCapsMutateResponse::Revoke(data) => json_response("revoke", data), @@ -375,9 +400,8 @@ impl ServerHandler for FrameworkDynamicCapsMcp { .with_server_info(Implementation::new("amber-framework-dynamic-caps", "dev")) .with_protocol_version(ProtocolVersion::V_2025_06_18) .with_instructions( - "Use `amber.v1.framework_dynamic_caps.inspect` for reads and \ - `amber.v1.framework_dynamic_caps.mutate` for share or revoke. Read \ - `amber://framework-dynamic-caps` only when you need more detail." + "Use the framework_dynamic_caps tools. Read the amber:// help resources only when \ + needed." .to_string(), ) } @@ -387,7 +411,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { request: InitializeRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; if context.peer.peer_info().is_none() { context.peer.set_peer_info(request); } @@ -399,7 +423,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; Ok(ListResourcesResult { resources: vec![ RawResource::new(HELP_RESOURCE_URI, "framework dynamic caps MCP").no_annotation(), @@ -414,7 +438,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; Ok(ListResourceTemplatesResult { resource_templates: vec![ RawResourceTemplate::new( @@ -433,19 +457,18 @@ impl ServerHandler for FrameworkDynamicCapsMcp { request: ReadResourceRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; let uri = request.uri; let text = if uri.as_str() == HELP_RESOURCE_URI { - self.render_help_resource() + self.help_resource() } else if let Some(name) = uri.as_str().strip_prefix(OPERATION_RESOURCE_PREFIX) { - self.render_operation_resource(name)? + self.operation_resource(name)? } else { return Err(McpError::resource_not_found( - format!("resource {} not found", uri), + format!("resource {uri} not found"), None, )); }; - Ok(ReadResourceResult::new(vec![ResourceContents::text( text, uri, )])) diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs new file mode 100644 index 00000000..6c11f41c --- /dev/null +++ b/runtime/site-controller/src/default_runtime.rs @@ -0,0 +1,2135 @@ +#[cfg(target_os = "linux")] +use std::net::Ipv4Addr; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + env, fs, + io::{Read as _, Write as _}, + net::{SocketAddr, TcpListener}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Arc, OnceLock}, + time::Duration, +}; + +use amber_compiler::{ + mesh::ProxyMetadata, + reporter::{ + direct::{DirectPlan, DirectRuntimeUrlSource}, + vm::VmPlan, + }, + run_plan::{RunLink, SiteKind}, +}; +use amber_manifest::NetworkProtocol; +use amber_mesh::{ + HttpRoutePlugin, InboundRoute, InboundTarget, MESH_CONFIG_FILENAME, MESH_IDENTITY_FILENAME, + MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, + MeshPeerTemplate, MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, + MeshProvisionTargetKind, OutboundRoute, component_route_id, router_dynamic_export_route_id, + router_export_route_id, +}; +use amber_proxy::{ + ControlEndpoint, apply_route_overlay_with_retry, load_output_proxy_metadata, + revoke_route_overlay_with_retry, +}; +use miette::{Context as _, IntoDiagnostic as _, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use sha2::{Digest as _, Sha256}; +use tokio::{ + sync::Mutex as AsyncMutex, + time::{Instant, sleep}, +}; +use url::Url; + +mod child_runtime; +mod compose_controller; +mod dynamic_routes; +mod kubernetes_controller; +mod site_artifacts; +mod site_runtime_support; + +pub(crate) use self::child_runtime::{ + default_site_controller_runtime, runtime_plan_for_site_from_controller_plan, +}; +pub(super) use self::site_artifacts::*; +pub use self::{ + child_runtime::cleanup_dynamic_site_children, + compose_controller::{ + inject_compose_site_controller, inject_compose_site_controller_with_mount_sources, + }, + kubernetes_controller::inject_kubernetes_site_controller, + site_artifacts::{ + SiteControllerPeerRouterRoute, inject_site_controller_peer_router_routes, + set_compose_router_published_mesh_port, set_site_artifact_mesh_identity_seed, + }, + site_runtime_support::{ + host_service_bind_addr_for_consumer, observability_endpoint_for_site, + prepare_kubernetes_artifact_namespace, reserve_host_port, reserve_loopback_port, + router_mesh_addr_for_consumer, site_controller_peer_router_url, walk_files, + }, +}; +use self::{ + child_runtime::{ + DynamicComposeChildMetadata, SiteControllerRuntimeApp, SiteControllerRuntimeChildRecord, + SiteControllerRuntimeState, StoredRouteOverlayPayload, dynamic_compose_child_metadata_path, + dynamic_route_overlay_path, load_dynamic_proxy_exports_metadata, + site_controller_runtime_child_runtime_root, site_controller_runtime_state_path, + write_dynamic_route_overlay_payload, + }, + dynamic_routes::*, + site_runtime_support::*, +}; +use super::*; +use crate::{ + http::{read_json, write_json}, + planner::{ + LocalChildRuntimeSpec, build_desired_site_artifact_files, build_local_child_runtime_spec, + }, + runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, + state::{FrameworkControlState, LiveChildRecord}, +}; + +const DYNAMIC_COMPOSE_CHILD_SCHEMA: &str = "amber.run.dynamic_compose_child"; +const DYNAMIC_COMPOSE_CHILD_VERSION: u32 = 1; +const DYNAMIC_COMPOSE_MESH_ROOT: &str = ".amber/mesh"; +const DYNAMIC_ROUTE_OVERLAY_FILENAME: &str = "site-router-overlay.json"; +const DYNAMIC_PROXY_EXPORTS_FILENAME: &str = "proxy-exports.json"; +const COMPONENT_CONTROL_SOCKET_PATH_IN_VOLUME: &str = "/router-control.sock"; +const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; +const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; +const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; +const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; +const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; +const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; +const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; +const KUBERNETES_ROUTER_COMPONENT_NAME: &str = "amber-router"; +const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; +const SITE_PLAN_VERSION: u32 = 2; +const SITE_CONTROLLER_RUNTIME_STATE_SCHEMA: &str = "amber.run.site_controller_runtime_state"; +const SITE_CONTROLLER_RUNTIME_STATE_VERSION: u32 = 1; +const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; +const DESIRED_LINKS_VERSION: u32 = 1; +const DEFAULT_EXTERNAL_ENV_FILE: &str = "router-external.env"; +const DEFAULT_K8S_OTEL_UPSTREAM: &str = "http://host.docker.internal:18890"; +const CONTAINER_HOST_ALIAS: &str = "host.docker.internal"; +const KUBERNETES_WORKLOAD_READY_TIMEOUT: Duration = Duration::from_secs(120); +const KUBERNETES_SITE_READY_BUFFER: Duration = Duration::from_secs(30); +const COMPOSE_EGRESS_SUBNET_COUNT: u32 = 1 << 18; + +static KUBERNETES_CONTAINER_HOST_IP: OnceLock> = OnceLock::new(); + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum SiteLifecycleStatus { + Starting, + Running, + Stopping, + Stopped, + Failed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct SiteManagerState { + schema: String, + version: u32, + run_id: String, + site_id: String, + kind: SiteKind, + status: SiteLifecycleStatus, + artifact_dir: String, + supervisor_pid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + process_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_identity_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + last_error: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct SiteSupervisorPlan { + schema: String, + version: u32, + run_id: String, + mesh_scope: String, + run_root: String, + coordinator_pid: u32, + site_id: String, + kind: SiteKind, + artifact_dir: String, + site_state_root: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_control_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + launch_env: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DesiredLinkState { + schema: String, + version: u32, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + external_slots: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + export_peers: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + external_slot_overlays: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + export_peer_overlays: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct DesiredExportPeer { + export_name: String, + peer_id: String, + peer_key_b64: String, + protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + route_id: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DirectRuntimeState { + #[serde(default)] + ready: bool, + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + dynamic_caps_port_by_component: BTreeMap, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct VmRuntimeState { + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + route_host_ports_by_component: BTreeMap>>, + #[serde(default)] + endpoint_forwards_by_component: BTreeMap>, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +fn required_str<'a>(value: Option<&'a str>, label: &str) -> Result<&'a str> { + value.ok_or_else(|| miette::miette!("missing {label}")) +} + +fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { + plan_root.join(".amber").join("direct-runtime.json") +} + +fn direct_current_control_socket_path(plan_root: &Path) -> PathBuf { + amber_mesh::stable_temp_socket_path("amber-direct-control", "current", plan_root) +} + +#[cfg(unix)] +fn ensure_direct_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + if fs::read_link(link) + .ok() + .is_some_and(|existing_target| existing_target == target) + { + return Ok(()); + } + if fs::symlink_metadata(link).is_ok() { + fs::remove_file(link) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale {description} {}", link.display()))?; + } + std::os::unix::fs::symlink(target, link) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create {description} {} -> {}", + link.display(), + target.display() + ) + }) +} + +#[cfg(not(unix))] +fn ensure_direct_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + let _ = (link, target, description); + Err(miette::miette!( + "direct runtime control sockets require unix symlink support" + )) +} + +fn vm_current_control_socket_path(plan_root: &Path) -> PathBuf { + amber_mesh::stable_temp_socket_path("amber-vm-control", "current", plan_root) +} + +#[cfg(unix)] +fn ensure_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + if fs::read_link(link) + .ok() + .is_some_and(|existing_target| existing_target == target) + { + return Ok(()); + } + if fs::symlink_metadata(link).is_ok() { + fs::remove_file(link) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale {description} {}", link.display()))?; + } + std::os::unix::fs::symlink(target, link) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create {description} {} -> {}", + link.display(), + target.display() + ) + }) +} + +#[cfg(not(unix))] +fn ensure_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + let _ = (link, target, description); + Err(miette::miette!( + "vm runtime control sockets require unix symlink support" + )) +} + +fn write_vm_runtime_state(plan_root: &Path, state: &VmRuntimeState) -> Result<()> { + let path = plan_root.join(".amber").join("vm-runtime.json"); + let parent = path + .parent() + .ok_or_else(|| miette::miette!("invalid vm runtime state path"))?; + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create vm runtime dir {}", parent.display()))?; + write_json(&path, state) +} + +const TCG_VM_STARTUP_TIMEOUT: Duration = Duration::from_secs(720); + +pub fn vm_uses_tcg_accel() -> bool { + #[cfg(target_os = "macos")] + { + env::var_os("AMBER_VM_FORCE_TCG").is_some() + } + + #[cfg(target_os = "linux")] + { + env::var_os("AMBER_VM_FORCE_TCG").is_some() || !Path::new("/dev/kvm").exists() + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + env::var_os("AMBER_VM_FORCE_TCG").is_some() + } +} + +pub fn vm_endpoint_forward_ready_timeout() -> Duration { + if vm_uses_tcg_accel() { + TCG_VM_STARTUP_TIMEOUT + } else { + Duration::from_secs(120) + } +} + +fn vm_endpoint_forward_ready_timeout_for_runtime_plan( + plan: &SiteControllerRuntimePlan, +) -> Duration { + plan.vm_endpoint_forward_ready_timeout_secs + .map(Duration::from_secs) + .unwrap_or_else(vm_endpoint_forward_ready_timeout) +} + +fn required_existing_mesh_peer_identities( + plan: &MeshProvisionPlan, + available_peer_identities_by_id: &BTreeMap, +) -> Result> { + let target_ids = plan + .targets + .iter() + .map(|target| target.config.identity.id.as_str()) + .collect::>(); + let required_peer_ids = plan + .targets + .iter() + .flat_map(|target| target.config.peers.iter()) + .filter(|peer| !target_ids.contains(peer.id.as_str())) + .map(|peer| peer.id.as_str()) + .collect::>(); + + required_peer_ids + .into_iter() + .map(|peer_id| { + let identity = available_peer_identities_by_id + .get(peer_id) + .cloned() + .ok_or_else(|| { + miette::miette!( + "mesh provision plan requires existing peer identity {peer_id}, but it is \ + not currently available" + ) + })?; + Ok((peer_id.to_string(), identity)) + }) + .collect() +} + +fn mesh_output_dir_for_target(root: &Path, target: &MeshProvisionTarget) -> Result { + match &target.output { + MeshProvisionOutput::Filesystem { dir } => { + let path = Path::new(dir); + if path.is_absolute() { + return Err(miette::miette!( + "mesh provision plan contains absolute filesystem output path {}", + path.display() + )); + } + Ok(root.join(path)) + } + MeshProvisionOutput::KubernetesSecret { name, .. } => Err(miette::miette!( + "local runtime does not support kubernetes provision target {name}" + )), + } +} + +fn provision_mesh_filesystem_with_peer_identities( + plan: &MeshProvisionPlan, + root: &Path, + existing_peer_identities_by_id: &BTreeMap, +) -> Result<()> { + let mut identities: HashMap = HashMap::new(); + for identity in existing_peer_identities_by_id.values() { + identities.insert( + identity.id.clone(), + MeshIdentity { + id: identity.id.clone(), + public_key: identity.public_key, + private_key: [0; 64], + mesh_scope: identity.mesh_scope.clone(), + }, + ); + } + for target in &plan.targets { + let id = target.config.identity.id.clone(); + let mesh_scope = target.config.identity.mesh_scope.clone(); + identities + .entry(id) + .or_insert_with(|| match plan.identity_seed.as_deref() { + Some(seed) => { + MeshIdentity::derive(target.config.identity.id.clone(), mesh_scope, seed) + } + None => MeshIdentity::generate(target.config.identity.id.clone(), mesh_scope), + }); + } + for target in &plan.targets { + let output_dir = mesh_output_dir_for_target(root, target)?; + fs::create_dir_all(&output_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create mesh output directory {}", + output_dir.display() + ) + })?; + let identity = identities + .get(&target.config.identity.id) + .ok_or_else(|| { + miette::miette!( + "missing generated identity for {}", + target.config.identity.id + ) + })? + .clone(); + let identity_secret = MeshIdentitySecret::from_identity(&identity); + let public_config = target.config.to_public_config(&identities).map_err(|err| { + miette::miette!( + "failed to render mesh config for {}: {err}", + target.config.identity.id + ) + })?; + write_json(&output_dir.join(MESH_IDENTITY_FILENAME), &identity_secret)?; + write_json(&output_dir.join(MESH_CONFIG_FILENAME), &public_config)?; + } + Ok(()) +} + +fn project_existing_peer_identities_into_mesh_config( + path: &Path, + existing_peer_identities_by_id: &BTreeMap, +) -> Result<()> { + if existing_peer_identities_by_id.is_empty() || !path.is_file() { + return Ok(()); + } + let mut config: MeshConfigPublic = read_json(path, "mesh config")?; + let canonical_mesh_scope = existing_peer_identities_by_id + .values() + .find_map(|identity| identity.mesh_scope.clone()); + let mut changed = false; + if let Some(identity) = existing_peer_identities_by_id.get(&config.identity.id) { + if config.identity.public_key != identity.public_key + || config.identity.mesh_scope != identity.mesh_scope + { + config.identity.public_key = identity.public_key; + config.identity.mesh_scope = identity.mesh_scope.clone(); + changed = true; + } + } else if let Some(mesh_scope) = canonical_mesh_scope.as_ref() + && config.identity.mesh_scope.as_deref() != Some(mesh_scope.as_str()) + { + config.identity.mesh_scope = Some(mesh_scope.clone()); + changed = true; + } + for peer in &mut config.peers { + let Some(identity) = existing_peer_identities_by_id.get(&peer.id) else { + continue; + }; + if peer.public_key != identity.public_key { + peer.public_key = identity.public_key; + changed = true; + } + } + if changed { + write_json(path, &config)?; + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn rewrite_peer_addr_for_slirp_gateway(peer_addr: &str) -> String { + let Ok(addr) = peer_addr.parse::() else { + return peer_addr.to_string(); + }; + if !addr.ip().is_loopback() { + return peer_addr.to_string(); + } + SocketAddr::from((Ipv4Addr::new(10, 0, 2, 2), addr.port())).to_string() +} + +#[derive(Debug, Deserialize)] +struct KubernetesSecretPayload { + #[serde(default)] + data: BTreeMap, +} +fn rewrite_dynamic_proxy_metadata( + artifact_root: &Path, + proxy_exports: &BTreeMap, + kind: SiteKind, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + if kind == SiteKind::Compose { + let path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); + let Some(x_amber) = root.get_mut(&x_amber_key) else { + return Err(miette::miette!( + "compose file {} is missing x-amber metadata", + path.display() + )); + }; + let exports = proxy_exports + .iter() + .map(|(name, export)| { + Ok(( + serde_yaml::Value::String(name.clone()), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([ + ( + serde_yaml::Value::String("component".to_string()), + serde_yaml::Value::String(export.component.clone()), + ), + ( + serde_yaml::Value::String("provide".to_string()), + serde_yaml::Value::String(export.provide.clone()), + ), + ( + serde_yaml::Value::String("protocol".to_string()), + serde_yaml::Value::String(export.protocol.clone()), + ), + ( + serde_yaml::Value::String("router_mesh_port".to_string()), + serde_yaml::Value::Number(0u64.into()), + ), + ( + serde_yaml::Value::String("route_id".to_string()), + serde_yaml::Value::String(dynamic_proxy_export_route_id(name, export)?), + ), + ])), + )) + }) + .collect::>()?; + let mut metadata = x_amber.as_mapping().cloned().ok_or_else(|| { + miette::miette!( + "compose file {} has non-mapping x-amber metadata", + path.display() + ) + })?; + metadata.insert( + serde_yaml::Value::String("exports".to_string()), + serde_yaml::Value::Mapping(exports), + ); + *x_amber = serde_yaml::Value::Mapping(metadata); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) + } else { + let exports = proxy_exports + .iter() + .map(|(name, export)| { + Ok(( + name.clone(), + JsonValue::Object(JsonMap::from_iter([ + ( + "component".to_string(), + JsonValue::String(export.component.clone()), + ), + ( + "provide".to_string(), + JsonValue::String(export.provide.clone()), + ), + ( + "protocol".to_string(), + JsonValue::String(export.protocol.clone()), + ), + ("router_mesh_port".to_string(), JsonValue::from(0u64)), + ( + "route_id".to_string(), + JsonValue::String(dynamic_proxy_export_route_id(name, export)?), + ), + ])), + )) + }) + .collect::>>()?; + let path = artifact_root.join("amber-proxy.json"); + let mut metadata: JsonValue = read_json(&path, "proxy metadata")?; + let Some(object) = metadata.as_object_mut() else { + return Err(miette::miette!( + "proxy metadata {} is not a JSON object", + path.display() + )); + }; + object.insert("exports".to_string(), JsonValue::Object(exports)); + write_json(&path, &metadata) + } +} + +fn rewrite_dynamic_compose_proxy_metadata( + artifact_root: &Path, + compose_project: &str, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); + let Some(x_amber) = root.get_mut(&x_amber_key) else { + return Err(miette::miette!( + "compose file {} is missing x-amber metadata", + path.display() + )); + }; + let mut metadata: ProxyMetadata = serde_yaml::from_value(x_amber.clone()) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "compose file {} has invalid x-amber metadata", + path.display() + ) + })?; + if let Some(router) = metadata.router.as_mut() { + router.compose_project = Some(compose_project.to_string()); + } + *x_amber = serde_yaml::to_value(&metadata) + .into_diagnostic() + .wrap_err("failed to serialize compose proxy metadata")?; + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +pub(super) fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + +fn compose_services<'a>( + document: &'a serde_yaml::Value, + path: &Path, +) -> Result<&'a serde_yaml::Mapping> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing a services mapping", + path.display() + ) + }) +} + +pub(super) fn compose_services_mut<'a>( + document: &'a mut serde_yaml::Value, + path: &Path, +) -> Result<&'a mut serde_yaml::Mapping> { + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing a services mapping", + path.display() + ) + }) +} + +fn compose_networks_mut<'a>( + document: &'a mut serde_yaml::Value, + path: &Path, +) -> Result> { + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a mapping", + path.display() + )); + }; + Ok(root + .get_mut(yaml_string("networks")) + .and_then(serde_yaml::Value::as_mapping_mut)) +} + +pub fn assign_compose_egress_network_subnets( + artifact_dir: &Path, + run_id: &str, + site_id: &str, +) -> Result<()> { + let compose_path = artifact_dir.join("compose.yaml"); + if !compose_path.is_file() { + return Ok(()); + } + + let mut document = read_compose_document(&compose_path)?; + let Some(networks) = compose_networks_mut(&mut document, &compose_path)? else { + return Ok(()); + }; + + let mut used_subnets = networks + .values() + .filter_map(compose_network_subnet) + .map(str::to_string) + .collect::>(); + let mut changed = false; + + for (name, network) in networks.iter_mut() { + let Some(name) = name.as_str() else { + continue; + }; + if !name.starts_with("amber_egress_") || compose_network_subnet(network).is_some() { + continue; + } + let subnet = next_compose_egress_subnet(run_id, site_id, name, &used_subnets); + set_compose_network_subnet(network, &subnet)?; + used_subnets.insert(subnet); + changed = true; + } + + if !changed { + return Ok(()); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +fn compose_network_subnet(network: &serde_yaml::Value) -> Option<&str> { + network + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("ipam"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|ipam| ipam.get(yaml_string("config"))) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|configs| configs.first()) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(yaml_string("subnet"))) + .and_then(serde_yaml::Value::as_str) +} + +fn set_compose_network_subnet(network: &mut serde_yaml::Value, subnet: &str) -> Result<()> { + let Some(mapping) = network.as_mapping_mut() else { + return Err(miette::miette!( + "compose network definition is not a mapping" + )); + }; + mapping.insert( + yaml_string("ipam"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("config"), + serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( + serde_yaml::Mapping::from_iter([(yaml_string("subnet"), yaml_string(subnet))]), + )]), + )])), + ); + Ok(()) +} + +fn next_compose_egress_subnet( + run_id: &str, + site_id: &str, + network_name: &str, + used_subnets: &BTreeSet, +) -> String { + let digest = Sha256::digest(format!("{run_id}:{site_id}:{network_name}")); + let mut bytes = [0u8; 4]; + bytes.copy_from_slice(&digest[..4]); + let base = u32::from_be_bytes(bytes) % COMPOSE_EGRESS_SUBNET_COUNT; + for offset in 0..COMPOSE_EGRESS_SUBNET_COUNT { + let candidate = + compose_egress_subnet_from_index((base + offset) % COMPOSE_EGRESS_SUBNET_COUNT); + if !used_subnets.contains(&candidate) { + return candidate; + } + } + unreachable!("compose egress subnet pool exhausted") +} + +fn compose_egress_subnet_from_index(index: u32) -> String { + let second_octet = 64 + ((index >> 12) & 0x3f); + let third_octet = (index >> 4) & 0xff; + let fourth_octet = (index & 0x0f) << 4; + format!("100.{second_octet}.{third_octet}.{fourth_octet}/28") +} + +fn compose_service_names(document: &serde_yaml::Value, path: &Path) -> Result> { + Ok(compose_services(document, path)? + .keys() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()) +} + +fn compose_service_dependency_names(service: &serde_yaml::Value) -> Result> { + let Some(mapping) = service.as_mapping() else { + return Err(miette::miette!( + "compose service definition is not a mapping" + )); + }; + let Some(depends_on) = mapping.get(yaml_string("depends_on")) else { + return Ok(Vec::new()); + }; + if let Some(sequence) = depends_on.as_sequence() { + return Ok(sequence + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()); + } + if let Some(depends_on_map) = depends_on.as_mapping() { + return Ok(depends_on_map + .keys() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()); + } + Err(miette::miette!( + "compose service has invalid depends_on declaration" + )) +} + +fn retain_compose_service_dependencies( + service: &mut serde_yaml::Value, + keep_services: &BTreeSet, +) -> Result<()> { + let Some(mapping) = service.as_mapping_mut() else { + return Err(miette::miette!( + "compose service definition is not a mapping" + )); + }; + let depends_on_key = yaml_string("depends_on"); + let Some(depends_on) = mapping.get_mut(&depends_on_key) else { + return Ok(()); + }; + match depends_on { + serde_yaml::Value::Sequence(sequence) => { + sequence.retain(|value| { + value + .as_str() + .is_some_and(|name| keep_services.contains(name)) + }); + if sequence.is_empty() { + mapping.remove(&depends_on_key); + } + } + serde_yaml::Value::Mapping(depends_on_map) => { + depends_on_map.retain(|key, _| { + key.as_str() + .is_some_and(|name| keep_services.contains(name)) + }); + if depends_on_map.is_empty() { + mapping.remove(&depends_on_key); + } + } + _ => { + return Err(miette::miette!( + "compose service has invalid depends_on declaration" + )); + } + } + Ok(()) +} + +fn compose_dynamic_root_service_names( + artifact_root: &Path, + assigned_components: &[String], +) -> Result> { + let compose_path = artifact_root.join("compose.yaml"); + let document = read_compose_document(&compose_path)?; + let service_names = compose_service_names(&document, &compose_path)?; + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut out = BTreeSet::new(); + for target in plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) + || !assigned.contains(target.config.identity.id.as_str()) + { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = target.output else { + return Err(miette::miette!( + "compose component {} does not use filesystem mesh output", + target.config.identity.id + )); + }; + let sidecar = Path::new(&dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose component {} has invalid mesh output dir {}", + target.config.identity.id, + dir + ) + })? + .to_string(); + if service_names.contains(&sidecar) { + out.insert(sidecar.clone()); + } + if let Some(program) = sidecar.strip_suffix("-net") + && service_names.contains(program) + { + out.insert(program.to_string()); + } + } + if out.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not contain services for {:?}", + compose_path.display(), + assigned_components + )); + } + Ok(out) +} + +fn compose_live_service_names( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_compose_path = Path::new(&plan.artifact_dir).join("compose.yaml"); + let site_document = read_compose_document(&site_compose_path)?; + let mut live_services = compose_service_names(&site_document, &site_compose_path)?; + for child in published_children { + let child_compose_path = Path::new(&child.artifact_root).join("compose.yaml"); + let child_document = read_compose_document(&child_compose_path)?; + live_services.extend(compose_service_names(&child_document, &child_compose_path)?); + } + Ok(live_services) +} + +fn compose_service_closure( + document: &serde_yaml::Value, + compose_path: &Path, + roots: &BTreeSet, +) -> Result> { + let services = compose_services(document, compose_path)?; + let mut closure = BTreeSet::new(); + let mut queue = roots.iter().cloned().collect::>(); + while let Some(service_name) = queue.pop() { + if !closure.insert(service_name.clone()) { + continue; + } + let service = services.get(yaml_string(&service_name)).ok_or_else(|| { + miette::miette!( + "compose file {} is missing service {}", + compose_path.display(), + service_name + ) + })?; + for dependency in compose_service_dependency_names(service)? { + queue.push(dependency); + } + } + Ok(closure) +} + +fn replace_artifact_snapshot(root: &Path, files: &BTreeMap) -> Result<()> { + fs::create_dir_all(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", root.display()))?; + let expected = files.keys().map(PathBuf::from).collect::>(); + for path in walk_files(root)? { + let relative = path + .strip_prefix(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))? + .to_path_buf(); + if !expected.contains(&relative) { + fs::remove_file(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale artifact {}", path.display()))?; + } + } + for (relative, contents) in files { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + fs::write(&path, contents) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display()))?; + } + Ok(()) +} + +fn wait_for_named_kubernetes_resource_deleted( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + name: &str, + label: &str, +) -> Result<()> { + let resource = format!("{resource_kind}/{name}"); + let timeout = format!("{}s", KUBERNETES_WORKLOAD_READY_TIMEOUT.as_secs().max(1)); + let output = kubectl_command(context) + .args([ + "-n", + namespace, + "wait", + "--for=delete", + "--timeout", + timeout.as_str(), + resource.as_str(), + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: wait for {resource} deletion"))?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.contains("not found") { + return Ok(()); + } + let detail = if stderr.is_empty() { + format!("status {}", output.status) + } else { + stderr + }; + Err(miette::miette!("{label} failed: {detail}")) +} + +fn wait_for_kubernetes_artifact_workloads_deleted( + context: Option<&str>, + namespace: &str, + workloads: &KubernetesArtifactWorkloads, + site_id: &str, +) -> Result<()> { + let label = format!("wait for kubernetes child workload deletion on site `{site_id}`"); + for job in &workloads.jobs { + wait_for_named_kubernetes_resource_deleted(context, namespace, "job", job, &label)?; + } + for deployment in &workloads.deployments { + wait_for_named_kubernetes_resource_deleted( + context, + namespace, + "deployment", + deployment, + &label, + )?; + } + Ok(()) +} + +async fn wait_for_kubernetes_site_router_ready( + plan: &SiteControllerRuntimePlan, + timeout: Duration, +) -> Result<()> { + debug_assert_eq!(plan.kind, SiteKind::Kubernetes); + + let deadline = Instant::now() + timeout; + loop { + let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); + let manager_state = if state_path.is_file() { + let state: SiteManagerState = read_json(&state_path, "site manager state")?; + if matches!(state.status, SiteLifecycleStatus::Failed) { + return Err(miette::miette!( + "kubernetes site `{}` failed while waiting for router recovery: {}", + plan.site_id, + state + .last_error + .unwrap_or_else(|| "unknown failure".to_string()) + )); + } + Some(state) + } else { + None + }; + + if let Some((control_target, mesh_target)) = + kubernetes_router_ready_targets(plan, manager_state.as_ref())? + && probe_kubernetes_router_control_ready(&control_target, Duration::from_millis(250)) + .await? + && router_mesh_listener_ready_target(&mesh_target, Duration::from_millis(250)).await + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for kubernetes site `{}` router recovery", + plan.site_id + )); + } + sleep(Duration::from_millis(200)).await; + } +} + +fn kubernetes_router_ready_targets( + plan: &SiteControllerRuntimePlan, + manager_state: Option<&SiteManagerState>, +) -> Result> { + if let Some(control_target) = plan.local_router_control.as_deref() { + return Ok(Some(( + control_target.to_string(), + kubernetes_local_router_mesh_target(plan)?, + ))); + } + + let Some(state) = manager_state else { + return Ok(None); + }; + if !matches!(state.status, SiteLifecycleStatus::Running) { + return Ok(None); + } + let (Some(control), Some(mesh_addr)) = ( + state.router_control.as_deref(), + state.router_mesh_addr.as_deref(), + ) else { + return Ok(None); + }; + let control_addr: SocketAddr = control + .parse() + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes router control addr `{control}`"))?; + let mesh_addr: SocketAddr = mesh_addr + .parse() + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes router mesh addr `{mesh_addr}`"))?; + Ok(Some((control_addr.to_string(), mesh_addr.to_string()))) +} + +fn kubernetes_local_router_mesh_target(plan: &SiteControllerRuntimePlan) -> Result { + let control_target = plan.local_router_control.as_deref().ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its local router control endpoint", + plan.site_id + ) + })?; + let (host, _) = control_target.rsplit_once(':').ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` has invalid local router control endpoint `{control_target}`", + plan.site_id + ) + })?; + let mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its router mesh port", + plan.site_id + ) + })?; + Ok(format!("{host}:{mesh_port}")) +} + +async fn router_mesh_listener_ready_target(target: &str, timeout: Duration) -> bool { + matches!( + tokio::time::timeout(timeout, tokio::net::TcpStream::connect(target)).await, + Ok(Ok(_)) + ) +} + +async fn probe_kubernetes_router_control_ready(target: &str, timeout: Duration) -> Result { + let url = format!("http://{target}/identity"); + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .into_diagnostic() + .wrap_err("failed to build kubernetes router readiness client")?; + let response = match client.get(&url).send().await { + Ok(response) => response, + Err(err) if err.is_connect() || err.is_timeout() => return Ok(false), + Err(err) => { + return Err(miette::miette!( + "failed to probe kubernetes router control at {target}: {err}" + )); + } + }; + if !response.status().is_success() { + return Ok(false); + } + Ok(response.json::().await.is_ok()) +} + +fn filter_direct_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { + let keep = component_ids.iter().copied().collect::>(); + let plan_path = artifact_root.join("direct-plan.json"); + let mut plan: DirectPlan = read_json(&plan_path, "direct plan")?; + plan.components + .retain(|component| keep.contains(&component.id)); + plan.startup_order + .retain(|component_id| keep.contains(component_id)); + filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); + let keep_mesh_output_dirs = plan + .components + .iter() + .filter_map(|component| { + Path::new(&component.sidecar.mesh_config_path) + .parent() + .map(path_to_forward_slash_string) + }) + .collect::>(); + plan.router = None; + write_json(&plan_path, &plan)?; + filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) +} + +fn filter_vm_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { + let keep = component_ids.iter().copied().collect::>(); + let plan_path = artifact_root.join("vm-plan.json"); + let mut plan: VmPlan = read_json(&plan_path, "vm plan")?; + plan.components + .retain(|component| keep.contains(&component.id)); + plan.startup_order + .retain(|component_id| keep.contains(component_id)); + filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); + let keep_mesh_output_dirs = plan + .components + .iter() + .filter_map(|component| { + Path::new(&component.mesh_config_path) + .parent() + .map(path_to_forward_slash_string) + }) + .collect::>(); + plan.router = None; + write_json(&plan_path, &plan)?; + filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) +} + +fn filter_dynamic_runtime_addresses( + runtime_addresses: &mut amber_compiler::reporter::direct::DirectRuntimeAddressPlan, + keep: &BTreeSet, +) { + runtime_addresses + .slots_by_scope + .retain(|scope, _| keep.contains(scope)); + for sources in runtime_addresses.slots_by_scope.values_mut() { + sources.retain(|_, source| keep.contains(&dynamic_runtime_source_component_id(source))); + } + runtime_addresses + .slot_items_by_scope + .retain(|scope, _| keep.contains(scope)); + for sources in runtime_addresses.slot_items_by_scope.values_mut() { + sources.retain(|_, items| { + items.retain(|source| keep.contains(&dynamic_runtime_source_component_id(source))); + !items.is_empty() + }); + } +} + +fn dynamic_runtime_source_component_id(source: &DirectRuntimeUrlSource) -> usize { + match source { + DirectRuntimeUrlSource::Slot { component_id, .. } + | DirectRuntimeUrlSource::SlotItem { component_id, .. } => *component_id, + } +} + +fn path_to_forward_slash_string(path: &Path) -> String { + path.components() + .fold(String::new(), |mut rendered, component| { + if !rendered.is_empty() { + rendered.push('/'); + } + rendered.push_str(&component.as_os_str().to_string_lossy()); + rendered + }) +} + +fn filter_dynamic_mesh_provision_plan( + artifact_root: &Path, + keep_component_output_dirs: &BTreeSet, +) -> Result<()> { + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mut plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; + let mut kept_component_ids = BTreeSet::new(); + let mut router_target = None; + let mut filtered_targets = Vec::with_capacity(plan.targets.len()); + for target in plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "dynamic mesh target {} does not use filesystem output", + target.config.identity.id + )); + }; + if keep_component_output_dirs.contains(dir) { + kept_component_ids.insert(target.config.identity.id.clone()); + filtered_targets.push(target); + } + } + MeshProvisionTargetKind::Router => { + router_target = Some(target); + } + } + } + + let Some(mut router_target) = router_target else { + return Err(miette::miette!( + "dynamic mesh provision plan {} is missing a router target", + plan_path.display() + )); + }; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + filtered_targets.push(router_target); + plan.targets = filtered_targets; + write_json(&plan_path, &plan) +} + +#[derive(Clone, Copy)] +enum DetachedChildRuntimeReadiness { + DirectReady, + VmMaterialized, +} + +impl DetachedChildRuntimeReadiness { + fn description(self) -> &'static str { + match self { + Self::DirectReady => "direct child runtime state", + Self::VmMaterialized => "vm child runtime state", + } + } + + fn ready(self, state_path: &Path) -> Result { + match self { + Self::DirectReady => { + let state: DirectRuntimeState = read_json(state_path, self.description())?; + Ok(state.ready) + } + Self::VmMaterialized => { + let _: VmRuntimeState = read_json(state_path, self.description())?; + Ok(true) + } + } + } +} + +async fn wait_for_detached_child_runtime_state( + pid: u32, + state_path: &Path, + timeout: Duration, + log_path: &Path, + readiness: DetachedChildRuntimeReadiness, +) -> Result<()> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if state_path.is_file() && readiness.ready(state_path)? { + return Ok(()); + } + if !pid_is_alive(pid) { + let log = fs::read_to_string(log_path).unwrap_or_default(); + return Err(miette::miette!( + "dynamic child runtime exited before {} became ready\nlog ({}):\n{}", + readiness.description(), + log_path.display(), + log + )); + } + sleep(Duration::from_millis(100)).await; + } + let log = fs::read_to_string(log_path).unwrap_or_default(); + Err(miette::miette!( + "timed out waiting for dynamic {}\nstate ({}):\nlog ({}):\n{}", + readiness.description(), + state_path.display(), + log_path.display(), + log + )) +} + +fn wait_for_detached_vm_child_endpoints_ready( + pid: u32, + artifact_root: &Path, + runtime_root: &Path, + timeout: Duration, + log_path: &Path, +) -> Result<()> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let deadline = Instant::now() + timeout; + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + for route in config.inbound { + let InboundTarget::Local { port: host_port } = route.target else { + continue; + }; + let addr = SocketAddr::from(([127, 0, 0, 1], host_port)); + loop { + let ready = match route.protocol { + MeshProtocol::Http => { + endpoint_returns_http_response_blocking(addr, Duration::from_millis(250))? + } + MeshProtocol::Tcp => endpoint_accepts_stable_connection_blocking( + addr, + Duration::from_millis(250), + )?, + }; + if ready { + break; + } + if !pid_is_alive(pid) { + let log = fs::read_to_string(log_path).unwrap_or_default(); + return Err(miette::miette!( + "dynamic vm child runtime exited before component {} endpoint {} became \ + ready\nlog ({}):\n{}", + component.moniker, + addr, + log_path.display(), + log + )); + } + if Instant::now() >= deadline { + let log = fs::read_to_string(log_path).unwrap_or_default(); + let protocol = match route.protocol { + MeshProtocol::Http => "http", + MeshProtocol::Tcp => "tcp", + }; + return Err(miette::miette!( + "timed out waiting for dynamic vm child component {} {} endpoint {} to \ + become ready\nlog ({}):\n{}", + component.moniker, + protocol, + addr, + log_path.display(), + log + )); + } + std::thread::sleep(Duration::from_millis(50)); + } + } + } + Ok(()) +} + +fn local_direct_peer_ports( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_direct_peer_ports_for_children(plan, &children) +} + +fn local_direct_peer_ports_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + peers.insert( + plan.router_identity_id.clone(), + plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its router mesh port", + plan.site_id + ) + })?, + ); + peers.extend(direct_peer_ports_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(direct_peer_ports_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn local_direct_peer_identities( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_direct_peer_identities_for_children(plan, &children) +} + +fn local_direct_peer_identities_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + let router = + direct_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? + .ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its live router identity", + plan.site_id + ) + })?; + peers.insert(router.id.clone(), router); + peers.extend(direct_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(direct_peer_identities_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn direct_peer_ports_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id, config.mesh_listen.port()); + } + Ok(peers) +} + +fn direct_peer_identities_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn direct_router_identity_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let Some(router) = plan.router.as_ref() else { + return Ok(None); + }; + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + Ok(Some(config.identity)) +} + +fn local_vm_peer_ports( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_vm_peer_ports_for_children(plan, &children) +} + +fn local_vm_peer_ports_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + peers.insert( + plan.router_identity_id.clone(), + plan.router_mesh_port.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) + })?, + ); + peers.extend(vm_peer_ports_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(vm_peer_ports_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn local_vm_peer_identities( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_vm_peer_identities_for_children(plan, &children) +} + +fn local_vm_peer_identities_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + let router = vm_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? + .ok_or_else(|| { + miette::miette!( + "vm site `{}` is missing its live router identity", + plan.site_id + ) + })?; + peers.insert(router.id.clone(), router); + peers.extend(vm_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(vm_peer_identities_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn vm_peer_ports_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + let port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "vm runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + peers.insert(config.identity.id, port); + } + Ok(peers) +} + +fn load_vm_runtime_state_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result { + let state_path = artifact_root.join(".amber").join("vm-runtime.json"); + if state_path.is_file() { + return read_json(&state_path, "vm runtime state"); + } + + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut state = VmRuntimeState::default(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + state + .component_mesh_port_by_id + .insert(component.id, config.mesh_listen.port()); + } + if let Some(router) = &plan.router { + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + state.router_mesh_port = Some(config.mesh_listen.port()); + } + write_vm_runtime_state(artifact_root, &state)?; + Ok(state) +} + +fn vm_peer_identities_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn vm_router_identity_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let Some(router) = plan.router.as_ref() else { + return Ok(None); + }; + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + Ok(Some(config.identity)) +} + +fn terminate_pid(pid: u32, timeout: Duration) -> Result<()> { + #[cfg(unix)] + { + terminate_detached_runtime(pid, timeout) + } + + #[cfg(not(unix))] + { + send_sigterm(pid); + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if !pid_is_alive(pid) { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(()) + } +} + +#[cfg(unix)] +fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { + let mut tracked = process_tree_postorder(root_pid)? + .into_iter() + .filter(|pid| pid_is_alive(*pid)) + .collect::>(); + if tracked.is_empty() { + return Ok(()); + } + + send_signal_to_process_group(root_pid, libc::SIGTERM); + send_signal_to_pids(&tracked, libc::SIGTERM); + + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + tracked.retain(|pid| pid_is_alive(*pid)); + if tracked.is_empty() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + + send_signal_to_process_group(root_pid, libc::SIGKILL); + send_signal_to_pids(&tracked, libc::SIGKILL); + + let force_deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < force_deadline { + tracked.retain(|pid| pid_is_alive(*pid)); + if tracked.is_empty() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + + Err(miette::miette!( + "failed to terminate detached runtime rooted at pid {root_pid}; surviving processes: {}", + tracked + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") + )) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + static VM_ACCEL_ENV_LOCK: OnceLock> = OnceLock::new(); + + #[test] + fn kubernetes_router_ready_prefers_local_targets_over_manager_state() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/test/router".to_string(), + local_router_control: Some("amber-router:24100".to_string()), + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + let stale_manager_state = SiteManagerState { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + status: SiteLifecycleStatus::Running, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + port_forward_pid: None, + context: None, + router_control: Some("127.0.0.1:9".to_string()), + router_mesh_addr: Some("127.0.0.1:9".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_url: None, + last_error: None, + }; + + assert_eq!( + kubernetes_router_ready_targets(&plan, Some(&stale_manager_state)) + .expect("targets should resolve"), + Some(( + "amber-router:24100".to_string(), + "amber-router:24000".to_string() + )), + "embedded kubernetes controllers must prefer their local router service over stale \ + manager-state loopback endpoints", + ); + } + + #[test] + fn kubernetes_router_ready_falls_back_to_manager_state_when_local_target_is_absent() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/test/router".to_string(), + local_router_control: None, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + let manager_state = SiteManagerState { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + status: SiteLifecycleStatus::Running, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + port_forward_pid: None, + context: None, + router_control: Some("127.0.0.1:24100".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_url: None, + last_error: None, + }; + + assert_eq!( + kubernetes_router_ready_targets(&plan, Some(&manager_state)) + .expect("targets should resolve"), + Some(("127.0.0.1:24100".to_string(), "127.0.0.1:24000".to_string())), + "host-supervised kubernetes sites should still fall back to manager-state endpoints \ + when no embedded local router target is available", + ); + } + + #[test] + fn vm_endpoint_forward_ready_timeout_honors_forced_tcg() { + let _guard = VM_ACCEL_ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock should not be poisoned"); + let previous = env::var_os("AMBER_VM_FORCE_TCG"); + unsafe { + env::set_var("AMBER_VM_FORCE_TCG", "1"); + } + assert_eq!( + vm_endpoint_forward_ready_timeout(), + TCG_VM_STARTUP_TIMEOUT, + "forced TCG must extend VM readiness timeouts on every supported host platform", + ); + match previous { + Some(value) => unsafe { env::set_var("AMBER_VM_FORCE_TCG", value) }, + None => unsafe { env::remove_var("AMBER_VM_FORCE_TCG") }, + } + } + + #[test] + fn runtime_plan_vm_ready_timeout_uses_explicit_budget() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "vm_local".to_string(), + kind: SiteKind::Vm, + router_identity_id: "/site/vm_local/router".to_string(), + local_router_control: None, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: Some(721), + launch_env: BTreeMap::new(), + }; + + assert_eq!( + vm_endpoint_forward_ready_timeout_for_runtime_plan(&plan), + Duration::from_secs(721), + "dynamic VM child publication should consume the budget written into the runtime plan", + ); + } + + #[tokio::test] + async fn detached_direct_child_runtime_state_waits_for_ready_flag() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let state_path = temp.path().join("direct-runtime.json"); + let log_path = temp.path().join("site.log"); + write_json( + &state_path, + &DirectRuntimeState { + ready: false, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + + let err = wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_millis(1), + &log_path, + DetachedChildRuntimeReadiness::DirectReady, + ) + .await + .expect_err("not-ready direct runtime state should time out"); + assert!( + err.to_string().contains("direct child runtime state"), + "error should identify the direct child runtime state: {err}" + ); + + write_json( + &state_path, + &DirectRuntimeState { + ready: true, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_secs(1), + &log_path, + DetachedChildRuntimeReadiness::DirectReady, + ) + .await + .expect("ready direct runtime state should be accepted"); + } + + #[tokio::test] + async fn detached_vm_child_runtime_state_accepts_materialized_vm_state() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let state_path = temp.path().join("vm-runtime.json"); + let log_path = temp.path().join("site.log"); + write_json( + &state_path, + &VmRuntimeState { + router_mesh_port: Some(23000), + ..Default::default() + }, + ) + .expect("vm runtime state should write"); + + wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_secs(1), + &log_path, + DetachedChildRuntimeReadiness::VmMaterialized, + ) + .await + .expect("materialized VM runtime state should be accepted"); + } +} diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs new file mode 100644 index 00000000..9b4931d8 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -0,0 +1,1621 @@ +use super::*; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct SiteControllerRuntimeState { + pub(super) schema: String, + pub(super) version: u32, + pub(super) run_id: String, + pub(super) site_id: String, + pub(super) kind: SiteKind, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(super) children: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub(super) direct_input_overlay_providers: BTreeSet, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct SiteControllerRuntimeChildRecord { + pub(super) child_id: u64, + pub(super) artifact_root: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) assigned_components: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) controller_routes: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(super) proxy_exports: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) direct_inputs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) process_pid: Option, + pub(super) published: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct DynamicComposeChildMetadata { + pub(super) schema: String, + pub(super) version: u32, + pub(super) services: Vec, + pub(super) readiness_services: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct StoredRouteOverlayPayload { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) peers: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) inbound_routes: Vec, +} + +#[derive(Clone, Default)] +pub struct DefaultSiteControllerRuntime { + runtime_apps: Arc>>, +} + +pub(crate) fn default_site_controller_runtime() -> SharedSiteControllerRuntime { + Arc::new(DefaultSiteControllerRuntime::default()) +} + +impl SiteControllerRuntime for DefaultSiteControllerRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + self.runtime_apps + .lock() + .expect("site controller runtime app cache poisoned") + .clear(); + Ok(()) + }) + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async move { + resolve_link_external_url_for_output( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + .await + }) + } + + fn prepare_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let app = self.runtime_app_for_site(plan, site_id)?; + site_controller_runtime_prepare_child(&app, &state, &child).await + }) + } + + fn publish_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let app = self.runtime_app_for_site(plan, site_id)?; + site_controller_runtime_publish_child(&app, &state, &child).await + }) + } + + fn rollback_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let app = self.runtime_app_for_site(plan, site_id)?; + site_controller_runtime_rollback_child(&app, child_id).await + }) + } + + fn destroy_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let app = self.runtime_app_for_site(plan, site_id)?; + site_controller_runtime_destroy_child(&app, &state, &child).await + }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result> { + let _ = self.runtime_app_for_plan(plan)?; + collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result { + load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + + fn router_mesh_addr_for_component_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + router_mesh_addr_for_component_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> Result<()> { + update_desired_overlay_for_consumer(site_state_root, overlay_id, overlay) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> Result<()> { + update_desired_overlay_for_provider(site_state_root, overlay_id, overlay) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +impl DefaultSiteControllerRuntime { + fn runtime_app_for_site( + &self, + controller_plan: &SiteControllerPlan, + site_id: &str, + ) -> Result { + let runtime_plan = runtime_plan_for_site_from_controller_plan(controller_plan, site_id)?; + self.runtime_app_for_plan(&runtime_plan) + } + + fn runtime_app_for_plan( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result { + let key = Path::new(&plan.site_state_root).to_path_buf(); + let mut runtime_apps = self + .runtime_apps + .lock() + .expect("site controller runtime app cache poisoned"); + if let Some(app) = runtime_apps.get(&key) { + return Ok(app.clone()); + } + let app = build_site_controller_runtime_app(plan.clone())?; + runtime_apps.insert(key, app.clone()); + Ok(app) + } +} + +pub(super) fn build_site_controller_runtime_app( + plan: SiteControllerRuntimePlan, +) -> Result { + let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); + let initial_state = if state_path.is_file() { + read_json(&state_path, "site controller runtime state")? + } else { + let state = SiteControllerRuntimeState { + schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, + run_id: plan.run_id.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + children: BTreeMap::new(), + direct_input_overlay_providers: BTreeSet::new(), + }; + write_json(&state_path, &state)?; + state + }; + Ok(SiteControllerRuntimeApp { + plan, + state_path, + state: Arc::new(AsyncMutex::new(initial_state)), + }) +} + +pub(crate) fn runtime_plan_for_site_from_controller_plan( + controller_plan: &SiteControllerPlan, + site_id: &str, +) -> Result { + if site_id == controller_plan.site_id { + return Ok(site_controller_runtime_plan_from_controller_plan( + controller_plan, + )); + } + + let site_state_root = Path::new(&controller_plan.state_root).join(site_id); + let supervisor: SiteSupervisorPlan = read_json( + &site_supervisor_plan_path(&site_state_root), + "site supervisor plan", + )?; + let manager_state_path = + crate::runtime_api::site_state_path(Path::new(&controller_plan.state_root), site_id); + let manager = manager_state_path + .is_file() + .then(|| read_json::(&manager_state_path, "site manager state")) + .transpose()?; + let local_router_control = manager + .as_ref() + .and_then(|state| state.router_control.clone()) + .or_else(|| { + Some(site_runtime_local_router_control( + supervisor.kind, + Path::new(&supervisor.artifact_dir), + )) + }); + let router_identity_id = manager + .as_ref() + .and_then(|state| state.router_identity_id.clone()) + .or_else(|| { + controller_plan + .peer_router_identities + .get(site_id) + .map(|identity| identity.id.clone()) + }) + .ok_or_else(|| miette::miette!("site `{site_id}` is missing its router identity"))?; + let router_mesh_port = manager + .as_ref() + .and_then(|state| state.router_mesh_addr.as_deref()) + .and_then(|addr| addr.rsplit_once(':')) + .and_then(|(_, port)| port.parse::().ok()) + .or(supervisor.router_mesh_port) + .or(supervisor.port_forward_mesh_port); + + let mut runtime_plan = site_controller_runtime_plan_from_controller_plan(controller_plan); + runtime_plan.run_id = supervisor.run_id; + runtime_plan.mesh_scope = supervisor.mesh_scope; + runtime_plan.run_root = supervisor.run_root; + runtime_plan.site_id = supervisor.site_id; + runtime_plan.kind = supervisor.kind; + runtime_plan.router_identity_id = router_identity_id; + runtime_plan.local_router_control = local_router_control; + runtime_plan.artifact_dir = supervisor.artifact_dir; + runtime_plan.site_state_root = supervisor.site_state_root; + runtime_plan.storage_root = supervisor.storage_root; + runtime_plan.runtime_root = supervisor.runtime_root; + runtime_plan.router_mesh_port = router_mesh_port; + runtime_plan.compose_project = supervisor.compose_project; + runtime_plan.kubernetes_namespace = supervisor.kubernetes_namespace; + runtime_plan.context = supervisor.context; + runtime_plan.observability_endpoint = supervisor.observability_endpoint; + runtime_plan.launch_env = supervisor.launch_env; + Ok(runtime_plan) +} + +fn site_runtime_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { + match kind { + SiteKind::Direct => format!( + "unix://{}", + super::direct_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Vm => format!( + "unix://{}", + super::vm_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Compose => "unix:///amber/control/router-control.sock".to_string(), + SiteKind::Kubernetes => "amber-router:24100".to_string(), + } +} + +#[derive(Clone)] +pub(super) struct SiteControllerRuntimeApp { + pub(super) plan: SiteControllerRuntimePlan, + pub(super) state_path: PathBuf, + pub(super) state: Arc>, +} + +fn local_child_runtime_spec( + plan: &SiteControllerRuntimePlan, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result { + build_local_child_runtime_spec(state, child, &plan.site_id) + .map_err(|err| miette::miette!(err.message)) +} + +pub(super) fn site_controller_runtime_state_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-runtime-state.json") +} + +pub(super) fn site_controller_runtime_child_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + Path::new(&plan.site_state_root) + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +pub(super) fn site_controller_runtime_child_artifact_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("artifact") +} + +pub(super) fn site_controller_runtime_child_runtime_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("runtime") +} + +pub(super) fn site_controller_runtime_child_storage_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("storage") +} + +pub(super) fn site_controller_runtime_child_peer_ports_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-ports.json") +} + +pub(super) fn site_controller_runtime_child_peer_identities_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-identities.json") +} + +pub(super) fn dynamic_compose_child_metadata_path(artifact_root: &Path) -> PathBuf { + artifact_root.join(".amber").join("compose-child.json") +} + +pub(super) fn dynamic_route_overlay_path(artifact_root: &Path) -> PathBuf { + artifact_root + .join(".amber") + .join(DYNAMIC_ROUTE_OVERLAY_FILENAME) +} + +pub(super) fn write_dynamic_route_overlay_payload( + artifact_root: &Path, + payload: &StoredRouteOverlayPayload, +) -> Result<()> { + let path = dynamic_route_overlay_path(artifact_root); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + write_json(&path, payload) +} + +fn dynamic_proxy_exports_path(artifact_root: &Path) -> PathBuf { + artifact_root + .join(".amber") + .join(DYNAMIC_PROXY_EXPORTS_FILENAME) +} + +pub(super) fn write_dynamic_proxy_exports_metadata( + artifact_root: &Path, + proxy_exports: &BTreeMap, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + let path = dynamic_proxy_exports_path(artifact_root); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + write_json(&path, proxy_exports) +} + +pub(super) fn load_dynamic_proxy_exports_metadata( + artifact_root: &Path, +) -> Result> { + let path = dynamic_proxy_exports_path(artifact_root); + if !path.is_file() { + return Ok(BTreeMap::new()); + } + read_json(&path, "dynamic proxy exports") +} + +pub fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> Result<()> { + let state_path = site_controller_runtime_state_path(site_state_root); + if !state_path.is_file() { + return Ok(()); + } + let mut state: SiteControllerRuntimeState = + read_json(&state_path, "site controller runtime state")?; + let compose_supervisor_plan = matches!(kind, SiteKind::Compose) + .then(|| { + read_json::( + &site_supervisor_plan_path(site_state_root), + "site supervisor plan", + ) + }) + .transpose()?; + for child in state.children.values() { + if let Some(plan) = compose_supervisor_plan.as_ref() { + cleanup_dynamic_compose_child(plan, child)?; + } + if let Some(pid) = child.process_pid { + terminate_pid(pid, site_ready_timeout_for_kind(kind))?; + } + let child_root = crate::runtime_api::site_controller_runtime_child_root_for_site( + site_state_root, + child.child_id, + ); + remove_dynamic_child_root(kind, &child_root, Some(Path::new(&child.artifact_root)))?; + } + if state.children.is_empty() && state.direct_input_overlay_providers.is_empty() { + return Ok(()); + } + state.children.clear(); + state.direct_input_overlay_providers.clear(); + write_json(&state_path, &state) +} + +fn cleanup_dynamic_compose_child( + plan: &SiteSupervisorPlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let artifact_root = Path::new(&child.artifact_root); + if !dynamic_compose_child_metadata_path(artifact_root).is_file() { + return Ok(()); + } + let compose_project = plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + plan.site_id + ) + })?; + let status = compose_command(Some(compose_project), artifact_root) + .envs(plan.launch_env.clone()) + .arg("down") + .arg("-v") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to clean up dynamic compose child for site `{}`", + plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "dynamic compose child cleanup on site `{}` failed with status {status}", + plan.site_id + )); + } + Ok(()) +} + +fn compose_cleanup_image_candidates(artifact_root: &Path) -> Result> { + let compose_path = artifact_root.join("compose.yaml"); + if !compose_path.is_file() { + return Ok(vec![ + "docker:28-cli".to_string(), + "python:3.13-alpine".to_string(), + ]); + } + + let document = read_compose_document(&compose_path)?; + let Some(root) = document.as_mapping() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + compose_path.display() + )); + }; + let Some(services) = root + .get(yaml_string("services")) + .and_then(serde_yaml::Value::as_mapping) + else { + return Err(miette::miette!( + "compose file {} is missing a services mapping", + compose_path.display() + )); + }; + + let mut images = BTreeSet::from([ + "docker:28-cli".to_string(), + "python:3.13-alpine".to_string(), + ]); + for service in services.values() { + let Some(service_mapping) = service.as_mapping() else { + continue; + }; + let Some(image) = service_mapping + .get(yaml_string("image")) + .and_then(serde_yaml::Value::as_str) + else { + continue; + }; + images.insert(image.to_string()); + } + Ok(images.into_iter().collect()) +} + +fn docker_image_exists_locally(image: &str) -> bool { + Command::new("docker") + .arg("image") + .arg("inspect") + .arg(image) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) +} + +fn force_remove_dir_if_exists_via_local_container( + path: &Path, + artifact_root: &Path, +) -> Result { + if !path.exists() { + return Ok(true); + } + let Some(parent) = path.parent() else { + return Ok(false); + }; + + for image in compose_cleanup_image_candidates(artifact_root)? { + if !docker_image_exists_locally(&image) { + continue; + } + let status = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("--user") + .arg("0:0") + .arg("-v") + .arg(format!("{}:{}", parent.display(), parent.display())) + .arg("--entrypoint") + .arg("rm") + .arg(&image) + .arg("-rf") + .arg(path) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to launch fallback cleanup container `{image}` for {}", + path.display() + ) + })?; + if status.success() { + return Ok(true); + } + } + Ok(false) +} + +fn remove_dynamic_child_root( + kind: SiteKind, + child_root: &Path, + artifact_root: Option<&Path>, +) -> Result<()> { + match remove_dir_if_exists(child_root) { + Ok(()) => Ok(()), + Err(err) if matches!(kind, SiteKind::Compose) => { + let Some(artifact_root) = artifact_root else { + return Err(err.wrap_err(format!( + "compose child root {} is missing its artifact directory", + child_root.display() + ))); + }; + if force_remove_dir_if_exists_via_local_container(child_root, artifact_root)? { + return Ok(()); + } + Err(err.wrap_err(format!( + "compose child root {} could not be removed with a local cleanup container", + child_root.display() + ))) + } + Err(err) => Err(err), + } +} + +pub(super) async fn site_controller_runtime_prepare_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let runtime_spec = local_child_runtime_spec(&app.plan, state, child)?; + let artifact_root = site_controller_runtime_child_artifact_root(&app.plan, child.child_id); + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + let live_components = if runtime_spec.direct_inputs.is_empty() { + BTreeMap::new() + } else { + collect_live_component_runtime_metadata(&app.plan)? + }; + replace_artifact_snapshot(&artifact_root, &runtime_spec.child_artifact_files)?; + project_dynamic_child_mesh_scope(&artifact_root, Some(&app.plan.mesh_scope))?; + if app.plan.kind == SiteKind::Kubernetes { + let _ = prepare_kubernetes_artifact_namespace( + &app.plan.run_id, + &app.plan.site_id, + &artifact_root, + )?; + } + patch_site_artifacts( + &artifact_root, + &app.plan.run_id, + &app.plan.site_id, + app.plan.kind, + &app.plan.launch_env, + app.plan.observability_endpoint.as_deref(), + )?; + match app.plan.kind { + SiteKind::Direct => { + filter_direct_stage_plan(&artifact_root, &runtime_spec.component_ids)?; + ensure_dynamic_proxy_export_component_routes_in_artifact( + &artifact_root, + &runtime_spec.proxy_exports, + &app.plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs_in_artifact( + &artifact_root, + &runtime_spec.direct_inputs, + &live_components, + )?; + } + SiteKind::Vm => { + filter_vm_stage_plan(&artifact_root, &runtime_spec.component_ids)?; + ensure_dynamic_proxy_export_component_routes_in_artifact( + &artifact_root, + &runtime_spec.proxy_exports, + &app.plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs_in_artifact( + &artifact_root, + &runtime_spec.direct_inputs, + &live_components, + )?; + } + SiteKind::Compose => { + let existing_peer_identities = + local_compose_peer_identities(&app.plan, &published_children)?; + prepare_dynamic_compose_child_artifact( + &app.plan, + &runtime_spec, + &artifact_root, + &published_children, + &existing_peer_identities, + &live_components, + )? + } + SiteKind::Kubernetes => { + let existing_peer_identities = + local_kubernetes_peer_identities(&app.plan, &published_children)?; + prepare_dynamic_kubernetes_child_artifact( + &app.plan, + &runtime_spec, + &artifact_root, + &existing_peer_identities, + &live_components, + )? + } + } + write_dynamic_proxy_exports_metadata(&artifact_root, &runtime_spec.proxy_exports)?; + rewrite_dynamic_proxy_metadata(&artifact_root, &runtime_spec.proxy_exports, app.plan.kind)?; + if app.plan.kind == SiteKind::Compose + && let Some(compose_project) = app.plan.compose_project.as_deref() + { + rewrite_dynamic_compose_proxy_metadata(&artifact_root, compose_project)?; + } + let mut state = app.state.lock().await; + state.children.insert( + child.child_id, + SiteControllerRuntimeChildRecord { + child_id: child.child_id, + artifact_root: artifact_root.display().to_string(), + assigned_components: runtime_spec.assigned_components.clone(), + controller_routes: runtime_spec.controller_routes.clone(), + proxy_exports: runtime_spec.proxy_exports.clone(), + direct_inputs: runtime_spec.direct_inputs.clone(), + process_pid: None, + published: false, + }, + ); + write_json(&app.state_path, &*state) +} + +pub(super) async fn site_controller_runtime_publish_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let runtime_spec = local_child_runtime_spec(&app.plan, state, child)?; + let child_id = child.child_id; + let (child, published_children) = { + let state = app.state.lock().await; + let child = state.children.get(&child_id).cloned().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let published_children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + (child, published_children) + }; + if child.published { + return Ok(()); + } + + match app.plan.kind { + SiteKind::Direct => { + let state = app.state.lock().await; + let existing_peer_ports = local_direct_peer_ports(&app.plan, &state)?; + let existing_peer_identities = local_direct_peer_identities(&app.plan, &state)?; + drop(state); + write_json( + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), + &existing_peer_ports, + )?; + write_json( + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), + &existing_peer_identities, + )?; + project_dynamic_child_mesh_scope( + Path::new(&child.artifact_root), + existing_peer_identities + .get(&app.plan.router_identity_id) + .and_then(|identity| identity.mesh_scope.as_deref()), + )?; + if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + } + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + fs::create_dir_all(&runtime_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; + fs::create_dir_all(&storage_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; + let process = spawn_detached_child( + &child_root, + &child_root.join("site.log"), + &app.plan.launch_env, + |cmd| { + cmd.arg("run-direct-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("direct-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + }, + )?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } + wait_for_detached_child_runtime_state( + process.id(), + &direct_runtime_state_path(Path::new(&child.artifact_root)), + site_ready_timeout_for_kind(SiteKind::Direct), + &child_root.join("site.log"), + DetachedChildRuntimeReadiness::DirectReady, + ) + .await?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_ports = overlay_peer_addr_map_from_ports( + &local_direct_peer_ports_for_children(&app.plan, &live_children)?, + ); + let live_peer_identities = + local_direct_peer_identities_for_children(&app.plan, &live_children)?; + write_direct_vm_live_route_overlay_payload( + Path::new(&child.artifact_root), + &child.assigned_components, + &child.proxy_exports, + &live_peer_ports, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + { + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + record.published = true; + } + write_json(&app.state_path, &*state)?; + drop(state); + project_dynamic_direct_router_surface(&app.plan, &child)?; + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + reconcile_site_proxy_metadata( + Path::new(&app.plan.artifact_dir), + &runtime_spec.site_artifact_files, + )?; + } + SiteKind::Vm => { + let state = app.state.lock().await; + let existing_peer_ports = local_vm_peer_ports(&app.plan, &state)?; + let existing_peer_identities = local_vm_peer_identities(&app.plan, &state)?; + drop(state); + write_json( + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), + &existing_peer_ports, + )?; + write_json( + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), + &existing_peer_identities, + )?; + project_dynamic_child_mesh_scope( + Path::new(&child.artifact_root), + existing_peer_identities + .get(&app.plan.router_identity_id) + .and_then(|identity| identity.mesh_scope.as_deref()), + )?; + if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + } + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + fs::create_dir_all(&runtime_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; + fs::create_dir_all(&storage_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; + let process = spawn_detached_child( + &child_root, + &child_root.join("site.log"), + &app.plan.launch_env, + |cmd| { + cmd.arg("run-vm-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("vm-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + }, + )?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } + let vm_ready_timeout = vm_endpoint_forward_ready_timeout_for_runtime_plan(&app.plan); + wait_for_detached_child_runtime_state( + process.id(), + &Path::new(&child.artifact_root) + .join(".amber") + .join("vm-runtime.json"), + vm_ready_timeout, + &child_root.join("site.log"), + DetachedChildRuntimeReadiness::VmMaterialized, + ) + .await?; + wait_for_detached_vm_child_endpoints_ready( + process.id(), + Path::new(&child.artifact_root), + &runtime_root, + vm_ready_timeout, + &child_root.join("site.log"), + )?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_ports = overlay_peer_addr_map_from_ports( + &local_vm_peer_ports_for_children(&app.plan, &live_children)?, + ); + let live_peer_identities = + local_vm_peer_identities_for_children(&app.plan, &live_children)?; + write_direct_vm_live_route_overlay_payload( + Path::new(&child.artifact_root), + &child.assigned_components, + &child.proxy_exports, + &live_peer_ports, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + { + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + record.published = true; + } + write_json(&app.state_path, &*state)?; + drop(state); + project_dynamic_vm_router_surface(&app.plan, &child)?; + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + reconcile_site_proxy_metadata( + Path::new(&app.plan.artifact_dir), + &runtime_spec.site_artifact_files, + )?; + } + SiteKind::Compose => { + let mut issuer_children = published_children.clone(); + issuer_children.push(child.clone()); + reconcile_dynamic_site_router_overlays_for_children( + app, + &published_children, + &issuer_children, + ) + .await?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; + let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + app.plan.site_id + ) + })?; + let (sidecar_services, workload_services): (Vec<_>, Vec<_>) = metadata + .services + .iter() + .cloned() + .partition(|service| is_compose_component_sidecar_service(service)); + if !sidecar_services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("up") + .arg("-d") + .args(&sidecar_services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to publish compose child sidecars on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child sidecar publish on site `{}` failed with status {status}", + app.plan.site_id + )); + } + wait_for_compose_services_running( + compose_project, + Path::new(&child.artifact_root), + &sidecar_services, + site_ready_timeout_for_kind(SiteKind::Compose), + ) + .await?; + } + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + // Compose workloads can make one-shot startup calls through their sidecars. + // Mark the child publishable as soon as its sidecars are up so same-site + // direct-input grants are reconciled before the workload process starts. + record.published = true; + write_json(&app.state_path, &*state)?; + } + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + if !workload_services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("up") + .arg("-d") + .args(&workload_services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to publish compose child workloads on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child workload publish on site `{}` failed with status {status}", + app.plan.site_id + )); + } + } + wait_for_compose_services_running( + compose_project, + Path::new(&child.artifact_root), + &metadata.readiness_services, + site_ready_timeout_for_kind(SiteKind::Compose), + ) + .await?; + } + SiteKind::Kubernetes => { + let artifact_root = Path::new(&child.artifact_root); + let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; + ensure_kubernetes_namespace(&supervisor_plan)?; + let status = kubectl_command(app.plan.context.as_deref()) + .current_dir(artifact_root) + .arg("apply") + .arg("-k") + .arg(".") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to publish kubernetes site `{}`", app.plan.site_id) + })?; + if !status.success() { + return Err(miette::miette!( + "kubernetes site `{}` publish failed with status {status}", + app.plan.site_id + )); + } + ensure_kubernetes_workloads_ready(&supervisor_plan)?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_identities = local_kubernetes_peer_identities(&app.plan, &live_children)?; + write_kubernetes_live_route_overlay_payload( + artifact_root, + &child.assigned_components, + &child.proxy_exports, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.published = true; + write_json(&app.state_path, &*state)?; + drop(state); + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + wait_for_kubernetes_site_router_ready( + &app.plan, + site_ready_timeout_for_kind(SiteKind::Kubernetes), + ) + .await?; + } + } + + Ok(()) +} + +pub(super) async fn site_controller_runtime_rollback_child( + app: &SiteControllerRuntimeApp, + child_id: u64, +) -> Result<()> { + let child = { + let mut state = app.state.lock().await; + let removed = state.children.remove(&child_id); + write_json(&app.state_path, &*state)?; + removed + }; + let Some(child) = child else { + return Ok(()); + }; + if let Some(pid) = child.process_pid { + terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; + } + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + remove_dynamic_child_root( + app.plan.kind, + &child_root, + Some(Path::new(&child.artifact_root)), + ) +} + +pub(super) async fn site_controller_runtime_destroy_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let child_id = child.child_id; + let child = { + let state = app.state.lock().await; + state.children.get(&child_id).cloned() + }; + if child.is_none() { + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + let artifact_root = child_root.join("artifact"); + remove_dynamic_child_root(app.plan.kind, &child_root, Some(&artifact_root))?; + return Ok(()); + } + if let Some(child) = child.as_ref() + && matches!( + app.plan.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes + ) + { + revoke_dynamic_site_router_overlay(&app.plan, child).await?; + } + if let Some(pid) = child.as_ref().and_then(|child| child.process_pid) { + terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; + } + + match app.plan.kind { + SiteKind::Compose => { + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; + let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + app.plan.site_id + ) + })?; + if !metadata.services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("rm") + .arg("--stop") + .arg("--force") + .arg("-v") + .args(&metadata.services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to destroy compose child on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child destroy on site `{}` failed with status {status}", + app.plan.site_id + )); + } + } + } + SiteKind::Kubernetes => { + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let artifact_root = Path::new(&child.artifact_root); + let files = read_artifact_snapshot(artifact_root)?; + let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files(&files)?; + replace_artifact_snapshot(artifact_root, &destroy_bundle)?; + let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; + let workloads = kubernetes_expected_workloads(artifact_root)?; + let status = kubectl_command(app.plan.context.as_deref()) + .current_dir(artifact_root) + .arg("delete") + .arg("-k") + .arg(".") + .arg("--ignore-not-found=true") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to destroy kubernetes child on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "kubernetes child destroy on site `{}` failed with status {status}", + app.plan.site_id + )); + } + wait_for_kubernetes_artifact_workloads_deleted( + app.plan.context.as_deref(), + supervisor_plan + .kubernetes_namespace + .as_deref() + .expect("kubernetes supervisor plan should include a namespace"), + &workloads, + &app.plan.site_id, + )?; + wait_for_kubernetes_site_router_ready( + &app.plan, + site_ready_timeout_for_kind(SiteKind::Kubernetes), + ) + .await?; + } + SiteKind::Direct | SiteKind::Vm => { + let site_artifact_files = build_desired_site_artifact_files(state, &app.plan.site_id) + .map_err(|err| miette::miette!(err.message))?; + reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_artifact_files)?; + } + } + + let mut state = app.state.lock().await; + state.children.remove(&child_id); + write_json(&app.state_path, &*state)?; + drop(state); + if matches!( + app.plan.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes + ) { + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + } + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + remove_dynamic_child_root( + app.plan.kind, + &child_root, + child.as_ref().map(|child| Path::new(&child.artifact_root)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compose_cleanup_image_candidates_include_service_images_and_fallbacks() { + let temp = tempfile::tempdir().expect("temp dir"); + fs::write( + temp.path().join("compose.yaml"), + "services:\n app:\n image: python:3.13-alpine\n helper:\n image: \ + ghcr.io/rdi-foundation/amber-helper:test-tag\n", + ) + .expect("compose file should write"); + + let images = compose_cleanup_image_candidates(temp.path()) + .expect("compose cleanup images should load"); + + assert_eq!( + images, + vec![ + "docker:28-cli".to_string(), + "ghcr.io/rdi-foundation/amber-helper:test-tag".to_string(), + "python:3.13-alpine".to_string(), + ], + ); + } + + #[test] + fn runtime_plan_for_kubernetes_site_uses_port_forward_mesh_runtime_metadata() { + let temp = tempfile::tempdir().expect("temp dir"); + let state_root = temp.path().join("state"); + let compose_state_root = state_root.join("compose_local"); + let kind_state_root = state_root.join("kind_local"); + let compose_artifact = temp.path().join("artifact").join("compose_local"); + let kind_artifact = temp.path().join("artifact").join("kind_local"); + fs::create_dir_all(&compose_state_root).expect("compose state root"); + fs::create_dir_all(&kind_state_root).expect("kind state root"); + fs::create_dir_all(&compose_artifact).expect("compose artifact"); + fs::create_dir_all(&kind_artifact).expect("kind artifact"); + + write_json( + &site_supervisor_plan_path(&kind_state_root), + &serde_json::json!({ + "schema": "amber.run.site_supervisor_plan", + "version": 2, + "run_id": "test-run", + "mesh_scope": "test-mesh", + "run_root": temp.path().join("run").display().to_string(), + "coordinator_pid": 1u32, + "site_id": "kind_local", + "kind": "kubernetes", + "artifact_dir": kind_artifact.display().to_string(), + "site_state_root": kind_state_root.display().to_string(), + "kubernetes_namespace": "amber-test-kind", + "context": "kind-amber-test", + "port_forward_mesh_port": 24036u16, + "port_forward_control_port": 24037u16, + "launch_env": {"KUBECONFIG": "/tmp/kubeconfig"} + }), + ) + .expect("kubernetes supervisor plan should write"); + write_json( + &crate::runtime_api::site_state_path(&state_root, "kind_local"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "kind_local", + "kind": "kubernetes", + "status": "running", + "artifact_dir": kind_artifact.display().to_string(), + "supervisor_pid": 1u32, + "kubernetes_namespace": "amber-test-kind", + "context": "kind-amber-test", + "router_control": "127.0.0.1:24037", + "router_mesh_addr": "127.0.0.1:24036", + "router_identity_id": "/site/kind_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([8u8; 32]) + }), + ) + .expect("kubernetes manager state should write"); + + let controller_plan = SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: "/site/compose_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::from([( + "kind_local".to_string(), + MeshIdentityPublic { + id: "/site/kind_local/router".to_string(), + public_key: [8u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/compose-control.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24034".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + state_path: compose_state_root + .join("site-controller-state.json") + .display() + .to_string(), + run_root: temp.path().join("run").display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: compose_state_root.display().to_string(), + artifact_dir: compose_artifact.display().to_string(), + controller_identity_path: None, + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24034), + compose_project: Some("amber-test-compose".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + + let runtime_plan = + runtime_plan_for_site_from_controller_plan(&controller_plan, "kind_local") + .expect("kubernetes runtime plan should derive from the managed site"); + assert_eq!(runtime_plan.site_id, "kind_local"); + assert_eq!(runtime_plan.kind, SiteKind::Kubernetes); + assert_eq!( + runtime_plan.artifact_dir, + kind_artifact.display().to_string() + ); + assert_eq!( + runtime_plan.site_state_root, + kind_state_root.display().to_string() + ); + assert_eq!( + runtime_plan.local_router_control.as_deref(), + Some("127.0.0.1:24037") + ); + assert_eq!(runtime_plan.router_identity_id, "/site/kind_local/router"); + assert_eq!(runtime_plan.router_mesh_port, Some(24036)); + assert_eq!( + runtime_plan.kubernetes_namespace.as_deref(), + Some("amber-test-kind") + ); + assert_eq!(runtime_plan.context.as_deref(), Some("kind-amber-test")); + assert_eq!( + runtime_plan + .launch_env + .get("KUBECONFIG") + .map(String::as_str), + Some("/tmp/kubeconfig") + ); + } + + #[test] + fn runtime_plan_for_vm_site_uses_managed_vm_runtime_metadata() { + let temp = tempfile::tempdir().expect("temp dir"); + let state_root = temp.path().join("state"); + let direct_state_root = state_root.join("direct_local"); + let vm_state_root = state_root.join("vm_local"); + let direct_artifact = temp.path().join("artifact").join("direct_local"); + let vm_artifact = temp.path().join("artifact").join("vm_local"); + fs::create_dir_all(&direct_state_root).expect("direct state root"); + fs::create_dir_all(&vm_state_root).expect("vm state root"); + fs::create_dir_all(&direct_artifact).expect("direct artifact"); + fs::create_dir_all(&vm_artifact).expect("vm artifact"); + + write_json( + &site_supervisor_plan_path(&vm_state_root), + &serde_json::json!({ + "schema": "amber.run.site_supervisor_plan", + "version": 2, + "run_id": "test-run", + "mesh_scope": "test-mesh", + "run_root": temp.path().join("run").display().to_string(), + "coordinator_pid": 1u32, + "site_id": "vm_local", + "kind": "vm", + "artifact_dir": vm_artifact.display().to_string(), + "site_state_root": vm_state_root.display().to_string(), + "storage_root": temp.path().join("storage").join("vm_local").display().to_string(), + "runtime_root": temp.path().join("runtime").join("vm_local").display().to_string(), + "router_mesh_port": 24001u16, + "launch_env": {"AMBER_TEST": "1"} + }), + ) + .expect("vm supervisor plan should write"); + write_json( + &crate::runtime_api::site_state_path(&state_root, "vm_local"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "vm_local", + "kind": "vm", + "status": "running", + "artifact_dir": vm_artifact.display().to_string(), + "supervisor_pid": 1u32, + "router_control": "unix:///tmp/vm-control.sock", + "router_mesh_addr": "127.0.0.1:24001", + "router_identity_id": "/site/vm_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([9u8; 32]) + }), + ) + .expect("vm manager state should write"); + + let controller_plan = SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: "/site/direct_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::from([( + "vm_local".to_string(), + MeshIdentityPublic { + id: "/site/vm_local/router".to_string(), + public_key: [9u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/direct-control.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + state_path: direct_state_root + .join("site-controller-state.json") + .display() + .to_string(), + run_root: temp.path().join("run").display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: direct_state_root.display().to_string(), + artifact_dir: direct_artifact.display().to_string(), + controller_identity_path: None, + storage_root: Some( + temp.path() + .join("storage") + .join("direct_local") + .display() + .to_string(), + ), + runtime_root: Some( + temp.path() + .join("runtime") + .join("direct_local") + .display() + .to_string(), + ), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + + let runtime_plan = runtime_plan_for_site_from_controller_plan(&controller_plan, "vm_local") + .expect("vm runtime plan should derive from the managed site"); + assert_eq!(runtime_plan.site_id, "vm_local"); + assert_eq!(runtime_plan.kind, SiteKind::Vm); + assert_eq!(runtime_plan.artifact_dir, vm_artifact.display().to_string()); + assert_eq!( + runtime_plan.site_state_root, + vm_state_root.display().to_string() + ); + assert_eq!( + runtime_plan.local_router_control.as_deref(), + Some("unix:///tmp/vm-control.sock") + ); + assert_eq!(runtime_plan.router_identity_id, "/site/vm_local/router"); + assert_eq!(runtime_plan.router_mesh_port, Some(24001)); + assert_eq!( + runtime_plan + .launch_env + .get("AMBER_TEST") + .map(String::as_str), + Some("1") + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs new file mode 100644 index 00000000..647363ca --- /dev/null +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -0,0 +1,473 @@ +#[cfg(unix)] +use std::os::unix::fs::MetadataExt as _; +use std::{fs, path::Path}; + +use serde_json::json; + +use super::*; + +const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; +const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; +const COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; +const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; +const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; +const COMPOSE_ROUTER_RUNTIME_GID: u32 = 65532; +const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; +const COMPOSE_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; + +fn ensure_string_sequence_contains(sequence: &mut serde_yaml::Sequence, value: &str) { + if sequence + .iter() + .any(|entry| entry.as_str().is_some_and(|existing| existing == value)) + { + return; + } + sequence.push(yaml_string(value)); +} + +fn uses_service_network_mode(service: &serde_yaml::Mapping) -> bool { + service + .get(yaml_string("network_mode")) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|mode| mode.starts_with("service:")) +} + +fn compose_controller_user() -> String { + #[cfg(unix)] + { + format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }) + } + + #[cfg(not(unix))] + { + "0:0".to_string() + } +} + +#[cfg(unix)] +fn compose_controller_group_add() -> Vec { + compose_controller_group_add_for( + unsafe { libc::getgid() }, + fs::metadata(DOCKER_SOCK_PATH) + .ok() + .map(|metadata| metadata.gid()), + ) +} + +#[cfg(unix)] +fn compose_controller_group_add_for( + runtime_gid: u32, + docker_socket_gid: Option, +) -> Vec { + let mut groups = vec![COMPOSE_ROUTER_RUNTIME_GID]; + if let Some(docker_socket_gid) = docker_socket_gid + && docker_socket_gid != runtime_gid + && !groups.contains(&docker_socket_gid) + { + groups.push(docker_socket_gid); + } + groups.into_iter().map(|gid| gid.to_string()).collect() +} + +#[cfg(not(unix))] +fn compose_controller_group_add() -> Vec { + vec![COMPOSE_ROUTER_RUNTIME_GID.to_string()] +} + +fn ensure_controller_environment( + service: &mut serde_yaml::Mapping, + plan: &SiteControllerPlan, +) -> Result<()> { + if plan.launch_env.is_empty() { + return Ok(()); + } + match service + .entry(yaml_string("environment")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + { + serde_yaml::Value::Sequence(sequence) => { + for (key, value) in &plan.launch_env { + let entry = format!("{key}={value}"); + let already_present = sequence.iter().any(|existing| { + existing.as_str().is_some_and(|existing| { + existing == entry || existing.starts_with(&format!("{key}=")) + }) + }); + if !already_present { + sequence.push(yaml_string(&entry)); + } + } + Ok(()) + } + serde_yaml::Value::Mapping(mapping) => { + for (key, value) in &plan.launch_env { + mapping.insert(yaml_string(key), yaml_string(value)); + } + Ok(()) + } + _ => Err(miette::miette!( + "compose site controller service has an unsupported environment shape" + )), + } +} + +fn ensure_controller_depends_on( + service: &mut serde_yaml::Mapping, + available_services: &serde_yaml::Mapping, +) -> Result<()> { + let mut desired = serde_yaml::Mapping::new(); + for (service_name, condition) in [ + ( + COMPOSE_PROVISIONER_SERVICE_NAME, + "service_completed_successfully", + ), + ( + COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME, + "service_completed_successfully", + ), + (COMPOSE_ROUTER_SERVICE_NAME, "service_started"), + ] { + if available_services.contains_key(yaml_string(service_name)) { + desired.insert( + yaml_string(service_name), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("condition"), + yaml_string(condition), + )])), + ); + } + } + if desired.is_empty() { + return Ok(()); + } + + let current = service + .entry(yaml_string("depends_on")) + .or_insert_with(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + match current { + serde_yaml::Value::Mapping(mapping) => { + for (key, value) in desired { + mapping.insert(key, value); + } + Ok(()) + } + serde_yaml::Value::Sequence(sequence) => { + let mut mapping = serde_yaml::Mapping::new(); + for value in sequence.iter() { + let Some(name) = value.as_str() else { + return Err(miette::miette!( + "compose site controller service depends_on sequence contains a \ + non-string entry" + )); + }; + mapping.insert( + yaml_string(name), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("condition"), + yaml_string("service_started"), + )])), + ); + } + for (key, value) in desired { + mapping.insert(key, value); + } + *current = serde_yaml::Value::Mapping(mapping); + Ok(()) + } + _ => Err(miette::miette!( + "compose site controller service has an unsupported depends_on shape" + )), + } +} + +pub fn inject_compose_site_controller( + artifact_root: &Path, + plan: &SiteControllerPlan, + plan_path: &Path, + controller_image: &str, +) -> Result<()> { + let plan_mount_source = plan_path.display().to_string(); + inject_compose_site_controller_with_mount_sources( + artifact_root, + plan, + &plan.run_root, + &plan_mount_source, + controller_image, + ) +} + +pub fn inject_compose_site_controller_with_mount_sources( + artifact_root: &Path, + plan: &SiteControllerPlan, + run_root_mount_source: &str, + plan_mount_source: &str, + controller_image: &str, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&compose_path)?; + let services = compose_services_mut(&mut document, &compose_path)?; + let available_services = services.clone(); + let service = services + .get_mut(yaml_string(SITE_CONTROLLER_SERVICE_NAME)) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services.{SITE_CONTROLLER_SERVICE_NAME}", + compose_path.display() + ) + })?; + + service.insert(yaml_string("image"), yaml_string(controller_image)); + service.insert(yaml_string("user"), yaml_string(&compose_controller_user())); + service.insert( + yaml_string("group_add"), + serde_yaml::Value::Sequence( + compose_controller_group_add() + .into_iter() + .map(|gid| yaml_string(&gid)) + .collect(), + ), + ); + service.insert( + yaml_string("healthcheck"), + serde_yaml::to_value(json!({ + "test": [ + "CMD-SHELL", + format!( + "wget -qO- http://127.0.0.1:{}/healthz | grep -q '\"ok\":true'", + plan.listen_addr.port() + ) + ], + "interval": "2s", + "timeout": "2s", + "retries": 30, + "start_period": "1s" + })) + .into_diagnostic() + .wrap_err("failed to serialize compose site controller healthcheck")?, + ); + service.insert(yaml_string("restart"), yaml_string("unless-stopped")); + + if uses_service_network_mode(service) { + service.remove(yaml_string("extra_hosts")); + } else { + let extra_hosts = service + .entry(yaml_string("extra_hosts")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!( + "compose site controller service has a non-sequence extra_hosts field" + ) + })?; + ensure_string_sequence_contains(extra_hosts, "host.docker.internal:host-gateway"); + } + + let volumes = service + .entry(yaml_string("volumes")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!("compose site controller service has a non-sequence volumes field") + })?; + for volume in [ + format!("{run_root_mount_source}:{}", plan.run_root), + format!("{plan_mount_source}:{COMPOSE_CONTROLLER_PLAN_PATH}"), + format!("{COMPOSE_ROUTER_CONTROL_VOLUME_NAME}:{COMPOSE_ROUTER_CONTROL_SOCKET_DIR}"), + format!("{DOCKER_SOCK_PATH}:{DOCKER_SOCK_PATH}"), + ] { + ensure_string_sequence_contains(volumes, &volume); + } + + ensure_controller_environment(service, plan)?; + ensure_controller_depends_on(service, &available_services)?; + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, net::SocketAddr}; + + use amber_compiler::run_plan::SiteKind; + + use super::*; + + #[cfg(unix)] + #[test] + fn compose_controller_groups_include_router_and_docker_socket_groups() { + assert_eq!( + compose_controller_group_add_for(1000, Some(1000)), + vec!["65532"], + "the controller already has its primary runtime group" + ); + assert_eq!( + compose_controller_group_add_for(1000, Some(998)), + vec!["65532", "998"], + "the controller should join the Docker socket group without becoming root" + ); + assert_eq!( + compose_controller_group_add_for(1000, Some(COMPOSE_ROUTER_RUNTIME_GID)), + vec!["65532"], + "router runtime group should not be duplicated" + ); + } + + fn test_plan(run_root: &Path) -> SiteControllerPlan { + SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "run-test".to_string(), + mesh_scope: "scope".to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + authority_url: "http://amber-site-controller:4100".to_string(), + router_identity_id: "/site/compose_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/router.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + state_path: run_root.join("state.json").display().to_string(), + run_root: run_root.display().to_string(), + state_root: run_root.join("state-root").display().to_string(), + site_state_root: run_root.join("site-state").display().to_string(), + artifact_dir: run_root.join("artifact").display().to_string(), + controller_identity_path: None, + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: Some("compose-project".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::from([( + "AMBER_DEV_IMAGE_TAGS".to_string(), + "router=dev-tag,helper=dev-tag".to_string(), + )]), + } + } + + #[test] + fn inject_compose_site_controller_keeps_service_network_mode_compatible() { + let temp = tempfile::tempdir().expect("temp dir"); + let compose_path = temp.path().join("compose.yaml"); + fs::write( + &compose_path, + "services: + amber-site-controller: + image: __amber_internal/site-controller + network_mode: service:amber-site-controller-net + extra_hosts: + - host.docker.internal:host-gateway +", + ) + .expect("compose file should write"); + + let plan = test_plan(temp.path()); + let plan_path = temp.path().join("site-controller-plan.json"); + fs::write(&plan_path, "{}").expect("plan file should write"); + + inject_compose_site_controller( + temp.path(), + &plan, + &plan_path, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose site controller injection should succeed"); + + let document = read_compose_document(&compose_path).expect("compose should parse"); + let service = document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|services| services.get(yaml_string(SITE_CONTROLLER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping) + .expect("site controller service should exist"); + assert_eq!( + service + .get(yaml_string("network_mode")) + .and_then(serde_yaml::Value::as_str), + Some("service:amber-site-controller-net"), + "late bootstrap must preserve the lowered sidecar sharing topology", + ); + assert!( + service.get(yaml_string("extra_hosts")).is_none(), + "late bootstrap must not add host mappings to service-networked controller workloads", + ); + } + + #[test] + fn inject_compose_site_controller_propagates_launch_env() { + let temp = tempfile::tempdir().expect("temp dir"); + let compose_path = temp.path().join("compose.yaml"); + fs::write( + &compose_path, + "services:\n amber-router:\n image: ghcr.io/rdi-foundation/amber-router:test\n amber-site-controller:\n image: __amber_internal/site-controller\n environment:\n - AMBER_DYNAMIC_CAPS_API_URL=http://127.0.0.1:19000\n", + ) + .expect("compose file should write"); + + let plan = test_plan(temp.path()); + let plan_path = temp.path().join("site-controller-plan.json"); + fs::write(&plan_path, "{}").expect("plan file should write"); + + inject_compose_site_controller( + temp.path(), + &plan, + &plan_path, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose site controller injection should succeed"); + + let document = read_compose_document(&compose_path).expect("compose should parse"); + let service = document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|services| services.get(yaml_string(SITE_CONTROLLER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping) + .expect("site controller service should exist"); + let environment = service + .get(yaml_string("environment")) + .and_then(serde_yaml::Value::as_sequence) + .expect("site controller service should include environment entries"); + assert!( + environment + .iter() + .any(|value| value.as_str().is_some_and(|value| { + value == "AMBER_DEV_IMAGE_TAGS=router=dev-tag,helper=dev-tag" + })), + "launch env should be merged into the existing environment list: {service:?}" + ); + assert!( + environment + .iter() + .any(|value| value.as_str().is_some_and(|value| { + value == "AMBER_DYNAMIC_CAPS_API_URL=http://127.0.0.1:19000" + })), + "existing controller program env must be preserved: {service:?}" + ); + let volumes = service + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .expect("site controller service should include volumes"); + assert!( + volumes.iter().any(|value| value + .as_str() + .is_some_and(|value| { value == format!("{}:{}", plan.run_root, plan.run_root) })), + "compose controller bootstrap should mount the run root into the existing service: \ + {service:?}" + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs new file mode 100644 index 00000000..dd8f05d3 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -0,0 +1,2126 @@ +use super::*; + +pub(super) fn dynamic_compose_mesh_dir(service_name: &str) -> String { + format!("{DYNAMIC_COMPOSE_MESH_ROOT}/{service_name}") +} + +pub(super) fn filter_dynamic_router_target( + router_target: &mut amber_mesh::MeshProvisionTarget, + kept_component_ids: &BTreeSet, +) { + router_target + .config + .peers + .retain(|peer| kept_component_ids.contains(&peer.id)); + router_target + .config + .inbound + .retain(|route| match &route.target { + InboundTarget::MeshForward { peer_id, .. } => kept_component_ids.contains(peer_id), + _ => route + .allowed_issuers + .iter() + .any(|issuer| kept_component_ids.contains(issuer)), + }); +} + +pub(super) fn dynamic_proxy_export_mesh_protocol( + export: &DynamicProxyExportRecord, +) -> Result { + let protocol = export + .protocol + .parse::() + .map_err(|err| miette::miette!("invalid dynamic proxy export protocol: {err}"))?; + mesh_protocol(protocol) +} + +pub(super) fn dynamic_proxy_export_http_plugins( + export: &DynamicProxyExportRecord, + protocol: MeshProtocol, +) -> Vec { + matches!( + (export.capability_kind.as_str(), protocol), + ("a2a", MeshProtocol::Http) + ) + .then_some(HttpRoutePlugin::A2a) + .into_iter() + .collect() +} + +pub(super) fn dynamic_proxy_export_route_id( + export_name: &str, + export: &DynamicProxyExportRecord, +) -> Result { + Ok(router_dynamic_export_route_id( + &export.component, + export_name, + dynamic_proxy_export_mesh_protocol(export)?, + )) +} + +pub(super) fn dynamic_input_direct_mesh_protocol( + input: &DynamicInputDirectRecord, +) -> Result { + let protocol = input + .protocol + .parse::() + .map_err(|err| miette::miette!("invalid dynamic direct-input protocol: {err}"))?; + mesh_protocol(protocol) +} + +pub(super) fn dynamic_input_direct_route_id( + input: &DynamicInputDirectRecord, + protocol: MeshProtocol, +) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => { + component_route_id(&input.provider_component, provide, protocol) + } + } +} + +pub(super) fn dynamic_input_direct_capability(input: &DynamicInputDirectRecord) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), + } +} + +pub(super) fn is_compose_component_sidecar_service(service_name: &str) -> bool { + service_name.ends_with("-net") +} + +pub(super) fn dynamic_input_direct_http_plugins( + input: &DynamicInputDirectRecord, + protocol: MeshProtocol, +) -> Vec { + dynamic_proxy_export_http_plugins( + &DynamicProxyExportRecord { + component_id: 0, + component: input.provider_component.clone(), + provide: dynamic_input_direct_capability(input), + protocol: input.protocol.clone(), + capability_kind: input.capability_kind.clone(), + capability_profile: input.capability_profile.clone(), + target_port: 0, + }, + protocol, + ) +} + +pub(super) fn overlay_peer_addr_map_from_ports( + ports: &BTreeMap, +) -> BTreeMap { + ports + .iter() + .map(|(component, port)| (component.clone(), format!("127.0.0.1:{port}"))) + .collect() +} + +pub(super) fn overlay_upsert_peer( + peers: &mut Vec, + peer_identities: &BTreeMap, + peer_id: &str, +) -> Result<()> { + if peers.iter().any(|peer| peer.id == peer_id) { + return Ok(()); + } + let identity = peer_identities.get(peer_id).ok_or_else(|| { + miette::miette!("dynamic route overlay is missing mesh identity for peer {peer_id}") + })?; + peers.push(MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }); + Ok(()) +} + +pub(super) fn augment_route_overlay_payload( + payload: &mut StoredRouteOverlayPayload, + proxy_exports: &BTreeMap, + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result<()> { + for export in proxy_exports.values() { + overlay_upsert_peer(&mut payload.peers, peer_identities, &export.component)?; + } + add_dynamic_proxy_export_overlay_routes( + &mut payload.inbound_routes, + proxy_exports, + |export| { + provider_peer_addrs + .get(&export.component) + .cloned() + .ok_or_else(|| { + miette::miette!( + "dynamic route overlay is missing a live peer address for {}", + export.component + ) + }) + }, + )?; + + Ok(()) +} + +pub(super) fn rewrite_dynamic_direct_inputs( + mesh_plan: &mut MeshProvisionPlan, + direct_inputs: &[DynamicInputDirectRecord], + live_components: &BTreeMap, +) -> Result<()> { + if direct_inputs.is_empty() { + return Ok(()); + } + + for input in direct_inputs { + let provider_runtime = live_components + .get(&input.provider_component) + .ok_or_else(|| { + miette::miette!( + "dynamic direct input {}.{} references live provider {} that is unavailable", + input.component, + input.slot, + input.provider_component + ) + })?; + let protocol = dynamic_input_direct_mesh_protocol(input)?; + let component_target = mesh_plan + .targets + .iter_mut() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == input.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic direct input {}.{} is missing component {} in the mesh provision plan", + input.component, + input.slot, + input.component + ) + })?; + if !component_target + .config + .peers + .iter() + .any(|peer| peer.id == provider_runtime.mesh_config.identity.id) + { + component_target + .config + .peers + .push(amber_mesh::MeshPeerTemplate { + id: provider_runtime.mesh_config.identity.id.clone(), + }); + } + + let route_id = dynamic_input_direct_route_id(input, protocol); + let capability = dynamic_input_direct_capability(input); + let mut matched = false; + for route in component_target + .config + .outbound + .iter_mut() + .filter(|route| route.slot == input.slot) + { + matched = true; + route.route_id = route_id.clone(); + route.protocol = protocol; + route.peer_addr = provider_runtime.component_reachable_mesh_addr.clone(); + route.peer_id = provider_runtime.mesh_config.identity.id.clone(); + route.capability = capability.clone(); + route.capability_kind = Some(input.capability_kind.clone()); + route.capability_profile = input.capability_profile.clone(); + route.http_plugins = dynamic_input_direct_http_plugins(input, protocol); + } + if !matched { + return Err(miette::miette!( + "dynamic direct input {}.{} is missing an outbound route in the mesh provision \ + plan", + input.component, + input.slot + )); + } + } + + Ok(()) +} + +pub(super) fn rewrite_dynamic_direct_inputs_in_artifact( + artifact_root: &Path, + direct_inputs: &[DynamicInputDirectRecord], + live_components: &BTreeMap, +) -> Result<()> { + if direct_inputs.is_empty() { + return Ok(()); + } + let path = artifact_root.join("mesh-provision-plan.json"); + let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + rewrite_dynamic_direct_inputs(&mut mesh_plan, direct_inputs, live_components)?; + write_json(&path, &mesh_plan) +} + +pub(super) fn build_filesystem_route_overlay_base( + artifact_root: &Path, + assigned_components: &[String], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result { + let mesh_plan: MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )?; + let kept_component_ids = assigned_components.iter().cloned().collect::>(); + let mut router_target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "dynamic artifact {} is missing a router mesh target", + artifact_root.display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = provider_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + peer_identities.get(&peer.id).map(|identity| MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>() + .ok_or_else(|| { + miette::miette!( + "dynamic artifact {} is missing a live mesh identity for one of its route peers", + artifact_root.display() + ) + })?; + Ok(StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }) +} + +pub(super) fn write_direct_vm_live_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + proxy_exports: &BTreeMap, + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result<()> { + let mut payload = build_filesystem_route_overlay_base( + artifact_root, + assigned_components, + provider_peer_addrs, + peer_identities, + )?; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + provider_peer_addrs, + peer_identities, + )?; + write_dynamic_route_overlay_payload(artifact_root, &payload) +} + +pub(super) fn ensure_dynamic_proxy_export_component_routes( + mesh_plan: &mut MeshProvisionPlan, + proxy_exports: &BTreeMap, + router_identity_id: &str, +) -> Result<()> { + for export in proxy_exports.values() { + let protocol = dynamic_proxy_export_mesh_protocol(export)?; + let route_id = component_route_id(&export.component, &export.provide, protocol); + let component_target = mesh_plan + .targets + .iter_mut() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the mesh provision plan", + export.component + ) + })?; + if !component_target + .config + .peers + .iter() + .any(|peer| peer.id == router_identity_id) + { + component_target + .config + .peers + .push(amber_mesh::MeshPeerTemplate { + id: router_identity_id.to_string(), + }); + } + if let Some(route) = component_target + .config + .inbound + .iter_mut() + .find(|route| route.route_id == route_id) + { + if !route + .allowed_issuers + .iter() + .any(|issuer| issuer == router_identity_id) + { + route.allowed_issuers.push(router_identity_id.to_string()); + route.allowed_issuers.sort(); + route.allowed_issuers.dedup(); + } + continue; + } + component_target.config.inbound.push(InboundRoute { + route_id, + capability: export.provide.clone(), + capability_kind: Some(export.capability_kind.clone()), + capability_profile: export.capability_profile.clone(), + protocol, + http_plugins: dynamic_proxy_export_http_plugins(export, protocol), + target: InboundTarget::Local { + port: export.target_port, + }, + allowed_issuers: vec![router_identity_id.to_string()], + }); + } + Ok(()) +} + +pub(super) fn ensure_dynamic_proxy_export_component_routes_in_artifact( + artifact_root: &Path, + proxy_exports: &BTreeMap, + router_identity_id: &str, +) -> Result<()> { + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mut mesh_plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; + ensure_dynamic_proxy_export_component_routes( + &mut mesh_plan, + proxy_exports, + router_identity_id, + )?; + write_json(&plan_path, &mesh_plan) +} + +pub(super) fn add_dynamic_proxy_export_overlay_routes( + inbound_routes: &mut Vec, + proxy_exports: &BTreeMap, + mut peer_addr_for_export: impl FnMut(&DynamicProxyExportRecord) -> Result, +) -> Result<()> { + for (export_name, export) in proxy_exports { + let protocol = dynamic_proxy_export_mesh_protocol(export)?; + let route_id = router_dynamic_export_route_id(&export.component, export_name, protocol); + let route = InboundRoute { + route_id, + capability: export_name.clone(), + capability_kind: Some(export.capability_kind.clone()), + capability_profile: export.capability_profile.clone(), + protocol, + http_plugins: dynamic_proxy_export_http_plugins(export, protocol), + target: InboundTarget::MeshForward { + peer_addr: peer_addr_for_export(export)?, + peer_id: export.component.clone(), + route_id: component_route_id(&export.component, &export.provide, protocol), + capability: export.provide.clone(), + }, + allowed_issuers: Vec::new(), + }; + if let Some(existing) = inbound_routes + .iter_mut() + .find(|existing| existing.route_id == route.route_id) + { + *existing = route; + } else { + inbound_routes.push(route); + } + } + Ok(()) +} + +pub(super) struct DynamicComposeMeshPlan { + pub(super) mesh_plan: MeshProvisionPlan, + pub(super) mesh_dirs: BTreeMap, + pub(super) component_mesh_dirs: BTreeMap, +} + +pub(super) fn build_dynamic_compose_mesh_plan( + artifact_root: &Path, + assigned_components: &[String], +) -> Result { + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut filtered_targets = Vec::new(); + let mut mesh_dirs = BTreeMap::new(); + let mut component_mesh_dirs = BTreeMap::new(); + + for mut target in plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + if !assigned.contains(target.config.identity.id.as_str()) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &mut target.output else { + return Err(miette::miette!( + "compose component {} does not use filesystem mesh output", + target.config.identity.id + )); + }; + let sidecar = Path::new(dir.as_str()) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose component {} has invalid mesh output dir {}", + target.config.identity.id, + dir + ) + })? + .to_string(); + let relative_dir = dynamic_compose_mesh_dir(&sidecar); + dir.clear(); + dir.push_str(&relative_dir); + component_mesh_dirs.insert(target.config.identity.id.clone(), relative_dir.clone()); + mesh_dirs.insert(sidecar, relative_dir); + filtered_targets.push(target); + } + MeshProvisionTargetKind::Router => {} + } + } + + if filtered_targets.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not contain assigned child mesh targets", + artifact_root.join("compose.yaml").display() + )); + } + + Ok(DynamicComposeMeshPlan { + mesh_plan: MeshProvisionPlan { + version: plan.version, + identity_seed: plan.identity_seed, + existing_peer_identities: Vec::new(), + targets: filtered_targets, + }, + mesh_dirs, + component_mesh_dirs, + }) +} + +pub(super) fn project_dynamic_direct_router_surface( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let state_path = direct_runtime_state_path(Path::new(&child.artifact_root)); + let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its router mesh port", + plan.site_id + ) + })?; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let mut state: DirectRuntimeState = read_json(&state_path, "direct runtime state")?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + state.router_mesh_port = Some(router_mesh_port); + write_json(&state_path, &state)?; + std::thread::sleep(Duration::from_millis(100)); + let state: DirectRuntimeState = read_json(&state_path, "direct runtime state")?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out projecting direct child router mesh port into {}", + state_path.display() + )); + } + } + + let site_current = direct_current_control_socket_path(Path::new(&plan.artifact_dir)); + let child_current = direct_current_control_socket_path(Path::new(&child.artifact_root)); + if let Some(parent) = child_current.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_direct_control_socket_link( + &child_current, + &site_current, + "dynamic child router control symlink", + )?; + let artifact_link = Path::new(&child.artifact_root) + .join(".amber") + .join("control") + .join("router-control.sock"); + if let Some(parent) = artifact_link.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_direct_control_socket_link( + &artifact_link, + &child_current, + "dynamic child router control artifact symlink", + ) +} + +pub(super) fn project_dynamic_vm_router_surface( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let state_path = Path::new(&child.artifact_root) + .join(".amber") + .join("vm-runtime.json"); + let runtime_root = site_controller_runtime_child_runtime_root(plan, child.child_id); + let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) + })?; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let mut state = + load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + state.router_mesh_port = Some(router_mesh_port); + write_vm_runtime_state(Path::new(&child.artifact_root), &state)?; + std::thread::sleep(Duration::from_millis(100)); + let state = + load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out projecting vm child router mesh port into {}", + state_path.display() + )); + } + } + + let site_current = vm_current_control_socket_path(Path::new(&plan.artifact_dir)); + let child_current = vm_current_control_socket_path(Path::new(&child.artifact_root)); + if let Some(parent) = child_current.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_control_socket_link( + &child_current, + &site_current, + "dynamic child vm router control symlink", + )?; + let artifact_link = Path::new(&child.artifact_root) + .join(".amber") + .join("control") + .join("router-control.sock"); + if let Some(parent) = artifact_link.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_control_socket_link( + &artifact_link, + &child_current, + "dynamic child vm router control artifact symlink", + ) +} + +pub(super) fn dynamic_child_route_overlay_id( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> String { + format!("framework-child:{}:{child_id}", plan.site_id) +} + +pub(super) fn site_router_control_endpoint( + plan: &SiteControllerRuntimePlan, +) -> Result { + if plan.kind == SiteKind::Kubernetes + && let Some(raw) = plan.local_router_control.as_deref() + { + return parse_control_endpoint(raw); + } + let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); + if state_path.is_file() { + let state: SiteManagerState = read_json(&state_path, "site manager state")?; + if let Some(raw) = state.router_control { + return parse_control_endpoint(&raw); + } + } + + match plan.kind { + SiteKind::Direct => Ok(ControlEndpoint::Unix(direct_current_control_socket_path( + Path::new(&plan.artifact_dir), + ))), + SiteKind::Vm => Ok(ControlEndpoint::Unix(vm_current_control_socket_path( + Path::new(&plan.artifact_dir), + ))), + SiteKind::Compose | SiteKind::Kubernetes => Err(miette::miette!( + "site `{}` manager state is missing router control endpoint", + plan.site_id + )), + } +} + +async fn site_router_overlay_control_endpoint_with_timeout( + plan: &SiteControllerRuntimePlan, + timeout: Duration, +) -> Result { + if plan.kind == SiteKind::Kubernetes { + wait_for_kubernetes_site_router_ready(plan, timeout).await?; + } + site_router_control_endpoint(plan) +} + +async fn site_router_overlay_control_endpoint( + plan: &SiteControllerRuntimePlan, +) -> Result { + site_router_overlay_control_endpoint_with_timeout(plan, site_ready_timeout_for_kind(plan.kind)) + .await +} + +pub(super) fn child_router_overlay_payload( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, + runtime_root: &Path, +) -> Result<(Vec, Vec)> { + let overlay_path = dynamic_route_overlay_path(artifact_root); + if overlay_path.is_file() { + let payload: StoredRouteOverlayPayload = read_json(&overlay_path, "site router overlay")?; + return Ok((payload.peers, payload.inbound_routes)); + } + let provision: MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )?; + let Some(router_target) = provision + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + else { + return Ok((Vec::new(), Vec::new())); + }; + let proxy_exports = load_dynamic_proxy_exports_metadata(artifact_root)?; + let mut overlay_peer_ids = router_target + .config + .peers + .iter() + .map(|peer| peer.id.clone()) + .collect::>(); + overlay_peer_ids.extend( + proxy_exports + .values() + .map(|export| export.component.clone()), + ); + + let mut peers = Vec::new(); + let mut peer_addr_by_id = BTreeMap::new(); + for peer_id in &overlay_peer_ids { + let component_target = provision + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == *peer_id + }) + .ok_or_else(|| { + miette::miette!( + "router overlay peer {} is missing from mesh provision plan", + peer_id + ) + })?; + let (identity, runtime_config) = match &component_target.output { + MeshProvisionOutput::Filesystem { dir } => { + let identity: MeshIdentitySecret = read_json( + &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), + "mesh identity", + )?; + let runtime_config: MeshConfigPublic = read_json( + &runtime_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )?; + (identity, Some(runtime_config)) + } + MeshProvisionOutput::KubernetesSecret { name, namespace } => ( + load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())?, + None, + ), + }; + let public_key = identity.public_key().into_diagnostic()?; + if let Some(runtime_config) = runtime_config { + peer_addr_by_id.insert( + runtime_config.identity.id.clone(), + runtime_config.mesh_listen.to_string(), + ); + } + peers.push(MeshPeer { + id: identity.id, + public_key, + }); + } + let mut inbound_routes = router_target.config.inbound.clone(); + for route in &mut inbound_routes { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = peer_addr_by_id.get(peer_id) + { + peer_addr.clone_from(resolved); + } + } + if !proxy_exports.is_empty() { + add_dynamic_proxy_export_overlay_routes(&mut inbound_routes, &proxy_exports, |export| { + match plan.kind { + SiteKind::Kubernetes => { + dynamic_proxy_export_kubernetes_peer_addr(artifact_root, &provision, export) + } + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => peer_addr_by_id + .get(&export.component) + .map(ToString::to_string) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing a live mesh address", + export.component + ) + }), + } + })?; + } + Ok((peers, inbound_routes)) +} + +pub(super) fn child_overlay_runtime_root( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> PathBuf { + match plan.kind { + SiteKind::Direct | SiteKind::Vm => { + site_controller_runtime_child_runtime_root(plan, child.child_id) + } + SiteKind::Compose | SiteKind::Kubernetes => PathBuf::from(&child.artifact_root), + } +} + +pub(super) const SITE_CONTROLLER_INTERNAL_OVERLAY_ID: &str = "framework-site-controller"; + +fn local_site_controller_runtime( + live_components: &BTreeMap, +) -> Result> { + let mut controllers = live_components.values().filter(|runtime| { + runtime.mesh_config.inbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + }); + let Some(controller) = controllers.next() else { + return Ok(None); + }; + if controllers.next().is_some() { + return Err(miette::miette!( + "dynamic site controller overlay found multiple local framework.component controller \ + runtimes" + )); + } + Ok(Some(controller)) +} + +fn dynamic_site_controller_overlay_payload( + published_children: &[SiteControllerRuntimeChildRecord], + live_components: &BTreeMap, +) -> Result> { + let Some(controller_runtime) = local_site_controller_runtime(live_components)? else { + return Ok(None); + }; + + let mut routes_by_id = BTreeMap::::new(); + let mut issuer_ids = BTreeSet::::new(); + for child in published_children { + for route in &child.controller_routes { + if route.allowed_issuers.is_empty() { + continue; + } + issuer_ids.extend(route.allowed_issuers.iter().cloned()); + if let Some(existing) = routes_by_id.get_mut(&route.route_id) { + if !controller_overlay_routes_compatible(existing, route) { + return Err(miette::miette!( + "dynamic controller overlay route {} is defined with incompatible route \ + metadata", + route.route_id + )); + } + existing + .allowed_issuers + .extend(route.allowed_issuers.iter().cloned()); + existing.allowed_issuers.sort(); + existing.allowed_issuers.dedup(); + } else { + let mut route = route.clone(); + route.allowed_issuers.sort(); + route.allowed_issuers.dedup(); + routes_by_id.insert(route.route_id.clone(), route); + } + } + } + if routes_by_id.is_empty() { + return Ok(None); + } + + let static_peer_ids = controller_runtime + .mesh_config + .peers + .iter() + .map(|peer| peer.id.as_str()) + .collect::>(); + let mut peers = Vec::new(); + for issuer_id in issuer_ids { + if issuer_id == controller_runtime.mesh_config.identity.id + || static_peer_ids.contains(issuer_id.as_str()) + { + continue; + } + let runtime = live_components.get(&issuer_id).ok_or_else(|| { + miette::miette!( + "dynamic controller overlay route grants issuer {} but that peer is not live on \ + site {}", + issuer_id, + controller_runtime.moniker + ) + })?; + peers.push(MeshPeer { + id: runtime.mesh_config.identity.id.clone(), + public_key: runtime.mesh_config.identity.public_key, + }); + } + + Ok(Some(StoredRouteOverlayPayload { + peers, + inbound_routes: routes_by_id.into_values().collect(), + })) +} + +fn dynamic_site_controller_overlay_required( + published_children: &[SiteControllerRuntimeChildRecord], +) -> bool { + published_children.iter().any(|child| { + child + .controller_routes + .iter() + .any(|route| !route.allowed_issuers.is_empty()) + }) +} + +fn controller_overlay_routes_compatible(left: &InboundRoute, right: &InboundRoute) -> bool { + left.capability == right.capability + && left.capability_kind == right.capability_kind + && left.capability_profile == right.capability_profile + && left.protocol == right.protocol + && left.http_plugins == right.http_plugins + && left.target == right.target +} + +pub(super) fn dynamic_direct_input_overlay_id(component: &str) -> String { + format!( + "framework-direct-inputs:{}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(component.as_bytes()) + ) +} + +pub(super) fn dynamic_direct_input_grants( + children: &[SiteControllerRuntimeChildRecord], +) -> Result>>> { + let mut grants = BTreeMap::>>::new(); + for child in children { + for input in &child.direct_inputs { + let route_id = + dynamic_input_direct_route_id(input, dynamic_input_direct_mesh_protocol(input)?); + grants + .entry(input.provider_component.clone()) + .or_default() + .entry(route_id) + .or_default() + .insert(input.component.clone()); + } + } + Ok(grants) +} + +pub(super) async fn reconcile_dynamic_site_controller_overlay( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + let live_components = collect_live_component_runtime_metadata(&app.plan)?; + let Some(controller_runtime) = local_site_controller_runtime(&live_components)? else { + if !dynamic_site_controller_overlay_required(&published_children) { + return Ok(()); + } + return Err(miette::miette!( + "site {} has published dynamic children but no live framework.component controller \ + component", + app.plan.site_id + )); + }; + let Some(control_endpoint) = controller_runtime.control_endpoint.as_ref() else { + return Ok(()); + }; + let Some(overlay) = + dynamic_site_controller_overlay_payload(&published_children, &live_components)? + else { + revoke_route_overlay_with_retry( + control_endpoint, + SITE_CONTROLLER_INTERNAL_OVERLAY_ID, + Duration::from_secs(30), + ) + .await?; + return Ok(()); + }; + apply_route_overlay_with_retry( + control_endpoint, + SITE_CONTROLLER_INTERNAL_OVERLAY_ID, + &overlay.peers, + &overlay.inbound_routes, + Duration::from_secs(30), + ) + .await +} + +pub(super) async fn reconcile_dynamic_direct_input_overlays( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let (published_children, previous_overlay_providers) = { + let state = app.state.lock().await; + ( + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(), + state.direct_input_overlay_providers.clone(), + ) + }; + let live_components = collect_live_component_runtime_metadata(&app.plan)?; + let grants = dynamic_direct_input_grants(&published_children)?; + let current_overlay_providers = grants.keys().cloned().collect::>(); + let overlay_providers = current_overlay_providers + .union(&previous_overlay_providers) + .cloned() + .collect::>(); + for component in &overlay_providers { + let Some(runtime) = live_components.get(component) else { + continue; + }; + let Some(control_endpoint) = runtime.control_endpoint.as_ref() else { + continue; + }; + let overlay_id = dynamic_direct_input_overlay_id(component); + let Some(route_grants) = grants.get(component) else { + revoke_route_overlay_with_retry(control_endpoint, &overlay_id, Duration::from_secs(30)) + .await?; + continue; + }; + + let mut peers = Vec::new(); + let mut known_peer_ids = BTreeSet::new(); + let mut inbound_routes = Vec::new(); + for (route_id, issuers) in route_grants { + let base_route = runtime + .mesh_config + .inbound + .iter() + .find(|route| route.route_id == *route_id) + .cloned() + .ok_or_else(|| { + miette::miette!( + "provider component {} is missing inbound route {} for a direct dynamic \ + input", + component, + route_id + ) + })?; + let mut route = base_route; + route.allowed_issuers = issuers.iter().cloned().collect(); + for issuer in issuers { + let issuer_runtime = live_components.get(issuer).ok_or_else(|| { + miette::miette!( + "direct dynamic input issuer {} is not live on site {}", + issuer, + app.plan.site_id + ) + })?; + if known_peer_ids.insert(issuer_runtime.mesh_config.identity.id.clone()) { + peers.push(MeshPeer { + id: issuer_runtime.mesh_config.identity.id.clone(), + public_key: issuer_runtime.mesh_config.identity.public_key, + }); + } + } + inbound_routes.push(route); + } + apply_route_overlay_with_retry( + control_endpoint, + &overlay_id, + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await?; + } + let mut state = app.state.lock().await; + if state.direct_input_overlay_providers != current_overlay_providers { + state.direct_input_overlay_providers = current_overlay_providers; + write_json(&app.state_path, &*state)?; + } + Ok(()) +} + +pub(super) async fn reconcile_dynamic_site_router_overlays( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + reconcile_dynamic_site_router_overlays_for_children( + app, + &published_children, + &published_children, + ) + .await +} + +pub(super) async fn reconcile_dynamic_site_router_overlays_for_children( + app: &SiteControllerRuntimeApp, + overlay_children: &[SiteControllerRuntimeChildRecord], + issuer_children: &[SiteControllerRuntimeChildRecord], +) -> Result<()> { + if overlay_children.is_empty() { + return Ok(()); + } + let endpoint = site_router_overlay_control_endpoint(&app.plan).await?; + let _ = issuer_children; + for child in overlay_children { + let artifact_root = Path::new(&child.artifact_root); + let runtime_root = child_overlay_runtime_root(&app.plan, child); + let (peers, inbound_routes) = + child_router_overlay_payload(&app.plan, artifact_root, &runtime_root)?; + if inbound_routes.is_empty() { + continue; + } + apply_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(&app.plan, child.child_id), + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await?; + } + + Ok(()) +} + +pub(super) async fn apply_dynamic_site_router_overlay( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let endpoint = site_router_overlay_control_endpoint(plan).await?; + let artifact_root = Path::new(&child.artifact_root); + let runtime_root = child_overlay_runtime_root(plan, child); + let (peers, inbound_routes) = child_router_overlay_payload(plan, artifact_root, &runtime_root)?; + if inbound_routes.is_empty() { + return Ok(()); + } + apply_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(plan, child.child_id), + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await +} + +pub(super) async fn revoke_dynamic_site_router_overlay( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let endpoint = site_router_overlay_control_endpoint(plan).await?; + revoke_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(plan, child.child_id), + Duration::from_secs(30), + ) + .await +} + +#[cfg(test)] +mod direct_input_tests { + use std::{ + fs, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration as StdDuration, Instant as StdInstant}, + }; + + use amber_mesh::MeshConfigTemplate; + use axum::response::IntoResponse; + + use super::*; + + async fn spawn_mock_router_mesh_listener() -> (SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("mock router mesh listener should bind"); + let addr = listener + .local_addr() + .expect("mock router mesh listener addr should resolve"); + let handle = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + drop(stream); + } + }); + (addr, handle) + } + + async fn spawn_gated_router_control_listener( + ready: Arc, + saw_mutation: Arc, + mutated_before_ready: Arc, + ) -> (SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("mock router control listener should bind"); + let addr = listener + .local_addr() + .expect("mock router control listener addr should resolve"); + let router = axum::Router::new() + .route( + "/identity", + axum::routing::get({ + let ready = ready.clone(); + move || { + let ready = ready.clone(); + async move { + if ready.load(Ordering::SeqCst) { + axum::Json(MeshIdentityPublic { + id: "/site/kind_local/router".to_string(), + public_key: [9u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }) + .into_response() + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response() + } + } + } + }), + ) + .route( + "/overlays/{overlay_id}", + axum::routing::put({ + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + move || { + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + async move { + saw_mutation.store(true, Ordering::SeqCst); + if !ready.load(Ordering::SeqCst) { + mutated_before_ready.store(true, Ordering::SeqCst); + } + axum::http::StatusCode::NO_CONTENT + } + } + }) + .delete({ + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + move || { + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + async move { + saw_mutation.store(true, Ordering::SeqCst); + if !ready.load(Ordering::SeqCst) { + mutated_before_ready.store(true, Ordering::SeqCst); + } + axum::http::StatusCode::NO_CONTENT + } + } + }), + ); + let handle = tokio::spawn(async move { + axum::serve(listener, router.into_make_service()) + .await + .expect("mock router control server should run"); + }); + (addr, handle) + } + + fn kubernetes_overlay_test_plan( + temp: &tempfile::TempDir, + control_addr: SocketAddr, + mesh_addr: SocketAddr, + ) -> SiteControllerRuntimePlan { + let artifact_dir = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("kind_local"); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/kind_local/router".to_string(), + local_router_control: Some(control_addr.to_string()), + artifact_dir: artifact_dir.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(mesh_addr.port()), + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + } + } + + #[test] + fn compose_site_router_control_endpoint_prefers_manager_state_volume_socket() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let site_state_root = temp.path().join("state").join("compose_local"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "compose_local", + "kind": "compose", + "status": "running", + "artifact_dir": temp.path().join("artifact").display().to_string(), + "supervisor_pid": 1u32, + "router_control": "volume://demo_amber-router-control/router-control.sock", + "router_mesh_addr": "127.0.0.1:24000", + }), + ) + .expect("manager state should write"); + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + router_identity_id: "/site/compose_local/router".to_string(), + local_router_control: Some("unix:///amber/control/router-control.sock".to_string()), + artifact_dir: temp.path().join("artifact").display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: Some("demo".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + + assert!(matches!( + site_router_control_endpoint(&plan).expect("router control endpoint should resolve"), + ControlEndpoint::VolumeSocket { volume, socket_path } + if volume == "demo_amber-router-control" && socket_path == "/router-control.sock" + )); + } + + #[test] + fn site_router_control_endpoint_prefers_local_embedded_target_over_manager_state() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let site_state_root = temp.path().join("state").join("kind_local"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "kind_local", + "kind": "kubernetes", + "status": "running", + "artifact_dir": temp.path().join("artifact").display().to_string(), + "supervisor_pid": 1u32, + "router_control": "127.0.0.1:9", + "router_mesh_addr": "127.0.0.1:9", + }), + ) + .expect("manager state should write"); + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/kind_local/router".to_string(), + local_router_control: Some("amber-router:24100".to_string()), + artifact_dir: temp.path().join("artifact").display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + + assert!(matches!( + site_router_control_endpoint(&plan).expect("router control endpoint should resolve"), + ControlEndpoint::Tcp(addr) if addr == "amber-router:24100" + )); + } + + #[tokio::test] + async fn probe_kubernetes_router_control_ready_accepts_mock_identity_response() { + let ready = Arc::new(AtomicBool::new(true)); + let (control_addr, control_handle) = spawn_gated_router_control_listener( + ready, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + ) + .await; + + assert!( + probe_kubernetes_router_control_ready( + &control_addr.to_string(), + StdDuration::from_secs(1) + ) + .await + .expect("router control probe should succeed"), + "the mock router control server should satisfy the readiness probe", + ); + control_handle.abort(); + } + + #[tokio::test] + async fn router_mesh_listener_ready_target_accepts_mock_listener() { + let (mesh_addr, mesh_handle) = spawn_mock_router_mesh_listener().await; + assert!( + router_mesh_listener_ready_target(&mesh_addr.to_string(), StdDuration::from_secs(1)) + .await, + "the mock mesh listener should satisfy the mesh readiness probe", + ); + mesh_handle.abort(); + } + + #[tokio::test] + async fn site_router_overlay_control_endpoint_waits_for_kubernetes_router_readiness() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let ready = Arc::new(AtomicBool::new(false)); + let saw_mutation = Arc::new(AtomicBool::new(false)); + let mutated_before_ready = Arc::new(AtomicBool::new(false)); + let (mesh_addr, mesh_handle) = spawn_mock_router_mesh_listener().await; + let (control_addr, control_handle) = + spawn_gated_router_control_listener(ready.clone(), saw_mutation, mutated_before_ready) + .await; + let plan = kubernetes_overlay_test_plan(&temp, control_addr, mesh_addr); + + let gate = ready.clone(); + let gate_handle = tokio::spawn(async move { + tokio::time::sleep(StdDuration::from_millis(350)).await; + gate.store(true, Ordering::SeqCst); + }); + + let start = StdInstant::now(); + let endpoint = + site_router_overlay_control_endpoint_with_timeout(&plan, StdDuration::from_secs(6)) + .await + .expect("overlay endpoint should wait for router readiness and then resolve"); + + assert!( + start.elapsed() >= StdDuration::from_millis(300), + "overlay endpoint resolution should wait until the Kubernetes router is ready", + ); + assert!(matches!( + endpoint, + ControlEndpoint::Tcp(addr) if addr == control_addr.to_string() + )); + gate_handle.abort(); + control_handle.abort(); + mesh_handle.abort(); + } + + #[test] + fn rewrite_dynamic_direct_inputs_points_child_at_provider_sidecar() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("mesh-provision-plan.json"); + write_json( + &path, + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/consumer".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:0".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: vec![OutboundRoute { + route_id: "old".to_string(), + rewrite_route_id: None, + slot: "api".to_string(), + capability_kind: Some("http".to_string()), + capability_profile: None, + listen_port: 0, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:1".to_string(), + peer_id: "/router".to_string(), + capability: "old".to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/components/consumer".to_string(), + }, + }], + }, + ) + .expect("write mesh plan"); + + let mut live_components = BTreeMap::new(); + live_components.insert( + "/provider".to_string(), + LiveComponentRuntimeMetadata { + moniker: "/provider".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + component_reachable_mesh_addr: "10.0.0.20:24001".to_string(), + control_endpoint: None, + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [7; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:0".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ); + rewrite_dynamic_direct_inputs_in_artifact( + temp.path(), + &[DynamicInputDirectRecord { + component: "/consumer".to_string(), + slot: "api".to_string(), + provider_component: "/provider".to_string(), + protocol: "http".to_string(), + capability_kind: "http".to_string(), + capability_profile: None, + target: DynamicInputRouteTarget::ComponentProvide { + provide: "serve".to_string(), + }, + }], + &live_components, + ) + .expect("rewrite should succeed"); + + let plan: MeshProvisionPlan = read_json(&path, "mesh provision plan").expect("plan"); + let target = &plan.targets[0]; + assert_eq!(target.config.peers.len(), 1); + assert_eq!(target.config.peers[0].id, "/provider"); + let route = &target.config.outbound[0]; + assert_eq!(route.peer_id, "/provider"); + assert_eq!(route.peer_addr, "10.0.0.20:24001"); + assert_eq!(route.capability, "serve"); + assert_eq!( + route.route_id, + component_route_id("/provider", "serve", MeshProtocol::Http) + ); + } + + #[test] + fn dynamic_direct_input_overlay_id_is_path_safe_for_component_monikers() { + let overlay_id = dynamic_direct_input_overlay_id("/source"); + assert_eq!( + overlay_id, "framework-direct-inputs:L3NvdXJjZQ", + "overlay ids should encode component monikers so router control paths stay \ + single-segment", + ); + assert!( + !overlay_id.contains('/'), + "overlay ids must not contain path separators: {overlay_id}", + ); + } + + #[test] + fn dynamic_direct_input_overlay_targets_only_current_and_previous_providers() { + let current = BTreeSet::from(["/provider".to_string()]); + let previous = BTreeSet::from(["/provider".to_string(), "/stale".to_string()]); + + let overlay_providers = current.union(&previous).cloned().collect::>(); + + assert_eq!( + overlay_providers, + BTreeSet::from(["/provider".to_string(), "/stale".to_string()]), + "reconciliation should only touch providers with current grants or previously applied \ + overlays", + ); + } + + #[test] + fn dynamic_site_controller_overlay_uses_child_controller_route_grants() { + let controller_moniker = + amber_compiler::run_plan::FrameworkComponentControllerMoniker::for_site("site-a") + .into_string(); + let grant_provide = + amber_compiler::run_plan::FrameworkComponentSiteToken::for_site("site-a") + .controller_grant_provide_name(amber_scenario::ComponentId(0)); + let internal_route_id = component_route_id( + controller_moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ); + let grant_route_id = component_route_id( + controller_moniker.as_str(), + grant_provide.as_str(), + MeshProtocol::Http, + ); + let live_components = BTreeMap::from([ + ( + controller_moniker.clone(), + LiveComponentRuntimeMetadata { + moniker: controller_moniker.clone(), + router_reachable_mesh_addr: "127.0.0.1:24000".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24000".to_string(), + control_endpoint: Some(ControlEndpoint::Unix("/tmp/controller.sock".into())), + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: controller_moniker.clone(), + public_key: [1; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeer { + id: "/static".to_string(), + public_key: [2; 32], + }], + inbound: vec![InboundRoute { + route_id: internal_route_id.clone(), + capability: + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec![ + "/site/router".to_string(), + "/static".to_string(), + ], + }], + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ), + ( + "/job-dynamic".to_string(), + LiveComponentRuntimeMetadata { + moniker: "/job-dynamic".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + control_endpoint: Some(ControlEndpoint::Unix("/tmp/job.sock".into())), + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/job-dynamic".to_string(), + public_key: [3; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: Some( + "127.0.0.1:19001".parse().expect("dynamic caps listen"), + ), + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ), + ]); + + let overlay = dynamic_site_controller_overlay_payload( + &[SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: "/tmp/child".to_string(), + assigned_components: vec!["/job-dynamic".to_string()], + controller_routes: vec![InboundRoute { + route_id: grant_route_id.clone(), + capability: grant_provide, + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/job-dynamic".to_string()], + }], + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }], + &live_components, + ) + .expect("overlay payload should build") + .expect("dynamic child should require an overlay"); + + assert_eq!(overlay.peers.len(), 1); + assert_eq!(overlay.peers[0].id, "/job-dynamic"); + assert_eq!(overlay.peers[0].public_key, [3; 32]); + assert_eq!(overlay.inbound_routes.len(), 1); + assert_eq!(overlay.inbound_routes[0].route_id, grant_route_id); + assert_eq!( + overlay.inbound_routes[0].allowed_issuers, + vec!["/job-dynamic".to_string()], + "the overlay should contribute only the dynamic child issuer recorded for this \ + grant-specific controller route", + ); + } + + #[test] + fn dynamic_site_controller_overlay_is_required_only_for_granted_controller_routes() { + let child = |controller_routes| SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: "/tmp/child".to_string(), + assigned_components: vec!["/job-dynamic".to_string()], + controller_routes, + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }; + + assert!( + !dynamic_site_controller_overlay_required(&[child(Vec::new())]), + "VM sites controlled from a direct site can publish ordinary dynamic children without \ + a local site-controller component" + ); + assert!( + !dynamic_site_controller_overlay_required(&[child(vec![InboundRoute { + route_id: "empty".to_string(), + capability: "component".to_string(), + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: Vec::new(), + }])]), + "routes with no dynamic issuers do not require a controller overlay" + ); + assert!( + dynamic_site_controller_overlay_required(&[child(vec![InboundRoute { + route_id: "granted".to_string(), + capability: "component".to_string(), + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/job-dynamic".to_string()], + }])]), + "controller overlays are still required when a dynamic child receives a granted \ + framework.component route" + ); + } + + #[test] + fn project_dynamic_direct_router_surface_creates_child_control_aliases() { + let temp = tempfile::tempdir().expect("tempdir should be created"); + let site_artifact = temp.path().join("site-artifact"); + let site_state_root = temp.path().join("state").join("direct_local"); + let child_artifact = site_state_root + .join("framework-component") + .join("children") + .join("1") + .join("artifact"); + fs::create_dir_all(site_artifact.join(".amber").join("control")) + .expect("site artifact control dir should exist"); + fs::create_dir_all(child_artifact.join(".amber")).expect("child artifact dir should exist"); + + let site_current = super::super::direct_current_control_socket_path(&site_artifact); + if let Some(parent) = site_current.parent() { + fs::create_dir_all(parent).expect("site current control dir should exist"); + } + let site_runtime = temp.path().join("router-runtime.sock"); + fs::write(&site_runtime, []).expect("fake router runtime socket placeholder should exist"); + super::super::ensure_direct_control_socket_link( + &site_current, + &site_runtime, + "site current control symlink", + ) + .expect("site current control symlink should be created"); + super::super::ensure_direct_control_socket_link( + &site_artifact + .join(".amber") + .join("control") + .join("router-control.sock"), + &site_current, + "site artifact control symlink", + ) + .expect("site artifact control symlink should be created"); + write_json( + &super::super::direct_runtime_state_path(&child_artifact), + &super::super::DirectRuntimeState::default(), + ) + .expect("child direct runtime state should be written"); + + let plan = runtime_api::SiteControllerRuntimePlan { + schema: "test".to_string(), + version: 1, + run_id: "run-123".to_string(), + mesh_scope: "mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + router_identity_id: "/site/direct_local/router".to_string(), + local_router_control: None, + artifact_dir: site_artifact.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }; + let child = SiteControllerRuntimeChildRecord { + child_id: 1, + artifact_root: child_artifact.display().to_string(), + assigned_components: vec!["/job-1".to_string()], + controller_routes: Vec::new(), + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }; + + project_dynamic_direct_router_surface(&plan, &child) + .expect("direct child router surface should project"); + + let state: super::super::DirectRuntimeState = read_json( + &super::super::direct_runtime_state_path(&child_artifact), + "direct runtime state", + ) + .expect("projected direct runtime state should be readable"); + assert_eq!(state.router_mesh_port, Some(24000)); + + let child_current = super::super::direct_current_control_socket_path(&child_artifact); + assert_eq!( + fs::read_link(&child_current).expect("child current control alias should exist"), + site_current, + "child current alias should point at the site current control alias", + ); + assert_eq!( + fs::read_link( + child_artifact + .join(".amber") + .join("control") + .join("router-control.sock") + ) + .expect("child artifact control alias should exist"), + child_current, + "child artifact control alias should point at the child current alias", + ); + } + + #[test] + fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { + let temp = tempfile::tempdir().expect("tempdir should be created"); + let site_state_root = temp.path().join("state").join("direct_local"); + let child_root = + runtime_api::site_controller_runtime_child_root_for_site(&site_state_root, 7); + fs::create_dir_all(child_root.join("artifact")).expect("child artifact dir should exist"); + fs::write(child_root.join("artifact").join("marker.txt"), "marker") + .expect("child marker should be written"); + write_json( + &site_controller_runtime_state_path(&site_state_root), + &SiteControllerRuntimeState { + schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, + run_id: "run-123".to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + direct_input_overlay_providers: BTreeSet::new(), + children: BTreeMap::from([( + 7, + SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: child_root.join("artifact").display().to_string(), + assigned_components: Vec::new(), + controller_routes: Vec::new(), + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }, + )]), + }, + ) + .expect("site controller runtime state should be written"); + + cleanup_dynamic_site_children(&site_state_root, SiteKind::Direct) + .expect("dynamic site children should be cleaned"); + + let state: SiteControllerRuntimeState = read_json( + &site_controller_runtime_state_path(&site_state_root), + "site controller runtime state", + ) + .expect("site controller runtime state should be readable"); + assert!(state.children.is_empty()); + assert!(!child_root.exists()); + } + + #[test] + fn inject_site_controller_peer_router_routes_adds_local_inbound_and_peer_outbound_routes() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("mesh-provision-plan.json"); + write_json( + &path, + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![ + MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/controller".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: vec![InboundRoute { + route_id: amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/site/local/router".to_string()], + }], + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/controller".to_string(), + }, + }, + MeshProvisionTarget { + kind: MeshProvisionTargetKind::Router, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/router".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: Some("127.0.0.1:24100".parse().expect("control listen")), + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/router".to_string(), + }, + } + ], + }, + ) + .expect("write mesh plan"); + + inject_site_controller_peer_router_routes( + temp.path(), + "local", + &["/site/peer/router".to_string()], + &[SiteControllerPeerRouterRoute { + site_id: "peer".to_string(), + peer_router: MeshIdentityPublic { + id: "/site/peer/router".to_string(), + public_key: [7; 32], + mesh_scope: Some("amber.test".to_string()), + }, + peer_addr: "10.0.0.20:24000".to_string(), + listen_addr: "127.0.0.1".to_string(), + listen_port: 25001, + }], + ) + .expect("inject routes"); + + let plan: MeshProvisionPlan = read_json(&path, "mesh provision plan").expect("plan"); + let router = plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .expect("router target should remain present"); + assert!( + router.config.inbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("local") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && route.allowed_issuers == vec!["/site/peer/router".to_string()] + && matches!( + route.target, + InboundTarget::MeshForward { + ref peer_id, + ref peer_addr, + ref route_id, + ref capability, + } if peer_id == "/site/local/controller" + && peer_addr == "127.0.0.1:23001" + && route_id + == &amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ) + && capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + ) + }), + "router should expose a local inbound route for peer site-controller traffic", + ); + assert!( + router.config.outbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("peer") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && route.peer_id == "/site/peer/router" + && route.peer_addr == "10.0.0.20:24000" + && route.listen_addr.as_deref() == Some("127.0.0.1") + && route.listen_port == 25001 + }), + "router should publish a local outbound listener for the peer site controller", + ); + assert!( + plan.existing_peer_identities.iter().any(|identity| { + identity.id == "/site/peer/router" + && identity.public_key == [7; 32] + && identity.mesh_scope.as_deref() == Some("amber.test") + }), + "router mesh plan should carry the peer router identity needed to provision the route", + ); + } + + #[test] + fn compose_router_port_helpers_rewrite_mesh_publish() { + let temp = tempfile::tempdir().expect("tempdir"); + fs::write( + temp.path().join("compose.yaml"), + r#" +services: + amber-router: + image: ghcr.io/rdi-foundation/amber-router:v0.2.x + ports: + - "127.0.0.1::24000" +"#, + ) + .expect("compose yaml should write"); + + set_compose_router_published_mesh_port(temp.path(), 34000).expect("rewrite mesh port"); + + let rendered = + fs::read_to_string(temp.path().join("compose.yaml")).expect("compose yaml should read"); + assert!( + rendered.contains("0.0.0.0:34000:24000"), + "mesh port should be rewritten to a deterministic host publish:\n{rendered}" + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs new file mode 100644 index 00000000..0fe2a30d --- /dev/null +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -0,0 +1,1510 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::Write as _, + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use serde_json::json; + +use super::*; + +const KUBERNETES_ROUTER_NAME: &str = "amber-router"; +const KUBERNETES_ROUTER_MESH_PORT: u16 = 24000; +const KUBERNETES_ROUTER_CONTROL_PORT: u16 = 24100; +const KUBERNETES_ROUTER_SERVICE_PATH: &str = "04-services/amber-router.yaml"; +const KUBERNETES_ROUTER_NETPOL_PATH: &str = "05-networkpolicies/amber-router-netpol.yaml"; +const KUBERNETES_CONTROLLER_NETPOL_PATH: &str = + "05-networkpolicies/amber-site-controller-netpol.yaml"; +const KUBERNETES_CONTROLLER_SEED_SECRET_PATH: &str = "01-secrets/amber-site-controller-seed.yaml"; +const KUBERNETES_CONTROLLER_SEED_SECRET_PATH_PREFIX: &str = "01-secrets/amber-site-controller-seed"; +const KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT: usize = 64 * 1024; +const KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-site-controller-sa.yaml"; +const KUBERNETES_CONTROLLER_ROLE_PATH: &str = "02-rbac/amber-site-controller-role.yaml"; +const KUBERNETES_CONTROLLER_ROLE_BINDING_PATH: &str = + "02-rbac/amber-site-controller-rolebinding.yaml"; +const KUBERNETES_CONTROLLER_DEPLOYMENT_PATH: &str = "03-deployments/amber-site-controller.yaml"; +const KUBERNETES_CONTROLLER_STATE_PVC_PATH: &str = + "03-persistentvolumeclaims/amber-site-controller-state.yaml"; +#[cfg(test)] +const KUBERNETES_CONTROLLER_SERVICE_PATH: &str = "04-services/amber-site-controller.yaml"; +const KUBERNETES_CONTROLLER_SEED_VOLUME: &str = "controller-seed"; +const KUBERNETES_CONTROLLER_STATE_VOLUME: &str = "controller-state"; +const KUBERNETES_CONTROLLER_STATE_PVC_NAME: &str = "amber-site-controller-state"; +const KUBERNETES_CONTROLLER_STATE_STORAGE_REQUEST: &str = "1Gi"; +const KUBERNETES_CONTROLLER_SITE_ROOT: &str = "/amber/site"; +const KUBERNETES_CONTROLLER_STATE_ROOT: &str = "/amber/site/state"; +const KUBERNETES_CONTROLLER_ARTIFACT_ROOT: &str = "/amber/site/artifact"; +const KUBERNETES_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; +const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controller-state.json"; +const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; +const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; +const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME: &str = "amber-provisioner"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME: &str = "amber-provisioner-dynamic"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH: &str = + "02-rbac/amber-provisioner-dynamic-role.yaml"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH: &str = + "02-rbac/amber-provisioner-dynamic-rolebinding.yaml"; + +#[derive(Clone, Debug)] +struct KubernetesControllerSeedSecret { + path: String, + name: String, + items: Vec, + document: serde_json::Value, +} + +pub fn inject_kubernetes_site_controller( + artifact_root: &Path, + plan: &SiteControllerPlan, + controller_image: &str, +) -> Result<()> { + let embedded_plan = build_embedded_kubernetes_controller_plan(plan); + let seed_secrets = + build_kubernetes_controller_seed_secrets(artifact_root, plan, &embedded_plan)?; + let labels = kubernetes_controller_labels(); + + for secret in &seed_secrets { + write_yaml_artifact(artifact_root.join(&secret.path), &secret.document)?; + } + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_STATE_PVC_PATH), + &json!({ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": KUBERNETES_CONTROLLER_STATE_PVC_NAME, + "labels": labels, + }, + "spec": { + "accessModes": ["ReadWriteOnce"], + "resources": { + "requests": { + "storage": KUBERNETES_CONTROLLER_STATE_STORAGE_REQUEST, + }, + }, + }, + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH), + &json!({ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + } + }), + )?; + write_kubernetes_dynamic_provisioner_rbac(artifact_root)?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_ROLE_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "Role", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "rules": [ + { + "apiGroups": [""], + "resources": [ + "configmaps", + "persistentvolumeclaims", + "secrets", + "services" + ], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["apps"], + "resources": ["deployments"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["batch"], + "resources": ["jobs"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["networking.k8s.io"], + "resources": ["networkpolicies"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + } + ] + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_ROLE_BINDING_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "subjects": [{ + "kind": "ServiceAccount", + "name": SITE_CONTROLLER_SERVICE_NAME, + }], + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": SITE_CONTROLLER_SERVICE_NAME, + } + }), + )?; + patch_kubernetes_controller_deployment( + artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), + plan, + controller_image, + &seed_secrets, + )?; + let mut resource_paths = seed_secrets + .iter() + .map(|secret| secret.path.as_str()) + .collect::>(); + resource_paths.extend([ + KUBERNETES_CONTROLLER_STATE_PVC_PATH, + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, + KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH, + KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH, + KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, + KUBERNETES_CONTROLLER_ROLE_PATH, + KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, + ]); + add_kubernetes_resource_paths(artifact_root, &resource_paths)?; + let peer_router_route_ports = peer_router_route_ports(&embedded_plan)?; + ensure_kubernetes_router_allows_site_controller_ingress( + artifact_root, + &peer_router_route_ports, + )?; + ensure_kubernetes_site_controller_allows_router_egress(artifact_root, &peer_router_route_ports) +} + +fn upsert_named_sequence_entry( + sequence: &mut serde_yaml::Sequence, + name: &str, + value: serde_yaml::Value, +) { + if let Some(existing) = sequence.iter_mut().find(|entry| { + entry + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|existing| existing == name) + }) { + *existing = value; + } else { + sequence.push(value); + } +} + +fn merge_kubernetes_controller_env( + container: &mut serde_yaml::Mapping, + plan: &SiteControllerPlan, +) -> Result<()> { + if plan.launch_env.is_empty() { + return Ok(()); + } + let env = container + .entry(yaml_string("env")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment env must be a sequence"))?; + for (name, value) in &plan.launch_env { + upsert_named_sequence_entry( + env, + name, + serde_yaml::to_value(json!({ "name": name, "value": value })) + .into_diagnostic() + .wrap_err("failed to serialize controller env entry")?, + ); + } + Ok(()) +} + +fn ensure_kubernetes_volume_mount( + container: &mut serde_yaml::Mapping, + name: &str, + mount_path: &str, + read_only: Option, +) -> Result<()> { + let mounts = container + .entry(yaml_string("volumeMounts")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment volumeMounts must be a sequence"))?; + let mut mount = json!({ + "name": name, + "mountPath": mount_path, + }); + if let Some(read_only) = read_only { + mount["readOnly"] = serde_json::Value::Bool(read_only); + } + upsert_named_sequence_entry( + mounts, + name, + serde_yaml::to_value(mount) + .into_diagnostic() + .wrap_err("failed to serialize controller volume mount")?, + ); + Ok(()) +} + +fn patch_kubernetes_controller_deployment( + path: PathBuf, + plan: &SiteControllerPlan, + controller_image: &str, + seed_secrets: &[KubernetesControllerSeedSecret], +) -> Result<()> { + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut deployment: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let pod_spec = deployment + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("template"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|template| template.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "controller deployment {} is missing spec.template.spec", + path.display() + ) + })?; + pod_spec.insert( + yaml_string("serviceAccountName"), + yaml_string(SITE_CONTROLLER_SERVICE_NAME), + ); + pod_spec.insert( + yaml_string("automountServiceAccountToken"), + serde_yaml::Value::Bool(true), + ); + + let init_containers = pod_spec + .entry(yaml_string("initContainers")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!("controller deployment initContainers must be a sequence") + })?; + upsert_named_sequence_entry( + init_containers, + "seed-site-controller", + serde_yaml::to_value(json!({ + "name": "seed-site-controller", + "image": controller_image, + "command": [ + "sh", + "-lc", + format!( + "set -eu +seed_if_missing() {{ + src=\"$1\" + dest=\"$2\" + if [ ! -e \"$dest\" ]; then + cp \"$src\" \"$dest\" + fi +}} +mkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +cp {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ + {KUBERNETES_CONTROLLER_PLAN_PATH} +seed_if_missing {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ + {KUBERNETES_CONTROLLER_STATE_PATH} +seed_if_missing {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ + {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH} +if [ -d {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact ]; then + rm -rf {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} + mkdir -p {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} + cp -R {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact/. \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +fi +" + ) + ], + "volumeMounts": [ + { + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SEED_ROOT, + "readOnly": true, + }, + { + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, + } + ] + })) + .into_diagnostic() + .wrap_err("failed to serialize controller seed init container")?, + ); + + let containers = pod_spec + .get_mut(yaml_string("containers")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("controller deployment is missing spec.template.spec.containers") + })?; + let main_index = containers + .iter() + .position(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) + .unwrap_or(0); + let main = containers + .get_mut(main_index) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| miette::miette!("controller deployment has no mutable main container"))?; + main.insert(yaml_string("image"), yaml_string(controller_image)); + merge_kubernetes_controller_env(main, plan)?; + ensure_kubernetes_volume_mount( + main, + KUBERNETES_CONTROLLER_STATE_VOLUME, + KUBERNETES_CONTROLLER_SITE_ROOT, + None, + )?; + + let volumes = pod_spec + .entry(yaml_string("volumes")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment volumes must be a sequence"))?; + let seed_sources = seed_secrets + .iter() + .map(|secret| { + json!({ + "secret": { + "name": secret.name.clone(), + "items": secret.items.clone(), + } + }) + }) + .collect::>(); + upsert_named_sequence_entry( + volumes, + KUBERNETES_CONTROLLER_SEED_VOLUME, + serde_yaml::to_value(json!({ + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "projected": { + "sources": seed_sources, + } + })) + .into_diagnostic() + .wrap_err("failed to serialize controller seed volume")?, + ); + upsert_named_sequence_entry( + volumes, + KUBERNETES_CONTROLLER_STATE_VOLUME, + serde_yaml::to_value(json!({ + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "persistentVolumeClaim": { + "claimName": KUBERNETES_CONTROLLER_STATE_PVC_NAME, + } + })) + .into_diagnostic() + .wrap_err("failed to serialize controller state volume")?, + ); + + write_yaml_artifact(path, &deployment) +} + +fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteControllerPlan { + let mut embedded = plan.clone(); + let port = plan.listen_addr.port(); + embedded.listen_addr = SocketAddr::from(([0, 0, 0, 0], port)); + embedded.authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{port}"); + embedded.local_router_control = Some(format!( + "{KUBERNETES_ROUTER_NAME}:{KUBERNETES_ROUTER_CONTROL_PORT}" + )); + embedded.run_root = format!("{KUBERNETES_CONTROLLER_SITE_ROOT}/run"); + embedded.state_root = format!("{KUBERNETES_CONTROLLER_SITE_ROOT}/state-root"); + embedded.site_state_root = KUBERNETES_CONTROLLER_STATE_ROOT.to_string(); + embedded.artifact_dir = KUBERNETES_CONTROLLER_ARTIFACT_ROOT.to_string(); + embedded.state_path = KUBERNETES_CONTROLLER_STATE_PATH.to_string(); + embedded.context = None; + embedded.storage_root = None; + embedded.runtime_root = None; + embedded.compose_project = None; + embedded.kubernetes_namespace = plan.kubernetes_namespace.clone(); + embedded.router_mesh_port = Some(KUBERNETES_ROUTER_MESH_PORT); + embedded +} + +fn build_kubernetes_controller_seed_secrets( + artifact_root: &Path, + plan: &SiteControllerPlan, + embedded_plan: &SiteControllerPlan, +) -> Result> { + let seed_files = build_kubernetes_controller_seed_files(artifact_root, plan, embedded_plan)?; + let mut chunks = Vec::>::new(); + let mut current = Vec::<(String, String)>::new(); + let mut current_size = 0usize; + for (path, contents) in seed_files { + let entry_size = path.len() + contents.len(); + if !current.is_empty() + && current_size + entry_size > KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT + { + chunks.push(current); + current = Vec::new(); + current_size = 0; + } + current.push((path, contents)); + current_size += entry_size; + } + if !current.is_empty() { + chunks.push(current); + } + chunks + .into_iter() + .enumerate() + .map(|(index, files)| render_kubernetes_controller_seed_secret(index, files)) + .collect() +} + +fn build_kubernetes_controller_seed_files( + artifact_root: &Path, + plan: &SiteControllerPlan, + embedded_plan: &SiteControllerPlan, +) -> Result> { + let state_json = fs::read_to_string(&plan.state_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", plan.state_path))?; + let desired_links = super::desired_links_path(Path::new(&plan.site_state_root)); + let desired_links_json = fs::read_to_string(&desired_links) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", desired_links.display()))?; + let embedded_plan_json = serde_json::to_string_pretty(embedded_plan) + .into_diagnostic() + .wrap_err("failed to serialize embedded kubernetes site controller plan")?; + let mut files = BTreeMap::from([ + ("site-controller-plan.json".to_string(), embedded_plan_json), + ("site-controller-state.json".to_string(), state_json), + ("desired-links.json".to_string(), desired_links_json), + ]); + for path in walk_files(artifact_root)? { + let relative = path + .strip_prefix(artifact_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; + let relative = relative + .components() + .map(|component| component.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + let contents = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + files.insert(format!("artifact/{relative}"), contents); + } + Ok(files.into_iter().collect()) +} + +fn render_kubernetes_controller_seed_secret( + index: usize, + files: Vec<(String, String)>, +) -> Result { + let name = if index == 0 { + format!("{SITE_CONTROLLER_SERVICE_NAME}-seed") + } else { + format!("{SITE_CONTROLLER_SERVICE_NAME}-seed-{index}") + }; + let path = if index == 0 { + KUBERNETES_CONTROLLER_SEED_SECRET_PATH.to_string() + } else { + format!("{KUBERNETES_CONTROLLER_SEED_SECRET_PATH_PREFIX}-{index}.yaml") + }; + let mut string_data = serde_json::Map::new(); + let mut items = Vec::with_capacity(files.len()); + for (entry_index, (relative_path, contents)) in files.into_iter().enumerate() { + let key = format!("seed-{entry_index:04}"); + string_data.insert(key.clone(), serde_json::Value::String(contents)); + items.push(json!({ + "key": key, + "path": relative_path, + })); + } + Ok(KubernetesControllerSeedSecret { + path, + name: name.clone(), + items, + document: json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": name, + "labels": kubernetes_controller_labels(), + }, + "type": "Opaque", + "stringData": string_data, + }), + }) +} + +fn write_kubernetes_dynamic_provisioner_rbac(artifact_root: &Path) -> Result<()> { + let labels = kubernetes_provisioner_labels(); + write_yaml_artifact_if_missing( + artifact_root.join(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH), + &json!({ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME, + "labels": labels, + } + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "Role", + "metadata": { + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + "labels": kubernetes_provisioner_labels(), + }, + "rules": [{ + "apiGroups": [""], + "resources": ["secrets"], + "verbs": ["create", "get", "update"], + }] + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + "labels": kubernetes_provisioner_labels(), + }, + "subjects": [{ + "kind": "ServiceAccount", + "name": KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME, + }], + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + } + }), + ) +} + +fn add_kubernetes_resource_paths(artifact_root: &Path, paths: &[&str]) -> Result<()> { + let kustomization_path = artifact_root.join("kustomization.yaml"); + let raw = fs::read_to_string(&kustomization_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", kustomization_path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", kustomization_path.display()))?; + let resources = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("resources"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "kustomization {} is missing a resources sequence", + kustomization_path.display() + ) + })?; + for path in paths { + if resources + .iter() + .any(|value| value.as_str().is_some_and(|existing| existing == *path)) + { + continue; + } + resources.push(yaml_string(path)); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", kustomization_path.display()))?; + fs::write(&kustomization_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", kustomization_path.display())) +} + +fn peer_router_route_ports(plan: &SiteControllerPlan) -> Result> { + plan.peer_site_router_urls + .values() + .map(|url| { + let parsed = Url::parse(url) + .into_diagnostic() + .wrap_err_with(|| format!("invalid peer router url `{url}`"))?; + parsed + .port() + .ok_or_else(|| miette::miette!("peer router url `{url}` is missing a route port")) + }) + .collect() +} + +fn ensure_kubernetes_router_service_ports( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + if route_ports.is_empty() { + return Ok(()); + } + let path = artifact_root.join(KUBERNETES_ROUTER_SERVICE_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let ports = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("router service {} is missing spec.ports", path.display()) + })?; + let existing = ports + .iter() + .filter_map(|entry| { + entry + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in route_ports { + if existing.contains(port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "name": format!("controller-route-{port}"), + "port": port, + "targetPort": port, + "protocol": "TCP", + })) + .into_diagnostic() + .wrap_err("failed to serialize router service route port")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn ensure_kubernetes_router_allows_site_controller_ingress( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + ensure_kubernetes_router_service_ports(artifact_root, route_ports)?; + let path = artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let ingress = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("ingress"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "router network policy {} is missing spec.ingress", + path.display() + ) + })?; + let controller_selector = kubernetes_controller_selector(); + let controller_component = controller_selector + .get("amber.io/component") + .expect("selector must contain component"); + let mut required_ports = + BTreeSet::from([KUBERNETES_ROUTER_CONTROL_PORT, KUBERNETES_ROUTER_MESH_PORT]); + required_ports.extend(route_ports.iter().copied()); + if let Some(rule) = ingress.iter_mut().find(|rule| { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("from"))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|from| { + from.iter().any(|peer| { + peer.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector_value| selector_value.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .is_some_and(|labels| { + labels.get(yaml_string("amber.io/component")) + == Some(&yaml_string(controller_component)) + }) + }) + }) + }) { + let ports = rule + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "router network policy {} controller ingress rule is missing ports", + path.display() + ) + })?; + let existing_ports = ports + .iter() + .filter_map(|port| { + port.as_mapping() + .and_then(|entry| entry.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in required_ports { + if existing_ports.contains(&port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "protocol": "TCP", + "port": port, + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router ingress port")?, + ); + } + } else { + ingress.push( + serde_yaml::to_value(json!({ + "from": [{ + "podSelector": { + "matchLabels": controller_selector, + } + }], + "ports": required_ports.into_iter().map(|port| json!({ + "protocol": "TCP", + "port": port, + })).collect::>() + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router ingress rule")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn ensure_kubernetes_site_controller_allows_router_egress( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + let mut required_ports = route_ports.clone(); + required_ports.insert(KUBERNETES_ROUTER_CONTROL_PORT); + required_ports.insert(KUBERNETES_ROUTER_MESH_PORT); + let path = artifact_root.join(KUBERNETES_CONTROLLER_NETPOL_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let egress = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("egress"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "site controller network policy {} is missing spec.egress", + path.display() + ) + })?; + let router_selector = BTreeMap::from([( + "amber.io/component".to_string(), + KUBERNETES_ROUTER_NAME.to_string(), + )]); + if let Some(rule) = egress.iter_mut().find(|rule| { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("to"))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|to| { + to.iter().any(|peer| { + peer.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector_value| selector_value.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .is_some_and(|labels| { + labels.get(yaml_string("amber.io/component")) + == Some(&yaml_string(KUBERNETES_ROUTER_NAME)) + }) + }) + }) + }) { + let ports = rule + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "site controller network policy {} router egress rule is missing ports", + path.display() + ) + })?; + let existing_ports = ports + .iter() + .filter_map(|port| { + port.as_mapping() + .and_then(|entry| entry.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in &required_ports { + if existing_ports.contains(port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "protocol": "TCP", + "port": port, + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router egress port")?, + ); + } + } else { + egress.push( + serde_yaml::to_value(json!({ + "to": [{ + "podSelector": { + "matchLabels": router_selector, + } + }], + "ports": required_ports.iter().map(|port| json!({ + "protocol": "TCP", + "port": port, + })).collect::>() + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router egress rule")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn kubernetes_controller_labels() -> BTreeMap { + BTreeMap::from([ + ( + "app.kubernetes.io/managed-by".to_string(), + "amber".to_string(), + ), + ( + "amber.io/component".to_string(), + SITE_CONTROLLER_SERVICE_NAME.to_string(), + ), + ("amber.io/type".to_string(), "site-controller".to_string()), + ]) +} + +fn kubernetes_provisioner_labels() -> BTreeMap { + BTreeMap::from([ + ( + "app.kubernetes.io/managed-by".to_string(), + "amber".to_string(), + ), + ( + "amber.io/component".to_string(), + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME.to_string(), + ), + ("amber.io/type".to_string(), "provisioner".to_string()), + ]) +} + +fn kubernetes_controller_selector() -> BTreeMap { + BTreeMap::from([( + "amber.io/component".to_string(), + SITE_CONTROLLER_SERVICE_NAME.to_string(), + )]) +} + +fn write_yaml_artifact(path: PathBuf, value: &impl serde::Serialize) -> Result<()> { + let parent = path.parent().ok_or_else(|| { + miette::miette!("artifact path {} has no parent directory", path.display()) + })?; + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + let rendered = serde_yaml::to_string(value) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + let mut file = fs::File::create(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", path.display()))?; + file.write_all(rendered.as_bytes()) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn write_yaml_artifact_if_missing(path: PathBuf, value: &impl serde::Serialize) -> Result<()> { + if path.exists() { + return Ok(()); + } + write_yaml_artifact(path, value) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf}; + + use amber_compiler::run_plan::SiteKind; + + use super::*; + + fn test_plan(root: &Path) -> SiteControllerPlan { + let controller_port = 32123; + SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "run-test".to_string(), + mesh_scope: "scope".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + listen_addr: SocketAddr::from(([127, 0, 0, 1], controller_port)), + authority_url: format!("http://amber-site-controller:{controller_port}"), + router_identity_id: "/site/kind_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("amber-router:24100".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + state_path: root.join("state.json").display().to_string(), + run_root: root.join("run").display().to_string(), + state_root: root.join("state-root").display().to_string(), + site_state_root: root.join("site-state").display().to_string(), + artifact_dir: root.join("artifact").display().to_string(), + controller_identity_path: None, + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("test-ns".to_string()), + context: Some("test-context".to_string()), + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::from([( + "AMBER_DEV_IMAGE_TAGS".to_string(), + "router=dev-tag,helper=dev-tag".to_string(), + )]), + } + } + + fn write_kubernetes_controller_fixture(artifact_root: &Path) { + fs::create_dir_all(artifact_root.join("04-services")).expect("services dir"); + fs::create_dir_all(artifact_root.join("05-networkpolicies")).expect("netpol dir"); + fs::create_dir_all(artifact_root.join("03-deployments")).expect("deployments dir"); + fs::write( + artifact_root.join("kustomization.yaml"), + "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ + 04-services/amber-site-controller.yaml\n - 04-services/amber-router.yaml\n - \ + 05-networkpolicies/amber-router-netpol.yaml\n - \ + 05-networkpolicies/amber-site-controller-netpol.yaml\n", + ) + .expect("kustomization should write"); + fs::write( + artifact_root.join(KUBERNETES_ROUTER_SERVICE_PATH), + "apiVersion: v1\nkind: Service\nmetadata:\n name: amber-router\nspec:\n ports:\n \ + - name: mesh\n port: 24000\n targetPort: 24000\n protocol: TCP\n", + ) + .expect("router service should write"); + fs::write( + artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH), + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ + amber-router-netpol\nspec:\n ingress: []\n", + ) + .expect("router netpol should write"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_NETPOL_PATH), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-site-controller-netpol +spec: + egress: + - to: + - podSelector: + matchLabels: + amber.io/component: amber-router + ports: + - protocol: TCP + port: 24100 +"#, + ) + .expect("site controller netpol should write"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), + r#" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: amber-site-controller +spec: + template: + spec: + containers: + - name: main + image: __amber_internal/site-controller + args: + - --plan + - /amber/site/state/site-controller-plan.json + env: + - name: EXISTING_ENV + value: kept + - name: sidecar + image: ghcr.io/rdi-foundation/amber-router:test +"#, + ) + .expect("controller deployment should write"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_PATH), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-site-controller +spec: + ports: + - name: framework-component + port: 32123 + targetPort: 32123 + protocol: TCP +"#, + ) + .expect("controller service should write"); + } + + fn write_controller_state(plan: &SiteControllerPlan) { + fs::write(&plan.state_path, "{}").expect("state should write"); + fs::create_dir_all(&plan.site_state_root).expect("site state dir"); + fs::write( + super::desired_links_path(Path::new(&plan.site_state_root)), + "{}", + ) + .expect("desired links should write"); + } + + fn seed_secret_paths(artifact_root: &Path) -> Vec { + let mut paths = + walk_files(&artifact_root.join("01-secrets")).expect("seed secrets should list"); + paths.sort(); + paths + } + + fn controller_pod_spec(deployment: &serde_yaml::Value) -> &serde_yaml::Mapping { + deployment + .as_mapping() + .and_then(|root| root.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|spec| spec.get(yaml_string("template"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|template| template.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include a pod spec") + } + + fn rule_values<'a>(rule: &'a serde_yaml::Value, key: &str) -> Vec<&'a str> { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string(key))) + .and_then(serde_yaml::Value::as_sequence) + .expect("rule sequence should exist") + .iter() + .map(|value| value.as_str().expect("rule value should be a string")) + .collect() + } + + #[test] + fn inject_kubernetes_site_controller_propagates_launch_env() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let deployment_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH)) + .expect("controller deployment should exist"); + assert!( + !deployment_raw.contains("artifact.tar.b64"), + "kubernetes controller bootstrap should seed ordinary files instead of a tarball" + ); + assert!( + !deployment_raw.contains("base64 -d"), + "kubernetes controller bootstrap should not decode a tarball at startup" + ); + assert!( + deployment_raw.contains("cp -R /amber/seed/artifact/."), + "kubernetes controller bootstrap should copy seeded artifact files directly" + ); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); + let pod_spec = controller_pod_spec(&deployment); + let containers = pod_spec + .get(yaml_string("containers")) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller deployment should include containers"); + assert_eq!( + containers.len(), + 2, + "bootstrap should patch the rendered controller deployment instead of replacing it" + ); + let main = containers + .iter() + .find(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should keep the main container"); + let env = main + .get(yaml_string("env")) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller main container should include env"); + assert!(env.iter().any(|entry| { + entry.as_mapping().is_some_and(|mapping| { + mapping.get(yaml_string("name")) == Some(&yaml_string("AMBER_DEV_IMAGE_TAGS")) + && mapping.get(yaml_string("value")) + == Some(&yaml_string("router=dev-tag,helper=dev-tag")) + }) + })); + assert!(env.iter().any(|entry| { + entry.as_mapping().is_some_and(|mapping| { + mapping.get(yaml_string("name")) == Some(&yaml_string("EXISTING_ENV")) + && mapping.get(yaml_string("value")) == Some(&yaml_string("kept")) + }) + })); + + let volumes = pod_spec + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller deployment should include volumes"); + let seed_volume = volumes + .iter() + .find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_SEED_VOLUME) + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include the seed volume"); + let projected_sources = seed_volume + .get(yaml_string("projected")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|projected| projected.get(yaml_string("sources"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller seed volume should project secrets"); + let projected_paths = projected_sources + .iter() + .flat_map(|source| { + source + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("secret"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|secret| secret.get(yaml_string("items"))) + .and_then(serde_yaml::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|item| { + item.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("path"))) + .and_then(serde_yaml::Value::as_str) + .map(str::to_string) + }) + .collect::>() + }) + .collect::>(); + assert!( + projected_paths + .iter() + .any(|path| path == "site-controller-plan.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "site-controller-state.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "desired-links.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "artifact/kustomization.yaml") + ); + let state_volume = volumes + .iter() + .find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_STATE_VOLUME) + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include the state volume"); + assert_eq!( + state_volume + .get(yaml_string("persistentVolumeClaim")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|pvc| pvc.get(yaml_string("claimName"))) + .and_then(serde_yaml::Value::as_str), + Some(KUBERNETES_CONTROLLER_STATE_PVC_NAME), + "controller state must be backed by a PVC" + ); + assert!( + !state_volume.contains_key(yaml_string("emptyDir")), + "controller state must not be backed by emptyDir" + ); + assert!( + seed_secret_paths(artifact_root) + .iter() + .any(|path| path.file_name().and_then(|name| name.to_str()) + == Some("amber-site-controller-seed.yaml")), + "kustomization should include the primary controller seed secret" + ); + } + + #[test] + fn inject_kubernetes_site_controller_bootstraps_dynamic_provisioner_rbac() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let kustomization = + fs::read_to_string(artifact_root.join("kustomization.yaml")).expect("kustomization"); + assert!(kustomization.contains(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH)); + assert!(kustomization.contains(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH)); + assert!(kustomization.contains(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH)); + + let controller_role_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_ROLE_PATH)) + .expect("controller role should exist"); + let controller_role: serde_yaml::Value = + serde_yaml::from_str(&controller_role_raw).expect("controller role should parse"); + let controller_rules = controller_role["rules"] + .as_sequence() + .expect("controller role should have rules"); + assert!( + controller_rules.iter().all(|rule| { + !rule_values(rule, "apiGroups").contains(&"rbac.authorization.k8s.io") + && !rule_values(rule, "resources").contains(&"serviceaccounts") + && !rule_values(rule, "resources").contains(&"roles") + && !rule_values(rule, "resources").contains(&"rolebindings") + }), + "site-controller Role must not be able to rewrite Kubernetes RBAC objects" + ); + + let provisioner_role_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH)) + .expect("dynamic provisioner role should exist"); + let provisioner_role: serde_yaml::Value = + serde_yaml::from_str(&provisioner_role_raw).expect("provisioner role should parse"); + assert_eq!( + provisioner_role["metadata"]["name"].as_str(), + Some(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME) + ); + let provisioner_rules = provisioner_role["rules"] + .as_sequence() + .expect("provisioner role should have rules"); + assert_eq!(provisioner_rules.len(), 1); + let rule = &provisioner_rules[0]; + assert_eq!(rule_values(rule, "apiGroups"), vec![""]); + assert_eq!(rule_values(rule, "resources"), vec!["secrets"]); + let verbs = rule_values(rule, "verbs"); + assert!(verbs.contains(&"create")); + assert!(verbs.contains(&"get")); + assert!(verbs.contains(&"update")); + assert!( + !rule + .as_mapping() + .expect("provisioner rule should be a mapping") + .contains_key(yaml_string("resourceNames")), + "dynamic provisioner secret names are not all known at site bootstrap time" + ); + + let role_binding_raw = fs::read_to_string( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH), + ) + .expect("dynamic provisioner rolebinding should exist"); + let role_binding: serde_yaml::Value = + serde_yaml::from_str(&role_binding_raw).expect("rolebinding should parse"); + assert_eq!( + role_binding["subjects"][0]["name"].as_str(), + Some(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME) + ); + assert_eq!( + role_binding["roleRef"]["name"].as_str(), + Some(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME) + ); + + let router_netpol_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH)) + .expect("router network policy should exist"); + let router_netpol: serde_yaml::Value = + serde_yaml::from_str(&router_netpol_raw).expect("router network policy should parse"); + let router_ingress_ports = router_netpol["spec"]["ingress"] + .as_sequence() + .expect("router network policy should include ingress") + .iter() + .flat_map(|rule| rule["ports"].as_sequence().into_iter().flatten()) + .filter_map(|port| port["port"].as_u64()) + .collect::>(); + assert!( + router_ingress_ports.contains(&u64::from(KUBERNETES_ROUTER_MESH_PORT)), + "site-controller must be allowed to probe the router mesh listener" + ); + assert!( + router_ingress_ports.contains(&u64::from(KUBERNETES_ROUTER_CONTROL_PORT)), + "site-controller must be allowed to probe the router control listener" + ); + } + + #[test] + fn inject_kubernetes_site_controller_chunks_seed_secrets_to_avoid_object_limits() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + fs::create_dir_all(artifact_root.join("seed-data")).expect("seed data dir"); + fs::write( + artifact_root.join("seed-data/large-a.txt"), + "a".repeat(KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT / 2), + ) + .expect("large artifact A should write"); + fs::write( + artifact_root.join("seed-data/large-b.txt"), + "b".repeat(KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT / 2), + ) + .expect("large artifact B should write"); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let seed_paths = seed_secret_paths(artifact_root); + assert!( + seed_paths.len() > 1, + "large controller seeds should be split across multiple secrets" + ); + for seed_path in &seed_paths { + let raw = fs::read_to_string(seed_path).expect("seed secret should read"); + let document: serde_yaml::Value = + serde_yaml::from_str(&raw).expect("seed secret should parse"); + let data_size = document + .as_mapping() + .and_then(|root| root.get(yaml_string("stringData"))) + .and_then(serde_yaml::Value::as_mapping) + .expect("seed secret should have stringData") + .iter() + .map(|(key, value)| { + key.as_str().unwrap_or_default().len() + + value.as_str().unwrap_or_default().len() + }) + .sum::(); + assert!( + data_size <= KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT, + "seed secret {} should stay under the chunk limit", + seed_path.display() + ); + } + + let deployment_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH)) + .expect("controller deployment should exist"); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); + let pod_spec = controller_pod_spec(&deployment); + let projected_sources = pod_spec + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|volumes| { + volumes.iter().find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_SEED_VOLUME) + }) + }) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|mapping| mapping.get(yaml_string("projected"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|projected| projected.get(yaml_string("sources"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller deployment should project seed secrets"); + assert_eq!( + projected_sources.len(), + seed_paths.len(), + "deployment should project every generated seed secret" + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs new file mode 100644 index 00000000..8aea93a7 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -0,0 +1,2693 @@ +use super::*; + +fn compose_component_mesh_peer_addr( + artifact_root: &Path, + component_id: &str, + output: &MeshProvisionOutput, + mesh_port: u16, +) -> Result { + let MeshProvisionOutput::Filesystem { dir } = output else { + return Err(miette::miette!( + "compose artifact {} component {} does not use filesystem mesh output", + artifact_root.display(), + component_id + )); + }; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose artifact {} component {} has invalid mesh output dir {}", + artifact_root.display(), + component_id, + dir + ) + })?; + Ok(format!("{service_name}:{mesh_port}")) +} + +fn kubernetes_component_mesh_peer_addr( + artifact_root: &Path, + component_id: &str, + output: &MeshProvisionOutput, + mesh_port: u16, +) -> Result { + let MeshProvisionOutput::KubernetesSecret { name, .. } = output else { + return Err(miette::miette!( + "kubernetes artifact {} component {} does not use a kubernetes secret mesh output", + artifact_root.display(), + component_id + )); + }; + let service_name = name.strip_suffix("-mesh").ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} component {} uses invalid mesh secret name {}", + artifact_root.display(), + component_id, + name + ) + })?; + Ok(format!("{service_name}:{mesh_port}")) +} + +fn local_component_mesh_peer_addr(mesh_port: u16) -> String { + #[cfg(target_os = "linux")] + { + std::net::SocketAddr::from((std::net::Ipv4Addr::new(10, 0, 2, 2), mesh_port)).to_string() + } + #[cfg(not(target_os = "linux"))] + { + std::net::SocketAddr::from(([127, 0, 0, 1], mesh_port)).to_string() + } +} + +fn local_router_mesh_peer_addr(mesh_port: u16) -> String { + std::net::SocketAddr::from(([127, 0, 0, 1], mesh_port)).to_string() +} + +fn site_controller_component_target(plan: &MeshProvisionPlan) -> Result<&MeshProvisionTarget> { + plan.targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.inbound.iter().any(|route| { + route.capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + }) + .ok_or_else(|| { + miette::miette!( + "mesh provision plan is missing the lowered framework.component controller target" + ) + }) +} + +fn local_site_controller_inbound_target( + artifact_root: &Path, + plan: &MeshProvisionPlan, + embedded_kind: Option, +) -> Result { + let target = site_controller_component_target(plan)?; + let peer_id = target.config.identity.id.clone(); + let mesh_port = target.config.mesh_listen.port(); + let peer_addr = match embedded_kind { + Some(EmbeddedMeshPlanKind::Compose) => { + compose_component_mesh_peer_addr(artifact_root, &peer_id, &target.output, mesh_port)? + } + Some(EmbeddedMeshPlanKind::Kubernetes) => { + kubernetes_component_mesh_peer_addr(artifact_root, &peer_id, &target.output, mesh_port)? + } + None => local_router_mesh_peer_addr(mesh_port), + }; + Ok(InboundTarget::MeshForward { + peer_addr, + peer_id: peer_id.clone(), + route_id: component_route_id( + &peer_id, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + }) +} + +fn build_dynamic_compose_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + component_mesh_dirs: &BTreeMap, + proxy_exports: &BTreeMap, + existing_site_peer_identities: &BTreeMap, +) -> Result { + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let kept_component_ids = plan + .targets + .iter() + .filter(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && assigned.contains(target.config.identity.id.as_str()) + }) + .map(|target| target.config.identity.id.clone()) + .collect::>(); + let mut router_target = plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "compose child artifact {} is missing a router mesh target", + artifact_root.join("compose.yaml").display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + let component_peer_addrs = plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + Ok(( + target.config.identity.id.clone(), + compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + target.config.mesh_listen.port(), + )?, + )) + }) + .collect::>>()?; + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = component_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + + let component_mesh_scopes = plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + ( + target.config.identity.id.clone(), + target.config.identity.mesh_scope.clone(), + ) + }) + .collect::>(); + let mut peer_identities = existing_site_peer_identities.clone(); + for (component, relative_dir) in component_mesh_dirs { + let identity: MeshIdentitySecret = read_json( + &artifact_root + .join(relative_dir) + .join(MESH_IDENTITY_FILENAME), + "mesh identity", + )?; + peer_identities.insert( + component.clone(), + MeshIdentityPublic { + id: identity.id.clone(), + public_key: identity.public_key().into_diagnostic()?, + mesh_scope: component_mesh_scopes.get(component).cloned().flatten(), + }, + ); + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + let identity = peer_identities.get(&peer.id).ok_or_else(|| { + miette::miette!( + "compose child router overlay peer {} is missing a live mesh identity", + peer.id + ) + })?; + Ok(MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>()?; + + let mut payload = StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + &component_peer_addrs, + &peer_identities, + )?; + Ok(payload) +} + +fn rewrite_compose_mesh_bind_mounts( + artifact_root: &Path, + mesh_dirs: &BTreeMap, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&compose_path)?; + let services = compose_services_mut(&mut document, &compose_path)?; + for (service_name, relative_dir) in mesh_dirs { + let service = services.get_mut(yaml_string(service_name)).ok_or_else(|| { + miette::miette!( + "compose file {} is missing child sidecar service {}", + compose_path.display(), + service_name + ) + })?; + let Some(service_mapping) = service.as_mapping_mut() else { + return Err(miette::miette!( + "compose service {} is not a mapping", + service_name + )); + }; + let volumes_key = yaml_string("volumes"); + let volumes = service_mapping + .get_mut(&volumes_key) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "compose child sidecar {} is missing a volumes list", + service_name + ) + })?; + let expected_prefix = format!("{service_name}-mesh:/amber/mesh"); + let replacement = serde_yaml::Value::String(format!("./{relative_dir}:/amber/mesh:ro")); + let mut replaced = false; + for volume in volumes.iter_mut() { + if volume + .as_str() + .is_some_and(|value| value.starts_with(&expected_prefix)) + { + *volume = replacement.clone(); + replaced = true; + } + } + if !replaced { + volumes.push(replacement); + } + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +pub(super) fn load_dynamic_compose_child_metadata( + artifact_root: &Path, +) -> Result { + read_json( + &dynamic_compose_child_metadata_path(artifact_root), + "dynamic compose child metadata", + ) +} + +fn load_running_site_router_identity( + plan: &SiteControllerRuntimePlan, +) -> Result { + Ok(load_live_site_router_mesh_config(plan)?.identity) +} + +fn filesystem_component_peer_identities_for_artifact( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + artifact_kind: &str, + compose_project: Option<&str>, +) -> Result> { + let mut peers = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "{artifact_kind} artifact {} has non-filesystem mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let config: MeshConfigPublic = if Path::new(dir).is_absolute() { + let compose_project = compose_project.ok_or_else(|| { + miette::miette!( + "{artifact_kind} artifact {} uses absolute mesh output {} without a compose \ + project", + artifact_root.display(), + dir + ) + })?; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "{artifact_kind} artifact {} has invalid absolute mesh output {}", + artifact_root.display(), + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name)? + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )? + }; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn read_compose_volume_mesh_config( + compose_project: &str, + service_name: &str, +) -> Result { + let volume_name = format!("{compose_project}_{service_name}-mesh"); + let output = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("-v") + .arg(format!("{volume_name}:/amber/mesh:ro")) + .arg("busybox:1.36.1") + .arg("cat") + .arg(format!("/amber/mesh/{MESH_CONFIG_FILENAME}")) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to read compose mesh config from docker volume {volume_name}") + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to read compose mesh config from docker volume \ + {volume_name}\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + serde_json::from_slice(&output.stdout) + .into_diagnostic() + .wrap_err_with(|| format!("docker volume {volume_name} returned invalid mesh config json")) +} + +fn compose_peer_identities_for_artifact( + artifact_root: &Path, + compose_project: Option<&str>, +) -> Result> { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + filesystem_component_peer_identities_for_artifact( + artifact_root, + &mesh_plan, + "compose", + compose_project, + ) +} + +pub(super) fn local_compose_peer_identities( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let mut peers = BTreeMap::new(); + let router = load_running_site_router_identity(plan)?; + peers.insert(router.id.clone(), router); + peers.extend(compose_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + plan.compose_project.as_deref(), + )?); + for child in published_children { + peers.extend(compose_peer_identities_for_artifact( + Path::new(&child.artifact_root), + plan.compose_project.as_deref(), + )?); + } + Ok(peers) +} + +fn kubernetes_peer_identities_for_artifact( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, +) -> Result> { + let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let mut peers = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes artifact {} has non-secret mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +pub(super) fn local_kubernetes_peer_identities( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let mut peers = BTreeMap::new(); + let router = load_running_site_router_identity(plan)?; + peers.insert(router.id.clone(), router); + peers.extend(kubernetes_peer_identities_for_artifact( + plan, + Path::new(&plan.artifact_dir), + )?); + for child in published_children { + peers.extend(kubernetes_peer_identities_for_artifact( + plan, + Path::new(&child.artifact_root), + )?); + } + Ok(peers) +} + +pub(crate) fn collect_live_component_runtime_metadata( + plan: &SiteControllerRuntimePlan, +) -> Result> { + let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); + let published_children = if state_path.is_file() { + let state: SiteControllerRuntimeState = + read_json(&state_path, "site controller runtime state")?; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + } else { + Vec::new() + }; + let mut components = match plan.kind { + SiteKind::Direct => collect_direct_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?), + )?, + SiteKind::Vm => collect_vm_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?), + )?, + SiteKind::Compose => collect_compose_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + plan.compose_project.as_deref(), + )?, + SiteKind::Kubernetes => { + collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&plan.artifact_dir))? + } + }; + for child in &published_children { + let child_components = match plan.kind { + SiteKind::Direct => collect_direct_artifact_runtime_metadata( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?, + SiteKind::Vm => collect_vm_artifact_runtime_metadata( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?, + SiteKind::Compose => collect_compose_artifact_runtime_metadata( + Path::new(&child.artifact_root), + plan.compose_project.as_deref(), + )?, + SiteKind::Kubernetes => { + collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&child.artifact_root))? + } + }; + components.extend(child_components); + } + Ok(components) +} + +pub(crate) fn load_live_site_router_mesh_config( + plan: &SiteControllerRuntimePlan, +) -> Result { + let artifact_root = Path::new(&plan.artifact_dir); + match plan.kind { + SiteKind::Direct => { + let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let direct_plan: DirectPlan = + read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let router = direct_plan.router.ok_or_else(|| { + miette::miette!("direct site `{}` is missing its router plan", plan.site_id) + })?; + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") + } + SiteKind::Vm => { + let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let vm_plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let router = vm_plan.router.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router plan", plan.site_id) + })?; + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") + } + SiteKind::Compose => { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + let target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing a router mesh target", + plan.site_id + ) + })?; + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "compose site `{}` has non-filesystem mesh output for router {}", + plan.site_id, + target.config.identity.id + )); + }; + if Path::new(dir).is_absolute() { + let compose_project = plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project", + plan.site_id + ) + })?; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose site `{}` has invalid router mesh output {}", + plan.site_id, + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name) + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + ) + } + } + SiteKind::Kubernetes => { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing a router mesh target", + plan.site_id + ) + })?; + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes site `{}` has non-secret mesh output for router {}", + plan.site_id, + target.config.identity.id + )); + }; + load_kubernetes_mesh_config_public(plan, name, namespace.as_deref()) + } + } +} + +fn collect_direct_artifact_runtime_metadata( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state: DirectRuntimeState = read_json( + &direct_runtime_state_path(artifact_root), + "direct runtime state", + )?; + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut components = BTreeMap::new(); + for component in &plan.components { + let mesh_config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + let mesh_port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "direct runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + components.insert( + component.moniker.clone(), + LiveComponentRuntimeMetadata { + moniker: component.moniker.clone(), + router_reachable_mesh_addr: local_router_mesh_peer_addr(mesh_port), + component_reachable_mesh_addr: local_component_mesh_peer_addr(mesh_port), + control_endpoint: Some(ControlEndpoint::Unix( + direct_component_control_socket_path( + runtime_root.join(&component.program.work_dir), + component.id, + ), + )), + mesh_config, + }, + ); + } + Ok(components) +} + +fn collect_vm_artifact_runtime_metadata( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut components = BTreeMap::new(); + for component in &plan.components { + let mesh_config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + let mesh_port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "vm runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + components.insert( + component.moniker.clone(), + LiveComponentRuntimeMetadata { + moniker: component.moniker.clone(), + router_reachable_mesh_addr: local_router_mesh_peer_addr(mesh_port), + component_reachable_mesh_addr: local_component_mesh_peer_addr(mesh_port), + control_endpoint: Some(ControlEndpoint::Unix(vm_component_control_socket_path( + runtime_root + .join("work") + .join("sidecars") + .join(&component.log_name), + component.id, + ))), + mesh_config, + }, + ); + } + Ok(components) +} + +fn direct_component_control_socket_path(work_dir: PathBuf, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-direct-control", + &format!("sidecar-{component_id}"), + &work_dir, + ) +} + +fn vm_component_control_socket_path(work_dir: PathBuf, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-vm-control", + &format!("sidecar-{component_id}"), + &work_dir, + ) +} + +fn collect_compose_artifact_runtime_metadata( + artifact_root: &Path, + compose_project: Option<&str>, +) -> Result> { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + let mut components = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "compose artifact {} has non-filesystem mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose artifact {} has invalid mesh output {} for component {}", + artifact_root.display(), + dir, + target.config.identity.id + ) + })?; + let mesh_config = if Path::new(dir).is_absolute() { + let compose_project = compose_project.ok_or_else(|| { + miette::miette!( + "compose artifact {} requires a compose project to resolve mesh output {}", + artifact_root.display(), + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name)? + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )? + }; + components.insert( + target.config.identity.id.clone(), + LiveComponentRuntimeMetadata { + moniker: target.config.identity.id.clone(), + router_reachable_mesh_addr: compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + component_reachable_mesh_addr: compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + control_endpoint: compose_project.map(|compose_project| { + ControlEndpoint::VolumeSocket { + volume: compose_component_control_socket_volume_name( + compose_project, + service_name, + ), + socket_path: COMPONENT_CONTROL_SOCKET_PATH_IN_VOLUME.to_string(), + } + }), + mesh_config, + }, + ); + } + Ok(components) +} + +#[cfg(test)] +mod tests { + use amber_mesh::MeshConfigTemplate; + + use super::*; + + fn expected_local_component_mesh_peer_addr(mesh_port: u16) -> String { + local_component_mesh_peer_addr(mesh_port) + } + + fn expected_local_router_mesh_peer_addr(mesh_port: u16) -> String { + local_router_mesh_peer_addr(mesh_port) + } + + #[test] + fn direct_runtime_metadata_uses_bounded_sidecar_control_socket_paths() { + let work_dir = PathBuf::from( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + mixed-run-doc-example-detach-very-long/state/runs/run-123/state/direct_local/runtime/\ + work/components/c1-admin", + ); + let socket = direct_component_control_socket_path(work_dir.clone(), 7); + let rendered = socket.display().to_string(); + + assert_eq!( + socket, + amber_mesh::stable_temp_socket_path("amber-direct-control", "sidecar-7", &work_dir) + ); + assert!( + rendered.len() < 104, + "direct runtime metadata should use the hashed short socket path: {rendered}", + ); + } + + #[test] + fn vm_runtime_metadata_uses_bounded_sidecar_control_socket_paths() { + let work_dir = PathBuf::from( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + linux-vm-framework_component-very-long/state/runs/run-123/state/vm_local/runtime/\ + work/sidecars/c1-worker-sidecar", + ); + let socket = vm_component_control_socket_path(work_dir.clone(), 11); + let rendered = socket.display().to_string(); + + assert_eq!( + socket, + amber_mesh::stable_temp_socket_path("amber-vm-control", "sidecar-11", &work_dir) + ); + assert!( + rendered.len() < 104, + "vm runtime metadata should use the hashed short socket path: {rendered}", + ); + } + + #[test] + fn collect_direct_runtime_metadata_uses_hashed_sidecar_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(runtime_root.join("work/components/c1-provider")) + .expect("runtime work dir should create"); + + write_json( + &artifact_root.join("direct-plan.json"), + &serde_json::json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [1], + "components": [ + { + "id": 1, + "moniker": "/provider", + "log_name": "provider", + "sidecar": { + "log_name": "provider-sidecar", + "mesh_port": 24001, + "mesh_config_path": "provider-mesh.json", + "mesh_identity_path": "provider-identity.json", + }, + "program": { + "log_name": "provider-program", + "work_dir": "work/components/c1-provider", + "execution": { + "kind": "direct", + "entrypoint": ["/bin/true"], + }, + }, + } + ], + "router": { + "identity_id": "/site/direct/router", + "mesh_port": 24000, + "control_port": 24100, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json" + } + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_root), + &DirectRuntimeState { + component_mesh_port_by_id: BTreeMap::from([(1, 24001)]), + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + write_json( + &runtime_root.join("provider-mesh.json"), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [3; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("provider mesh config should write"); + + let metadata = collect_direct_artifact_runtime_metadata(&artifact_root, &runtime_root) + .expect("direct metadata should load"); + let provider = metadata.get("/provider").expect("provider metadata"); + let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { + panic!("direct metadata should expose a unix control socket"); + }; + assert_eq!( + provider.router_reachable_mesh_addr, + expected_local_router_mesh_peer_addr(24001), + "direct runtime metadata should expose the router-reachable mesh address", + ); + assert_eq!( + provider.component_reachable_mesh_addr, + expected_local_component_mesh_peer_addr(24001), + "direct runtime metadata should expose the sidecar-reachable mesh address", + ); + assert_eq!( + path, + &amber_mesh::stable_temp_socket_path( + "amber-direct-control", + "sidecar-1", + &runtime_root.join("work/components/c1-provider"), + ), + "direct runtime metadata should use the hashed short sidecar control socket path", + ); + } + + #[test] + fn collect_vm_runtime_metadata_uses_hashed_sidecar_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(runtime_root.join("work/sidecars/vm-provider")) + .expect("runtime sidecar dir should create"); + + write_json( + &artifact_root.join("vm-plan.json"), + &serde_json::json!({ + "version": "1", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [1], + "runtime_addresses": {}, + "components": [ + { + "id": 1, + "moniker": "/provider", + "log_name": "vm-provider", + "mesh_config_path": "provider-mesh.json", + "mesh_identity_path": "provider-identity.json", + "cpus": { "kind": "literal", "value": 1 }, + "memory_mib": { "kind": "literal", "value": 512 }, + "base_image": { "kind": "static", "path": "/tmp/base.img" }, + "egress": "none", + "storage_mounts": [], + } + ] + }), + ) + .expect("vm plan should write"); + write_vm_runtime_state( + &artifact_root, + &VmRuntimeState { + slot_ports_by_component: BTreeMap::new(), + slot_route_ports_by_component: BTreeMap::new(), + route_host_ports_by_component: BTreeMap::new(), + endpoint_forwards_by_component: BTreeMap::new(), + component_mesh_port_by_id: BTreeMap::from([(1, 24001)]), + router_mesh_port: None, + }, + ) + .expect("vm runtime state should write"); + write_json( + &runtime_root.join("provider-mesh.json"), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [5; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("provider mesh config should write"); + + let metadata = collect_vm_artifact_runtime_metadata(&artifact_root, &runtime_root) + .expect("vm metadata should load"); + let provider = metadata.get("/provider").expect("provider metadata"); + let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { + panic!("vm metadata should expose a unix control socket"); + }; + assert_eq!( + provider.router_reachable_mesh_addr, + expected_local_router_mesh_peer_addr(24001), + "vm runtime metadata should expose the router-reachable mesh address", + ); + assert_eq!( + provider.component_reachable_mesh_addr, + expected_local_component_mesh_peer_addr(24001), + "vm runtime metadata should expose the sidecar-reachable mesh address", + ); + assert_eq!( + path, + &amber_mesh::stable_temp_socket_path( + "amber-vm-control", + "sidecar-1", + &runtime_root.join("work/sidecars/vm-provider"), + ), + "vm runtime metadata should use the hashed short sidecar control socket path", + ); + } + + #[test] + fn collect_live_component_runtime_metadata_tolerates_missing_runtime_state_file() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + let site_state_root = temp.path().join("state"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(&runtime_root).expect("runtime root should create"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + + write_json( + &artifact_root.join("direct-plan.json"), + &serde_json::json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [], + "components": [], + "router": { + "identity_id": "/site/direct/router", + "mesh_port": 24000, + "control_port": 24100, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json" + } + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_root), + &DirectRuntimeState::default(), + ) + .expect("direct runtime state should write"); + + let metadata = collect_live_component_runtime_metadata(&SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run-123".to_string(), + mesh_scope: "amber.test".to_string(), + run_root: temp.path().display().to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + router_identity_id: "/site/direct/router".to_string(), + local_router_control: None, + artifact_dir: artifact_root.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:35000".parse().expect("listen addr should parse"), + storage_root: None, + runtime_root: Some(runtime_root.display().to_string()), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }) + .expect("metadata collection should succeed without a runtime state file"); + + assert!( + metadata.is_empty(), + "static-only sites should not require a site controller runtime state file", + ); + } + + #[test] + fn compose_runtime_metadata_uses_volume_root_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + fs::create_dir_all(artifact_root.join(".amber/mesh/provider-net")) + .expect("mesh artifact dir should create"); + + write_json( + &artifact_root.join("mesh-provision-plan.json"), + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/provider".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: ".amber/mesh/provider-net".to_string(), + }, + }], + }, + ) + .expect("mesh provision plan should write"); + write_json( + &artifact_root + .join(".amber/mesh/provider-net") + .join(MESH_CONFIG_FILENAME), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [9; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("mesh config should write"); + + let components = collect_compose_artifact_runtime_metadata(&artifact_root, Some("demo")) + .expect("compose metadata should load"); + let provider = components.get("/provider").expect("provider metadata"); + let endpoint = provider + .control_endpoint + .as_ref() + .expect("compose metadata should expose a control endpoint"); + let ControlEndpoint::VolumeSocket { + volume, + socket_path, + } = endpoint + else { + panic!("expected compose control endpoint to use a volume socket"); + }; + assert_eq!(volume, "demo_provider-net-control"); + assert_eq!( + socket_path, "/router-control.sock", + "compose volume sockets must use the volume-root path, not the in-container mount path", + ); + } +} + +fn collect_kubernetes_artifact_runtime_metadata( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, +) -> Result> { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let mut components = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes artifact {} has non-secret mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + name.strip_suffix("-mesh").ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} component {} uses invalid mesh secret name {}", + artifact_root.display(), + target.config.identity.id, + name + ) + })?; + let mesh_config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; + components.insert( + target.config.identity.id.clone(), + LiveComponentRuntimeMetadata { + moniker: target.config.identity.id.clone(), + router_reachable_mesh_addr: kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + component_reachable_mesh_addr: kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + control_endpoint: None, + mesh_config, + }, + ); + } + Ok(components) +} + +fn compose_component_control_socket_volume_name( + compose_project: &str, + service_name: &str, +) -> String { + format!("{compose_project}_{service_name}-control") +} + +pub(super) fn prepare_dynamic_compose_child_artifact( + plan: &SiteControllerRuntimePlan, + runtime_spec: &LocalChildRuntimeSpec, + artifact_root: &Path, + published_children: &[SiteControllerRuntimeChildRecord], + existing_site_peer_identities: &BTreeMap, + live_components: &BTreeMap, +) -> Result<()> { + project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; + let compose_path = artifact_root.join("compose.yaml"); + let mut desired_document = read_compose_document(&compose_path)?; + let root_services = + compose_dynamic_root_service_names(artifact_root, &runtime_spec.assigned_components)?; + let service_closure = + compose_service_closure(&desired_document, &compose_path, &root_services)?; + let live_services = compose_live_service_names(plan, published_children)?; + let mut kept_services = service_closure + .difference(&live_services) + .cloned() + .collect::>(); + kept_services.remove(COMPOSE_PROVISIONER_SERVICE_NAME); + if kept_services.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not retain any child-owned services after filtering", + compose_path.display() + )); + } + + let services = compose_services_mut(&mut desired_document, &compose_path)?; + services.retain(|name, _| { + name.as_str() + .is_some_and(|service_name| kept_services.contains(service_name)) + }); + for service in services.values_mut() { + retain_compose_service_dependencies(service, &kept_services)?; + } + let rendered = serde_yaml::to_string(&desired_document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display()))?; + + let DynamicComposeMeshPlan { + mesh_plan, + mesh_dirs, + component_mesh_dirs, + } = build_dynamic_compose_mesh_plan(artifact_root, &runtime_spec.assigned_components)?; + let mut mesh_plan = mesh_plan; + ensure_dynamic_proxy_export_component_routes( + &mut mesh_plan, + &runtime_spec.proxy_exports, + &plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs(&mut mesh_plan, &runtime_spec.direct_inputs, live_components)?; + let existing_peer_identities = + required_existing_mesh_peer_identities(&mesh_plan, existing_site_peer_identities)?; + mesh_plan.existing_peer_identities = existing_peer_identities.values().cloned().collect(); + write_json(&artifact_root.join("mesh-provision-plan.json"), &mesh_plan)?; + provision_mesh_filesystem_with_peer_identities( + &mesh_plan, + artifact_root, + &existing_peer_identities, + )?; + for relative_dir in mesh_dirs.values() { + project_existing_peer_identities_into_mesh_config( + &artifact_root.join(relative_dir).join(MESH_CONFIG_FILENAME), + &existing_peer_identities, + )?; + } + rewrite_compose_mesh_bind_mounts(artifact_root, &mesh_dirs)?; + let overlay_payload = build_dynamic_compose_route_overlay_payload( + artifact_root, + &runtime_spec.assigned_components, + &component_mesh_dirs, + &runtime_spec.proxy_exports, + existing_site_peer_identities, + )?; + write_json(&dynamic_route_overlay_path(artifact_root), &overlay_payload)?; + write_embedded_compose_mesh_provision_plan(artifact_root, &mesh_plan)?; + + write_json( + &dynamic_compose_child_metadata_path(artifact_root), + &DynamicComposeChildMetadata { + schema: DYNAMIC_COMPOSE_CHILD_SCHEMA.to_string(), + version: DYNAMIC_COMPOSE_CHILD_VERSION, + services: kept_services.iter().cloned().collect(), + readiness_services: root_services + .into_iter() + .filter(|service| kept_services.contains(service)) + .collect(), + }, + ) +} + +pub(super) fn read_compose_document(path: &Path) -> Result { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display())) +} + +pub(super) fn read_embedded_compose_mesh_provision_plan( + artifact_root: &Path, +) -> Result { + let path = artifact_root.join("compose.yaml"); + let document = read_compose_document(&path)?; + let Some(root) = document.as_mapping() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let configs_key = serde_yaml::Value::String("configs".to_string()); + let config_name = serde_yaml::Value::String("amber-mesh-provision-plan".to_string()); + let content_key = serde_yaml::Value::String("content".to_string()); + let content = root + .get(&configs_key) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|configs| configs.get(&config_name)) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(&content_key)) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing configs.amber-mesh-provision-plan.content", + path.display() + ) + })?; + serde_json::from_str(content).map_err(|err| { + miette::miette!( + "compose file {} has invalid embedded mesh provision plan: {err}", + path.display() + ) + }) +} + +fn write_embedded_compose_mesh_provision_plan( + artifact_root: &Path, + plan: &MeshProvisionPlan, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&path)?; + let plan_json = serde_json::to_string_pretty(plan) + .into_diagnostic() + .wrap_err("failed to serialize compose mesh provision plan")?; + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("configs"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|configs| configs.get_mut(yaml_string("amber-mesh-provision-plan"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing configs.amber-mesh-provision-plan", + path.display() + ) + })? + .insert(yaml_string("content"), serde_yaml::Value::String(plan_json)); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn read_embedded_kubernetes_mesh_provision_plan(artifact_root: &Path) -> Result { + let path = artifact_root + .join("01-configmaps") + .join("amber-mesh-provision.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let document: serde_yaml::Value = + serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "invalid kubernetes mesh provision configmap {}", + path.display() + ) + })?; + let mesh_plan = document + .as_mapping() + .and_then(|root| root.get(yaml_string("data"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|data| data.get(yaml_string("mesh-plan.json"))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + miette::miette!( + "kubernetes mesh provision configmap {} is missing data.mesh-plan.json", + path.display() + ) + })?; + serde_json::from_str(mesh_plan).map_err(|err| { + miette::miette!( + "kubernetes mesh provision configmap {} has invalid mesh plan: {err}", + path.display() + ) + }) +} + +fn read_kubernetes_runtime_mesh_provision_plan(artifact_root: &Path) -> Result { + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + return read_json(&path, "mesh provision plan"); + } + read_embedded_kubernetes_mesh_provision_plan(artifact_root) +} + +fn write_embedded_kubernetes_mesh_provision_plan( + artifact_root: &Path, + plan: &MeshProvisionPlan, +) -> Result<()> { + let path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "invalid kubernetes mesh provision configmap {}", + path.display() + ) + })?; + let plan_json = serde_json::to_string_pretty(plan) + .into_diagnostic() + .wrap_err("failed to serialize kubernetes mesh provision plan")?; + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("data"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "kubernetes mesh provision configmap {} is missing a data mapping", + path.display() + ) + })? + .insert( + yaml_string("mesh-plan.json"), + serde_yaml::Value::String(plan_json), + ); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +#[derive(Clone, Debug)] +pub struct SiteControllerPeerRouterRoute { + pub site_id: String, + pub peer_router: MeshIdentityPublic, + pub peer_addr: String, + pub listen_addr: String, + pub listen_port: u16, +} + +pub fn inject_site_controller_peer_router_routes( + artifact_root: &Path, + local_site_id: &str, + allowed_issuers: &[String], + routes: &[SiteControllerPeerRouterRoute], +) -> Result<()> { + let mut write_embedded = None; + let mut plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else if artifact_root.join("compose.yaml").is_file() { + write_embedded = Some(EmbeddedMeshPlanKind::Compose); + read_embedded_compose_mesh_provision_plan(artifact_root)? + } else if artifact_root + .join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH) + .is_file() + { + write_embedded = Some(EmbeddedMeshPlanKind::Kubernetes); + read_embedded_kubernetes_mesh_provision_plan(artifact_root)? + } else { + return Err(miette::miette!( + "site artifact {} is missing a mesh provision plan", + artifact_root.display() + )); + }; + + let local_inbound_target = + local_site_controller_inbound_target(artifact_root, &plan, write_embedded)?; + inject_site_controller_peer_router_routes_into_plan( + &mut plan, + local_inbound_target, + local_site_id, + allowed_issuers, + routes, + )?; + + match write_embedded { + None => write_json(&artifact_root.join("mesh-provision-plan.json"), &plan), + Some(EmbeddedMeshPlanKind::Compose) => { + write_embedded_compose_mesh_provision_plan(artifact_root, &plan) + } + Some(EmbeddedMeshPlanKind::Kubernetes) => { + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan) + } + } +} + +pub fn set_compose_router_published_mesh_port(artifact_root: &Path, host_port: u16) -> Result<()> { + set_compose_router_published_port(artifact_root, "0.0.0.0", host_port, 24000, false) +} + +fn set_compose_router_published_port( + artifact_root: &Path, + host_ip: &str, + host_port: u16, + container_port: u16, + append_if_missing: bool, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&path)?; + let router_ports = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|services| services.get_mut(yaml_string(COMPOSE_ROUTER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|router| router.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services.{COMPOSE_ROUTER_SERVICE_NAME}.ports", + path.display() + ) + })?; + let desired = format!("{host_ip}:{host_port}:{container_port}"); + let mut updated = false; + for port in &mut *router_ports { + let Some(raw) = port.as_str() else { + continue; + }; + if raw == desired { + updated = true; + continue; + } + if raw.ends_with(&format!("::{container_port}")) { + *port = serde_yaml::Value::String(desired.clone()); + updated = true; + } + } + if !updated && append_if_missing { + router_ports.push(serde_yaml::Value::String(desired)); + updated = true; + } + if !updated { + return Err(miette::miette!( + "compose file {} has no router publish to rewrite for container port {container_port}", + path.display(), + )); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +#[derive(Clone, Copy, Debug)] +enum EmbeddedMeshPlanKind { + Compose, + Kubernetes, +} + +fn inject_site_controller_peer_router_routes_into_plan( + plan: &mut MeshProvisionPlan, + local_inbound_target: InboundTarget, + local_site_id: &str, + allowed_issuers: &[String], + routes: &[SiteControllerPeerRouterRoute], +) -> Result<()> { + let router = plan + .targets + .iter_mut() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| miette::miette!("mesh provision plan is missing a router target"))?; + let inbound_route_id = site_controller_internal_route_id(local_site_id); + let local_target_peer_id = match &local_inbound_target { + InboundTarget::MeshForward { peer_id, .. } => Some(peer_id.clone()), + _ => None, + }; + router + .config + .inbound + .retain(|route| route.route_id != inbound_route_id); + router.config.inbound.push(InboundRoute { + route_id: inbound_route_id, + capability: SITE_CONTROLLER_INTERNAL_CAPABILITY.to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: local_inbound_target, + allowed_issuers: allowed_issuers.to_vec(), + }); + router + .config + .inbound + .sort_by(|left, right| left.route_id.cmp(&right.route_id)); + router + .config + .outbound + .retain(|route| !route.route_id.starts_with("site-controller:")); + let mut existing_peer_identities = plan + .existing_peer_identities + .iter() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect::>(); + if let Some(peer_id) = local_target_peer_id + && !router.config.peers.iter().any(|peer| peer.id == peer_id) + { + router.config.peers.push(MeshPeerTemplate { id: peer_id }); + } + for route in routes { + existing_peer_identities.insert(route.peer_router.id.clone(), route.peer_router.clone()); + if !router + .config + .peers + .iter() + .any(|peer| peer.id == route.peer_router.id) + { + router.config.peers.push(MeshPeerTemplate { + id: route.peer_router.id.clone(), + }); + } + router.config.outbound.push(OutboundRoute { + route_id: site_controller_internal_route_id(&route.site_id), + rewrite_route_id: None, + slot: route.site_id.clone(), + capability_kind: None, + capability_profile: None, + listen_port: route.listen_port, + listen_addr: Some(route.listen_addr.clone()), + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: route.peer_addr.clone(), + peer_id: route.peer_router.id.clone(), + capability: SITE_CONTROLLER_INTERNAL_CAPABILITY.to_string(), + }); + } + plan.existing_peer_identities = existing_peer_identities.into_values().collect(); + router + .config + .outbound + .sort_by(|left, right| left.route_id.cmp(&right.route_id)); + router + .config + .peers + .sort_by(|left, right| left.id.cmp(&right.id)); + Ok(()) +} + +pub fn set_site_artifact_mesh_identity_seed( + artifact_root: &Path, + identity_seed: &str, +) -> Result<()> { + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_json(&path, &plan)?; + } + return Ok(()); + } + + let compose_path = artifact_root.join("compose.yaml"); + if compose_path.is_file() { + let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; + } + return Ok(()); + } + + let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + if configmap_path.is_file() { + let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; + } + return Ok(()); + } + + Err(miette::miette!( + "site artifact {} is missing a mesh provision plan", + artifact_root.display() + )) +} + +pub(super) fn project_kubernetes_dynamic_child_destroy_artifact_files( + artifact_files: &BTreeMap, +) -> Result> { + let kustomization_path = "kustomization.yaml"; + let raw = artifact_files.get(kustomization_path).ok_or_else(|| { + miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") + })?; + let mut document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is not a YAML mapping") + })?; + let resources = root + .get_mut(yaml_string("resources")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is missing a resources sequence") + })?; + let mut projected = artifact_files + .iter() + .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) + .map(|(path, contents)| (path.clone(), contents.clone())) + .collect::>(); + let shared_paths = [ + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH, + KUBERNETES_PROVISIONER_ROLE_PATH, + KUBERNETES_PROVISIONER_ROLEBINDING_PATH, + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, + ] + .into_iter() + .collect::>(); + let mut kept_resources = Vec::new(); + for resource in resources + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_owned) + { + if shared_paths.contains(resource.as_str()) { + continue; + } + let raw = artifact_files + .get(&resource) + .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; + projected.insert(resource.clone(), raw.clone()); + kept_resources.push(serde_yaml::Value::String(resource)); + } + *resources = kept_resources; + + if let Some(generators) = root + .get_mut(yaml_string("secretGenerator")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + generators.retain(|generator| { + generator + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) + }); + } + + if let Some(replacements) = root + .get_mut(yaml_string("replacements")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + replacements.clear(); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; + projected.insert(kustomization_path.to_string(), rendered); + Ok(projected) +} + +pub(super) fn read_artifact_snapshot(root: &Path) -> Result> { + walk_files(root)? + .into_iter() + .map(|path| { + let relative = path + .strip_prefix(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; + let relative = path_to_forward_slash_string(relative); + let contents = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + Ok((relative, contents)) + }) + .collect() +} + +fn rewrite_dynamic_kubernetes_apply_bundle( + artifact_root: &Path, + component_ids: &[usize], +) -> Result<()> { + let files = read_artifact_snapshot(artifact_root)?; + let projected = + crate::runtime_api::project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; + replace_artifact_snapshot(artifact_root, &projected) +} + +pub(super) fn prepare_dynamic_kubernetes_child_artifact( + plan: &SiteControllerRuntimePlan, + runtime_spec: &LocalChildRuntimeSpec, + artifact_root: &Path, + existing_site_peer_identities: &BTreeMap, + live_components: &BTreeMap, +) -> Result<()> { + project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let assigned = runtime_spec + .assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut kept_component_ids = BTreeSet::new(); + let mut router_target = None; + let mut overlay_targets = Vec::with_capacity(mesh_plan.targets.len()); + let mut provision_targets = Vec::new(); + for target in mesh_plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + if assigned.contains(target.config.identity.id.as_str()) { + kept_component_ids.insert(target.config.identity.id.clone()); + provision_targets.push(target.clone()); + overlay_targets.push(target); + } + } + MeshProvisionTargetKind::Router => { + router_target = Some(target); + } + } + } + + let Some(mut router_target) = router_target else { + return Err(miette::miette!( + "dynamic mesh provision plan {} is missing a router target", + plan_path.display() + )); + }; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + overlay_targets.push(router_target); + let mut overlay_plan = MeshProvisionPlan { + version: mesh_plan.version.clone(), + identity_seed: mesh_plan.identity_seed.clone(), + existing_peer_identities: Vec::new(), + targets: overlay_targets, + }; + let overlay_existing_peer_identities = + required_existing_mesh_peer_identities(&overlay_plan, existing_site_peer_identities)?; + overlay_plan.existing_peer_identities = + overlay_existing_peer_identities.values().cloned().collect(); + write_json(&plan_path, &overlay_plan)?; + let mut provision_plan = MeshProvisionPlan { + version: mesh_plan.version, + identity_seed: mesh_plan.identity_seed, + existing_peer_identities: Vec::new(), + targets: provision_targets, + }; + ensure_dynamic_proxy_export_component_routes( + &mut provision_plan, + &runtime_spec.proxy_exports, + &plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs( + &mut provision_plan, + &runtime_spec.direct_inputs, + live_components, + )?; + let provision_existing_peer_identities = + required_existing_mesh_peer_identities(&provision_plan, existing_site_peer_identities)?; + provision_plan.existing_peer_identities = provision_existing_peer_identities + .values() + .cloned() + .collect(); + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &provision_plan)?; + project_dynamic_kubernetes_proxy_export_resources( + artifact_root, + &provision_plan, + &runtime_spec.proxy_exports, + )?; + rewrite_dynamic_kubernetes_apply_bundle(artifact_root, &runtime_spec.component_ids) +} + +fn kubernetes_peer_addrs_for_artifact(artifact_root: &Path) -> Result> { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + mesh_plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + Ok(( + target.config.identity.id.clone(), + kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + target.config.mesh_listen.port(), + )?, + )) + }) + .collect() +} + +fn build_kubernetes_route_overlay_base( + artifact_root: &Path, + assigned_components: &[String], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let kept_component_ids = assigned_components.iter().cloned().collect::>(); + let mut router_target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} is missing a router mesh target", + artifact_root.display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = provider_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + peer_identities.get(&peer.id).map(|identity| MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>() + .ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} is missing a live mesh identity for one of its route peers", + artifact_root.display() + ) + })?; + Ok(StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }) +} + +pub(super) fn write_kubernetes_live_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + proxy_exports: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result<()> { + let provider_peer_addrs = kubernetes_peer_addrs_for_artifact(artifact_root)?; + let mut payload = build_kubernetes_route_overlay_base( + artifact_root, + assigned_components, + &provider_peer_addrs, + peer_identities, + )?; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + &provider_peer_addrs, + peer_identities, + )?; + write_dynamic_route_overlay_payload(artifact_root, &payload) +} + +fn kubernetes_network_policy_paths_by_component_label( + artifact_root: &Path, +) -> Result> { + let netpol_root = artifact_root.join("05-networkpolicies"); + if !netpol_root.is_dir() { + return Ok(BTreeMap::new()); + } + let mut netpol_paths = BTreeMap::new(); + for path in walk_files(&netpol_root)? { + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; + let Some(root) = document.as_mapping() else { + continue; + }; + if root + .get(yaml_string("kind")) + .and_then(serde_yaml::Value::as_str) + != Some("NetworkPolicy") + { + continue; + } + let Some(component_label) = root + .get(yaml_string("metadata")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("labels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) + .and_then(serde_yaml::Value::as_str) + else { + continue; + }; + netpol_paths.insert(component_label.to_string(), path); + } + Ok(netpol_paths) +} + +fn project_dynamic_kubernetes_proxy_export_resources( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + proxy_exports: &BTreeMap, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + + let netpol_paths = kubernetes_network_policy_paths_by_component_label(artifact_root)?; + let exported_mesh_ports = proxy_exports + .values() + .map(|export| { + let component_target = mesh_plan + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the kubernetes mesh plan", + export.component + ) + })?; + Ok(( + format!("c{}", export.component_id), + component_target.config.mesh_listen.port(), + )) + }) + .collect::>>()?; + + for (component_label, mesh_port) in exported_mesh_ports { + let path = netpol_paths.get(&component_label).ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {component_label} is missing a kubernetes network \ + policy in {}", + artifact_root.join("05-networkpolicies").display() + ) + })?; + ensure_kubernetes_network_policy_router_ingress(path, mesh_port)?; + } + + Ok(()) +} + +fn ensure_kubernetes_network_policy_router_ingress(path: &Path, mesh_port: u16) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} is not a YAML mapping", + path.display() + ) + })?; + let spec = root + .get_mut(yaml_string("spec")) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} is missing a spec mapping", + path.display() + ) + })?; + let ingress = spec + .entry(yaml_string("ingress")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} has non-sequence spec.ingress", + path.display() + ) + })?; + + let router_peer = kubernetes_router_network_policy_peer(); + let mut matched_port_rule = false; + let mut changed = false; + for rule in ingress.iter_mut() { + let Some(rule_mapping) = rule.as_mapping_mut() else { + continue; + }; + let matches_port = rule_mapping + .get(yaml_string("ports")) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|ports| { + ports + .iter() + .any(|port| network_policy_port_matches(port, mesh_port)) + }); + if !matches_port { + continue; + } + matched_port_rule = true; + let Some(from) = rule_mapping + .get_mut(yaml_string("from")) + .and_then(serde_yaml::Value::as_sequence_mut) + else { + break; + }; + if from.iter().any(network_policy_peer_is_router) { + break; + } + from.push(router_peer.clone()); + changed = true; + break; + } + + if !matched_port_rule { + ingress.push(serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter( + [ + ( + yaml_string("from"), + serde_yaml::Value::Sequence(vec![router_peer.clone()]), + ), + ( + yaml_string("ports"), + serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( + serde_yaml::Mapping::from_iter([ + (yaml_string("protocol"), yaml_string("TCP")), + ( + yaml_string("port"), + serde_yaml::Value::Number(u64::from(mesh_port).into()), + ), + ]), + )]), + ), + ], + ))); + changed = true; + } + + if !changed { + return Ok(()); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn kubernetes_router_network_policy_peer() -> serde_yaml::Value { + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("podSelector"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("matchLabels"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("amber.io/component"), + yaml_string(KUBERNETES_ROUTER_COMPONENT_NAME), + )])), + )])), + )])) +} + +fn network_policy_port_matches(port: &serde_yaml::Value, expected_port: u16) -> bool { + port.as_mapping() + .and_then(|port| port.get(yaml_string("port"))) + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) + }) + == Some(u64::from(expected_port)) +} + +fn network_policy_peer_is_router(peer: &serde_yaml::Value) -> bool { + peer.as_mapping() + .and_then(|peer| peer.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector| selector.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component"))) + .and_then(serde_yaml::Value::as_str) + == Some(KUBERNETES_ROUTER_COMPONENT_NAME) +} + +pub(super) fn dynamic_proxy_export_kubernetes_peer_addr( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + export: &DynamicProxyExportRecord, +) -> Result { + let component_target = mesh_plan + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the kubernetes mesh plan", + export.component + ) + })?; + kubernetes_component_mesh_peer_addr( + artifact_root, + &export.component, + &component_target.output, + component_target.config.mesh_listen.port(), + ) +} + +pub(super) fn project_dynamic_child_mesh_scope( + artifact_root: &Path, + mesh_scope: Option<&str>, +) -> Result<()> { + let Some(mesh_scope) = mesh_scope else { + return Ok(()); + }; + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_json(&path, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + let compose_path = artifact_root.join("compose.yaml"); + if compose_path.is_file() { + let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + if configmap_path.is_file() { + let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + Err(miette::miette!( + "dynamic artifact {} is missing a mesh provision plan", + artifact_root.display() + )) +} + +fn project_mesh_provision_plan_scope(plan: &mut MeshProvisionPlan, mesh_scope: &str) -> bool { + let mut changed = false; + for target in &mut plan.targets { + if target.config.identity.mesh_scope.as_deref() == Some(mesh_scope) { + continue; + } + target.config.identity.mesh_scope = Some(mesh_scope.to_string()); + changed = true; + } + changed +} + +fn mesh_provision_plan_scopes(plan: &MeshProvisionPlan) -> BTreeSet { + let mut scopes = BTreeSet::new(); + for target in &plan.targets { + if let Some(scope) = target.config.identity.mesh_scope.as_deref() { + scopes.insert(scope.to_string()); + } + } + for identity in &plan.existing_peer_identities { + if let Some(scope) = identity.mesh_scope.as_deref() { + scopes.insert(scope.to_string()); + } + } + scopes +} + +fn rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root: &Path, + existing_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let rewrite_scopes = existing_scopes + .iter() + .filter(|scope| scope.as_str() != mesh_scope) + .cloned() + .collect::>(); + if rewrite_scopes.is_empty() { + return Ok(()); + } + + for path in walk_files(artifact_root)? { + match path.extension().and_then(|extension| extension.to_str()) { + Some("json") => rewrite_json_scope_literals(&path, &rewrite_scopes, mesh_scope)?, + Some("yaml" | "yml") => { + rewrite_yaml_scope_literals(&path, &rewrite_scopes, mesh_scope)? + } + Some("env") => rewrite_env_scope_literals(&path, &rewrite_scopes, mesh_scope)?, + _ => {} + } + } + Ok(()) +} + +fn rewrite_json_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_json::Value = serde_json::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid json {}", path.display()))?; + if !rewrite_scope_json_value(&mut document, rewrite_scopes, mesh_scope) { + return Ok(()); + } + let rendered = serde_json::to_string_pretty(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_yaml_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid yaml {}", path.display()))?; + if !rewrite_scope_yaml_value(&mut document, rewrite_scopes, mesh_scope) { + return Ok(()); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_env_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut changed = false; + let mut rendered = raw + .lines() + .map(|line| { + let Some((name, value)) = line.split_once('=') else { + return line.to_string(); + }; + if !rewrite_scopes.contains(value) { + return line.to_string(); + } + changed = true; + format!("{name}={mesh_scope}") + }) + .collect::>() + .join("\n"); + if !changed { + return Ok(()); + } + if raw.ends_with('\n') { + rendered.push('\n'); + } + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_scope_json_value( + value: &mut serde_json::Value, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + match value { + serde_json::Value::String(string) => { + rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) + } + serde_json::Value::Array(values) => { + let mut changed = false; + for value in values { + changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); + } + changed + } + serde_json::Value::Object(map) => { + let mut changed = false; + for value in map.values_mut() { + changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); + } + changed + } + _ => false, + } +} + +fn rewrite_scope_yaml_value( + value: &mut serde_yaml::Value, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + match value { + serde_yaml::Value::String(string) => { + rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) + } + serde_yaml::Value::Sequence(values) => { + let mut changed = false; + for value in values { + changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); + } + changed + } + serde_yaml::Value::Mapping(map) => { + let mut changed = false; + for (_, value) in map.iter_mut() { + changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); + } + changed + } + _ => false, + } +} + +fn rewrite_scope_string_value( + string: &mut String, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + if rewrite_scopes.contains(string) { + *string = mesh_scope.to_string(); + return true; + } + let Some((name, value)) = string.split_once('=') else { + return false; + }; + if !rewrite_scopes.contains(value) { + return false; + } + *string = format!("{name}={mesh_scope}"); + true +} + +fn load_kubernetes_mesh_secret_payload( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result<(String, KubernetesSecretPayload)> { + let namespace = namespace + .or(plan.kubernetes_namespace.as_deref()) + .ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its namespace", + plan.site_id + ) + })? + .to_string(); + let output = kubectl_command(plan.context.as_deref()) + .arg("-n") + .arg(&namespace) + .arg("get") + .arg("secret") + .arg(name) + .arg("-o") + .arg("json") + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to read kubernetes mesh secret {} in namespace {}", + name, namespace + ) + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to read kubernetes mesh secret {} in namespace {}: {}", + name, + namespace, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let payload: KubernetesSecretPayload = + serde_json::from_slice(&output.stdout).map_err(|err| { + miette::miette!( + "invalid kubernetes secret payload for {} in namespace {}: {err}", + name, + namespace + ) + })?; + Ok((namespace, payload)) +} + +fn decode_kubernetes_mesh_secret_json( + payload: &KubernetesSecretPayload, + namespace: &str, + name: &str, + key: &str, + description: &str, +) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let encoded = payload.data.get(key).ok_or_else(|| { + miette::miette!( + "kubernetes mesh secret {} in namespace {} is missing {}", + name, + namespace, + key + ) + })?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.as_bytes()) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to decode kubernetes {description} {} in namespace {}", + name, namespace + ) + })?; + serde_json::from_slice(&bytes).map_err(|err| { + miette::miette!( + "invalid kubernetes {description} {} in namespace {}: {err}", + name, + namespace + ) + }) +} + +pub(super) fn load_kubernetes_mesh_identity_secret( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result { + let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; + decode_kubernetes_mesh_secret_json( + &payload, + &namespace, + name, + MESH_IDENTITY_FILENAME, + "mesh identity", + ) +} + +fn load_kubernetes_mesh_config_public( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result { + let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; + decode_kubernetes_mesh_secret_json( + &payload, + &namespace, + name, + MESH_CONFIG_FILENAME, + "mesh config", + ) +} + +pub(super) fn reconcile_site_proxy_metadata( + site_artifact_root: &Path, + site_artifact_files: &BTreeMap, +) -> Result<()> { + let Some(proxy_metadata) = site_artifact_files.get("amber-proxy.json") else { + return Ok(()); + }; + let path = site_artifact_root.join("amber-proxy.json"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + fs::write(&path, proxy_metadata) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs new file mode 100644 index 00000000..b4f729d9 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -0,0 +1,1737 @@ +use super::*; + +pub(super) fn remove_dir_if_exists(path: &Path) -> Result<()> { + if !path.exists() { + return Ok(()); + } + fs::remove_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove {}", path.display())) +} + +pub fn prepare_kubernetes_artifact_namespace( + run_id: &str, + site_id: &str, + artifact_dir: &Path, +) -> Result { + let namespace = kubernetes_namespace_name(run_id, site_id); + let kustomization = artifact_dir.join("kustomization.yaml"); + let contents = fs::read_to_string(&kustomization) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", kustomization.display()))?; + + let mut saw_namespace = false; + let mut out = String::new(); + for line in contents.lines() { + if line.trim_start().starts_with("namespace:") { + saw_namespace = true; + out.push_str("namespace: "); + out.push_str(&namespace); + } else { + out.push_str(line); + } + out.push('\n'); + } + if !saw_namespace { + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out.push_str("namespace: "); + out.push_str(&namespace); + out.push('\n'); + } + if out != contents { + fs::write(&kustomization, out) + .into_diagnostic() + .wrap_err_with(|| format!("failed to update {}", kustomization.display()))?; + } + + Ok(namespace) +} + +pub(super) fn site_supervisor_plan_for_site_runtime( + plan: &SiteControllerRuntimePlan, + artifact_dir: &Path, + kubernetes_namespace: Option, +) -> SiteSupervisorPlan { + SiteSupervisorPlan { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + coordinator_pid: 0, + site_id: plan.site_id.clone(), + kind: plan.kind, + artifact_dir: artifact_dir.display().to_string(), + site_state_root: plan.site_state_root.clone(), + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace, + context: plan.context.clone(), + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: plan.observability_endpoint.clone(), + site_controller_url: None, + launch_env: plan.launch_env.clone(), + } +} + +pub(super) fn prepare_kubernetes_artifact_for_apply( + plan: &SiteControllerRuntimePlan, + artifact_dir: &Path, +) -> Result { + debug_assert_eq!(plan.kind, SiteKind::Kubernetes); + let namespace = + prepare_kubernetes_artifact_namespace(&plan.run_id, &plan.site_id, artifact_dir)?; + patch_site_artifacts( + artifact_dir, + &plan.run_id, + &plan.site_id, + plan.kind, + &plan.launch_env, + plan.observability_endpoint.as_deref(), + )?; + Ok(site_supervisor_plan_for_site_runtime( + plan, + artifact_dir, + Some(namespace), + )) +} + +pub(super) fn kubernetes_namespace_name(run_id: &str, site_id: &str) -> String { + let raw = format!("amber-{run_id}-{site_id}"); + let mut out = String::with_capacity(raw.len().min(63)); + let mut last_was_dash = false; + + for ch in raw.chars() { + let next = if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '-' + }; + if next == '-' { + if out.is_empty() || last_was_dash { + continue; + } + last_was_dash = true; + } else { + last_was_dash = false; + } + out.push(next); + if out.len() == 63 { + break; + } + } + while out.ends_with('-') { + out.pop(); + } + if out.is_empty() { + "amber".to_string() + } else { + out + } +} + +pub(super) async fn wait_for_compose_services_running( + compose_project: &str, + artifact_root: &Path, + services: &[String], + timeout: Duration, +) -> Result<()> { + if services.is_empty() { + return Ok(()); + } + let deadline = Instant::now() + timeout; + loop { + let output = compose_command(Some(compose_project), artifact_root) + .arg("ps") + .arg("--services") + .arg("--status") + .arg("running") + .args(services) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to query compose child services in {}", + artifact_root.display() + ) + })?; + if output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let running = stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::>(); + if services + .iter() + .all(|service| running.contains(service.as_str())) + { + return Ok(()); + } + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for compose child services {:?} in {}", + services, + artifact_root.display() + )); + } + sleep(Duration::from_millis(200)).await; + } +} + +pub fn walk_files(root: &Path) -> Result> { + let mut files = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to list {}", path.display()))? + { + let entry = entry.into_diagnostic()?; + let entry_path = entry.path(); + if entry_path.is_dir() { + stack.push(entry_path); + } else { + files.push(entry_path); + } + } + } + Ok(files) +} + +pub fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result { + if !matches!(kind, SiteKind::Compose | SiteKind::Kubernetes) { + return Ok(endpoint.to_string()); + } + + let mut url = Url::parse(endpoint) + .into_diagnostic() + .wrap_err_with(|| format!("invalid observability endpoint {endpoint}"))?; + let should_rewrite = url.host_str().is_some_and(|host| { + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .map(|addr| addr.is_loopback() || addr.is_unspecified()) + .unwrap_or(false) + }); + if should_rewrite { + let host = container_host_for_consumer(SiteKind::Direct, kind); + url.set_host(Some(&host)) + .map_err(|_| miette::miette!("failed to rewrite observability endpoint {endpoint}"))?; + } + Ok(url.to_string()) +} + +#[derive(Clone, Copy)] +enum PortBindScope { + Loopback, + Host, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct PortRange { + start: u16, + end: u16, +} + +impl PortRange { + fn contains(self, port: u16) -> bool { + self.start <= port && port <= self.end + } +} + +pub fn reserve_loopback_port() -> Result { + reserve_port(PortBindScope::Loopback) +} + +pub fn reserve_host_port() -> Result { + reserve_port(PortBindScope::Host) +} + +fn reserve_port(scope: PortBindScope) -> Result { + const LOOPBACK_PORT_RANGE_START: u16 = 30000; + const LOOPBACK_PORT_RANGE_END: u16 = 65000; + static RESERVED_LOOPBACK_PORTS: OnceLock>> = OnceLock::new(); + + let reserved = RESERVED_LOOPBACK_PORTS.get_or_init(|| std::sync::Mutex::new(BTreeSet::new())); + let mut reserved = reserved + .lock() + .expect("loopback port allocator should not be poisoned"); + let ephemeral_range = host_ephemeral_port_range(); + let span = u32::from(LOOPBACK_PORT_RANGE_END - LOOPBACK_PORT_RANGE_START); + let mut next = + LOOPBACK_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; + for _ in 0..usize::from(LOOPBACK_PORT_RANGE_END - LOOPBACK_PORT_RANGE_START) { + if next >= LOOPBACK_PORT_RANGE_END { + next = LOOPBACK_PORT_RANGE_START; + } + let port = next; + next += 1; + if reserved.contains(&port) { + continue; + } + // These ports become listeners after allocation; avoid ports the kernel may choose first + // for local outbound clients while the runtime is still starting. + if !port_allowed_for_runtime_listener(port, ephemeral_range) { + continue; + } + if port_available(port, scope) { + reserved.insert(port); + return Ok(port); + } + } + + Err(miette::miette!( + "failed to allocate a unique local port in {}-{}", + LOOPBACK_PORT_RANGE_START, + LOOPBACK_PORT_RANGE_END - 1 + )) +} + +fn port_allowed_for_runtime_listener(port: u16, ephemeral_range: Option) -> bool { + !ephemeral_range.is_some_and(|range| range.contains(port)) +} + +#[cfg(target_os = "linux")] +fn host_ephemeral_port_range() -> Option { + parse_linux_ephemeral_port_range( + &fs::read_to_string("/proc/sys/net/ipv4/ip_local_port_range").ok()?, + ) +} + +#[cfg(target_os = "linux")] +fn parse_linux_ephemeral_port_range(raw: &str) -> Option { + let mut parts = raw.split_whitespace(); + let start = parts.next()?.parse::().ok()?; + let end = parts.next()?.parse::().ok()?; + (start <= end).then_some(PortRange { start, end }) +} + +#[cfg(target_os = "macos")] +fn host_ephemeral_port_range() -> Option { + let start = sysctl_port("net.inet.ip.portrange.first")?; + let end = sysctl_port("net.inet.ip.portrange.last")?; + (start <= end).then_some(PortRange { start, end }) +} + +#[cfg(target_os = "macos")] +fn sysctl_port(name: &str) -> Option { + let name = std::ffi::CString::new(name).ok()?; + let mut value: libc::c_int = 0; + let mut len = std::mem::size_of_val(&value); + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr(), + (&mut value as *mut libc::c_int).cast(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + (rc == 0 && (0..=u16::MAX as libc::c_int).contains(&value)).then_some(value as u16) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn host_ephemeral_port_range() -> Option { + None +} + +fn port_available(port: u16, scope: PortBindScope) -> bool { + let addr = match scope { + PortBindScope::Loopback => "127.0.0.1", + PortBindScope::Host => "0.0.0.0", + }; + TcpListener::bind((addr, port)).is_ok() +} + +pub(super) fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-supervisor-plan.json") +} + +pub(super) fn desired_links_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("desired-links.json") +} + +pub(super) fn empty_desired_link_state() -> DesiredLinkState { + DesiredLinkState { + schema: DESIRED_LINKS_SCHEMA.to_string(), + version: DESIRED_LINKS_VERSION, + external_slots: BTreeMap::new(), + export_peers: Vec::new(), + external_slot_overlays: BTreeMap::new(), + export_peer_overlays: BTreeMap::new(), + } +} + +pub(super) fn update_desired_overlay_for_consumer( + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + empty_desired_link_state() + }; + state + .external_slot_overlays + .insert(overlay_id.to_string(), overlay); + write_json(&path, &state) +} + +pub(super) fn update_desired_overlay_for_provider( + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + empty_desired_link_state() + }; + state + .export_peer_overlays + .insert(overlay_id.to_string(), overlay); + write_json(&path, &state) +} + +pub(super) fn clear_desired_overlay_for_consumer( + site_state_root: &Path, + overlay_id: &str, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + return Ok(()); + }; + state.external_slot_overlays.remove(overlay_id); + write_json(&path, &state) +} + +pub(super) fn clear_desired_overlay_for_provider( + site_state_root: &Path, + overlay_id: &str, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + return Ok(()); + }; + state.export_peer_overlays.remove(overlay_id); + write_json(&path, &state) +} + +pub(super) fn patch_site_artifacts( + artifact_dir: &Path, + run_id: &str, + site_id: &str, + kind: SiteKind, + launch_env: &BTreeMap, + observability_endpoint: Option<&str>, +) -> Result<()> { + if matches!(kind, SiteKind::Compose) { + assign_compose_egress_network_subnets(artifact_dir, run_id, site_id)?; + } + if matches!(kind, SiteKind::Kubernetes) { + for env_file_name in [ + DEFAULT_EXTERNAL_ENV_FILE, + "component-sidecar.env", + "root-config.env", + "root-config-secret.env", + ] { + let env_file = artifact_dir.join(env_file_name); + if env_file.is_file() { + patch_generated_env_file(&env_file, launch_env)?; + } + } + + if let Some(endpoint) = observability_endpoint { + let upstream = observability_endpoint_for_site(kind, endpoint)?; + for path in walk_files(artifact_dir)? { + if path.extension().and_then(|ext| ext.to_str()) != Some("yaml") { + continue; + } + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + if !raw.contains(DEFAULT_K8S_OTEL_UPSTREAM) { + continue; + } + fs::write(&path, raw.replace(DEFAULT_K8S_OTEL_UPSTREAM, &upstream)) + .into_diagnostic() + .wrap_err_with(|| format!("failed to patch {}", path.display()))?; + } + } + } + Ok(()) +} + +pub(super) fn patch_generated_env_file( + path: &Path, + launch_env: &BTreeMap, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut out = String::new(); + for line in raw.lines() { + if let Some((key, _)) = line.split_once('=') + && let Some(value) = launch_env.get(key.trim()) + { + out.push_str(key.trim()); + out.push('='); + out.push_str(value); + out.push('\n'); + continue; + } + out.push_str(line); + out.push('\n'); + } + fs::write(path, out) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +pub(super) fn compose_command(project_name: Option<&str>, artifact_dir: &Path) -> Command { + let mut cmd = Command::new("docker"); + cmd.arg("compose") + .arg("-f") + .arg(artifact_dir.join("compose.yaml")); + if let Some(project_name) = project_name { + cmd.arg("-p").arg(project_name); + } + cmd.current_dir(artifact_dir); + cmd +} + +pub(super) fn kubectl_command(context: Option<&str>) -> Command { + let mut cmd = Command::new("kubectl"); + if let Some(context) = context { + cmd.arg("--context").arg(context); + } + cmd +} + +pub(super) fn ensure_kubernetes_namespace(plan: &SiteSupervisorPlan) -> Result<()> { + if !should_prepare_kubernetes_namespace(plan, is_in_cluster_kubernetes_runtime()) { + return Ok(()); + } + let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; + let deadline = Instant::now() + Duration::from_secs(60); + let context = plan.context.as_deref(); + let mut last_error = None::; + loop { + let output = kubectl_command(context) + .arg("get") + .arg("namespace") + .arg(namespace) + .arg("-o") + .arg("json") + .output() + .into_diagnostic() + .wrap_err_with(|| format!("failed to query kubernetes namespace `{namespace}`"))?; + if output.status.success() { + let namespace_json: serde_json::Value = serde_json::from_slice(&output.stdout) + .into_diagnostic() + .wrap_err_with(|| format!("failed to parse kubernetes namespace `{namespace}`"))?; + let is_terminating = namespace_json + .pointer("/metadata/deletionTimestamp") + .is_some_and(|value| !value.is_null()); + if !is_terminating { + return Ok(()); + } + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.contains("(NotFound)") || stderr.contains("not found") { + let create_output = kubectl_command(context) + .arg("create") + .arg("namespace") + .arg(namespace) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to create kubernetes namespace `{namespace}`") + })?; + if create_output.status.success() { + return Ok(()); + } + last_error = Some( + String::from_utf8_lossy(&create_output.stderr) + .trim() + .to_string(), + ); + } else if !stderr.is_empty() { + last_error = Some(stderr); + } + } + if Instant::now() >= deadline { + let detail = last_error + .as_deref() + .filter(|detail| !detail.is_empty()) + .map(|detail| format!(": {detail}")) + .unwrap_or_default(); + return Err(miette::miette!( + "failed to prepare kubernetes namespace `{namespace}` within 60s{detail}" + )); + } + std::thread::sleep(Duration::from_millis(500)); + } +} + +fn should_prepare_kubernetes_namespace( + plan: &SiteSupervisorPlan, + in_cluster_runtime: bool, +) -> bool { + plan.context.is_some() || !in_cluster_runtime +} + +fn is_in_cluster_kubernetes_runtime() -> bool { + env::var_os("KUBERNETES_SERVICE_HOST").is_some() +} + +pub(super) fn ensure_kubernetes_workloads_ready(plan: &SiteSupervisorPlan) -> Result<()> { + let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; + let context = plan.context.as_deref(); + let expected = kubernetes_expected_workloads(Path::new(&plan.artifact_dir))?; + wait_for_named_kubernetes_resources( + context, + namespace, + "job", + "condition=complete", + &expected.jobs, + &format!("wait for kubernetes jobs for site `{}`", plan.site_id), + )?; + wait_for_named_kubernetes_resources( + context, + namespace, + "deployment", + "condition=available", + &expected.deployments, + &format!( + "wait for kubernetes deployments for site `{}`", + plan.site_id + ), + )?; + Ok(()) +} + +#[derive(Default)] +pub(super) struct KubernetesArtifactWorkloads { + pub(super) jobs: Vec, + pub(super) deployments: Vec, +} + +pub(super) fn kubernetes_expected_workloads( + artifact_dir: &Path, +) -> Result { + let mut jobs = BTreeSet::new(); + let mut deployments = BTreeSet::new(); + for path in walk_files(artifact_dir)? { + if !matches!( + path.extension().and_then(|ext| ext.to_str()), + Some("yaml" | "yml") + ) { + continue; + } + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read kubernetes artifact {}", path.display()))?; + for document in serde_yaml::Deserializer::from_str(&raw) { + use serde::Deserialize as _; + + let value = serde_yaml::Value::deserialize(document) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to parse kubernetes artifact {}", path.display()) + })?; + let kind = value + .get("kind") + .and_then(serde_yaml::Value::as_str) + .unwrap_or_default(); + let name = value + .get("metadata") + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(serde_yaml::Value::String("name".to_string()))) + .and_then(serde_yaml::Value::as_str) + .unwrap_or_default(); + if name.is_empty() { + continue; + } + match kind { + "Job" => { + jobs.insert(name.to_string()); + } + "Deployment" => { + deployments.insert(name.to_string()); + } + _ => {} + } + } + } + Ok(KubernetesArtifactWorkloads { + jobs: jobs.into_iter().collect(), + deployments: deployments.into_iter().collect(), + }) +} + +pub(super) fn wait_for_named_kubernetes_resources( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + condition: &str, + names: &[String], + label: &str, +) -> Result<()> { + for name in names { + wait_for_named_kubernetes_resource( + context, + namespace, + resource_kind, + condition, + name, + label, + )?; + } + Ok(()) +} + +pub(super) fn wait_for_named_kubernetes_resource( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + condition: &str, + name: &str, + label: &str, +) -> Result<()> { + let resource = format!("{resource_kind}/{name}"); + let deadline = Instant::now() + KUBERNETES_WORKLOAD_READY_TIMEOUT; + loop { + let get_output = kubectl_command(context) + .args(["-n", namespace, "get", resource.as_str()]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: query {resource}"))?; + if !get_output.status.success() { + if Instant::now() >= deadline { + let stderr = String::from_utf8_lossy(&get_output.stderr) + .trim() + .to_string(); + let detail = if stderr.is_empty() { + format!("status {}", get_output.status) + } else { + stderr + }; + return Err(miette::miette!("{label} failed: {detail}")); + } + std::thread::sleep(Duration::from_millis(250)); + continue; + } + + let remaining = deadline.saturating_duration_since(Instant::now()); + let timeout = format!("{}s", remaining.as_secs().max(1)); + let wait_output = kubectl_command(context) + .args([ + "-n", + namespace, + "wait", + "--for", + condition, + "--timeout", + timeout.as_str(), + resource.as_str(), + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: wait for {resource}"))?; + if wait_output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&wait_output.stderr) + .trim() + .to_string(); + if stderr.contains("not found") && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(250)); + continue; + } + let detail = if stderr.is_empty() { + format!("status {}", wait_output.status) + } else { + stderr + }; + return Err(miette::miette!("{label} failed: {detail}")); + } +} + +pub(super) fn mesh_protocol(protocol: NetworkProtocol) -> Result { + Ok(match protocol { + NetworkProtocol::Http | NetworkProtocol::Https => MeshProtocol::Http, + NetworkProtocol::Tcp => MeshProtocol::Tcp, + _ => { + return Err(miette::miette!( + "mixed-site mesh links do not support protocol `{protocol}`" + )); + } + }) +} + +pub(crate) fn site_ready_timeout_for_kind(kind: SiteKind) -> Duration { + match kind { + SiteKind::Kubernetes => KUBERNETES_WORKLOAD_READY_TIMEOUT + KUBERNETES_SITE_READY_BUFFER, + SiteKind::Direct | SiteKind::Compose | SiteKind::Vm => Duration::from_secs(120), + } +} + +pub(super) fn amber_cli_executable() -> Result { + if let Some(path) = env::var_os("CARGO_BIN_EXE_amber") { + let path = PathBuf::from(path); + if path.is_file() { + return Ok(path); + } + } + + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve amber executable path")?; + let exe_name = format!("amber{}", std::env::consts::EXE_SUFFIX); + for dir in [current.parent(), current.parent().and_then(Path::parent)] + .into_iter() + .flatten() + { + let candidate = dir.join(&exe_name); + if candidate.is_file() { + return Ok(candidate); + } + } + Err(miette::miette!( + "failed to locate the amber executable next to {}", + current.display() + )) +} + +pub(crate) fn spawn_detached_child( + work_dir: &Path, + log_path: &Path, + extra_env: &BTreeMap, + build: impl FnOnce(&mut Command), +) -> Result { + let exe = amber_cli_executable()?; + spawn_detached_child_with_executable(&exe, work_dir, log_path, extra_env, build) +} + +fn spawn_detached_child_with_executable( + executable: &Path, + work_dir: &Path, + log_path: &Path, + extra_env: &BTreeMap, + build: impl FnOnce(&mut Command), +) -> Result { + #[cfg(unix)] + use std::os::unix::process::CommandExt as _; + + let log = fs::File::create(log_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create log {}", log_path.display()))?; + let log_err = log + .try_clone() + .into_diagnostic() + .wrap_err("failed to clone log handle")?; + let mut cmd = Command::new(executable); + cmd.current_dir(work_dir); + cmd.envs(extra_env); + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::from(log)); + cmd.stderr(Stdio::from(log_err)); + #[cfg(unix)] + unsafe { + cmd.pre_exec(|| { + if libc::setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + build(&mut cmd); + cmd.spawn().into_diagnostic().wrap_err_with(|| { + format!( + "failed to spawn background amber process in {}", + work_dir.display() + ) + }) +} + +pub(super) fn endpoint_returns_http_response_blocking( + addr: SocketAddr, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if let Ok(mut stream) = + std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(100)) + { + let _ = + stream.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); + let mut buf = [0u8; 32]; + if let Ok(read) = stream.read(&mut buf) + && read > 0 + && buf[..read].starts_with(b"HTTP/") + { + return Ok(true); + } + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(false) +} + +pub(super) fn endpoint_accepts_stable_connection_blocking( + addr: SocketAddr, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(100)).is_ok() { + return Ok(true); + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(false) +} + +pub(super) fn pid_is_alive(pid: u32) -> bool { + #[cfg(unix)] + { + let alive = if unsafe { libc::kill(pid as i32, 0) } == 0 { + true + } else { + std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + }; + alive && process_status_code(pid) != Some('Z') + } + + #[cfg(not(unix))] + { + let _ = pid; + true + } +} + +#[cfg(unix)] +fn process_status_code(pid: u32) -> Option { + let output = Command::new("ps") + .arg("-o") + .arg("stat=") + .arg("-p") + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) +} + +#[cfg(unix)] +fn parse_process_status_code(raw: &str) -> Option { + raw.split_whitespace() + .next()? + .chars() + .next() + .map(|state| state.to_ascii_uppercase()) +} + +#[cfg(not(unix))] +pub(super) fn send_sigterm(_pid: u32) {} + +#[cfg(unix)] +pub(super) fn send_signal_to_pids(pids: &[u32], signal: i32) { + for pid in pids { + unsafe { + libc::kill(*pid as i32, signal); + } + } +} + +#[cfg(not(unix))] +pub(super) fn send_signal_to_pids(_pids: &[u32], _signal: i32) {} + +#[cfg(unix)] +pub(super) fn send_signal_to_process_group(root_pid: u32, signal: i32) { + unsafe { + libc::kill(-(root_pid as i32), signal); + } +} + +#[cfg(not(unix))] +pub(super) fn send_signal_to_process_group(_root_pid: u32, _signal: i32) {} + +#[cfg(unix)] +fn parse_process_tree_line(raw: &str) -> Option<(u32, u32)> { + let mut fields = raw.split_ascii_whitespace(); + let pid = fields.next()?.parse::().ok()?; + let ppid = fields.next()?.parse::().ok()?; + Some((pid, ppid)) +} + +#[cfg(unix)] +fn process_tree_postorder_from_ps(raw: &str, root_pid: u32) -> Vec { + fn visit( + pid: u32, + children_by_parent: &std::collections::BTreeMap>, + seen: &mut std::collections::BTreeSet, + out: &mut Vec, + ) { + if !seen.insert(pid) { + return; + } + if let Some(children) = children_by_parent.get(&pid) { + for child in children { + visit(*child, children_by_parent, seen, out); + } + } + out.push(pid); + } + + let mut children_by_parent = std::collections::BTreeMap::>::new(); + for line in raw.lines() { + let Some((pid, ppid)) = parse_process_tree_line(line) else { + continue; + }; + children_by_parent.entry(ppid).or_default().push(pid); + } + for children in children_by_parent.values_mut() { + children.sort_unstable(); + } + + let mut out = Vec::new(); + visit( + root_pid, + &children_by_parent, + &mut std::collections::BTreeSet::new(), + &mut out, + ); + out +} + +pub(super) fn process_tree_postorder(root_pid: u32) -> Result> { + #[cfg(unix)] + { + let output = Command::new("ps") + .args(["-axo", "pid=,ppid="]) + .output() + .into_diagnostic() + .wrap_err("failed to query process tree via `ps`")?; + if !output.status.success() { + return Err(miette::miette!( + "`ps -axo pid=,ppid=` failed with status {}", + output.status + )); + } + Ok(process_tree_postorder_from_ps( + &String::from_utf8_lossy(&output.stdout), + root_pid, + )) + } + + #[cfg(not(unix))] + { + Ok(vec![root_pid]) + } +} + +pub(crate) async fn resolve_link_external_url_for_output( + provider: &LaunchedSite, + provider_output_dir: &Path, + link: &RunLink, + consumer_kind: SiteKind, + _run_root: &Path, +) -> Result { + external_slot_url(provider, provider_output_dir, link, consumer_kind) +} + +pub fn site_controller_peer_router_url(controller_site_kind: SiteKind, route_port: u16) -> String { + let host = match controller_site_kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1", + SiteKind::Compose | SiteKind::Kubernetes => KUBERNETES_ROUTER_COMPONENT_NAME, + }; + format!("http://{host}:{route_port}") +} + +pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { + matches!( + consumer_kind, + SiteKind::Compose | SiteKind::Kubernetes | SiteKind::Vm + ) || (cfg!(target_os = "linux") && matches!(consumer_kind, SiteKind::Direct)) +} + +pub fn host_service_bind_addr_for_consumer(consumer_kind: SiteKind, port: u16) -> SocketAddr { + host_proxy_bind_addr(consumer_needs_host_wide_listener(consumer_kind), port) +} + +pub fn router_mesh_addr_for_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, +) -> Result { + match consumer_kind { + SiteKind::Compose | SiteKind::Kubernetes => { + let addr = router_mesh_addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| { + format!("invalid live router mesh address `{router_mesh_addr}`") + })?; + let host = container_host_for_consumer(provider_kind, consumer_kind); + Ok(format!("{host}:{}", addr.port())) + } + SiteKind::Direct => Ok(router_mesh_addr.to_string()), + SiteKind::Vm => Ok(router_mesh_addr.to_string()), + } +} + +pub fn router_mesh_addr_for_component_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, +) -> Result { + match consumer_kind { + SiteKind::Compose | SiteKind::Kubernetes => { + let addr = router_mesh_addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| { + format!("invalid live router mesh address `{router_mesh_addr}`") + })?; + let host = container_host_for_consumer(provider_kind, consumer_kind); + Ok(format!("{host}:{}", addr.port())) + } + SiteKind::Direct | SiteKind::Vm => { + #[cfg(target_os = "linux")] + { + Ok(rewrite_peer_addr_for_slirp_gateway(router_mesh_addr)) + } + #[cfg(not(target_os = "linux"))] + { + Ok(router_mesh_addr.to_string()) + } + } + } +} + +pub(super) fn host_proxy_bind_addr(needs_host_wide_listener: bool, port: u16) -> SocketAddr { + if needs_host_wide_listener { + SocketAddr::from(([0, 0, 0, 0], port)) + } else { + SocketAddr::from(([127, 0, 0, 1], port)) + } +} + +pub(super) fn external_slot_url( + provider: &LaunchedSite, + provider_output_dir: &Path, + link: &RunLink, + consumer_kind: SiteKind, +) -> Result { + let router_mesh_addr = if let Some(router_mesh_addr) = + crate::runtime_api::published_router_mesh_addr_for_consumer_kind( + &provider.receipt, + consumer_kind, + ) { + router_mesh_addr.to_string() + } else { + router_mesh_addr_for_consumer( + provider.receipt.kind, + consumer_kind, + provider + .receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("provider site is missing router mesh addr"))?, + )? + }; + let route_id = provider_export_route_id(provider_output_dir, link)?; + let mut mesh_url = Url::parse(&format!("mesh://{router_mesh_addr}")) + .into_diagnostic() + .wrap_err("failed to build mesh link url")?; + let peer_key = + base64::engine::general_purpose::STANDARD.encode(provider.router_identity.public_key); + mesh_url + .query_pairs_mut() + .append_pair("peer_id", &provider.router_identity.id) + .append_pair("peer_key", &peer_key) + .append_pair("route_id", &route_id) + .append_pair("capability", &link.export_name); + Ok(mesh_url.to_string()) +} + +pub(super) fn provider_export_route_id( + provider_output_dir: &Path, + link: &RunLink, +) -> Result { + if let Some(route_id) = load_output_proxy_metadata(provider_output_dir)? + .exports + .get(&link.export_name) + .and_then(|export| export.route_id.clone()) + { + return Ok(route_id); + } + Ok(router_export_route_id( + &link.export_name, + mesh_protocol(link.protocol)?, + )) +} + +pub(super) fn container_host_for_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, +) -> String { + let container_host_ip = container_host_ip(); + container_host_from_resolved_ip(provider_kind, consumer_kind, container_host_ip.as_deref()) +} + +pub(super) fn container_host_from_resolved_ip( + _provider_kind: SiteKind, + consumer_kind: SiteKind, + container_host_ip: Option<&str>, +) -> String { + match consumer_kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), + SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), + SiteKind::Kubernetes => container_host_ip + .unwrap_or(CONTAINER_HOST_ALIAS) + .to_string(), + } +} + +pub(super) fn container_host_ip() -> Option { + KUBERNETES_CONTAINER_HOST_IP + .get_or_init(resolve_container_host_ip) + .clone() +} + +pub(super) fn resolve_container_host_ip() -> Option { + if cfg!(target_os = "linux") { + return resolve_linux_container_host_ip(); + } + resolve_desktop_container_host_ip() +} + +pub(super) fn resolve_linux_container_host_ip() -> Option { + let output = Command::new("docker") + .arg("network") + .arg("inspect") + .arg("bridge") + .arg("--format") + .arg("{{(index .IPAM.Config 0).Gateway}}") + .output() + .ok()?; + if !output.status.success() { + return None; + } + + let host = String::from_utf8(output.stdout).ok()?.trim().to_string(); + host.parse::().ok()?; + Some(host) +} + +pub(super) fn resolve_desktop_container_host_ip() -> Option { + let output = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("busybox:1.36.1") + .arg("nslookup") + .arg(CONTAINER_HOST_ALIAS) + .output() + .ok()?; + if !output.status.success() { + return None; + } + + String::from_utf8(output.stdout) + .ok()? + .lines() + .flat_map(str::split_whitespace) + .filter_map(|token| token.parse::().ok()) + .map(|ip| ip.to_string()) + .next_back() +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + fs, + net::{SocketAddr, TcpListener}, + }; + + use amber_compiler::{ + mesh::{PROXY_METADATA_FILENAME, PROXY_METADATA_VERSION}, + run_plan::SiteKind, + }; + use amber_manifest::NetworkProtocol; + use amber_mesh::{MeshIdentity, MeshIdentityPublic, router_export_route_id}; + use amber_proxy::ControlEndpoint; + use base64::Engine as _; + use tempfile::tempdir; + + #[cfg(target_os = "linux")] + use super::parse_linux_ephemeral_port_range; + use super::{ + LaunchedSite, PortBindScope, PortRange, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, + SiteSupervisorPlan, external_slot_url, port_allowed_for_runtime_listener, port_available, + resolve_link_external_url_for_output, router_mesh_addr_for_consumer, + should_prepare_kubernetes_namespace, + }; + #[cfg(unix)] + use super::{parse_process_status_code, process_tree_postorder_from_ps}; + + #[test] + fn runtime_port_allocator_rejects_ephemeral_client_ports() { + let linux_default = Some(PortRange { + start: 32768, + end: 60999, + }); + + assert!( + !port_allowed_for_runtime_listener(46274, linux_default), + "runtime listeners must not use ports the kernel can choose for outbound clients" + ); + assert!(port_allowed_for_runtime_listener(30000, linux_default)); + assert!(port_allowed_for_runtime_listener(61000, linux_default)); + } + + #[cfg(target_os = "linux")] + #[test] + fn parses_linux_ephemeral_port_range() { + assert_eq!( + parse_linux_ephemeral_port_range("32768\t60999\n"), + Some(PortRange { + start: 32768, + end: 60999 + }) + ); + assert_eq!(parse_linux_ephemeral_port_range("60999 32768"), None); + } + + #[test] + fn host_port_availability_rejects_ports_bound_on_non_primary_loopback() { + let Ok(listener) = TcpListener::bind(("127.0.0.2", 0)) else { + return; + }; + let port = listener + .local_addr() + .expect("listener local addr should be available") + .port(); + if !port_available(port, PortBindScope::Loopback) { + return; + } + + assert!( + !port_available(port, PortBindScope::Host), + "host-port reservations must reject ports that would make a 0.0.0.0 bind fail" + ); + } + + fn kubernetes_supervisor_plan(context: Option<&str>) -> SiteSupervisorPlan { + SiteSupervisorPlan { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run-1".to_string(), + mesh_scope: "test".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 0, + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: context.map(str::to_string), + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + } + } + + #[cfg(unix)] + #[test] + fn parse_process_status_code_normalizes_unix_ps_output() { + assert_eq!(parse_process_status_code("S+\n"), Some('S')); + assert_eq!(parse_process_status_code("z\n"), Some('Z')); + assert_eq!(parse_process_status_code(""), None); + } + + #[cfg(unix)] + #[test] + fn process_tree_postorder_from_ps_includes_descendants_before_root() { + let tree = process_tree_postorder_from_ps( + "\ +1 0 +42 1 +7 42 +8 42 +9 8 +100 1 +", + 42, + ); + + assert_eq!(tree, vec![7, 9, 8, 42]); + } + + #[cfg(unix)] + #[test] + fn spawn_detached_child_propagates_launch_env() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = tempdir().expect("temp dir should create"); + let executable = temp.path().join("amber-stub"); + let log_path = temp.path().join("child.log"); + let env_output_path = temp.path().join("env.txt"); + fs::write( + &executable, + r#"#!/bin/sh +set -eu +printf '%s' "${AMBER_VM_FORCE_TCG:-missing}" > "$1" +"#, + ) + .expect("stub executable should write"); + let mut permissions = fs::metadata(&executable) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&executable, permissions).expect("stub should chmod"); + + let launch_env = BTreeMap::from([("AMBER_VM_FORCE_TCG".to_string(), "1".to_string())]); + let mut child = super::spawn_detached_child_with_executable( + &executable, + temp.path(), + &log_path, + &launch_env, + |cmd| { + cmd.arg(&env_output_path); + }, + ) + .expect("detached child should spawn"); + let status = child.wait().expect("stub child should exit"); + assert!(status.success(), "stub child should succeed: {status}"); + assert_eq!( + fs::read_to_string(&env_output_path).expect("env output should read"), + "1", + "detached children must inherit launch_env so VM child runtimes see TCG forcing", + ); + } + + #[test] + fn in_cluster_kubernetes_controller_skips_namespace_bootstrap() { + let plan = kubernetes_supervisor_plan(None); + assert!( + !should_prepare_kubernetes_namespace(&plan, true), + "in-cluster site controllers should not perform cluster-scoped namespace probes" + ); + } + + #[test] + fn external_kubernetes_supervisor_still_prepares_namespace() { + let plan = kubernetes_supervisor_plan(Some("kind-test")); + assert!( + should_prepare_kubernetes_namespace(&plan, true), + "external site supervisors still need to prepare the namespace before applying \ + artifacts" + ); + } + + #[test] + fn direct_consumers_keep_router_mesh_addr_verbatim() { + let peer_addr = + router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Direct, "127.0.0.1:24077") + .expect("direct consumers should keep direct peer mesh addresses"); + assert_eq!(peer_addr, "127.0.0.1:24077"); + } + + #[test] + fn vm_site_consumers_keep_router_mesh_addr_verbatim() { + let peer_addr = + router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Vm, "127.0.0.1:24077") + .expect("vm site consumers should keep peer router mesh addresses"); + assert_eq!(peer_addr, "127.0.0.1:24077"); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_component_consumers_rewrite_loopback_router_mesh_addr_to_slirp_gateway() { + let peer_addr = super::router_mesh_addr_for_component_consumer( + SiteKind::Direct, + SiteKind::Direct, + "127.0.0.1:24077", + ) + .expect("direct component consumers should rewrite loopback peer mesh addresses"); + assert_eq!(peer_addr, "10.0.2.2:24077"); + } + + #[test] + fn host_service_bind_addr_matches_component_reachability() { + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Compose, 24077), + SocketAddr::from(([0, 0, 0, 0], 24077)) + ); + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 24078), + SocketAddr::from(([0, 0, 0, 0], 24078)) + ); + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Vm, 24079), + SocketAddr::from(([0, 0, 0, 0], 24079)) + ); + + let direct = super::host_service_bind_addr_for_consumer(SiteKind::Direct, 24080); + if cfg!(target_os = "linux") { + assert_eq!(direct, SocketAddr::from(([0, 0, 0, 0], 24080))); + } else { + assert_eq!(direct, SocketAddr::from(([127, 0, 0, 1], 24080))); + } + } + + #[test] + fn compose_consumers_reach_kubernetes_peers_via_host_alias() { + let peer_addr = super::router_mesh_addr_for_consumer( + SiteKind::Kubernetes, + SiteKind::Compose, + "127.0.0.1:24077", + ) + .expect("compose consumers should rewrite kubernetes peers to a host-reachable address"); + assert_eq!(peer_addr, "host.docker.internal:24077"); + } + + #[test] + fn kubernetes_consumers_resolve_cross_site_links_as_mesh_router_urls() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + SiteKind::Kubernetes, + temp.path(), + )) + .expect("kubernetes consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://"), + "cross-site kubernetes consumers should route through mesh, got {resolved}" + ); + assert!( + !resolved.starts_with("http://"), + "cross-site kubernetes consumers should not bounce through an HTTP bridge proxy: \ + {resolved}" + ); + assert!(resolved.contains("peer_id=%2Fsite%2Fcompose_local%2Frouter")); + } + + #[test] + fn external_slot_url_prefers_host_published_mesh_addr_for_kubernetes_consumers() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: Some("unix:///tmp/router.sock".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: Some( + base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), + ), + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = external_slot_url(&provider, temp.path(), &link, SiteKind::Kubernetes) + .expect("external slot url should resolve"); + + assert!( + resolved.starts_with("mesh://192.168.65.254:24000"), + "container consumers should use the host-published mesh address, got {resolved}" + ); + } + + #[test] + fn external_slot_url_keeps_vm_router_mesh_addr_loopback() { + let temp = tempdir().expect("tempdir"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-vm/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: Some("unix:///tmp/router.sock".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: Some( + base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), + ), + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "vm_local".to_string(), + provider_component: "/job-vm/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-vm/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = external_slot_url(&provider, temp.path(), &link, SiteKind::Vm).expect( + "external slot url should resolve for vm site consumers without slirp rewriting", + ); + + assert!( + resolved.starts_with("mesh://127.0.0.1:24000"), + "vm site consumers should use the live router mesh address, got {resolved}" + ); + } +} diff --git a/cli/src/framework_component/dynamic_caps.rs b/runtime/site-controller/src/dynamic_caps.rs similarity index 93% rename from cli/src/framework_component/dynamic_caps.rs rename to runtime/site-controller/src/dynamic_caps.rs index 8cac4f89..a9a0e35b 100644 --- a/cli/src/framework_component/dynamic_caps.rs +++ b/runtime/site-controller/src/dynamic_caps.rs @@ -77,6 +77,8 @@ pub(crate) enum DynamicCapabilityShareOutcome { #[derive(Clone, Debug)] pub(crate) struct DynamicCapabilityRevokeOutcome { + // Production code only needs to know whether revocation changed anything; tests assert the + // exact grant set to keep cascade revocation behavior pinned down. #[cfg_attr(not(test), allow(dead_code))] pub(crate) revoked_grant_ids: Vec, } @@ -115,12 +117,28 @@ pub(crate) struct ControlDynamicShareRequest { pub(crate) options: serde_json::Value, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicGrantAuthoritySyncRequest { + pub(crate) authority_sites: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicGrantAuthoritySyncResponse { + pub(crate) synced: usize, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicInspectRefRequest { pub(crate) holder_component_id: String, pub(crate) r#ref: String, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicResolveRefRequest { + pub(crate) holder_component_id: String, + pub(crate) r#ref: String, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicRevokeRequest { pub(crate) caller_component_id: String, @@ -157,6 +175,16 @@ pub(crate) struct ControlDynamicResolveOriginRequest { pub(crate) source: DynamicCapabilityControlSourceRequest, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct InternalDynamicResolveOriginRequest { + pub(crate) holder_component_id: String, + #[serde(flatten)] + pub(crate) source: DynamicCapabilityControlSourceRequest, + pub(crate) holder_peer_id: String, + pub(crate) holder_peer_key_b64: String, + pub(crate) holder_site_kind: SiteKind, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicResolveOriginResponse { pub(crate) held_id: String, @@ -169,6 +197,17 @@ pub(crate) struct ControlDynamicResolveOriginResponse { pub(crate) origin_peer_addr: String, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicResolveRefResponse { + #[serde(flatten)] + pub(crate) origin: ControlDynamicResolveOriginResponse, + pub(crate) relative_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) query: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) fragment: Option, +} + fn dynamic_grant_live_default() -> bool { true } @@ -212,6 +251,39 @@ pub(crate) fn root_authority_key(selector: &RootAuthoritySelectorIr) -> String { .encode(serde_json::to_vec(selector).expect("root authority selector should serialize")) } +fn origin_overlay_suffix( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&serde_json::json!({ + "holder_component_id": holder_component_id, + "root_authority_selector": root_authority_selector, + })) + .expect("dynamic capability origin overlay key should serialize"), + ) +} + +pub(crate) fn origin_overlay_id( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + format!( + "dynamic-cap-origin-{}", + origin_overlay_suffix(holder_component_id, root_authority_selector) + ) +} + +pub(crate) fn origin_route_id( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + format!( + "dynamic-cap-origin-route-{}", + origin_overlay_suffix(holder_component_id, root_authority_selector) + ) +} + pub(crate) fn held_id_for_root(selector: &RootAuthoritySelectorIr) -> String { format!( "{DYNAMIC_CAPABILITY_ROOT_HELD_PREFIX}{}", @@ -257,7 +329,7 @@ pub(crate) fn next_dynamic_grant_id(state: &mut FrameworkControlState) -> String "{DYNAMIC_CAPABILITY_GRANT_ID_PREFIX}{:016x}", state.next_dynamic_capability_grant_id ); - state.next_dynamic_capability_grant_id += 1; + state.next_dynamic_capability_grant_id += state.id_stride.max(1); grant_id } @@ -1032,19 +1104,18 @@ pub(crate) fn mint_dynamic_capability_ref( state: &FrameworkControlState, grant: &DynamicGrantRecord, ) -> std::result::Result { - let signing_key = amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { + let identity = dynamic_ref_issuer_identity(state)?; + let signing_key = identity.signing_key().map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("dynamic capability signing key is invalid: {err}"), + &format!("site controller mesh identity is invalid: {err}"), ) })?; amber_mesh::dynamic_caps::build_dynamic_capability_ref_url( amber_mesh::dynamic_caps::DynamicCapabilityRefClaims { version: amber_mesh::dynamic_caps::DYNAMIC_CAPS_REF_VERSION, run_id: state.run_id.clone(), + issuer_id: identity.id.clone(), grant_id: grant.grant_id.clone(), holder_component_id: grant.holder_component_id.clone(), descriptor_hint: Some(grant.descriptor.label.clone()), @@ -1062,20 +1133,22 @@ pub(crate) fn mint_dynamic_capability_ref( }) } +fn dynamic_ref_issuer_identity( + state: &FrameworkControlState, +) -> std::result::Result<&amber_mesh::MeshIdentitySecret, ProtocolErrorResponse> { + state.controller_identity.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "site controller mesh identity is not loaded", + ) + }) +} + pub(crate) fn inspect_dynamic_ref( state: &FrameworkControlState, holder_component_id: &str, raw_ref: &str, ) -> std::result::Result { - let signing_key = amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!("dynamic capability signing key is invalid: {err}"), - ) - })?; let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) .map_err(|err| { protocol_error( @@ -1104,6 +1177,19 @@ pub(crate) fn inspect_dynamic_ref( "dynamic capability ref is bound to a different holder", )); } + let identity = dynamic_ref_issuer_identity(state)?; + if parsed.claims.issuer_id != identity.id { + return Err(protocol_error( + ProtocolErrorCode::MalformedRef, + "dynamic capability ref issuer is not this site controller", + )); + } + let signing_key = identity.signing_key().map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site controller mesh identity is invalid: {err}"), + ) + })?; amber_mesh::dynamic_caps::verify_dynamic_capability_ref(&parsed, &signing_key.verifying_key()) .map_err(|err| { protocol_error( @@ -1315,6 +1401,15 @@ pub(crate) fn share_dynamic_capability( }) } +pub(crate) fn sync_dynamic_capability_grant_authority_sites( + state: &mut FrameworkControlState, + authority_sites: &BTreeMap, +) { + state + .dynamic_capability_grant_authority_sites + .extend(authority_sites.clone()); +} + fn caller_has_revoke_authority( state: &FrameworkControlState, roots: &BTreeMap, diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs new file mode 100644 index 00000000..44be2691 --- /dev/null +++ b/runtime/site-controller/src/http.rs @@ -0,0 +1,237 @@ +use std::{ + fs, + io::Write as _, + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; + +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use serde::{Deserialize, Serialize}; + +use super::{orchestration::ProtocolApiError, planner::ControlStateApp, *}; + +pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { + app.runtime.cleanup().await +} + +pub(super) fn required_header( + headers: &HeaderMap, + name: &str, +) -> std::result::Result { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + ProtocolApiError::unauthorized(format!( + "missing authenticated control-state request header `{name}`" + )) + }) +} + +const REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const REMOTE_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); + +fn should_retry_remote_controller_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT + ) +} + +pub(super) async fn post_json_with_retry( + client: &ReqwestClient, + url: &str, + body: &TReq, +) -> std::result::Result { + let deadline = tokio::time::Instant::now() + REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT; + loop { + match client.post(url).json(body).send().await { + Ok(response) + if should_retry_remote_controller_status(response.status()) + && tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Ok(response) => return Ok(response), + Err(_) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Err(err) => return Err(err), + } + } +} + +pub(super) async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("framework service should install Ctrl-C handler"); + }; + + #[cfg(unix)] + let terminate = async { + use tokio::signal::unix::{SignalKind, signal}; + + signal(SignalKind::terminate()) + .expect("framework service should install SIGTERM handler") + .recv() + .await; + }; + + #[cfg(unix)] + tokio::select! { + _ = ctrl_c => {} + _ = terminate => {} + } + + #[cfg(not(unix))] + ctrl_c.await; +} + +pub(super) fn write_json(path: &Path, value: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; + write_bytes_atomic(path, &bytes) +} + +pub(super) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("tmp"); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let tmp_path = path.with_file_name(format!(".{file_name}.tmp-{}-{nonce}", std::process::id())); + + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", tmp_path.display()))?; + if let Err(err) = file.write_all(bytes) { + let _ = fs::remove_file(&tmp_path); + return Err(miette::miette!( + "failed to write {}: {err}", + tmp_path.display() + )); + } + if let Err(err) = file.sync_all() { + let _ = fs::remove_file(&tmp_path); + return Err(miette::miette!( + "failed to sync {}: {err}", + tmp_path.display() + )); + } + drop(file); + + fs::rename(&tmp_path, path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to replace {} with {}", + path.display(), + tmp_path.display() + ) + })?; + sync_parent_directory(path)?; + Ok(()) +} + +pub(super) fn sync_parent_directory(path: &Path) -> Result<()> { + #[cfg(unix)] + { + if let Some(parent) = path.parent() { + fs::File::open(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to open parent directory {}", parent.display()))? + .sync_all() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to sync parent directory {}", parent.display()) + })?; + } + } + #[cfg(not(unix))] + { + let _ = path; + } + Ok(()) +} + +pub(super) fn read_json Deserialize<'de>>(path: &Path, label: &str) -> Result { + let bytes = fs::read(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {label} {}", path.display()))?; + serde_json::from_slice(&bytes) + .map_err(|err| miette::miette!("invalid {label} {}: {err}", path.display())) +} + +#[cfg(test)] +mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use axum::{ + Json, Router, + extract::State, + http::StatusCode, + response::{IntoResponse as _, Response}, + routing::post, + }; + use serde_json::json; + use tokio::net::TcpListener; + + use super::*; + + #[tokio::test] + async fn post_json_with_retry_tolerates_transient_service_unavailable() { + async fn handler(State(attempts): State>) -> Response { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + return (StatusCode::SERVICE_UNAVAILABLE, Json(json!({"ok": false}))) + .into_response(); + } + (StatusCode::OK, Json(json!({"ok": true}))).into_response() + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route("/retry", post(handler)) + .with_state(attempts.clone()); + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let serve = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("test server should run"); + }); + + let response = post_json_with_retry( + &ReqwestClient::new(), + &format!("http://{addr}/retry"), + &json!({"hello": "world"}), + ) + .await + .expect("request should eventually succeed"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + + serve.abort(); + let _ = serve.await; + } +} diff --git a/cli/src/framework_component/mod.rs b/runtime/site-controller/src/lib.rs similarity index 54% rename from cli/src/framework_component/mod.rs rename to runtime/site-controller/src/lib.rs index 2da16e1c..9ea7175a 100644 --- a/cli/src/framework_component/mod.rs +++ b/runtime/site-controller/src/lib.rs @@ -1,13 +1,11 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fs, future::Future, - io::Write as _, - net::{Ipv4Addr, SocketAddr}, + net::SocketAddr, path::{Path, PathBuf}, pin::Pin, sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::Duration, }; use amber_compiler::{ @@ -24,7 +22,7 @@ use amber_manifest::{ Manifest, ManifestRef, ManifestSpans, NetworkProtocol, RawBinding, RawExportTarget, SlotDecl, }; use amber_mesh::{ - MeshIdentity, MeshProtocol, + MeshProtocol, component_protocol::{ BindingInputDescription, ChildDescribeResponse, ChildHandle, ChildListResponse, ChildState, ChildSummary, ConfigFieldDescription, CreateChildRequest, CreateChildResponse, InputState, @@ -41,7 +39,7 @@ use amber_mesh::{ use amber_proxy::{ apply_route_overlay_with_retry, clear_external_slot_with_retry, register_export_peer_with_retry, register_external_slot_with_retry, - unregister_export_peer_with_retry, + revoke_route_overlay_with_retry, unregister_export_peer_with_retry, }; use amber_resolver::{Backend, RemoteResolver, Resolution, Resolver}; use amber_scenario::{ @@ -50,54 +48,67 @@ use amber_scenario::{ ir::{BindingFromIr, BindingIr, ComponentExportTargetIr, ComponentIr, ManifestCatalogEntryIr}, }; use axum::{ - Json, Router, - extract::{Path as AxumPath, State}, + Json, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, - routing::{get, post}, }; use base64::Engine as _; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use reqwest::Client as ReqwestClient; use serde::{Deserialize, Serialize}; use serde_json::json; -use tokio::{net::TcpListener, signal, sync::Mutex}; - -use crate::mixed_run::{ - BridgeProxyHandle, BridgeProxyKey, DesiredExportPeerOverlay, DesiredExternalSlotOverlay, - LaunchedSite, LiveComponentRuntimeMetadata, SiteActuatorPlan, SiteReceipt, - clear_desired_overlay_for_consumer, clear_desired_overlay_for_provider, - collect_live_component_runtime_metadata, host_service_bind_addr_for_consumer, - host_service_host_for_consumer, launched_site_from_receipt, load_live_site_router_mesh_config, - parse_control_endpoint, project_kubernetes_dynamic_child_artifact_files, - read_json as read_run_json, resolve_link_external_url_for_output, - router_mesh_addr_for_consumer, site_actuator_child_root_for_site, site_state_path, - stop_bridge_proxies, update_desired_overlay_for_consumer, update_desired_overlay_for_provider, -}; +use tokio::{signal, sync::Mutex}; mod api; mod ccs_api; mod control_state_api; mod control_state_mcp; +mod default_runtime; mod dynamic_caps; mod http; mod mcp; mod mcp_common; mod orchestration; mod planner; +mod runtime_api; +mod site_controller; mod state; #[cfg(test)] mod tests; -pub(crate) use self::{ - http::{run_framework_ccs, run_framework_control_state}, - orchestration::{ - SiteActuatorDestroyRequest, SiteActuatorPrepareRequest, SiteActuatorPublishRequest, +pub use self::{ + default_runtime::{ + SiteControllerPeerRouterRoute, assign_compose_egress_network_subnets, + cleanup_dynamic_site_children, host_service_bind_addr_for_consumer, + inject_compose_site_controller, inject_compose_site_controller_with_mount_sources, + inject_kubernetes_site_controller, inject_site_controller_peer_router_routes, + observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, reserve_host_port, + reserve_loopback_port, router_mesh_addr_for_consumer, + set_compose_router_published_mesh_port, set_site_artifact_mesh_identity_seed, + site_controller_peer_router_url, vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, + walk_files, + }, + runtime_api::{ + DesiredExportPeerOverlay, DesiredExternalSlotOverlay, DesiredRouteOverlay, LaunchedSite, + LiveComponentRuntimeMetadata, SiteControllerRuntimeFuture, SiteControllerRuntimePlan, + SiteReceipt, launched_site_from_receipt, parse_control_endpoint, + project_kubernetes_dynamic_child_artifact_files, site_controller_plan_path, + site_controller_runtime_child_root_for_site, + site_controller_runtime_plan_from_controller_plan, site_state_path, }, state::{ - DynamicInputRouteRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - DynamicSitePlanRecord, build_control_state, ccs_listen_addr_for_site, ccs_url_for_site, - control_state_service_url, generate_framework_auth_token, write_control_state, - write_control_state_service_plan, write_framework_ccs_plan, + DynamicInputDirectRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, + SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_MESH_IDENTITY_PATH, + SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, + build_site_controller_state, site_controller_internal_route_id, write_control_state, + write_site_controller_plan, }, }; + +pub async fn run_site_controller_default(plan_path: PathBuf) -> Result<()> { + site_controller::run_site_controller( + plan_path, + default_runtime::default_site_controller_runtime(), + ) + .await +} diff --git a/cli/src/framework_component/mcp.rs b/runtime/site-controller/src/mcp.rs similarity index 91% rename from cli/src/framework_component/mcp.rs rename to runtime/site-controller/src/mcp.rs index cb0ae3d1..0e7571b1 100644 --- a/cli/src/framework_component/mcp.rs +++ b/runtime/site-controller/src/mcp.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use amber_manifest::ManifestRef; +use amber_mesh::component_protocol::{CreateChildRequest, TemplateResolveRequest}; use axum::http::request::Parts; use rmcp::{ ErrorData as McpError, Json, RoleServer, ServerHandler, @@ -23,21 +24,22 @@ use serde_json::Value; use super::{ ccs_api::{ - self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, + FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, FrameworkComponentMutateRequest, FrameworkComponentMutateResponse, }, - http::*, - mcp_common::{McpOperationResponse, json_response, map_protocol_api_error, map_protocol_error}, - planner::CcsApp, - state::{CapabilityInstanceRecord, FrameworkControlState}, - *, + mcp_common::{McpOperationResponse, json_response, map_protocol_api_error}, + planner::SiteControllerApp, + site_controller::{ + authorize_public_request, execute_site_controller_framework_inspect, + execute_site_controller_framework_mutate, + }, }; const HELP_RESOURCE_URI: &str = "amber://framework-component"; const OPERATION_RESOURCE_PREFIX: &str = "amber://framework-component/op/"; pub(crate) fn service( - app: CcsApp, + app: SiteControllerApp, ) -> StreamableHttpService { StreamableHttpService::new( move || Ok(FrameworkComponentMcp::new(app.clone())), @@ -48,12 +50,12 @@ pub(crate) fn service( #[derive(Clone)] pub(crate) struct FrameworkComponentMcp { - app: CcsApp, + app: SiteControllerApp, tool_router: ToolRouter, } impl FrameworkComponentMcp { - fn new(app: CcsApp) -> Self { + fn new(app: SiteControllerApp) -> Self { Self { app, tool_router: Self::tool_router(), @@ -63,17 +65,23 @@ impl FrameworkComponentMcp { async fn authorize( &self, context: &RequestContext, - ) -> Result<(CapabilityInstanceRecord, FrameworkControlState), McpError> { + ) -> Result< + ( + super::state::CapabilityInstanceRecord, + super::state::FrameworkControlState, + ), + McpError, + > { let parts = context .extensions .get::() .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - authorize_request(&self.app, &parts.headers) + authorize_public_request(&self.app, &parts.headers) .await .map_err(map_protocol_api_error) } - fn render_help_resource(&self) -> String { + fn help_resource(&self) -> String { let mut out = String::from("# framework.component MCP\n\n"); out.push_str("Tools:\n"); out.push_str("- `amber.v1.framework_component.inspect`\n"); @@ -106,7 +114,7 @@ impl FrameworkComponentMcp { out } - fn render_operation_resource(&self, name: &str) -> Result { + fn operation_resource(&self, name: &str) -> Result { let doc = match name { "list_templates" => { "# `list_templates`\n\nTool: `amber.v1.framework_component.inspect`\n\nArguments:\n\ @@ -259,9 +267,10 @@ impl FrameworkComponentMcp { context: RequestContext, ) -> Result, McpError> { let (record, state) = self.authorize(&context).await?; - let response = ccs_api::execute_framework_component_inspect( + let response = execute_site_controller_framework_inspect( + &self.app, + &record, &state, - record.authority_realm_id, match args { InspectArgs::ListTemplates => FrameworkComponentInspectRequest::ListTemplates, InspectArgs::GetTemplate { template } => { @@ -281,7 +290,7 @@ impl FrameworkComponentMcp { }, ) .await - .map_err(map_protocol_error)?; + .map_err(map_protocol_api_error)?; match response { FrameworkComponentInspectResponse::ListTemplates(data) => { json_response("list_templates", data) @@ -311,10 +320,11 @@ impl FrameworkComponentMcp { Parameters(args): Parameters, context: RequestContext, ) -> Result, McpError> { - let (record, _) = self.authorize(&context).await?; - let response = ccs_api::execute_framework_component_mutate( + let (record, state) = self.authorize(&context).await?; + let response = execute_site_controller_framework_mutate( &self.app, - &record.cap_instance_id, + &record, + &state, match args { MutateArgs::CreateChild { template, @@ -374,7 +384,7 @@ impl ServerHandler for FrameworkComponentMcp { request: InitializeRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; if context.peer.peer_info().is_none() { context.peer.set_peer_info(request); } @@ -386,7 +396,7 @@ impl ServerHandler for FrameworkComponentMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; Ok(ListResourcesResult { resources: vec![ RawResource::new(HELP_RESOURCE_URI, "framework.component MCP").no_annotation(), @@ -401,7 +411,7 @@ impl ServerHandler for FrameworkComponentMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; Ok(ListResourceTemplatesResult { resource_templates: vec![ RawResourceTemplate::new( @@ -420,19 +430,18 @@ impl ServerHandler for FrameworkComponentMcp { request: ReadResourceRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; let uri = request.uri; let text = if uri.as_str() == HELP_RESOURCE_URI { - self.render_help_resource() + self.help_resource() } else if let Some(name) = uri.as_str().strip_prefix(OPERATION_RESOURCE_PREFIX) { - self.render_operation_resource(name)? + self.operation_resource(name)? } else { return Err(McpError::resource_not_found( - format!("resource {} not found", uri), + format!("resource {uri} not found"), None, )); }; - Ok(ReadResourceResult::new(vec![ResourceContents::text( text, uri, )])) diff --git a/cli/src/framework_component/mcp_common.rs b/runtime/site-controller/src/mcp_common.rs similarity index 100% rename from cli/src/framework_component/mcp_common.rs rename to runtime/site-controller/src/mcp_common.rs diff --git a/cli/src/framework_component/orchestration.rs b/runtime/site-controller/src/orchestration.rs similarity index 52% rename from cli/src/framework_component/orchestration.rs rename to runtime/site-controller/src/orchestration.rs index ad5f3ae9..85ecf85c 100644 --- a/cli/src/framework_component/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -1,37 +1,10 @@ use amber_mesh::{ - InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, OutboundRoute, - router_external_route_id, + InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, router_external_route_id, }; +use amber_proxy::{ControlEndpoint, fetch_router_identity}; use super::{http::*, planner::*, state::*, *}; -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPrepareRequest { - pub(crate) site_plan: DynamicSitePlanRecord, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPublishRequest { - pub(crate) site_plan: DynamicSitePlanRecord, -} - -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct SiteActuatorDestroyRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) desired_site_plan: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct ControlCreateChildRequest { - pub(super) cap_instance_id: String, - pub(super) request: CreateChildRequest, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct ControlDestroyChildRequest { - pub(super) cap_instance_id: String, -} - #[derive(Debug)] pub(super) struct ProtocolApiError(pub(super) ProtocolErrorResponse); @@ -111,6 +84,88 @@ impl IntoResponse for ProtocolApiError { } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ResolveExternalLinkUrlRequest { + pub(crate) child_id: u64, + pub(crate) link: RunLink, + pub(crate) consumer_kind: SiteKind, + pub(crate) provider_in_child: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ResolveExternalLinkUrlResponse { + pub(crate) external_url: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PublishExternalSlotOverlayRequest { + pub(crate) site_id: String, + pub(crate) overlay_id: String, + pub(crate) slot_name: String, + pub(crate) url: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ClearExternalSlotOverlayRequest { + pub(crate) site_id: String, + pub(crate) overlay_id: String, + pub(crate) slot_name: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PublishExportPeerOverlayRequest { + pub(crate) site_id: String, + pub(crate) overlay_id: String, + pub(crate) export_name: String, + pub(crate) peer_id: String, + pub(crate) peer_key_b64: String, + pub(crate) protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) route_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ClearExportPeerOverlayRequest { + pub(crate) site_id: String, + pub(crate) overlay_id: String, + pub(crate) export_name: String, + pub(crate) peer_id: String, + pub(crate) peer_key_b64: String, + pub(crate) protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) route_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RevokeDynamicCapabilityOriginOverlaysRequest { + pub(crate) site_id: String, + pub(crate) overlay_ids: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RemoteChildRuntimeRequest { + pub(crate) site_id: String, + pub(crate) state: FrameworkControlState, + pub(crate) child: LiveChildRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RemoteChildRollbackRequest { + pub(crate) site_id: String, + pub(crate) child_id: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RouterIdentityRequest { + pub(crate) site_id: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RouterIdentityResponse { + pub(crate) id: String, + pub(crate) public_key_b64: String, +} + pub(super) fn control_state_step_error( step: &str, err: impl std::fmt::Display, @@ -121,7 +176,7 @@ pub(super) fn control_state_step_error( ) } -pub(super) fn actuator_protocol_error( +pub(super) fn controller_protocol_error( code: ProtocolErrorCode, site_id: &str, action: &str, @@ -133,22 +188,6 @@ pub(super) fn actuator_protocol_error( ) } -pub(super) fn site_state_root_for(app: &ControlStateApp, site_id: &str) -> PathBuf { - Path::new(&app.state_root).join(site_id) -} - -pub(super) fn site_actuator_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { - site_state_root_for(app, site_id).join("site-actuator-plan.json") -} - -pub(super) fn framework_ccs_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { - site_state_root_for(app, site_id).join("framework-ccs-plan.json") -} - -pub(super) fn site_actuator_base_url(plan: &SiteActuatorPlan) -> String { - format!("http://{}", plan.listen_addr) -} - pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> SiteReceipt { SiteReceipt { kind: state.kind, @@ -161,144 +200,639 @@ pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> S context: state.context.clone(), router_control: state.router_control.clone(), router_mesh_addr: state.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: state.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: state.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), + site_controller_url: state.site_controller_url.clone(), } } -pub(super) fn full_site_plan_record( +pub(super) fn load_site_manager_state( + app: &ControlStateApp, site_id: &str, - site_plan: &amber_compiler::run_plan::RunSitePlan, -) -> DynamicSitePlanRecord { - DynamicSitePlanRecord { - site_id: site_id.to_string(), - kind: site_plan.site.kind, - router_identity_id: site_plan.router_identity_id.clone(), - component_ids: site_plan - .scenario_ir - .components - .iter() - .map(|component| component.id) - .collect(), - assigned_components: site_plan.assigned_components.clone(), - artifact_files: site_plan.artifact_files.clone(), - desired_artifact_files: site_plan.artifact_files.clone(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), +) -> std::result::Result { + if site_id == app.controller_plan.site_id { + return Ok(local_site_manager_state_view(app)); } + let state_path = site_state_path(&app.state_root, site_id); + if state_path.is_file() { + return read_json(&state_path, "site manager state").map_err(|err| { + protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!("site `{site_id}` manager state is unavailable: {err}"), + ) + }); + } + Err(protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!( + "site `{site_id}` manager state is unavailable: {}", + state_path.display() + ), + )) } -pub(super) fn desired_site_plan_map( - state: &FrameworkControlState, - site_ids: &BTreeSet, -) -> std::result::Result, ProtocolErrorResponse> { - let planned = build_site_plan_subset( - &decode_live_scenario(state)?, - &placement_file_from_state(state), - &run_plan_activation_from_state(state), - &live_assignment_map(state), - site_ids, - ) - .map_err(|err| { +fn local_site_manager_state_view(app: &ControlStateApp) -> SiteManagerStateView { + let runtime_plan = site_controller_runtime_plan_from_controller_plan(&app.controller_plan); + let persisted_state = + load_site_manager_state_at(Path::new(&app.controller_plan.site_state_root)).ok(); + let router_identity = app + .runtime + .load_live_site_router_mesh_config(&runtime_plan) + .ok() + .map(|router_mesh| { + ( + router_mesh.identity.id, + base64::engine::general_purpose::STANDARD.encode(router_mesh.identity.public_key), + ) + }); + SiteManagerStateView { + status: "running".to_string(), + kind: app.controller_plan.kind, + artifact_dir: app.controller_plan.artifact_dir.clone(), + supervisor_pid: 0, + process_pid: None, + compose_project: app.controller_plan.compose_project.clone(), + kubernetes_namespace: app.controller_plan.kubernetes_namespace.clone(), + port_forward_pid: None, + context: app.controller_plan.context.clone(), + router_control: app + .controller_plan + .local_router_control + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_control.clone()) + }), + router_mesh_addr: app + .controller_plan + .published_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_mesh_addr.clone()) + }), + compose_consumer_router_mesh_addr: app + .controller_plan + .compose_consumer_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.compose_consumer_router_mesh_addr.clone()) + }), + kubernetes_consumer_router_mesh_addr: app + .controller_plan + .kubernetes_consumer_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.kubernetes_consumer_router_mesh_addr.clone()) + }), + router_identity_id: router_identity + .as_ref() + .map(|(id, _)| id.clone()) + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_identity_id.clone()) + }), + router_public_key_b64: router_identity + .map(|(_, public_key_b64)| public_key_b64) + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_public_key_b64.clone()) + }), + site_controller_url: Some(app.controller_plan.authority_url.clone()), + } +} + +pub(super) fn load_launched_site( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + let state = load_site_manager_state(app, site_id)?; + if state.status != "running" { + return Err(protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!("site `{site_id}` is not running"), + )); + } + let receipt = site_receipt_from_manager_state(&state); + launched_site_from_receipt(&receipt, &app.mesh_scope).map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("failed to rebuild desired site plans for affected sites: {err}"), + &format!("site `{site_id}` routing metadata is invalid: {err}"), ) - })?; - Ok(planned - .iter() - .map(|(site_id, site_plan)| (site_id.clone(), full_site_plan_record(site_id, site_plan))) - .collect()) + }) } -pub(super) fn load_site_manager_state( +fn site_router_control_endpoint( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - read_run_json( - &site_state_path(&app.state_root, site_id), - "site manager state", - ) - .map_err(|err| { +) -> std::result::Result { + if site_id == app.controller_plan.site_id + && let Some(raw) = app.controller_plan.local_router_control.as_deref() + { + return parse_control_endpoint(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` router control endpoint is invalid: {err}"), + ) + }); + } + let state = load_site_manager_state(app, site_id)?; + let raw = state.router_control.ok_or_else(|| { protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` manager state is unavailable: {err}"), + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` router control endpoint is unavailable"), + ) + })?; + parse_control_endpoint(&raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` router control endpoint is invalid: {err}"), ) }) } -pub(super) fn load_launched_site( +pub(super) async fn router_identity_for_overlay( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { +) -> std::result::Result { + let execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, site_id)?.to_string() + }; + if execution_site != app.controller_plan.site_id { + let identity = remote_controller_post_json::<_, RouterIdentityResponse>( + app, + &execution_site, + "/v1/internal/router-identity", + &RouterIdentityRequest { + site_id: site_id.to_string(), + }, + ProtocolErrorCode::ControlStateUnavailable, + "read router identity", + ) + .await?; + return mesh_peer_from_router_identity_response(site_id, identity); + } + local_router_identity_for_overlay_site(app, site_id).await +} + +pub(super) async fn local_router_identity_for_overlay_site( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { let state = load_site_manager_state(app, site_id)?; - if state.status != "running" { - return Err(protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` is not running"), - )); + if let (Some(id), Some(public_key_b64)) = + (state.router_identity_id, state.router_public_key_b64) + && let Ok(decoded) = + base64::engine::general_purpose::STANDARD.decode(public_key_b64.as_bytes()) + && let Ok(public_key) = decoded.as_slice().try_into() + { + return Ok(MeshPeer { id, public_key }); } - let receipt = site_receipt_from_manager_state(&state); - launched_site_from_receipt(&receipt, &app.mesh_scope).map_err(|err| { + if let Ok(router_mesh) = app + .runtime + .load_live_site_router_mesh_config(&site_runtime_plan_from_control_app(app, site_id)?) + { + return Ok(MeshPeer { + id: router_mesh.identity.id, + public_key: router_mesh.identity.public_key, + }); + } + let identity = fetch_router_identity(&site_router_control_endpoint(app, site_id)?) + .await + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to read site `{site_id}` router identity: {err}"), + ) + })?; + Ok(MeshPeer { + id: identity.id, + public_key: identity.public_key, + }) +} + +fn mesh_peer_from_router_identity_response( + site_id: &str, + identity: RouterIdentityResponse, +) -> std::result::Result { + if let Ok(public_key) = + base64::engine::general_purpose::STANDARD.decode(identity.public_key_b64.as_bytes()) + && let Ok(public_key) = public_key.as_slice().try_into() + { + return Ok(MeshPeer { + id: identity.id, + public_key, + }); + } + Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` router identity public key is invalid"), + )) +} + +fn site_state_root_for_site<'a>( + app: &'a ControlStateApp, + site_id: &str, +) -> std::borrow::Cow<'a, Path> { + if site_id == app.controller_plan.site_id { + return std::borrow::Cow::Borrowed(Path::new(&app.controller_plan.site_state_root)); + } + std::borrow::Cow::Owned(Path::new(&app.state_root).join(site_id)) +} + +fn site_runtime_plan_from_control_app( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + crate::default_runtime::runtime_plan_for_site_from_controller_plan( + &app.controller_plan, + site_id, + ) + .map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("site `{site_id}` routing metadata is invalid: {err}"), + &format!("site `{site_id}` runtime plan is unavailable: {err}"), + ) + }) +} + +fn local_site_kind_from_state( + state: &FrameworkControlState, + site_id: &str, +) -> std::result::Result { + state + .placement + .offered_sites + .get(site_id) + .map(|site| site.kind) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` is missing from offered sites"), + ) + }) +} + +pub(super) fn site_execution_site_from_state<'a>( + state: &'a FrameworkControlState, + site_id: &'a str, +) -> std::result::Result<&'a str, ProtocolErrorResponse> { + let site = state.placement.offered_sites.get(site_id).ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` is missing from offered sites"), + ) + })?; + match site.kind { + SiteKind::Vm => site.controller_site.as_deref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("vm site `{site_id}` is missing its controller_site"), + ) + }), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + +pub(super) async fn resolve_external_link_url_local( + app: &ControlStateApp, + request: &ResolveExternalLinkUrlRequest, +) -> std::result::Result { + let provider = load_launched_site(app, &request.link.provider_site)?; + let provider_output_dir = provider_output_dir_for_request( + site_state_root_for_site(app, &request.link.provider_site).as_ref(), + request.child_id, + Path::new(&provider.receipt.artifact_dir), + request.provider_in_child, + ); + let external_url = app + .runtime + .resolve_link_external_url( + &provider, + &provider_output_dir, + &request.link, + request.consumer_kind, + &app.run_root, + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &request.link.provider_site, + "compute external slot overlay", + err, + ) + })?; + Ok(ResolveExternalLinkUrlResponse { external_url }) +} + +pub(super) async fn publish_external_slot_overlay_local( + app: &ControlStateApp, + request: &PublishExternalSlotOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + register_external_slot_with_retry( + &site_router_control_endpoint(app, &request.site_id)?, + &request.slot_name, + &request.url, + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &request.site_id, + "publish external slot overlay", + err, + ) + })?; + app.runtime + .update_desired_overlay_for_consumer( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + DesiredExternalSlotOverlay { + slot_name: request.slot_name.clone(), + url: request.url.clone(), + }, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "persist desired external slot overlay", + err, + ) + }) +} + +pub(super) async fn clear_external_slot_overlay_local( + app: &ControlStateApp, + request: &ClearExternalSlotOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .clear_desired_overlay_for_consumer( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "persist external slot overlay removal", + err, + ) + })?; + clear_external_slot_with_retry( + &site_router_control_endpoint(app, &request.site_id)?, + &request.slot_name, + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "retract external slot overlay", + err, + ) + }) +} + +pub(super) async fn publish_export_peer_overlay_local( + app: &ControlStateApp, + request: &PublishExportPeerOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + register_export_peer_with_retry( + &site_router_control_endpoint(app, &request.site_id)?, + &request.export_name, + &request.peer_id, + &request.peer_key_b64, + &request.protocol, + request.route_id.as_deref(), + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &request.site_id, + "publish export-peer overlay", + err, + ) + })?; + app.runtime + .update_desired_overlay_for_provider( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + DesiredExportPeerOverlay { + export_name: request.export_name.clone(), + peer_id: request.peer_id.clone(), + peer_key_b64: request.peer_key_b64.clone(), + protocol: request.protocol.clone(), + route_id: request.route_id.clone(), + }, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "persist desired export-peer overlay", + err, + ) + }) +} + +pub(super) async fn clear_export_peer_overlay_local( + app: &ControlStateApp, + request: &ClearExportPeerOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .clear_desired_overlay_for_provider( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "persist export-peer overlay removal", + err, + ) + })?; + unregister_export_peer_with_retry( + &site_router_control_endpoint(app, &request.site_id)?, + &request.export_name, + &request.peer_id, + &request.peer_key_b64, + &request.protocol, + request.route_id.as_deref(), + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "retract export-peer overlay", + err, ) }) } -pub(super) fn load_site_actuator_plan( +pub(super) async fn revoke_dynamic_capability_origin_overlays_local( + app: &ControlStateApp, + request: &RevokeDynamicCapabilityOriginOverlaysRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + let endpoint = site_router_control_endpoint(app, &request.site_id)?; + for overlay_id in &request.overlay_ids { + revoke_route_overlay_with_retry(&endpoint, overlay_id, Duration::from_secs(30)) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &request.site_id, + "retract dynamic capability origin overlays", + err, + ) + })?; + } + Ok(()) +} + +fn remote_controller_base_url( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - let path = site_actuator_plan_path_for_site(app, site_id); - read_json(&path, "site actuator plan").map_err(|err| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` actuator plan is unavailable: {err}"), - ) - }) +) -> std::result::Result { + if site_id == app.controller_plan.site_id { + return Ok(app.controller_plan.authority_url.clone()); + } + app.controller_plan + .peer_site_router_urls + .get(site_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site controller `{}` has no router-local controller route to site `{site_id}`", + app.controller_plan.site_id + ), + ) + }) } -pub(super) fn load_framework_ccs_plan( +async fn remote_controller_post_json Deserialize<'de>>( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - let path = framework_ccs_plan_path_for_site(app, site_id); - read_json(&path, "framework CCS plan").map_err(|err| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` framework CCS plan is unavailable: {err}"), - ) - }) + path: &str, + body: &TReq, + code: ProtocolErrorCode, + action: &str, +) -> std::result::Result { + let response = post_json_with_retry( + &app.client, + &format!( + "{}{}", + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), + path + ), + body, + ) + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if response.status().is_success() { + return response + .json() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err)); + } + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error); + } + Err(controller_protocol_error( + code, + site_id, + action, + format!("remote controller returned {status}"), + )) } -pub(super) fn framework_ccs_base_url(plan: &FrameworkCcsPlan) -> String { - format!("http://{}", plan.listen_addr) +async fn remote_controller_post_no_content( + app: &ControlStateApp, + site_id: &str, + path: &str, + body: &TReq, + code: ProtocolErrorCode, + action: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + let response = post_json_with_retry( + &app.client, + &format!( + "{}{}", + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), + path + ), + body, + ) + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if response.status().is_success() { + return Ok(()); + } + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error); + } + Err(controller_protocol_error( + code, + site_id, + action, + format!("remote controller returned {status}"), + )) } -pub(super) fn load_site_actuator_plan_at( +pub(super) fn load_site_runtime_plan_at( site_state_root: &Path, -) -> std::result::Result { - read_json( - &site_state_root.join("site-actuator-plan.json"), - "site actuator plan", +) -> std::result::Result { + let plan: SiteControllerPlan = read_json( + &site_controller_plan_path(site_state_root), + "site controller plan", ) .map_err(|err| { protocol_error( ProtocolErrorCode::SiteNotActive, &format!( - "site actuator plan under `{}` is unavailable: {err}", + "site controller plan under `{}` is unavailable: {err}", site_state_root.display() ), ) - }) + })?; + Ok(site_controller_runtime_plan_from_controller_plan(&plan)) } pub(super) fn load_site_manager_state_at( site_state_root: &Path, ) -> std::result::Result { - read_run_json( + read_json( &site_state_root.join("manager-state.json"), "site manager state", ) @@ -319,111 +853,32 @@ pub(super) async fn publish_dynamic_capability_origin( request: &dynamic_caps::PublishDynamicCapabilityOriginRequest, ) -> std::result::Result { - let plan = load_framework_ccs_plan(app, site_id)?; - let url = format!( - "{}/v1/internal/dynamic-caps/origins/publish", - framework_ccs_base_url(&plan).trim_end_matches('/') - ); - let response = app - .client - .post(url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .json(request) - .send() - .await - .map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to reach framework CCS on site `{site_id}` while publishing dynamic \ - capability origin: {err}" - ), - ) - })?; - if response.status().is_success() { - return response.json().await.map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "framework CCS on site `{site_id}` returned invalid JSON while publishing \ - dynamic capability origin: {err}" - ), - ) - }); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to read framework CCS error response on site `{site_id}` while publishing \ - dynamic capability origin: {err}" - ), + if site_id == app.controller_plan.site_id { + return publish_dynamic_capability_origin_local( + &LocalDynamicCapabilityOriginApp { + site_state_root: PathBuf::from(&app.controller_plan.site_state_root), + runtime: app.runtime.clone(), + }, + request.clone(), ) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error); + .await + .map_err(|err| err.0); } - Err(protocol_error( + remote_controller_post_json( + app, + site_id, + "/v1/internal/dynamic-caps/origins/publish", + request, ProtocolErrorCode::OriginUnavailable, - &format!( - "framework CCS on site `{site_id}` returned {status} while publishing dynamic \ - capability origin" - ), - )) -} - -#[derive(Clone, Debug)] -pub(super) struct DynamicCapabilityRuntimeEndpoint { - pub(super) site_id: String, - pub(super) runtime: LiveComponentRuntimeMetadata, -} - -pub(super) fn dynamic_capability_component_runtime_endpoint( - app: &ControlStateApp, - state: &FrameworkControlState, - logical_component_id: &str, -) -> std::result::Result { - let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; - let assignments = live_assignment_map(state); - let site_id = assignments.get(moniker).ok_or_else(|| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!("live component `{logical_component_id}` is not assigned to a live site"), - ) - })?; - let site_plan = load_site_actuator_plan(app, site_id)?; - let runtime = collect_live_component_runtime_metadata(&site_plan) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to resolve live runtime metadata for component \ - `{logical_component_id}` on site `{site_id}`: {err}" - ), - ) - })? - .remove(moniker) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "live runtime metadata for component `{logical_component_id}` is unavailable \ - on site `{site_id}`" - ), - ) - })?; - Ok(DynamicCapabilityRuntimeEndpoint { - site_id: site_id.clone(), - runtime, - }) + "publish dynamic capability origin", + ) + .await } pub(super) fn dynamic_capability_origin_route_surface( runtime: &LiveComponentRuntimeMetadata, site_components: &BTreeMap, site_router: &MeshConfigPublic, - site_kind: SiteKind, route_id: &str, root_authority_selector: &RootAuthoritySelectorIr, allowed_issuers: Vec, @@ -455,7 +910,7 @@ pub(super) fn dynamic_capability_origin_route_surface( protocol: static_route.protocol, http_plugins: static_route.http_plugins.clone(), target: InboundTarget::MeshForward { - peer_addr: runtime.host_mesh_addr.clone(), + peer_addr: runtime.router_reachable_mesh_addr.clone(), peer_id: runtime.mesh_config.identity.id.clone(), route_id: static_route.route_id.clone(), capability: static_route.capability.clone(), @@ -505,8 +960,15 @@ pub(super) fn dynamic_capability_origin_route_surface( ), })? .expect("self-provide roots should be handled before outbound route resolution"); - let peer_addr = - dynamic_capability_origin_mesh_peer_addr(site_components, site_kind, route); + let peer_addr = if route.peer_id == runtime.mesh_config.identity.id { + runtime.router_reachable_mesh_addr.clone() + } else { + site_components + .values() + .find(|component| component.mesh_config.identity.id == route.peer_id) + .map(|component| component.router_reachable_mesh_addr.clone()) + .unwrap_or_else(|| route.peer_addr.clone()) + }; Ok(( InboundRoute { route_id: route_id.to_string(), @@ -560,28 +1022,6 @@ pub(super) fn dynamic_capability_origin_route_surface( } } -fn dynamic_capability_origin_mesh_peer_addr( - site_components: &BTreeMap, - site_kind: SiteKind, - route: &OutboundRoute, -) -> String { - if let Some(peer_runtime) = site_components - .values() - .find(|component| component.mesh_config.identity.id == route.peer_id) - { - return peer_runtime.host_mesh_addr.clone(); - } - if matches!(site_kind, SiteKind::Direct | SiteKind::Vm) { - #[cfg(target_os = "linux")] - if let Ok(addr) = route.peer_addr.parse::() - && addr.ip() == Ipv4Addr::new(10, 0, 2, 2) - { - return SocketAddr::from((Ipv4Addr::LOCALHOST, addr.port())).to_string(); - } - } - route.peer_addr.clone() -} - pub(super) fn dynamic_capability_origin_target_mesh_peer( runtime: &LiveComponentRuntimeMetadata, site_components: &BTreeMap, @@ -654,10 +1094,10 @@ pub(super) fn dynamic_capability_allowed_mesh_peers( } pub(super) async fn publish_dynamic_capability_origin_local( - app: &CcsApp, + app: &LocalDynamicCapabilityOriginApp, request: dynamic_caps::PublishDynamicCapabilityOriginRequest, ) -> std::result::Result { - let site_plan = load_site_actuator_plan_at(&app.site_state_root)?; + let site_plan = load_site_runtime_plan_at(&app.site_state_root)?; let holder_component_id = match &request.root_authority_selector { RootAuthoritySelectorIr::SelfProvide { component_id, .. } => component_id.clone(), RootAuthoritySelectorIr::Binding { @@ -670,8 +1110,10 @@ pub(super) async fn publish_dynamic_capability_origin_local( } => consumer_component_id.clone(), }; let holder_moniker = dynamic_caps::moniker_from_logical_component_id(&holder_component_id)?; - let mut site_components = - collect_live_component_runtime_metadata(&site_plan).map_err(|err| { + let mut site_components = app + .runtime + .collect_live_component_runtime_metadata(&site_plan) + .map_err(|err| { protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( @@ -680,15 +1122,18 @@ pub(super) async fn publish_dynamic_capability_origin_local( ), ) })?; - let site_router = load_live_site_router_mesh_config(&site_plan).map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to collect live router metadata on site `{}`: {err}", - site_plan.site_id - ), - ) - })?; + let site_router = app + .runtime + .load_live_site_router_mesh_config(&site_plan) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "failed to collect live router metadata on site `{}`: {err}", + site_plan.site_id + ), + ) + })?; let runtime = site_components.remove(holder_moniker).ok_or_else(|| { protocol_error( ProtocolErrorCode::OriginUnavailable, @@ -727,7 +1172,6 @@ pub(super) async fn publish_dynamic_capability_origin_local( &runtime, &site_components, &site_router, - site_plan.kind, &request.route_id, &request.root_authority_selector, allowed_issuers, @@ -767,142 +1211,220 @@ pub(super) async fn publish_dynamic_capability_origin_local( }) } -pub(super) async fn call_site_actuator( +pub(super) async fn prepare_child_on_site( app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, site_id: &str, - path: &str, - body: Option<&B>, - error_code: ProtocolErrorCode, - action: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let plan = load_site_actuator_plan(app, site_id)?; - let url = format!("{}{}", site_actuator_base_url(&plan), path); - let request = app.client.post(url); - let request = if let Some(body) = body { - request.json(body) - } else { - request - }; - let response = request - .send() - .await - .map_err(|err| actuator_protocol_error(error_code, site_id, action, err))?; - if response.status().is_success() { - return Ok(()); + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( + app, + execution_site, + "/v1/internal/children/prepare", + &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), + state: state.clone(), + child: child.clone(), + }, + ProtocolErrorCode::PrepareFailed, + "prepare child", + ) + .await; } - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - Err(actuator_protocol_error( - error_code, - site_id, - action, - format!("HTTP {status}: {}", body.trim()), - )) + prepare_child_on_local_site(app, state, child, site_id).await } -pub(super) async fn prepare_child_on_site( +pub(super) async fn prepare_child_on_local_site( app: &ControlStateApp, - child_id: u64, - site_plan: &DynamicSitePlanRecord, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/prepare"); - call_site_actuator( - app, - &site_plan.site_id, - &path, - Some(&SiteActuatorPrepareRequest { - site_plan: site_plan.clone(), - }), - ProtocolErrorCode::PrepareFailed, - "prepare child", - ) - .await + app.runtime + .prepare_child(&app.controller_plan, state.clone(), child.clone(), site_id) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PrepareFailed, + site_id, + "prepare child", + err, + ) + }) } pub(super) async fn publish_child_on_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( + app, + execution_site, + "/v1/internal/children/publish", + &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), + state: state.clone(), + child: child.clone(), + }, + ProtocolErrorCode::PublishFailed, + "publish child", + ) + .await; + } + publish_child_on_local_site(app, state, child, site_id).await +} + +pub(super) async fn publish_child_on_local_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .publish_child(&app.controller_plan, state.clone(), child.clone(), site_id) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + site_id, + "publish child", + err, + ) + }) +} + +fn site_controller_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u64) -> bool { + err.code == ProtocolErrorCode::PublishFailed + && err + .message + .contains(&format!("child {child_id} is not prepared")) +} + +pub(super) async fn publish_child_on_site_with_prepare_retry( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + match publish_child_on_site(app, state, child, site_id).await { + Ok(()) => Ok(()), + Err(err) if site_controller_child_needs_prepare(&err, child.child_id) => { + prepare_child_on_site(app, state, child, site_id).await?; + publish_child_on_site(app, state, child, site_id).await + } + Err(err) => Err(err), + } +} + +pub(super) async fn rollback_child_on_site( app: &ControlStateApp, child_id: u64, - site_plan: &DynamicSitePlanRecord, + site_id: &str, +) -> Result<()> { + let state = app.control_state.lock().await.clone(); + let execution_site = site_execution_site_from_state(&state, site_id) + .map_err(|err| miette::miette!(err.message))?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( + app, + execution_site, + "/v1/internal/children/rollback", + &RemoteChildRollbackRequest { + site_id: site_id.to_string(), + child_id, + }, + ProtocolErrorCode::ControlStateUnavailable, + "rollback child", + ) + .await + .map_err(|err| miette::miette!(err.message)); + } + app.runtime + .rollback_child(&app.controller_plan, child_id, site_id) + .await +} + +pub(super) async fn destroy_child_on_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/publish"); - call_site_actuator( - app, - &site_plan.site_id, - &path, - Some(&SiteActuatorPublishRequest { - site_plan: site_plan.clone(), - }), - ProtocolErrorCode::PublishFailed, - "publish child", - ) - .await + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( + app, + execution_site, + "/v1/internal/children/destroy", + &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), + state: state.clone(), + child: child.clone(), + }, + ProtocolErrorCode::ControlStateUnavailable, + "destroy child", + ) + .await; + } + destroy_child_on_local_site(app, state, child, site_id).await } -fn site_actuator_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u64) -> bool { - err.code == ProtocolErrorCode::PublishFailed - && err - .message - .contains(&format!("site actuator child {child_id} is not prepared")) +pub(super) async fn destroy_child_on_local_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .destroy_child(&app.controller_plan, state.clone(), child.clone(), site_id) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + site_id, + "destroy child", + err, + ) + }) } -pub(super) async fn publish_child_on_site_with_prepare_retry( +pub(super) async fn prepare_child_on_sites( app: &ControlStateApp, - child_id: u64, - site_plan: &DynamicSitePlanRecord, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - match publish_child_on_site(app, child_id, site_plan).await { - Ok(()) => Ok(()), - Err(err) if site_actuator_child_needs_prepare(&err, child_id) => { - prepare_child_on_site(app, child_id, site_plan).await?; - publish_child_on_site(app, child_id, site_plan).await - } - Err(err) => Err(err), + for site_id in child_runtime_site_ids(child)? { + prepare_child_on_site(app, state, child, &site_id).await?; } + Ok(()) } -pub(super) async fn rollback_child_on_site( +pub(super) async fn publish_child_on_sites_with_prepare_retry( app: &ControlStateApp, - child_id: u64, - site_id: &str, -) -> Result<()> { - let path = format!("/v1/children/{child_id}/rollback"); - let plan = load_site_actuator_plan(app, site_id) - .map_err(|err| miette::miette!("failed to load site actuator plan: {}", err.message))?; - let url = format!("{}{}", site_actuator_base_url(&plan), path); - let response = app - .client - .post(url) - .send() - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to rollback child on site `{site_id}`"))?; - if response.status().is_success() { - return Ok(()); + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + for site_id in child_runtime_site_ids(child)? { + publish_child_on_site_with_prepare_retry(app, state, child, &site_id).await?; } - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - Err(miette::miette!( - "failed to rollback child on site `{site_id}`: HTTP {status}: {}", - body.trim() - )) + Ok(()) } -pub(super) async fn destroy_child_on_site( +pub(super) async fn destroy_child_on_sites( app: &ControlStateApp, - child_id: u64, - site_id: &str, - desired_site_plan: Option, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/destroy"); - call_site_actuator( - app, - site_id, - &path, - Some(&SiteActuatorDestroyRequest { desired_site_plan }), - ProtocolErrorCode::ControlStateUnavailable, - "destroy child", - ) - .await + for site_id in child_runtime_site_ids(child)? { + destroy_child_on_site(app, state, child, &site_id).await?; + } + Ok(()) } pub(super) async fn publish_external_slot_overlay( @@ -913,61 +1435,55 @@ pub(super) async fn publish_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let provider_output_dir = - provider_output_dir_for_link(app, child, Path::new(&provider.receipt.artifact_dir), link); - let external_url = { - let mut bridge_proxies = app.bridge_proxies.lock().await; - resolve_link_external_url_for_output( - &provider, - &provider_output_dir, - link, - consumer.receipt.kind, - &app.run_root, - &mut bridge_proxies, + let (consumer_kind, provider_execution_site, consumer_execution_site) = { + let state = app.control_state.lock().await; + ( + local_site_kind_from_state(&state, &link.consumer_site)?, + site_execution_site_from_state(&state, &link.provider_site)?.to_string(), + site_execution_site_from_state(&state, &link.consumer_site)?.to_string(), ) - .await - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::PublishFailed, - &link.consumer_site, - "compute external slot overlay", - err, - ) - })? }; - register_external_slot_with_retry( - &consumer.router_control, - &link.external_slot_name, - &external_url, - Duration::from_secs(30), - ) - .await - .map_err(|err| { - actuator_protocol_error( + let resolve_request = ResolveExternalLinkUrlRequest { + child_id: child.child_id, + link: link.clone(), + consumer_kind, + provider_in_child: provider_in_child_for_link(child, link), + }; + let external_url = if provider_execution_site == app.controller_plan.site_id { + resolve_external_link_url_local(app, &resolve_request) + .await? + .external_url + } else { + remote_controller_post_json::<_, ResolveExternalLinkUrlResponse>( + app, + &provider_execution_site, + "/v1/internal/link-overlays/external-url", + &resolve_request, ProtocolErrorCode::PublishFailed, - &link.consumer_site, - "publish external slot overlay", - err, + "resolve provider link url", ) - })?; - update_desired_overlay_for_consumer( - &site_state_root_for(app, &link.consumer_site), - overlay_id, - DesiredExternalSlotOverlay { - slot_name: link.external_slot_name.clone(), - url: external_url, - }, - ) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, - "persist desired external slot overlay", - err, + .await? + .external_url + }; + let publish_request = PublishExternalSlotOverlayRequest { + site_id: link.consumer_site.clone(), + overlay_id: overlay_id.to_string(), + slot_name: link.external_slot_name.clone(), + url: external_url, + }; + if consumer_execution_site == app.controller_plan.site_id { + publish_external_slot_overlay_local(app, &publish_request).await + } else { + remote_controller_post_no_content( + app, + &consumer_execution_site, + "/v1/internal/link-overlays/external-slot/publish", + &publish_request, + ProtocolErrorCode::PublishFailed, + "publish external slot overlay", ) - }) + .await + } } pub(super) async fn publish_export_peer_overlay( @@ -975,51 +1491,41 @@ pub(super) async fn publish_export_peer_overlay( child: &LiveChildRecord, link: &RunLink, ) -> std::result::Result<(), ProtocolErrorResponse> { + if link.provider_site == link.consumer_site { + return Ok(()); + } let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let consumer_key = - base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); let route_id = export_peer_route_id(child, link)?; - register_export_peer_with_retry( - &provider.router_control, - &link.export_name, - &consumer.router_identity.id, - &consumer_key, - &link.protocol.to_string(), - Some(&route_id), - Duration::from_secs(30), - ) - .await - .map_err(|err| { - actuator_protocol_error( + let provider_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.provider_site)?.to_string() + }; + let consumer_identity = router_identity_for_overlay(app, &link.consumer_site).await?; + let publish_request = PublishExportPeerOverlayRequest { + site_id: link.provider_site.clone(), + overlay_id: overlay_id.to_string(), + export_name: link.export_name.clone(), + peer_id: consumer_identity.id, + peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(consumer_identity.public_key), + protocol: link.protocol.to_string(), + route_id: Some(route_id), + }; + if provider_execution_site == app.controller_plan.site_id { + publish_export_peer_overlay_local(app, &publish_request).await + } else { + remote_controller_post_no_content( + app, + &provider_execution_site, + "/v1/internal/link-overlays/export-peer/publish", + &publish_request, ProtocolErrorCode::PublishFailed, - &link.provider_site, "publish export-peer overlay", - err, - ) - })?; - update_desired_overlay_for_provider( - &site_state_root_for(app, &link.provider_site), - overlay_id, - DesiredExportPeerOverlay { - export_name: link.export_name.clone(), - peer_id: consumer.router_identity.id, - peer_key_b64: consumer_key, - protocol: link.protocol.to_string(), - route_id: Some(route_id), - }, - ) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, - "persist desired export-peer overlay", - err, ) - }) + .await + } } pub(super) fn child_link_records(child: &LiveChildRecord) -> Vec { @@ -1066,26 +1572,25 @@ pub(super) fn link_still_required( }) } -pub(super) fn provider_output_dir_for_link( - app: &ControlStateApp, - child: &LiveChildRecord, +pub(super) fn provider_output_dir_for_request( + site_state_root: &Path, + child_id: u64, provider_artifact_dir: &Path, - link: &RunLink, + provider_in_child: bool, ) -> PathBuf { - let provider_in_child = child.fragment.as_ref().is_some_and(|fragment| { + if !provider_in_child { + return provider_artifact_dir.to_path_buf(); + } + site_controller_runtime_child_root_for_site(site_state_root, child_id).join("artifact") +} + +pub(super) fn provider_in_child_for_link(child: &LiveChildRecord, link: &RunLink) -> bool { + child.fragment.as_ref().is_some_and(|fragment| { fragment .components .iter() .any(|component| component.moniker == link.provider_component) - }); - if !provider_in_child { - return provider_artifact_dir.to_path_buf(); - } - site_actuator_child_root_for_site( - &site_state_root_for(app, &link.provider_site), - child.child_id, - ) - .join("artifact") + }) } pub(super) fn export_peer_route_id( @@ -1101,12 +1606,7 @@ pub(super) fn export_peer_route_id( ), ) })?; - let provider_in_child = child.fragment.as_ref().is_some_and(|fragment| { - fragment - .components - .iter() - .any(|component| component.moniker == link.provider_component) - }); + let provider_in_child = provider_in_child_for_link(child, link); Ok(if provider_in_child { router_dynamic_export_route_id(&link.provider_component, &link.export_name, protocol) } else { @@ -1176,36 +1676,34 @@ pub(super) async fn clear_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - clear_desired_overlay_for_consumer(&site_state_root_for(app, &link.consumer_site), overlay_id) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, - "persist external slot overlay removal", - err, - ) - })?; { let state = app.control_state.lock().await; if link_still_required(&state, child_id, link) { return Ok(()); } } - let consumer = load_launched_site(app, &link.consumer_site)?; - clear_external_slot_with_retry( - &consumer.router_control, - &link.external_slot_name, - Duration::from_secs(30), - ) - .await - .map_err(|err| { - actuator_protocol_error( + let consumer_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.consumer_site)?.to_string() + }; + let clear_request = ClearExternalSlotOverlayRequest { + site_id: link.consumer_site.clone(), + overlay_id: overlay_id.to_string(), + slot_name: link.external_slot_name.clone(), + }; + if consumer_execution_site == app.controller_plan.site_id { + clear_external_slot_overlay_local(app, &clear_request).await + } else { + remote_controller_post_no_content( + app, + &consumer_execution_site, + "/v1/internal/link-overlays/external-slot/clear", + &clear_request, ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, "retract external slot overlay", - err, ) - }) + .await + } } pub(super) async fn clear_export_peer_overlay( @@ -1214,57 +1712,47 @@ pub(super) async fn clear_export_peer_overlay( child: &LiveChildRecord, link: &RunLink, ) -> std::result::Result<(), ProtocolErrorResponse> { + if link.provider_site == link.consumer_site { + return Ok(()); + } let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; - clear_desired_overlay_for_provider(&site_state_root_for(app, &link.provider_site), overlay_id) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, - "persist export-peer overlay removal", - err, - ) - })?; { let state = app.control_state.lock().await; if link_still_required(&state, child_id, link) { return Ok(()); } } - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let consumer_key = - base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); let route_id = export_peer_route_id(child, link)?; - unregister_export_peer_with_retry( - &provider.router_control, - &link.export_name, - &consumer.router_identity.id, - &consumer_key, - &link.protocol.to_string(), - Some(&route_id), - Duration::from_secs(30), - ) - .await - .map_err(|err| { - actuator_protocol_error( + let provider_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.provider_site)?.to_string() + }; + let consumer_identity = router_identity_for_overlay(app, &link.consumer_site).await?; + let clear_request = ClearExportPeerOverlayRequest { + site_id: link.provider_site.clone(), + overlay_id: overlay_id.to_string(), + export_name: link.export_name.clone(), + peer_id: consumer_identity.id, + peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(consumer_identity.public_key), + protocol: link.protocol.to_string(), + route_id: Some(route_id), + }; + if provider_execution_site == app.controller_plan.site_id { + clear_export_peer_overlay_local(app, &clear_request).await + } else { + remote_controller_post_no_content( + app, + &provider_execution_site, + "/v1/internal/link-overlays/export-peer/clear", + &clear_request, ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, "retract export-peer overlay", - err, ) - }) -} - -pub(super) async fn publish_child_overlays( - app: &ControlStateApp, - child: &LiveChildRecord, -) -> std::result::Result<(), ProtocolErrorResponse> { - for link in child_link_records(child) { - publish_link_overlays(app, child, &link).await?; + .await } - Ok(()) } pub(super) async fn retract_child_overlays( @@ -1277,77 +1765,73 @@ pub(super) async fn retract_child_overlays( Ok(()) } -pub(super) fn child_site_publish_waves(child: &LiveChildRecord) -> Vec> { - let site_ids = child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect::>(); - let mut incoming = site_ids - .iter() - .map(|site_id| (site_id.clone(), BTreeSet::::new())) - .collect::>(); - let mut outgoing = site_ids - .iter() - .map(|site_id| (site_id.clone(), BTreeSet::::new())) - .collect::>(); - for link in child_link_records(child) { - if link.weak || link.provider_site == link.consumer_site { - continue; - } - if !site_ids.contains(&link.provider_site) || !site_ids.contains(&link.consumer_site) { - continue; - } - incoming - .get_mut(&link.consumer_site) - .expect("consumer site should be tracked") - .insert(link.provider_site.clone()); - outgoing - .get_mut(&link.provider_site) - .expect("provider site should be tracked") - .insert(link.consumer_site.clone()); - } - - let mut ready = incoming - .iter() - .filter(|(_, deps)| deps.is_empty()) - .map(|(site_id, _)| site_id.clone()) - .collect::>(); - let mut waves = Vec::new(); - let mut scheduled = BTreeSet::new(); - while !ready.is_empty() { - let wave = ready.iter().cloned().collect::>(); - let mut next_ready = BTreeSet::new(); - for site_id in &wave { - if !scheduled.insert(site_id.clone()) { +pub(super) async fn retract_dynamic_capability_origin_overlays( + app: &ControlStateApp, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + let overlays_by_site = { + let state = app.control_state.lock().await; + let Some(fragment) = child.fragment.as_ref() else { + return Ok(()); + }; + let mut overlays_by_site = BTreeMap::>::new(); + for component in &fragment.components { + if component.program.is_none() { continue; } - for consumer in outgoing - .get(site_id) - .into_iter() - .flat_map(|sites| sites.iter()) - { - let deps = incoming - .get_mut(consumer) - .expect("consumer dependencies should be tracked"); - deps.remove(site_id); - if deps.is_empty() { - next_ready.insert(consumer.clone()); - } + let holder_component_id = dynamic_caps::logical_component_id(&component.moniker); + for held in dynamic_caps::live_held_entries(&state, &holder_component_id)? { + let selector = if let Some(selector) = held.root_authority_selector.clone() { + selector + } else if let Some(grant_id) = held.grant_id.as_deref() { + state + .dynamic_capability_grants + .get(grant_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("dynamic grant `{grant_id}` is missing from state"), + ) + })? + .root_authority_selector + .clone() + } else { + continue; + }; + let site_id = site_id_for_root_authority_selector(&state, &selector)?; + overlays_by_site.entry(site_id).or_default().insert( + dynamic_caps::origin_overlay_id(&holder_component_id, &selector), + ); } } - waves.push(wave); - ready = next_ready; - } + overlays_by_site + }; - let remaining = site_ids - .into_iter() - .filter(|site_id| scheduled.insert(site_id.clone())) - .collect::>(); - if !remaining.is_empty() { - waves.push(remaining); + for (site_id, overlay_ids) in overlays_by_site { + let execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &site_id)?.to_string() + }; + let revoke_request = RevokeDynamicCapabilityOriginOverlaysRequest { + site_id: site_id.clone(), + overlay_ids: overlay_ids.into_iter().collect(), + }; + if execution_site == app.controller_plan.site_id { + revoke_dynamic_capability_origin_overlays_local(app, &revoke_request).await?; + } else { + remote_controller_post_no_content( + app, + &execution_site, + "/v1/internal/dynamic-caps/origins/revoke", + &revoke_request, + ProtocolErrorCode::ControlStateUnavailable, + "retract dynamic capability origin overlays", + ) + .await?; + } } - waves + + Ok(()) } pub(super) fn cloned_child_record( @@ -1365,13 +1849,13 @@ pub(super) fn cloned_child_record( }) } -pub(super) async fn rollback_prepared_sites( - app: &ControlStateApp, - child_id: u64, - prepared_sites: &[String], -) -> Result<()> { - for site_id in prepared_sites { - rollback_child_on_site(app, child_id, site_id).await?; +pub(super) async fn rollback_prepared_sites(app: &ControlStateApp, child_id: u64) -> Result<()> { + let child = { + let state = app.control_state.lock().await; + cloned_child_record(&state, child_id).map_err(|err| miette::miette!(err.message))? + }; + for site_id in child_runtime_site_ids(&child).map_err(|err| miette::miette!(err.message))? { + rollback_child_on_site(app, child_id, &site_id).await?; } Ok(()) } @@ -1403,63 +1887,67 @@ pub(super) async fn continue_create_committed_hidden( child }; - let site_plans = child - .site_plans - .iter() - .map(|site_plan| (site_plan.site_id.clone(), site_plan.clone())) - .collect::>(); - let child_sites = site_plans.keys().cloned().collect::>(); - let links = child_link_records(&child); - let mut published_child_sites = BTreeSet::new(); - for wave in child_site_publish_waves(&child) { - for site_id in &wave { - for link in links.iter().filter(|link| link.consumer_site == *site_id) { - let provider_ready = !child_sites.contains(&link.provider_site) - || published_child_sites.contains(&link.provider_site); - if provider_ready { - publish_link_overlays(app, &child, link).await?; - } - } + let state = app.control_state.lock().await.clone(); + if let Err(err) = publish_child_on_sites_with_prepare_retry(app, &state, &child).await { + let cleanup_error = destroy_child_on_sites(app, &state, &child).await.err(); + if let Some(cleanup_error) = cleanup_error { + return Err(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), + )); } - let mut publish_tasks = tokio::task::JoinSet::new(); - for site_id in &wave { - let app = app.clone(); - let child_id = child.child_id; - let site_plan = site_plans - .get(site_id) - .expect("site plan should exist for wave site") - .clone(); - let site_id = site_id.clone(); - publish_tasks.spawn(async move { - publish_child_on_site_with_prepare_retry(&app, child_id, &site_plan) - .await - .map(|_| site_id) - }); + let mut state = app.control_state.lock().await; + if child_record_location(&state, child.child_id).is_ok() { + let tx_id = child_create_tx_id(&state, child.child_id)?; + persist_control_state_update(&mut state, &app.state_path, "create_aborted", |state| { + append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); + remove_child_record(state, child.child_id)?; + Ok(()) + })?; } - let mut first_error = None; - let mut published_wave_sites = Vec::new(); - while let Some(result) = publish_tasks.join_next().await { - match result { - Ok(Ok(site_id)) => published_wave_sites.push(site_id), - Ok(Err(err)) if first_error.is_none() => first_error = Some(err), - Ok(Err(_)) => {} - Err(err) if first_error.is_none() => { - first_error = Some(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("site publish task failed: {err}"), - )); + return Err(err); + } + let published_links = child_link_records(&child); + let mut applied_links = Vec::new(); + for link in &published_links { + if let Err(err) = publish_link_overlays(app, &child, link).await { + let mut cleanup_error = None; + for published_link in applied_links.iter().rev() { + if let Err(retract_err) = retract_link_overlays(app, &child, published_link).await + && cleanup_error.is_none() + { + cleanup_error = Some(retract_err); } - Err(_) => {} } - } - if let Some(err) = first_error { + if let Err(destroy_err) = destroy_child_on_sites(app, &state, &child).await + && cleanup_error.is_none() + { + cleanup_error = Some(destroy_err); + } + if let Some(cleanup_error) = cleanup_error { + return Err(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), + )); + } + let mut state = app.control_state.lock().await; + if child_record_location(&state, child.child_id).is_ok() { + let tx_id = child_create_tx_id(&state, child.child_id)?; + persist_control_state_update( + &mut state, + &app.state_path, + "create_aborted", + |state| { + append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); + remove_child_record(state, child.child_id)?; + Ok(()) + }, + )?; + } return Err(err); } - for site_id in published_wave_sites { - published_child_sites.insert(site_id); - } + applied_links.push(link.clone()); } - publish_child_overlays(app, &child).await?; let mut state = app.control_state.lock().await; let child = cloned_child_record(&state, child_id)?; @@ -1489,43 +1977,8 @@ pub(super) async fn continue_destroy_retracted( } child }; - let desired_site_plans = { - let state = app.control_state.lock().await; - desired_site_plan_map( - &state, - &child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect(), - )? - }; - let mut destroy_tasks = tokio::task::JoinSet::new(); - for site_plan in child.site_plans.clone() { - let app = app.clone(); - let desired_site_plan = desired_site_plans.get(&site_plan.site_id).cloned(); - destroy_tasks.spawn(async move { - destroy_child_on_site(&app, child.child_id, &site_plan.site_id, desired_site_plan).await - }); - } - let mut first_error = None; - while let Some(result) = destroy_tasks.join_next().await { - match result { - Ok(Ok(())) => {} - Ok(Err(err)) if first_error.is_none() => first_error = Some(err), - Ok(Err(_)) => {} - Err(err) if first_error.is_none() => { - first_error = Some(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("site destroy task failed: {err}"), - )); - } - Err(_) => {} - } - } - if let Some(err) = first_error { - return Err(err); - } + let state = app.control_state.lock().await.clone(); + destroy_child_on_sites(app, &state, &child).await?; let mut state = app.control_state.lock().await; let child = cloned_child_record(&state, child_id)?; @@ -1553,6 +2006,7 @@ pub(super) async fn continue_destroy_requested( child }; retract_child_overlays(app, &child).await?; + retract_dynamic_capability_origin_overlays(app, &child).await?; { let mut state = app.control_state.lock().await; @@ -1595,10 +2049,10 @@ pub(super) async fn execute_create_child( }; let (tx_id, child) = child; - let mut prepared_sites = Vec::new(); - for site_plan in &child.site_plans { - if let Err(err) = prepare_child_on_site(app, child.child_id, site_plan).await { - let rollback_err = rollback_prepared_sites(app, child.child_id, &prepared_sites).await; + let state = app.control_state.lock().await.clone(); + if let Err(err) = prepare_child_on_sites(app, &state, &child).await { + let rollback_err = rollback_prepared_sites(app, child.child_id).await; + { let mut state = app.control_state.lock().await; if state .pending_creates @@ -1617,17 +2071,16 @@ pub(super) async fn execute_create_child( }, )?; } - let err = if let Err(rollback_err) = rollback_err { - protocol_error( - ProtocolErrorCode::PrepareFailed, - &format!("{}; rollback failed: {rollback_err}", err.message), - ) - } else { - err - }; - return Err(err.into()); } - prepared_sites.push(site_plan.site_id.clone()); + let err = if let Err(rollback_err) = rollback_err { + protocol_error( + ProtocolErrorCode::PrepareFailed, + &format!("{}; rollback failed: {rollback_err}", err.message), + ) + } else { + err + }; + return Err(err.into()); } { @@ -1746,12 +2199,7 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { } } ChildState::CreatePrepared => { - let prepared_sites = child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect::>(); - rollback_prepared_sites(app, child.child_id, &prepared_sites) + rollback_prepared_sites(app, child.child_id) .await .wrap_err_with(|| { format!( diff --git a/cli/src/framework_component/planner.rs b/runtime/site-controller/src/planner.rs similarity index 78% rename from cli/src/framework_component/planner.rs rename to runtime/site-controller/src/planner.rs index 3581b969..e55177da 100644 --- a/cli/src/framework_component/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -1,4 +1,16 @@ use super::{api::*, state::*, *}; +use crate::runtime_api::SharedSiteControllerRuntime; + +#[derive(Clone, Debug)] +pub(crate) struct LocalChildRuntimeSpec { + pub(crate) component_ids: Vec, + pub(crate) assigned_components: Vec, + pub(crate) child_artifact_files: BTreeMap, + pub(crate) site_artifact_files: BTreeMap, + pub(crate) controller_routes: Vec, + pub(crate) proxy_exports: BTreeMap, + pub(crate) direct_inputs: Vec, +} pub(super) async fn prepare_child_record( state: &mut FrameworkControlState, @@ -57,6 +69,19 @@ pub(super) async fn prepare_child_record( BTreeMap::from([(wrapper_url.to_string(), wrapper_manifest)]), ) .await?; + let compiled = amber_compiler::run_plan::lower_framework_component_bindings_for_placement( + &compiled, + &placement_file_from_state(state), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "failed to lower framework.component bindings in dynamic child `{}`: {err}", + request.name + ), + ) + })?; let (fragment, outputs) = extract_live_child_fragment( state, &compiled, @@ -76,7 +101,6 @@ pub(super) async fn prepare_child_record( fragment: Some(fragment), input_bindings: child_input_binding_records(&live_scenario, &resolved_bindings), assignments: BTreeMap::new(), - site_plans: Vec::new(), overlay_ids: Vec::new(), overlays: Vec::new(), outputs, @@ -130,14 +154,6 @@ pub(super) fn rebuild_live_child_runtime_metadata( child.overlays = dynamic_overlay_records(&planned.incident_links, fragment); let mut live_assignments = existing_assignments.clone(); live_assignments.extend(child.assignments.clone()); - child.site_plans = dynamic_site_plans( - &planned.site_plans, - &child.assignments, - fragment, - &child.outputs, - &child.overlays, - &dynamic_input_route_records(&live_assignments, fragment, &child.input_bindings), - )?; child.overlay_ids = child .overlays .iter() @@ -1031,24 +1047,12 @@ pub(super) fn build_wrapper_manifest( synthetic_sources.insert( synthetic_name.clone(), SyntheticSourceRecord { - slot_name: binding.slot_name.clone(), actual_source, source_child_id, weak: source.weak, }, ); - if let BindingFrom::Framework(framework) = &source.from { - bindings.push(raw_binding( - &format!("#{child_name}"), - binding.slot_name.clone(), - "framework", - framework.capability.to_string(), - source.weak, - )?); - continue; - } - slots.insert( synthetic_name.clone(), SlotDecl::builder() @@ -1154,12 +1158,12 @@ pub(super) fn wrapper_manifest_url(authority_realm_id: usize, child_id: u64) -> } pub(super) fn allocate_child_id(state: &mut FrameworkControlState) -> u64 { - state.next_child_id += 1; + state.next_child_id += state.id_stride.max(1); state.next_child_id } pub(super) fn allocate_tx_id(state: &mut FrameworkControlState) -> u64 { - state.next_tx_id += 1; + state.next_tx_id += state.id_stride.max(1); state.next_tx_id } @@ -1300,14 +1304,194 @@ pub(super) fn remove_child_record( Ok(()) } -pub(super) fn dynamic_site_plans( - desired_site_plans: &BTreeMap, +struct LocalChildRuntimeSpecInputs<'a> { + state: &'a FrameworkControlState, + outputs: &'a BTreeMap, + overlays: &'a [DynamicOverlayRecord], + direct_inputs: &'a [DynamicInputDirectRecord], +} + +fn parse_mesh_provision_plan_artifact( + artifact_files: &BTreeMap, +) -> std::result::Result, ProtocolErrorResponse> { + if let Some(raw) = artifact_files.get("mesh-provision-plan.json") { + return serde_json::from_str(raw).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("mesh-provision-plan.json is not a valid mesh provision plan: {err}"), + ) + }); + } + + if let Some(raw) = artifact_files.get("compose.yaml") { + let document: serde_yaml::Value = serde_yaml::from_str(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("compose.yaml is not valid YAML: {err}"), + ) + })?; + let content = document + .as_mapping() + .and_then(|root| root.get(serde_yaml::Value::String("configs".to_string()))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|configs| { + configs.get(serde_yaml::Value::String( + "amber-mesh-provision-plan".to_string(), + )) + }) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(serde_yaml::Value::String("content".to_string()))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "compose.yaml is missing configs.amber-mesh-provision-plan.content", + ) + })?; + return serde_json::from_str(content).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("compose.yaml has an invalid embedded mesh provision plan: {err}"), + ) + }); + } + + if let Some(raw) = artifact_files.get("01-configmaps/amber-mesh-provision.yaml") { + let document: serde_yaml::Value = serde_yaml::from_str(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("01-configmaps/amber-mesh-provision.yaml is not valid YAML: {err}"), + ) + })?; + let content = document + .as_mapping() + .and_then(|root| root.get(serde_yaml::Value::String("data".to_string()))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|data| data.get(serde_yaml::Value::String("mesh-plan.json".to_string()))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "01-configmaps/amber-mesh-provision.yaml is missing data.mesh-plan.json", + ) + })?; + return serde_json::from_str(content).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "01-configmaps/amber-mesh-provision.yaml has an invalid embedded mesh \ + provision plan: {err}" + ), + ) + }); + } + + Ok(None) +} + +fn framework_controller_route_grants_for_child( + state: &FrameworkControlState, + site_id: &str, + child_moniker_set: &BTreeSet<&str>, +) -> BTreeMap> { + let mut grants = BTreeMap::>::new(); + for record in state.capability_instances.values() { + if record.controller_site_id != site_id { + continue; + } + if record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + continue; + } + if !child_moniker_set.contains(record.recipient_component_moniker.as_str()) { + continue; + } + grants + .entry(record.route_id.clone()) + .or_default() + .insert(record.recipient_peer_id.clone()); + } + grants +} + +fn controller_routes_for_child( + state: &FrameworkControlState, + site_id: &str, + desired_site_plan: &amber_compiler::run_plan::RunSitePlan, + child_moniker_set: &BTreeSet<&str>, +) -> std::result::Result, ProtocolErrorResponse> { + let grants = framework_controller_route_grants_for_child(state, site_id, child_moniker_set); + let Some(mesh_plan) = parse_mesh_provision_plan_artifact(&desired_site_plan.artifact_files)? + else { + if grants.is_empty() { + return Ok(Vec::new()); + } + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "site `{site_id}` has dynamic framework.component grants but no mesh provision \ + plan artifact" + ), + )); + }; + + let mut routes = Vec::new(); + for target in mesh_plan.targets { + if !matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Component) { + continue; + } + if !target.config.inbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) { + continue; + } + for mut route in target.config.inbound { + if route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + { + route + .allowed_issuers + .retain(|issuer| child_moniker_set.contains(issuer.as_str())); + if !route.allowed_issuers.is_empty() { + routes.push(route); + } + continue; + } + let Some(allowed_issuers) = grants.get(&route.route_id) else { + continue; + }; + route.allowed_issuers = allowed_issuers.iter().cloned().collect(); + routes.push(route); + } + } + + let planned_route_ids = routes + .iter() + .map(|route| route.route_id.as_str()) + .collect::>(); + let missing = grants + .keys() + .filter(|route_id| !planned_route_ids.contains(route_id.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "site `{site_id}` mesh plan is missing dynamic framework.component controller \ + routes: {}", + missing.join(", ") + ), + )); + } + Ok(routes) +} + +fn build_local_child_runtime_spec_from_site_plan( + site_id: &str, + desired_site_plan: &amber_compiler::run_plan::RunSitePlan, assignments: &BTreeMap, fragment: &LiveScenarioFragment, - outputs: &BTreeMap, - overlays: &[DynamicOverlayRecord], - routed_inputs: &[DynamicInputRouteRecord], -) -> std::result::Result, ProtocolErrorResponse> { + inputs: LocalChildRuntimeSpecInputs<'_>, +) -> std::result::Result { let child_moniker_set = fragment .components .iter() @@ -1318,145 +1502,293 @@ pub(super) fn dynamic_site_plans( .iter() .map(|component| (component.id, component.moniker.as_str())) .collect::>(); - let mut site_plans = Vec::new(); - for (site_id, desired_site_plan) in desired_site_plans { - let component_ids = desired_site_plan - .scenario_ir - .components - .iter() - .filter(|component| child_moniker_set.contains(component.moniker.as_str())) - .map(|component| component.id) - .collect::>(); - if component_ids.is_empty() { - continue; - } - let assigned_components = desired_site_plan - .assigned_components - .iter() - .filter(|moniker| { - fragment - .components - .iter() - .any(|component| component.moniker == **moniker) - }) - .cloned() - .collect::>(); - let mut proxy_exports = BTreeMap::new(); - for (name, output) in outputs { - let Some((component_id, component_moniker, provide_name)) = - output.sources.iter().find_map(|source| { - let BindingFromIr::Component { component, provide } = &source.from else { - return None; - }; - let moniker = child_monikers.get(component)?; - (assignments.get(*moniker)? == site_id).then_some(( - *component, - *moniker, - provide.as_str(), - )) - }) - else { - continue; - }; - let protocol = match output.decl.kind.transport() { - CapabilityTransport::Http => "http", - CapabilityTransport::NonNetwork => continue, - _ => continue, - }; - let export = - dynamic_proxy_export_record(fragment, component_id, provide_name, protocol) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "dynamic export `{name}` on component `{component_moniker}` could \ - not be resolved to a concrete network endpoint" - ), - ) - })?; - proxy_exports.insert(name.clone(), export); - } - for overlay in overlays { - let DynamicOverlayAction::ExportPeer { link } = &overlay.action else { - continue; - }; - if link.provider_site != *site_id - || !child_moniker_set.contains(link.provider_component.as_str()) - { - continue; - } - if proxy_exports.contains_key(&link.export_name) { - continue; - } - let component = fragment + let component_ids = desired_site_plan + .scenario_ir + .components + .iter() + .filter(|component| child_moniker_set.contains(component.moniker.as_str())) + .map(|component| component.id) + .collect::>(); + if component_ids.is_empty() { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` has no child-owned components in the desired site plan"), + )); + } + let assigned_components = desired_site_plan + .assigned_components + .iter() + .filter(|moniker| { + fragment .components .iter() - .find(|component| component.moniker == link.provider_component) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "dynamic export provider `{}` is missing from the live child fragment", - link.provider_component - ), - ) - })?; - let export = dynamic_proxy_export_record( - fragment, - component.id, - &link.provide, - &link.protocol.to_string(), - ) + .any(|component| component.moniker == **moniker) + }) + .cloned() + .collect::>(); + let mut proxy_exports = BTreeMap::new(); + for (name, output) in inputs.outputs { + let Some((component_id, component_moniker, provide_name)) = + output.sources.iter().find_map(|source| { + let BindingFromIr::Component { component, provide } = &source.from else { + return None; + }; + let moniker = child_monikers.get(component)?; + (assignments.get(*moniker)? == site_id).then_some(( + *component, + *moniker, + provide.as_str(), + )) + }) + else { + continue; + }; + let protocol = match output.decl.kind.transport() { + CapabilityTransport::Http => "http", + CapabilityTransport::NonNetwork => continue, + _ => continue, + }; + let export = dynamic_proxy_export_record(fragment, component_id, provide_name, protocol) .ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, &format!( - "dynamic export `{}` on component `{}` could not be resolved to a \ - concrete network endpoint", - link.export_name, link.provider_component + "dynamic export `{name}` on component `{component_moniker}` could not be \ + resolved to a concrete network endpoint" ), ) })?; - proxy_exports.insert(link.export_name.clone(), export); + proxy_exports.insert(name.clone(), export); + } + for overlay in inputs.overlays { + let DynamicOverlayAction::ExportPeer { link } = &overlay.action else { + continue; + }; + if link.provider_site != site_id + || !child_moniker_set.contains(link.provider_component.as_str()) + { + continue; } - let artifact_files = if desired_site_plan.site.kind == SiteKind::Kubernetes { - project_kubernetes_dynamic_child_artifact_files( - &desired_site_plan.artifact_files, - &component_ids, - ) - .map_err(|err| { + if proxy_exports.contains_key(&link.export_name) { + continue; + } + let component = fragment + .components + .iter() + .find(|component| component.moniker == link.provider_component) + .ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, &format!( - "failed to project kubernetes child artifact for site `{site_id}`: {err}" + "dynamic export provider `{}` is missing from the live child fragment", + link.provider_component ), ) - })? - } else { - desired_site_plan.artifact_files.clone() - }; - site_plans.push(DynamicSitePlanRecord { - site_id: site_id.clone(), - kind: desired_site_plan.site.kind, - router_identity_id: desired_site_plan.router_identity_id.clone(), - component_ids, - assigned_components, - artifact_files, - desired_artifact_files: desired_site_plan.artifact_files.clone(), - proxy_exports, - routed_inputs: routed_inputs - .iter() - .filter(|input| input.component == child_monikers[&fragment.root_component_id]) - .filter(|input| { - assignments - .get(input.component.as_str()) - .is_some_and(|assigned_site| assigned_site == site_id) - }) - .cloned() - .collect(), - }); + })?; + let export = dynamic_proxy_export_record( + fragment, + component.id, + &link.provide, + &link.protocol.to_string(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic export `{}` on component `{}` could not be resolved to a concrete \ + network endpoint", + link.export_name, link.provider_component + ), + ) + })?; + proxy_exports.insert(link.export_name.clone(), export); } - Ok(site_plans) + let child_artifact_files = if desired_site_plan.site.kind == SiteKind::Kubernetes { + project_kubernetes_dynamic_child_artifact_files( + &desired_site_plan.artifact_files, + &component_ids, + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to project kubernetes child artifact for site `{site_id}`: {err}"), + ) + })? + } else { + desired_site_plan.artifact_files.clone() + }; + Ok(LocalChildRuntimeSpec { + component_ids, + assigned_components, + child_artifact_files, + site_artifact_files: desired_site_plan.artifact_files.clone(), + controller_routes: controller_routes_for_child( + inputs.state, + site_id, + desired_site_plan, + &child_moniker_set, + )?, + proxy_exports, + direct_inputs: inputs + .direct_inputs + .iter() + .filter(|input| input.component == child_monikers[&fragment.root_component_id]) + .filter(|input| { + assignments + .get(input.component.as_str()) + .is_some_and(|assigned_site| assigned_site == site_id) + }) + .cloned() + .collect(), + }) +} + +pub(super) fn child_runtime_site_ids( + child: &LiveChildRecord, +) -> std::result::Result, ProtocolErrorResponse> { + let fragment = child.fragment.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` is missing the authoritative fragment required for placement", + child.name + ), + ) + })?; + let site_ids = fragment + .components + .iter() + .filter_map(|component| child.assignments.get(component.moniker.as_str())) + .cloned() + .collect::>(); + if site_ids.is_empty() { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` has no site assignment for any fragment component", + child.name + ), + )); + } + Ok(site_ids.into_iter().collect()) +} + +pub(super) fn child_runtime_site_id( + child: &LiveChildRecord, +) -> std::result::Result { + let site_ids = child_runtime_site_ids(child)?; + match site_ids.len() { + 1 => Ok(site_ids + .into_iter() + .next() + .expect("single child site id should be present")), + _ => Err(protocol_error( + ProtocolErrorCode::PlacementUnsatisfied, + &format!( + "dynamic child `{}` spans multiple sites ({}) but site controllers only create \ + children within a single site", + child.name, + site_ids.into_iter().collect::>().join(", ") + ), + )), + } +} + +pub(super) fn live_assignment_map_with_child( + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> BTreeMap { + let mut assignments = live_assignment_map(state); + assignments.extend(child.assignments.clone()); + assignments +} + +pub(super) fn build_local_child_runtime_spec( + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result { + let fragment = child.fragment.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` is missing the authoritative fragment required for runtime \ + preparation", + child.name + ), + ) + })?; + let scenario = scenario_with_fragment(&live_scenario_ir(state)?, fragment)?; + let live_assignments = live_assignment_map_with_child(state, child); + let desired_site_plan = build_site_plan_subset( + &scenario, + &placement_file_from_state(state), + &run_plan_activation_from_state(state), + &live_assignments, + &BTreeSet::from([site_id.to_string()]), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "failed to build desired site plan for local child `{}` on site `{site_id}`: {err}", + child.name + ), + ) + })? + .remove(site_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "desired site plan for local child `{}` is missing site `{site_id}`", + child.name + ), + ) + })?; + build_local_child_runtime_spec_from_site_plan( + site_id, + &desired_site_plan, + &live_assignments, + fragment, + LocalChildRuntimeSpecInputs { + state, + outputs: &child.outputs, + overlays: &child.overlays, + direct_inputs: &dynamic_input_direct_records( + &live_assignments, + fragment, + &child.input_bindings, + ), + }, + ) +} + +pub(super) fn build_desired_site_artifact_files( + state: &FrameworkControlState, + site_id: &str, +) -> std::result::Result, ProtocolErrorResponse> { + build_site_plan_subset( + &decode_live_scenario(state)?, + &placement_file_from_state(state), + &run_plan_activation_from_state(state), + &live_assignment_map(state), + &BTreeSet::from([site_id.to_string()]), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to build desired site artifacts for site `{site_id}`: {err}"), + ) + })? + .remove(site_id) + .map(|site_plan| site_plan.artifact_files) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("desired site artifact snapshot is missing site `{site_id}`"), + ) + }) } pub(super) fn child_input_binding_records( @@ -1486,11 +1818,11 @@ pub(super) fn child_input_binding_records( .collect() } -pub(super) fn dynamic_input_route_records( +pub(super) fn dynamic_input_direct_records( assignments: &BTreeMap, fragment: &LiveScenarioFragment, input_bindings: &[ChildInputBindingRecord], -) -> Vec { +) -> Vec { let Some(root_component) = fragment .components .iter() @@ -1525,7 +1857,7 @@ pub(super) fn dynamic_input_route_records( CapabilityTransport::NonNetwork => return None, _ => return None, }; - (provider_site == child_site).then(|| DynamicInputRouteRecord { + (provider_site == child_site).then(|| DynamicInputDirectRecord { component: root_component.moniker.clone(), slot: binding.slot.clone(), provider_component: provider_component.clone(), @@ -1653,6 +1985,54 @@ pub(super) fn create_child_response(child: &LiveChildRecord) -> CreateChildRespo } } +fn merge_framework_controller_component_ir( + existing: &mut ComponentIr, + fragment_controller: &ComponentIr, + id_map: &BTreeMap, +) -> std::result::Result<(), ProtocolErrorResponse> { + let mut fragment_metadata = amber_compiler::run_plan::framework_component_controller_metadata( + fragment_controller.metadata.as_ref(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "dynamic child fragment controller is missing its framework metadata", + ) + })?; + let mut existing_metadata = amber_compiler::run_plan::framework_component_controller_metadata( + existing.metadata.as_ref(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "live framework.component controller is missing its framework metadata", + ) + })?; + for grant in fragment_metadata.grants.values_mut() { + if let Some(remapped) = id_map.get(&grant.authority_realm_id) { + grant.authority_realm_id = *remapped; + } + } + for (slot, decl) in &fragment_controller.slots { + existing + .slots + .entry(slot.clone()) + .or_insert_with(|| decl.clone()); + } + for (provide, decl) in &fragment_controller.provides { + existing + .provides + .entry(provide.clone()) + .or_insert_with(|| decl.clone()); + } + existing_metadata.grants.extend(fragment_metadata.grants); + existing.metadata = Some( + serde_json::to_value(existing_metadata) + .expect("framework controller metadata should serialize"), + ); + Ok(()) +} + pub(super) fn extract_live_child_fragment( state: &mut FrameworkControlState, compiled: &CompiledScenario, @@ -1678,20 +2058,73 @@ pub(super) fn extract_live_child_fragment( ) })? .0; + let existing_controllers_by_site = state + .base_scenario + .components + .iter() + .filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (metadata.execution_site, component.clone())) + }) + .collect::>(); let mut id_map = BTreeMap::new(); + let mut reused_controller_component_ids = BTreeSet::new(); + let mut reused_controller_fragments = BTreeMap::>::new(); for component in &compiled.scenario_ir().components { if component.id == wrapper_root { continue; } + if let Some(metadata) = amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) && let Some(existing_controller) = + existing_controllers_by_site.get(&metadata.execution_site) + { + id_map.insert(component.id, existing_controller.id); + reused_controller_component_ids.insert(component.id); + reused_controller_fragments + .entry(existing_controller.id) + .or_default() + .push(component.clone()); + continue; + } id_map.insert(component.id, state.next_component_id); state.next_component_id += 1; } + let mut controller_replacements = BTreeMap::::new(); + for (existing_controller_id, fragments) in reused_controller_fragments { + let existing_controller = state + .base_scenario + .components + .iter() + .find(|component| component.id == existing_controller_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "live framework.component controller {} disappeared while preparing a \ + dynamic child fragment", + existing_controller_id + ), + ) + })?; + let replacement = controller_replacements + .entry(existing_controller_id) + .or_insert(existing_controller); + for fragment_controller in &fragments { + merge_framework_controller_component_ir(replacement, fragment_controller, &id_map)?; + } + } let mut components = compiled .scenario_ir() .components .iter() - .filter(|component| component.id != wrapper_root) + .filter(|component| { + component.id != wrapper_root && !reused_controller_component_ids.contains(&component.id) + }) .map(|component| { let mut component = component.clone(); component.id = *id_map @@ -1720,6 +2153,7 @@ pub(super) fn extract_live_child_fragment( component }) .collect::>(); + components.extend(controller_replacements.into_values()); components.sort_by(|left, right| left.id.cmp(&right.id)); let mut bindings = Vec::new(); @@ -1768,25 +2202,6 @@ pub(super) fn extract_live_child_fragment( rewritten.weak = synthetic.weak; synthetic.source_child_id } - BindingFromIr::Framework { - authority_realm, .. - } if *authority_realm == wrapper_root => { - let Some(synthetic) = synthetic_sources.values().find(|source| { - source.slot_name == binding.to.slot - && matches!(source.actual_source, BindingFrom::Framework(_)) - }) else { - return Err(protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "missing framework synthetic source mapping for slot `{}`", - binding.to.slot - ), - )); - }; - rewritten.from = BindingFromIr::from(&synthetic.actual_source); - rewritten.weak = synthetic.weak; - synthetic.source_child_id - } BindingFromIr::Framework { authority_realm, .. } => { @@ -1934,6 +2349,17 @@ pub(super) fn collect_capability_instances( .iter() .map(|component| (component.id, component.moniker.clone())) .collect::>(); + let mut controller_metadata_by_id = state + .base_scenario + .components + .iter() + .filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (component.id, metadata)) + }) + .collect::>(); for child in &active_children { let Some(fragment) = child.fragment.as_ref() else { continue; @@ -1944,6 +2370,12 @@ pub(super) fn collect_capability_instances( .iter() .map(|component| (component.id, component.moniker.clone())), ); + controller_metadata_by_id.extend(fragment.components.iter().filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (component.id, metadata)) + })); } let mut site_by_moniker = state.placement.assignments.clone(); @@ -1958,6 +2390,7 @@ pub(super) fn collect_capability_instances( binding, &moniker_by_id, &site_by_moniker, + &controller_metadata_by_id, state.generation, )?; } @@ -1971,6 +2404,7 @@ pub(super) fn collect_capability_instances( &binding.binding, &moniker_by_id, &site_by_moniker, + &controller_metadata_by_id, state.generation, )?; } @@ -1978,73 +2412,142 @@ pub(super) fn collect_capability_instances( Ok(records) } +fn framework_capability_recipient_peer_id( + recipient_component_moniker: &str, + recipient_site_id: &str, + controller_site_id: &str, +) -> String { + if recipient_site_id == controller_site_id { + recipient_component_moniker.to_string() + } else { + format!("/site/{controller_site_id}/router") + } +} + pub(super) fn collect_capability_instance_from_binding( records: &mut BTreeMap, binding: &BindingIr, moniker_by_id: &BTreeMap, site_by_moniker: &BTreeMap, + controller_metadata_by_id: &BTreeMap< + usize, + amber_compiler::run_plan::FrameworkComponentControllerMetadata, + >, generation: u64, ) -> Result<()> { - let BindingFromIr::Framework { - authority_realm, - capability, - } = &binding.from - else { - return Ok(()); - }; - if capability != "component" { - return Ok(()); - } - - let authority_realm_moniker = moniker_by_id.get(authority_realm).cloned().ok_or_else(|| { - miette::miette!( - "framework.component authority realm id {authority_realm} is missing from the \ - authoritative live graph" - ) - })?; let recipient_component_moniker = moniker_by_id .get(&binding.to.component) .cloned() .ok_or_else(|| { miette::miette!( - "framework.component recipient component id {} is missing from the authoritative \ - live graph", - binding.to.component - ) + "framework.component recipient component id {} is missing from the \ + authoritative live graph", + binding.to.component + ) })?; let recipient_site_id = site_by_moniker .get(&recipient_component_moniker) .cloned() .ok_or_else(|| { miette::miette!( - "framework.component recipient `{recipient_component_moniker}` is missing a site \ - assignment in the authoritative live graph" - ) + "framework.component recipient `{recipient_component_moniker}` is missing a \ + site assignment in the authoritative live graph" + ) })?; - let cap_instance_id = framework_cap_instance_id( - authority_realm_moniker.as_str(), - recipient_component_moniker.as_str(), - &binding.to.component.to_string(), - &binding.to.slot, - capability, - ); - records.insert( - cap_instance_id.clone(), - CapabilityInstanceRecord { - cap_instance_id: cap_instance_id.clone(), - route_id: cap_instance_id, - authority_realm_id: *authority_realm, - authority_realm_moniker, - recipient_component_id: binding.to.component, - recipient_component_moniker: recipient_component_moniker.clone(), - recipient_peer_id: recipient_component_moniker, - recipient_site_id, - capability: capability.clone(), - slot: binding.to.slot.clone(), - generation, - }, - ); - Ok(()) + + match &binding.from { + BindingFromIr::Framework { capability, .. } => { + if capability != "component" { + return Ok(()); + } + Err(miette::miette!( + "framework.component binding {}.{} must be lowered before capability refresh", + recipient_component_moniker, + binding.to.slot, + )) + } + BindingFromIr::Component { component, provide } => { + let Some(metadata) = controller_metadata_by_id.get(component) else { + return Ok(()); + }; + let provider_moniker = moniker_by_id.get(component).cloned().ok_or_else(|| { + miette::miette!( + "framework.component controller component id {component} is missing from the authoritative live graph" + ) + })?; + let route_id = amber_mesh::component_route_id( + provider_moniker.as_str(), + provide, + amber_mesh::MeshProtocol::Http, + ); + if let Some(grant) = metadata.grants.get(provide) { + let cap_instance_id = framework_cap_instance_id( + grant.authority_realm_moniker.as_str(), + recipient_component_moniker.as_str(), + &binding.to.component.to_string(), + &binding.to.slot, + "component", + ); + records.insert( + cap_instance_id.clone(), + CapabilityInstanceRecord { + cap_instance_id, + route_id, + authority_realm_id: grant.authority_realm_id, + authority_realm_moniker: grant.authority_realm_moniker.clone(), + recipient_component_id: binding.to.component, + recipient_component_moniker: recipient_component_moniker.clone(), + recipient_peer_id: framework_capability_recipient_peer_id( + recipient_component_moniker.as_str(), + recipient_site_id.as_str(), + metadata.execution_site.as_str(), + ), + recipient_site_id, + controller_site_id: metadata.execution_site.clone(), + managed_site_id: grant.managed_site.clone(), + capability: "component".to_string(), + slot: binding.to.slot.clone(), + generation, + }, + ); + return Ok(()); + } + if provide != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + return Ok(()); + } + let cap_instance_id = framework_cap_instance_id( + provider_moniker.as_str(), + recipient_component_moniker.as_str(), + &binding.to.component.to_string(), + &binding.to.slot, + provide, + ); + records.insert( + cap_instance_id.clone(), + CapabilityInstanceRecord { + cap_instance_id, + route_id, + authority_realm_id: *component, + authority_realm_moniker: provider_moniker, + recipient_component_id: binding.to.component, + recipient_component_moniker: recipient_component_moniker.clone(), + recipient_peer_id: framework_capability_recipient_peer_id( + recipient_component_moniker.as_str(), + recipient_site_id.as_str(), + metadata.execution_site.as_str(), + ), + recipient_site_id, + controller_site_id: metadata.execution_site.clone(), + managed_site_id: String::new(), + capability: provide.clone(), + slot: binding.to.slot.clone(), + generation, + }, + ); + Ok(()) + } + _ => Ok(()), + } } pub(super) fn template_mode(template: &ChildTemplate) -> TemplateMode { @@ -3047,18 +3550,21 @@ pub(super) struct ControlStateApp { pub(super) run_root: PathBuf, pub(super) state_root: PathBuf, pub(super) mesh_scope: Arc, - pub(super) control_state_auth_token: Arc, + pub(super) controller_plan: Arc, pub(super) authority_locks: Arc>>>>, - pub(super) bridge_proxies: Arc>>, + pub(super) runtime: SharedSiteControllerRuntime, } #[derive(Clone)] -pub(super) struct CcsApp { - pub(super) client: ReqwestClient, +pub(super) struct SiteControllerApp { + pub(super) control: ControlStateApp, + pub(super) ready: Arc, +} + +#[derive(Clone)] +pub(super) struct LocalDynamicCapabilityOriginApp { pub(super) site_state_root: PathBuf, - pub(super) control_state_url: Arc, - pub(super) router_auth_token: Arc, - pub(super) control_state_auth_token: Arc, + pub(super) runtime: SharedSiteControllerRuntime, } #[derive(Clone, Debug, Deserialize)] @@ -3082,7 +3588,13 @@ pub(super) struct SiteManagerStateView { #[serde(default)] pub(super) router_mesh_addr: Option, #[serde(default)] + pub(super) compose_consumer_router_mesh_addr: Option, + #[serde(default)] + pub(super) kubernetes_consumer_router_mesh_addr: Option, + #[serde(default)] pub(super) router_identity_id: Option, #[serde(default)] pub(super) router_public_key_b64: Option, + #[serde(default)] + pub(super) site_controller_url: Option, } diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs new file mode 100644 index 00000000..7b24bb96 --- /dev/null +++ b/runtime/site-controller/src/runtime_api.rs @@ -0,0 +1,648 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + future::Future, + net::SocketAddr, + path::{Path, PathBuf}, + pin::Pin, + sync::Arc, +}; + +use amber_compiler::run_plan::{RunLink, SiteKind}; +use amber_mesh::{InboundRoute, MeshConfigPublic, MeshIdentityPublic, MeshPeer}; +use amber_proxy::ControlEndpoint; +use base64::Engine as _; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use serde::{Deserialize, Serialize}; + +use super::state::{FrameworkControlState, LiveChildRecord, SiteControllerPlan}; + +const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; +const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; +const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; +const SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA: &str = "amber.run.site_controller_runtime_plan"; +const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; + +pub type SiteControllerRuntimeFuture<'a, T> = Pin> + Send + 'a>>; + +pub(crate) trait SiteControllerRuntime: Send + Sync { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()>; + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String>; + + fn prepare_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn publish_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn rollback_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn destroy_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result>; + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result; + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result; + + fn router_mesh_addr_for_component_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + self.router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> Result<()>; + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> Result<()>; + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()>; + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()>; +} + +pub(crate) type SharedSiteControllerRuntime = Arc; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SiteReceipt { + pub kind: SiteKind, + pub artifact_dir: String, + pub supervisor_pid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port_forward_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_identity_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub site_controller_url: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SiteControllerRuntimePlan { + pub schema: String, + pub version: u32, + pub run_id: String, + pub mesh_scope: String, + pub run_root: String, + pub site_id: String, + pub kind: SiteKind, + pub router_identity_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_router_control: Option, + pub artifact_dir: String, + pub site_state_root: String, + pub listen_addr: SocketAddr, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_endpoint_forward_ready_timeout_secs: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub launch_env: BTreeMap, +} + +#[derive(Clone, Debug)] +pub struct LiveComponentRuntimeMetadata { + pub moniker: String, + pub router_reachable_mesh_addr: String, + pub component_reachable_mesh_addr: String, + pub control_endpoint: Option, + pub mesh_config: MeshConfigPublic, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DesiredExternalSlotOverlay { + pub slot_name: String, + pub url: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DesiredExportPeerOverlay { + pub export_name: String, + pub peer_id: String, + pub peer_key_b64: String, + pub protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DesiredRouteOverlay { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub peers: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inbound_routes: Vec, +} + +#[derive(Clone, Debug)] +pub struct LaunchedSite { + pub receipt: SiteReceipt, + pub router_control: ControlEndpoint, + pub router_identity: MeshIdentityPublic, + pub router_addr: SocketAddr, +} + +pub fn site_controller_runtime_plan_from_controller_plan( + plan: &SiteControllerPlan, +) -> SiteControllerRuntimePlan { + SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + router_identity_id: plan.router_identity_id.clone(), + local_router_control: plan.local_router_control.clone(), + artifact_dir: plan.artifact_dir.clone(), + site_state_root: plan.site_state_root.clone(), + listen_addr: plan.listen_addr, + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace: plan.kubernetes_namespace.clone(), + context: plan.context.clone(), + observability_endpoint: plan.observability_endpoint.clone(), + vm_endpoint_forward_ready_timeout_secs: plan.vm_endpoint_forward_ready_timeout_secs, + launch_env: plan.launch_env.clone(), + } +} + +pub fn site_controller_runtime_child_root_for_site( + site_state_root: &Path, + child_id: u64, +) -> PathBuf { + site_state_root + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +pub fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { + state_root.join(site_id).join("manager-state.json") +} + +pub fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-plan.json") +} + +fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + +fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) +} + +fn kubernetes_dynamic_apply_resource_kept_from_contents( + resource: &str, + raw: &str, + child_component_labels: &BTreeSet, +) -> Result { + if matches!( + resource, + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH | KUBERNETES_PROVISIONER_JOB_PATH + ) || resource.starts_with("03-persistentvolumeclaims/") + { + return Ok(true); + } + + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + Ok(document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("labels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|component_id| child_component_labels.contains(component_id))) +} + +pub fn project_kubernetes_dynamic_child_artifact_files( + artifact_files: &BTreeMap, + component_ids: &[usize], +) -> Result> { + let child_component_labels = component_ids + .iter() + .map(|component_id| format!("c{component_id}")) + .collect::>(); + let kustomization_path = "kustomization.yaml"; + let raw = artifact_files.get(kustomization_path).ok_or_else(|| { + miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") + })?; + let mut document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is not a YAML mapping") + })?; + let resources = root + .get_mut(yaml_string("resources")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is missing a resources sequence") + })?; + let mut projected = artifact_files + .iter() + .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) + .map(|(path, contents)| (path.clone(), contents.clone())) + .collect::>(); + let mut kept_resources = Vec::new(); + let mut kept_resource_names = BTreeSet::new(); + for resource in resources + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_owned) + { + let raw = artifact_files + .get(&resource) + .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; + if !kubernetes_dynamic_apply_resource_kept_from_contents( + &resource, + raw, + &child_component_labels, + )? { + continue; + } + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + if let Some(name) = kubernetes_resource_name(&document) { + kept_resource_names.insert(name.to_string()); + } + projected.insert(resource.clone(), raw.clone()); + kept_resources.push(serde_yaml::Value::String(resource)); + } + *resources = kept_resources; + + if let Some(generators) = root + .get_mut(yaml_string("secretGenerator")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + generators.retain(|generator| { + generator + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) + }); + } + + if let Some(replacements) = root + .get_mut(yaml_string("replacements")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + replacements.retain_mut(|replacement| { + let Some(targets) = replacement + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) + .and_then(serde_yaml::Value::as_sequence_mut) + else { + return false; + }; + targets.retain(|target| { + target + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("select"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|select| select.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| kept_resource_names.contains(name)) + }); + !targets.is_empty() + }); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; + projected.insert(kustomization_path.to_string(), rendered); + Ok(projected) +} + +pub fn parse_control_endpoint(raw: &str) -> Result { + if let Some(path) = raw.strip_prefix("unix://") { + return Ok(ControlEndpoint::Unix(PathBuf::from(path))); + } + if let Some(rest) = raw.strip_prefix("volume://") + && let Some((volume, socket_path)) = rest.split_once('/') + { + return Ok(ControlEndpoint::VolumeSocket { + volume: volume.to_string(), + socket_path: format!("/{}", socket_path), + }); + } + Ok(ControlEndpoint::Tcp(raw.to_string())) +} + +pub fn decode_public_key(value: &str) -> Result<[u8; 32]> { + let bytes = base64::engine::general_purpose::STANDARD + .decode(value) + .into_diagnostic() + .wrap_err("invalid base64 router public key")?; + bytes + .as_slice() + .try_into() + .map_err(|_| miette::miette!("invalid router public key length")) +} + +pub fn launched_site_from_receipt( + site_receipt: &SiteReceipt, + mesh_scope: &str, +) -> Result { + let router_control = parse_control_endpoint( + site_receipt + .router_control + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router control"))?, + )?; + let router_addr = site_receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router mesh addr"))? + .parse() + .into_diagnostic() + .wrap_err("invalid router mesh addr in site receipt")?; + let router_identity = MeshIdentityPublic { + id: site_receipt + .router_identity_id + .clone() + .ok_or_else(|| miette::miette!("site receipt is missing router identity id"))?, + public_key: decode_public_key( + site_receipt + .router_public_key_b64 + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router public key"))?, + )?, + mesh_scope: Some(mesh_scope.to_string()), + }; + Ok(LaunchedSite { + receipt: site_receipt.clone(), + router_control, + router_identity, + router_addr, + }) +} + +pub fn published_router_mesh_addr_for_consumer_kind( + site_receipt: &SiteReceipt, + consumer_kind: SiteKind, +) -> Option<&str> { + match consumer_kind { + SiteKind::Compose => site_receipt.compose_consumer_router_mesh_addr.as_deref(), + SiteKind::Kubernetes => site_receipt.kubernetes_consumer_router_mesh_addr.as_deref(), + SiteKind::Direct | SiteKind::Vm => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn yaml_sequence_strings<'a>(document: &'a serde_yaml::Value, key: &str) -> Vec<&'a str> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string(key))) + .and_then(serde_yaml::Value::as_sequence) + .expect("sequence should exist") + .iter() + .map(|value| value.as_str().expect("sequence entry should be a string")) + .collect() + } + + #[test] + fn dynamic_kubernetes_child_projection_drops_provisioner_rbac() { + let files = BTreeMap::from([ + ( + "kustomization.yaml".to_string(), + r#" +resources: + - 01-configmaps/amber-mesh-provision.yaml + - 02-rbac/amber-provisioner-sa.yaml + - 02-rbac/amber-provisioner-role.yaml + - 02-rbac/amber-provisioner-rolebinding.yaml + - 02-rbac/amber-provisioner-job.yaml + - 03-persistentvolumeclaims/child-state.yaml + - 03-deployments/child.yaml + - 03-deployments/other.yaml +secretGenerator: + - name: amber-router-external + - name: child-secret +replacements: + - source: + kind: ConfigMap + name: amber-mesh-provision + targets: + - select: + name: child + - select: + name: other +"# + .to_string(), + ), + ( + "01-configmaps/amber-mesh-provision.yaml".to_string(), + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-sa.yaml".to_string(), + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-role.yaml".to_string(), + "apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: \ + amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-rolebinding.yaml".to_string(), + "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: \ + amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-job.yaml".to_string(), + "apiVersion: batch/v1\nkind: Job\nmetadata:\n name: amber-provisioner\n" + .to_string(), + ), + ( + "03-persistentvolumeclaims/child-state.yaml".to_string(), + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: child-state\n" + .to_string(), + ), + ( + "03-deployments/child.yaml".to_string(), + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: child\n labels:\n \ + amber.io/component-id: c1\n" + .to_string(), + ), + ( + "03-deployments/other.yaml".to_string(), + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: other\n labels:\n \ + amber.io/component-id: c2\n" + .to_string(), + ), + ("README.md".to_string(), "operator notes".to_string()), + ]); + + let projected = project_kubernetes_dynamic_child_artifact_files(&files, &[1]) + .expect("dynamic child projection should succeed"); + + assert!(projected.contains_key("01-configmaps/amber-mesh-provision.yaml")); + assert!(projected.contains_key("02-rbac/amber-provisioner-job.yaml")); + assert!(projected.contains_key("03-persistentvolumeclaims/child-state.yaml")); + assert!(projected.contains_key("03-deployments/child.yaml")); + assert_eq!( + projected.get("README.md").map(String::as_str), + Some("operator notes") + ); + assert!(!projected.contains_key("02-rbac/amber-provisioner-sa.yaml")); + assert!(!projected.contains_key("02-rbac/amber-provisioner-role.yaml")); + assert!(!projected.contains_key("02-rbac/amber-provisioner-rolebinding.yaml")); + assert!(!projected.contains_key("03-deployments/other.yaml")); + + let kustomization: serde_yaml::Value = + serde_yaml::from_str(&projected["kustomization.yaml"]) + .expect("projected kustomization should parse"); + assert_eq!( + yaml_sequence_strings(&kustomization, "resources"), + vec![ + "01-configmaps/amber-mesh-provision.yaml", + "02-rbac/amber-provisioner-job.yaml", + "03-persistentvolumeclaims/child-state.yaml", + "03-deployments/child.yaml", + ] + ); + let generators = kustomization + .as_mapping() + .and_then(|root| root.get(yaml_string("secretGenerator"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("secret generators should remain"); + assert_eq!(generators.len(), 1); + assert_eq!( + generators[0] + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str), + Some("child-secret") + ); + let replacement_targets = kustomization + .as_mapping() + .and_then(|root| root.get(yaml_string("replacements"))) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|replacements| replacements.first()) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|replacement| replacement.get(yaml_string("targets"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("kept replacement should have targets"); + assert_eq!(replacement_targets.len(), 1); + assert_eq!( + replacement_targets[0] + .as_mapping() + .and_then(|target| target.get(yaml_string("select"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|select| select.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str), + Some("child") + ); + } +} diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs new file mode 100644 index 00000000..615b89d0 --- /dev/null +++ b/runtime/site-controller/src/site_controller.rs @@ -0,0 +1,1553 @@ +use std::sync::Arc; + +use amber_mesh::dynamic_caps::HeldListResponse; +use amber_proxy::{ControlEndpoint, fetch_router_identity}; +use axum::{ + Json, Router, + extract::{Path as AxumPath, State}, + http::{HeaderMap, StatusCode}, + routing::{get, post}, +}; +use base64::Engine as _; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use reqwest::Client as ReqwestClient; +use serde::{Serialize, de::DeserializeOwned}; +use tokio::net::TcpListener; + +use super::{ + ccs_api::{self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse}, + control_state_api::{ + self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, + DynamicCapsMutateResponse, resolve_dynamic_capability_origin_internal, + }, + dynamic_caps::{ + self, ControlDynamicGrantAuthoritySyncRequest, ControlDynamicGrantAuthoritySyncResponse, + ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, + ControlDynamicInspectRefRequest, ControlDynamicResolveOriginRequest, + ControlDynamicResolveRefRequest, ControlDynamicRevokeRequest, ControlDynamicShareRequest, + InternalDynamicResolveOriginRequest, + }, + http::{ + cleanup_dynamic_bridge_proxies, post_json_with_retry, read_json, required_header, + shutdown_signal, + }, + orchestration::{ + ClearExportPeerOverlayRequest, ClearExternalSlotOverlayRequest, ProtocolApiError, + PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, + RemoteChildRollbackRequest, RemoteChildRuntimeRequest, ResolveExternalLinkUrlRequest, + ResolveExternalLinkUrlResponse, RevokeDynamicCapabilityOriginOverlaysRequest, + RouterIdentityRequest, RouterIdentityResponse, clear_export_peer_overlay_local, + clear_external_slot_overlay_local, destroy_child_on_local_site, execute_create_child, + execute_destroy_child, local_router_identity_for_overlay_site, prepare_child_on_local_site, + publish_child_on_local_site, publish_dynamic_capability_origin_local, + publish_export_peer_overlay_local, publish_external_slot_overlay_local, + recover_control_state, resolve_external_link_url_local, + revoke_dynamic_capability_origin_overlays_local, rollback_child_on_site, + }, + planner::{ + ControlStateApp, LocalDynamicCapabilityOriginApp, SiteControllerApp, protocol_error, + }, + state::*, + *, +}; +use crate::runtime_api::SharedSiteControllerRuntime; + +pub(crate) const CONTROLLER_LOCAL_ONLY_HEADER: &str = "x-amber-site-controller-local-only"; + +pub(crate) async fn run_site_controller( + plan_path: PathBuf, + runtime: SharedSiteControllerRuntime, +) -> Result<()> { + let plan: SiteControllerPlan = read_json(plan_path.as_path(), "site controller plan")?; + let mut control_state: FrameworkControlState = + read_json(Path::new(&plan.state_path), "site controller state file")?; + if let Some(identity_path) = plan.controller_identity_path.as_deref() { + control_state.controller_identity = Some(read_json( + Path::new(identity_path), + "site controller mesh identity", + )?); + } + persist_control_state(Path::new(&plan.state_path), &mut control_state)?; + let control = ControlStateApp { + control_state: Arc::new(Mutex::new(control_state)), + client: ReqwestClient::new(), + state_path: PathBuf::from(&plan.state_path), + run_root: PathBuf::from(&plan.run_root), + state_root: PathBuf::from(&plan.state_root), + mesh_scope: Arc::::from(plan.mesh_scope.clone()), + controller_plan: Arc::new(plan.clone()), + authority_locks: Arc::new(Mutex::new(BTreeMap::new())), + runtime, + }; + let ready = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let app_state = SiteControllerApp { + control: control.clone(), + ready: ready.clone(), + }; + let app = site_controller_router(app_state.clone()); + let listener = TcpListener::bind(plan.listen_addr) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to bind site controller on {}", plan.listen_addr))?; + let serve_task = tokio::spawn(async move { + axum::serve(listener, app.into_make_service()) + .with_graceful_shutdown(shutdown_signal()) + .await + .into_diagnostic() + }); + if let Err(err) = recover_control_state(&control).await { + serve_task.abort(); + let _ = serve_task.await; + let cleanup_result = cleanup_dynamic_bridge_proxies(&control).await; + return match cleanup_result { + Ok(()) => Err(err).wrap_err("site controller recovery failed"), + Err(cleanup_err) => Err(miette::miette!( + "site controller recovery failed: {err}\nbridge proxy cleanup failed: \ + {cleanup_err}" + )), + }; + } + ready.store(true, std::sync::atomic::Ordering::SeqCst); + let serve_result = serve_task + .await + .into_diagnostic() + .wrap_err("site controller task failed")?; + let cleanup_result = cleanup_dynamic_bridge_proxies(&control).await; + match (serve_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), Ok(())) => Err(err).wrap_err("site controller failed"), + (Ok(()), Err(err)) => Err(err).wrap_err("site controller failed to stop bridge proxies"), + (Err(serve_err), Err(cleanup_err)) => Err(miette::miette!( + "site controller failed: {serve_err}\nbridge proxy cleanup failed: {cleanup_err}" + )), + } +} + +pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { + Router::new() + .nest_service("/mcp", super::mcp::service(app_state.clone())) + .nest_service( + "/v1/controller/dynamic-caps/mcp", + super::control_state_mcp::service(app_state.clone()), + ) + .route("/", get(controller_healthz)) + .route("/healthz", get(controller_healthz)) + .route( + SITE_CONTROLLER_STATE_PATH, + get(get_site_controller_state_route), + ) + .route( + "/v1/controller/router-identity", + get(get_router_identity_route), + ) + .route("/v1/templates", get(list_templates)) + .route("/v1/templates/{template}", get(describe_template)) + .route("/v1/templates/{template}/resolve", post(resolve_template)) + .route("/v1/children", get(list_children).post(create_child)) + .route( + "/v1/children/{child}", + get(describe_child).delete(destroy_child), + ) + .route("/v1/snapshot", post(snapshot)) + .route( + "/v1/internal/dynamic-caps/origins/publish", + post(publish_dynamic_origin), + ) + .route( + "/v1/internal/dynamic-caps/resolve-origin", + post(resolve_dynamic_origin_internal_route), + ) + .route( + "/v1/internal/dynamic-caps/origins/revoke", + post(revoke_dynamic_origin_overlays), + ) + .route( + "/v1/internal/link-overlays/external-url", + post(resolve_external_link_url), + ) + .route( + "/v1/internal/router-identity", + post(get_internal_router_identity_route), + ) + .route( + "/v1/internal/children/prepare", + post(prepare_child_runtime_route), + ) + .route( + "/v1/internal/children/publish", + post(publish_child_runtime_route), + ) + .route( + "/v1/internal/children/rollback", + post(rollback_child_runtime_route), + ) + .route( + "/v1/internal/children/destroy", + post(destroy_child_runtime_route), + ) + .route( + "/v1/internal/link-overlays/external-slot/publish", + post(publish_external_slot_overlay_route), + ) + .route( + "/v1/internal/link-overlays/external-slot/clear", + post(clear_external_slot_overlay_route), + ) + .route( + "/v1/internal/link-overlays/export-peer/publish", + post(publish_export_peer_overlay_route), + ) + .route( + "/v1/internal/link-overlays/export-peer/clear", + post(clear_export_peer_overlay_route), + ) + .route( + "/v1/controller/dynamic-caps/held", + post(control_dynamic_held_list_route), + ) + .route( + "/v1/controller/dynamic-caps/held/detail", + post(control_dynamic_held_detail_route), + ) + .route( + "/v1/controller/dynamic-caps/share", + post(control_dynamic_share_route), + ) + .route( + "/v1/controller/dynamic-caps/grant-authorities/sync", + post(control_dynamic_grant_authorities_sync_route), + ) + .route( + "/v1/controller/dynamic-caps/inspect-ref", + post(control_dynamic_inspect_ref_route), + ) + .route( + "/v1/controller/dynamic-caps/resolve-ref", + post(control_dynamic_resolve_ref_route), + ) + .route( + "/v1/controller/dynamic-caps/revoke", + post(control_dynamic_revoke_route), + ) + .route( + "/v1/controller/dynamic-caps/resolve-origin", + post(control_dynamic_resolve_origin_route), + ) + .with_state(app_state) +} + +async fn controller_healthz(State(app): State) -> Response { + let ready = app.ready.load(std::sync::atomic::Ordering::SeqCst); + let status = if ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + (status, Json(json!({ "ok": ready }))).into_response() +} + +async fn get_site_controller_state_route( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + authorize_local_controller_request(&app, &headers).await?; + Ok(Json(app.control.control_state.lock().await.clone())) +} + +fn controller_local_only(headers: &HeaderMap) -> bool { + headers + .get(CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "1") +} + +pub(super) async fn authorize_local_controller_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + if !controller_local_only(headers) { + return Err(ProtocolApiError::unauthorized( + "local controller endpoint requires router-local forwarding".to_string(), + )); + } + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let expected_route_id = { + let state = app.control.control_state.lock().await; + let scenario = amber_scenario::Scenario::try_from(state.base_scenario.clone()).map_err( + |err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read lowered scenario while authorizing a local controller request: {err}" + )) + }, + )?; + let controller_peer_id = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| { + metadata.execution_site == app.control.controller_plan.site_id + }) + }) + .map(|(_, component)| component.moniker.as_str().to_string()) + .ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "site controller component for site `{}` is missing from the lowered scenario", + app.control.controller_plan.site_id, + )) + })?; + amber_mesh::component_route_id( + controller_peer_id.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; + if route_id != expected_route_id { + return Err(ProtocolApiError::unauthorized(format!( + "local controller endpoint requires internal route `{expected_route_id}`" + ))); + } + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let local_router_id = app.control.controller_plan.router_identity_id.as_str(); + let known_peer_router = app + .control + .controller_plan + .peer_router_identities + .values() + .any(|identity| identity.id == peer_id); + if peer_id != local_router_id && !known_peer_router { + return Err(ProtocolApiError::unauthorized(format!( + "local controller endpoint received an unknown router peer `{peer_id}`" + ))); + } + Ok(()) +} + +fn local_router_control_endpoint( + app: &SiteControllerApp, +) -> std::result::Result { + if let Some(raw) = app.control.controller_plan.local_router_control.as_deref() { + return parse_control_endpoint(raw).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller local router control endpoint is invalid: {err}" + )) + }); + } + let manager_state_path = + Path::new(&app.control.controller_plan.site_state_root).join("manager-state.json"); + if manager_state_path.is_file() { + let state: super::planner::SiteManagerStateView = + read_json(&manager_state_path, "site manager state").map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site manager state is unavailable: {err}" + )) + })?; + if let Some(raw) = state.router_control { + return parse_control_endpoint(&raw).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site manager state has an invalid router control endpoint: {err}" + )) + }); + } + } + Err(ProtocolApiError::control_state_unavailable( + "site controller local router control endpoint is unavailable".to_string(), + )) +} + +async fn get_router_identity_route( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + authorize_remote_controller_request(&app, &headers).await?; + let endpoint = local_router_control_endpoint(&app)?; + let identity = fetch_router_identity(&endpoint).await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read local router identity: {err}" + )) + })?; + Ok(Json(RouterIdentityResponse { + id: identity.id, + public_key_b64: base64::engine::general_purpose::STANDARD.encode(identity.public_key), + })) +} + +async fn get_internal_router_identity_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_remote_controller_request(&app, &headers).await?; + let execution_site = { + let state = app.control.control_state.lock().await; + super::orchestration::site_execution_site_from_state(&state, &request.site_id)?.to_string() + }; + if execution_site != app.control.controller_plan.site_id { + return Err(ProtocolApiError::control_state_unavailable(format!( + "site `{}` is controlled by `{execution_site}`, not `{}`", + request.site_id, app.control.controller_plan.site_id, + ))); + } + let identity = local_router_identity_for_overlay_site(&app.control, &request.site_id).await?; + Ok(Json(RouterIdentityResponse { + id: identity.id, + public_key_b64: base64::engine::general_purpose::STANDARD.encode(identity.public_key), + })) +} + +pub(super) async fn authorize_public_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(CapabilityInstanceRecord, FrameworkControlState), ProtocolApiError> { + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let state = app.control.control_state.lock().await.clone(); + let record = super::api::authorize_capability_route(&state, &route_id, &peer_id) + .map_err(ProtocolApiError::from)? + .clone(); + Ok((record, state)) +} + +pub(super) async fn authorize_remote_controller_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + let (record, state) = authorize_public_request(app, headers).await?; + if record.capability != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + return Err(ProtocolApiError::unauthorized( + "remote site controller requests must arrive over the ordinary controller internal \ + capability" + .to_string(), + )); + } + let is_controller = state + .base_scenario + .components + .iter() + .find(|component| component.id == record.recipient_component_id) + .is_some_and(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some() + }); + if !is_controller { + return Err(ProtocolApiError::unauthorized( + "remote site controller requests must target a synthetic controller component" + .to_string(), + )); + } + Ok(()) +} +pub(super) fn authorize_dynamic_caps_sidecar_request( + headers: &HeaderMap, + expected_component_id: &str, +) -> std::result::Result<(), ProtocolApiError> { + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + if !route_id.ends_with(&format!( + ":{}:http", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + )) { + return Err(ProtocolApiError::unauthorized( + "dynamic capability control requests must arrive over the site controller internal \ + route" + .to_string(), + )); + } + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let expected_peer_id = dynamic_caps::moniker_from_logical_component_id(expected_component_id) + .map_err(ProtocolApiError::from)?; + if peer_id != expected_peer_id { + return Err(ProtocolApiError::unauthorized(format!( + "dynamic capability control request for `{expected_component_id}` came from \ + `{peer_id}`" + ))); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum DynamicCapsRequestAuth { + Sidecar, + RemoteController, +} + +pub(super) async fn authorize_dynamic_caps_request( + app: &SiteControllerApp, + headers: &HeaderMap, + expected_component_id: &str, +) -> std::result::Result { + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let expected_sidecar_peer = + dynamic_caps::moniker_from_logical_component_id(expected_component_id) + .map_err(ProtocolApiError::from)?; + if peer_id == expected_sidecar_peer { + authorize_dynamic_caps_sidecar_request(headers, expected_component_id)?; + return Ok(DynamicCapsRequestAuth::Sidecar); + } + authorize_remote_controller_request(app, headers).await?; + Ok(DynamicCapsRequestAuth::RemoteController) +} + +pub(super) async fn authorize_dynamic_caps_mcp_session_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + let _route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + if authorize_remote_controller_request(app, headers) + .await + .is_ok() + { + return Ok(()); + } + + let component_id = dynamic_caps::logical_component_id(&peer_id); + let state = app.control.control_state.lock().await.clone(); + if !dynamic_caps::live_component_ids(&state) + .map_err(ProtocolApiError::from)? + .contains(&component_id) + { + return Err(ProtocolApiError::unauthorized(format!( + "dynamic capability MCP request came from non-component peer `{peer_id}`" + ))); + } + authorize_dynamic_caps_sidecar_request(headers, &component_id) +} + +fn ensure_controller_ready(app: &SiteControllerApp) -> std::result::Result<(), ProtocolApiError> { + if app.ready.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + Err(ProtocolApiError::control_state_unavailable( + "site controller is still recovering", + )) +} + +fn remote_controller_urls(app: &SiteControllerApp) -> BTreeMap { + app.control + .controller_plan + .peer_site_router_urls + .iter() + .filter(|(site_id, _)| site_id.as_str() != app.control.controller_plan.site_id) + .map(|(site_id, url)| (site_id.clone(), url.clone())) + .collect() +} + +fn remote_controller_base_url( + app: &SiteControllerApp, + site_id: &str, +) -> std::result::Result { + remote_controller_urls(app).remove(site_id).ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "site controller `{}` has no router-local controller route to site `{site_id}`", + app.control.controller_plan.site_id + )) + }) +} + +async fn remote_controller_post( + app: &SiteControllerApp, + site_id: &str, + path: &str, + body: &TReq, +) -> std::result::Result { + let url = format!( + "{}{}", + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), + path + ); + let response = post_json_with_retry(&app.control.client, &url, body) + .await + .map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to reach site controller for site `{site_id}` through its router-local \ + controller route: {err}" + )) + })?; + if response.status().is_success() { + return response.json().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned invalid JSON through its \ + router-local controller route: {err}" + )) + }); + } + let status = response.status(); + let body = response.bytes().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read site controller error response for site `{site_id}`: {err}" + )) + })?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error.into()); + } + Err(ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned {status} through its router-local \ + controller route" + ))) +} + +async fn local_held_list( + app: &SiteControllerApp, + request: ControlDynamicHeldListRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::HeldList(request), + ) + .await? + { + DynamicCapsInspectResponse::HeldList(response) => Ok(response), + _ => unreachable!("held_list should return held list"), + } +} + +async fn local_held_detail( + app: &SiteControllerApp, + request: ControlDynamicHeldDetailRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::HeldDetail(request), + ) + .await? + { + DynamicCapsInspectResponse::HeldDetail(response) => Ok(response), + _ => unreachable!("held_detail should return held detail"), + } +} + +async fn local_inspect_ref( + app: &SiteControllerApp, + request: ControlDynamicInspectRefRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::InspectRef(request), + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => Ok(response), + _ => unreachable!("inspect_ref should return inspect response"), + } +} + +async fn local_resolve_origin( + app: &SiteControllerApp, + request: ControlDynamicResolveOriginRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::ResolveOrigin(request), + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(response), + _ => unreachable!("resolve_origin should return origin resolution"), + } +} + +pub(super) async fn execute_site_controller_dynamic_caps_resolve_ref( + app: &SiteControllerApp, + request: ControlDynamicResolveRefRequest, +) -> std::result::Result { + let inspected = match execute_site_controller_dynamic_caps_inspect( + app, + DynamicCapsInspectRequest::InspectRef(ControlDynamicInspectRefRequest { + holder_component_id: request.holder_component_id.clone(), + r#ref: request.r#ref.clone(), + }), + false, + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => response, + _ => unreachable!("inspect_ref should return inspect response"), + }; + let origin = match execute_site_controller_dynamic_caps_inspect( + app, + DynamicCapsInspectRequest::ResolveOrigin(ControlDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id: inspected.grant_id.clone(), + }, + }), + false, + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => response, + _ => unreachable!("resolve_origin should return origin resolution"), + }; + let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(&request.r#ref) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::MalformedRef, + &format!("dynamic capability ref is malformed: {err}"), + )) + })?; + Ok(dynamic_caps::ControlDynamicResolveRefResponse { + origin, + relative_path: parsed.relative_path, + query: parsed.query, + fragment: parsed.fragment, + }) +} + +async fn local_share( + app: &SiteControllerApp, + request: ControlDynamicShareRequest, +) -> std::result::Result { + let response = match control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Share(request.clone()), + ) + .await? + { + DynamicCapsMutateResponse::Share(response) => response, + _ => unreachable!("share should return share response"), + }; + if let Err(err) = sync_shared_grant_authority_site(app, &response).await { + if response.outcome == "created" + && let Some(grant_id) = response.grant_id.as_ref() + && let Err(rollback_err) = control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Revoke(ControlDynamicRevokeRequest { + caller_component_id: request.caller_component_id, + target: dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id: grant_id.clone(), + }, + }), + ) + .await + { + return Err(ProtocolApiError::control_state_unavailable(format!( + "{}; failed to roll back shared grant `{grant_id}` after sync failure: {}", + err.0.message, rollback_err.0.message + ))); + } + return Err(err); + } + Ok(response) +} + +async fn local_revoke( + app: &SiteControllerApp, + request: ControlDynamicRevokeRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Revoke(request), + ) + .await? + { + DynamicCapsMutateResponse::Revoke(response) => Ok(response), + _ => unreachable!("revoke should return revoke response"), + } +} + +fn site_for_dynamic_source( + state: &FrameworkControlState, + source: &dynamic_caps::DynamicCapabilityControlSourceRequest, +) -> std::result::Result { + match source { + dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector, + } => site_id_for_root_authority_selector(state, root_authority_selector), + dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { grant_id } => { + site_id_for_dynamic_grant(state, grant_id) + } + } + .map_err(ProtocolApiError::from) +} + +async fn sync_shared_grant_authority_site( + app: &SiteControllerApp, + response: &amber_mesh::dynamic_caps::ShareResponse, +) -> std::result::Result<(), ProtocolApiError> { + let Some(grant_id) = response.grant_id.as_ref() else { + return Ok(()); + }; + let holder_site_id = { + let state = app.control.control_state.lock().await; + let grant = state + .dynamic_capability_grants + .get(grant_id) + .ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "dynamic capability share reported grant `{grant_id}` but it is missing \ + from site controller `{}` state", + app.control.controller_plan.site_id + )) + })?; + site_id_for_logical_component(&state, &grant.holder_component_id).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "shared grant `{grant_id}` holder `{}` has no live site assignment: {}", + grant.holder_component_id, err.message + )) + })? + }; + let authority_site_id = app.control.controller_plan.site_id.as_str(); + if holder_site_id == authority_site_id { + return Ok(()); + } + let _: ControlDynamicGrantAuthoritySyncResponse = remote_controller_post( + app, + &holder_site_id, + "/v1/controller/dynamic-caps/grant-authorities/sync", + &ControlDynamicGrantAuthoritySyncRequest { + authority_sites: BTreeMap::from([(grant_id.clone(), authority_site_id.to_string())]), + }, + ) + .await?; + Ok(()) +} + +async fn local_sync_dynamic_grant_authorities( + app: &SiteControllerApp, + request: ControlDynamicGrantAuthoritySyncRequest, +) -> std::result::Result { + let synced = request.authority_sites.len(); + { + let mut state = app.control.control_state.lock().await; + persist_control_state_update( + &mut state, + &app.control.state_path, + "dynamic capability grant authority sync", + |state| { + dynamic_caps::sync_dynamic_capability_grant_authority_sites( + state, + &request.authority_sites, + ); + Ok(()) + }, + )?; + } + Ok(ControlDynamicGrantAuthoritySyncResponse { synced }) +} + +fn site_for_held_id( + state: &FrameworkControlState, + held_id: &str, +) -> std::result::Result { + match dynamic_caps::parse_held_entry_key(held_id).map_err(ProtocolApiError::from)? { + dynamic_caps::HeldEntryKey::RootAuthority(selector) => { + site_id_for_root_authority_selector(state, &selector) + } + dynamic_caps::HeldEntryKey::Grant(grant_id) => site_id_for_dynamic_grant(state, &grant_id), + } + .map_err(ProtocolApiError::from) +} + +fn site_for_dynamic_ref( + state: &FrameworkControlState, + raw_ref: &str, +) -> std::result::Result { + let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::MalformedRef, + &format!("dynamic capability ref is malformed: {err}"), + )) + })?; + site_id_for_dynamic_grant(state, &parsed.claims.grant_id).map_err(ProtocolApiError::from) +} + +fn framework_request_site_id( + _state: &FrameworkControlState, + record: &CapabilityInstanceRecord, +) -> std::result::Result { + if !record.controller_site_id.is_empty() { + return Ok(record.controller_site_id.clone()); + } + Err(ProtocolApiError::from(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "framework.component capability instance `{}` is missing its controller_site_id", + record.cap_instance_id + ), + ))) +} + +pub(super) async fn execute_site_controller_framework_inspect( + app: &SiteControllerApp, + record: &CapabilityInstanceRecord, + state: &FrameworkControlState, + request: FrameworkComponentInspectRequest, +) -> std::result::Result { + ensure_controller_ready(app)?; + let authority_site = framework_request_site_id(state, record)?; + if authority_site != app.control.controller_plan.site_id { + return Err(ProtocolApiError::control_state_unavailable(format!( + "framework.component request is bound to controller site `{authority_site}` but \ + reached site controller `{}`", + app.control.controller_plan.site_id + ))); + } + + ccs_api::execute_framework_component_inspect(state, record.authority_realm_id, request) + .await + .map_err(ProtocolApiError::from) +} + +pub(super) async fn execute_site_controller_framework_mutate( + app: &SiteControllerApp, + record: &CapabilityInstanceRecord, + state: &FrameworkControlState, + request: ccs_api::FrameworkComponentMutateRequest, +) -> std::result::Result { + ensure_controller_ready(app)?; + let authority_site = framework_request_site_id(state, record)?; + if authority_site != app.control.controller_plan.site_id { + return Err(ProtocolApiError::control_state_unavailable(format!( + "framework.component request is bound to controller site `{authority_site}` but \ + reached site controller `{}`", + app.control.controller_plan.site_id + ))); + } + + match request { + ccs_api::FrameworkComponentMutateRequest::CreateChild(request) => { + Ok(ccs_api::FrameworkComponentMutateResponse::CreateChild( + execute_create_child(&app.control, record.authority_realm_id, request).await?, + )) + } + ccs_api::FrameworkComponentMutateRequest::DestroyChild { child } => { + execute_destroy_child(&app.control, record.authority_realm_id, &child).await?; + Ok(ccs_api::FrameworkComponentMutateResponse::DestroyChild( + ccs_api::DestroyChildResponse { + child, + destroyed: true, + }, + )) + } + } +} + +pub(super) async fn execute_site_controller_dynamic_caps_inspect( + app: &SiteControllerApp, + request: DynamicCapsInspectRequest, + local_only: bool, +) -> std::result::Result { + ensure_controller_ready(app)?; + match request { + DynamicCapsInspectRequest::HeldList(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::HeldList( + local_held_list(app, request).await?, + )); + } + let mut held = local_held_list(app, request.clone()).await?.held; + for site_id in remote_controller_urls(app).into_keys() { + held.extend( + remote_controller_post::<_, HeldListResponse>( + app, + &site_id, + "/v1/controller/dynamic-caps/held", + &request, + ) + .await? + .held, + ); + } + held.sort_by(|left, right| left.held_id.cmp(&right.held_id)); + held.dedup_by(|left, right| left.held_id == right.held_id); + Ok(DynamicCapsInspectResponse::HeldList(HeldListResponse { + held, + })) + } + DynamicCapsInspectRequest::HeldDetail(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::HeldDetail( + local_held_detail(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_held_id(&state, &request.held_id)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::HeldDetail( + local_held_detail(app, request).await?, + )); + } + Ok(DynamicCapsInspectResponse::HeldDetail( + remote_controller_post( + app, + &site_id, + "/v1/controller/dynamic-caps/held/detail", + &request, + ) + .await?, + )) + } + DynamicCapsInspectRequest::InspectRef(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::InspectRef( + local_inspect_ref(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_ref(&state, &request.r#ref)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::InspectRef( + local_inspect_ref(app, request).await?, + )); + } + Ok(DynamicCapsInspectResponse::InspectRef( + remote_controller_post( + app, + &site_id, + "/v1/controller/dynamic-caps/inspect-ref", + &request, + ) + .await?, + )) + } + DynamicCapsInspectRequest::ResolveOrigin(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::ResolveOrigin( + local_resolve_origin(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.source)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::ResolveOrigin( + local_resolve_origin(app, request).await?, + )); + } + let holder_peer = control_state_api::local_component_runtime( + &app.control, + &state, + &request.holder_component_id, + )? + .mesh_config + .identity; + Ok(DynamicCapsInspectResponse::ResolveOrigin( + remote_controller_post( + app, + &site_id, + "/v1/internal/dynamic-caps/resolve-origin", + &InternalDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: request.source, + holder_peer_id: holder_peer.id, + holder_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(holder_peer.public_key), + holder_site_kind: app.control.controller_plan.kind, + }, + ) + .await?, + )) + } + } +} + +pub(super) async fn execute_site_controller_dynamic_caps_mutate( + app: &SiteControllerApp, + request: DynamicCapsMutateRequest, + local_only: bool, +) -> std::result::Result { + ensure_controller_ready(app)?; + match request { + DynamicCapsMutateRequest::Share(request) => { + if local_only { + return Ok(DynamicCapsMutateResponse::Share( + local_share(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.source)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsMutateResponse::Share( + local_share(app, request).await?, + )); + } + Ok(DynamicCapsMutateResponse::Share( + remote_controller_post( + app, + &site_id, + "/v1/controller/dynamic-caps/share", + &request, + ) + .await?, + )) + } + DynamicCapsMutateRequest::Revoke(request) => { + if local_only { + return Ok(DynamicCapsMutateResponse::Revoke( + local_revoke(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.target)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsMutateResponse::Revoke( + local_revoke(app, request).await?, + )); + } + Ok(DynamicCapsMutateResponse::Revoke( + remote_controller_post( + app, + &site_id, + "/v1/controller/dynamic-caps/revoke", + &request, + ) + .await?, + )) + } + } +} + +async fn list_templates( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await? + { + FrameworkComponentInspectResponse::ListTemplates(response) => Ok(Json(response)), + _ => unreachable!("list_templates should return template list"), + } +} + +async fn describe_template( + State(app): State, + headers: HeaderMap, + AxumPath(template): AxumPath, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetTemplate { template }, + ) + .await? + { + FrameworkComponentInspectResponse::GetTemplate(response) => Ok(Json(response)), + _ => unreachable!("get_template should return template description"), + } +} + +async fn resolve_template( + State(app): State, + headers: HeaderMap, + AxumPath(template): AxumPath, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ResolveTemplate { template, request }, + ) + .await? + { + FrameworkComponentInspectResponse::ResolveTemplate(response) => Ok(Json(response)), + _ => unreachable!("resolve_template should return resolved template"), + } +} + +async fn list_children( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ListChildren, + ) + .await? + { + FrameworkComponentInspectResponse::ListChildren(response) => Ok(Json(response)), + _ => unreachable!("list_children should return child list"), + } +} + +async fn create_child( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_mutate( + &app, + &record, + &state, + ccs_api::FrameworkComponentMutateRequest::CreateChild(request), + ) + .await? + { + ccs_api::FrameworkComponentMutateResponse::CreateChild(response) => Ok(Json(response)), + _ => unreachable!("create_child should return create response"), + } +} + +async fn describe_child( + State(app): State, + headers: HeaderMap, + AxumPath(child): AxumPath, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetChild { child }, + ) + .await? + { + FrameworkComponentInspectResponse::GetChild(response) => Ok(Json(response)), + _ => unreachable!("get_child should return child description"), + } +} + +async fn snapshot( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetSnapshot, + ) + .await? + { + FrameworkComponentInspectResponse::GetSnapshot(response) => Ok(Json(response)), + _ => unreachable!("get_snapshot should return snapshot"), + } +} + +async fn destroy_child( + State(app): State, + headers: HeaderMap, + AxumPath(child): AxumPath, +) -> std::result::Result { + let (record, state) = authorize_public_request(&app, &headers).await?; + let _ = execute_site_controller_framework_mutate( + &app, + &record, + &state, + ccs_api::FrameworkComponentMutateRequest::DestroyChild { child }, + ) + .await?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_dynamic_origin( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + authorize_remote_controller_request(&app, &headers).await?; + let ccs_app = LocalDynamicCapabilityOriginApp { + site_state_root: PathBuf::from(&app.control.controller_plan.site_state_root), + runtime: app.control.runtime.clone(), + }; + Ok(Json( + publish_dynamic_capability_origin_local(&ccs_app, request).await?, + )) +} + +async fn resolve_dynamic_origin_internal_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + authorize_remote_controller_request(&app, &headers).await?; + Ok(Json( + resolve_dynamic_capability_origin_internal(&app.control, request).await?, + )) +} + +async fn revoke_dynamic_origin_overlays( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + revoke_dynamic_capability_origin_overlays_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn resolve_external_link_url( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_remote_controller_request(&app, &headers).await?; + Ok(Json( + resolve_external_link_url_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?, + )) +} + +async fn prepare_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + ensure_controller_ready(&app)?; + prepare_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + ensure_controller_ready(&app)?; + publish_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn rollback_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + ensure_controller_ready(&app)?; + rollback_child_on_site(&app.control, request.child_id, &request.site_id) + .await + .map_err(|err| ProtocolApiError::control_state_unavailable(err.to_string()))?; + Ok(StatusCode::NO_CONTENT) +} + +async fn destroy_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + ensure_controller_ready(&app)?; + destroy_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_external_slot_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + publish_external_slot_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn clear_external_slot_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + clear_external_slot_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_export_peer_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + publish_export_peer_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn clear_export_peer_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_remote_controller_request(&app, &headers).await?; + clear_export_peer_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn control_dynamic_held_list_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::HeldList(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsInspectResponse::HeldList(response) => Ok(Json(response)), + _ => unreachable!("held_list should return held list"), + } +} + +async fn control_dynamic_held_detail_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::HeldDetail(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsInspectResponse::HeldDetail(response) => Ok(Json(response)), + _ => unreachable!("held_detail should return held detail"), + } +} + +async fn control_dynamic_share_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.caller_component_id).await?; + match execute_site_controller_dynamic_caps_mutate( + &app, + DynamicCapsMutateRequest::Share(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsMutateResponse::Share(response) => Ok(Json(response)), + _ => unreachable!("share should return share response"), + } +} + +async fn control_dynamic_grant_authorities_sync_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_remote_controller_request(&app, &headers).await?; + Ok(Json( + local_sync_dynamic_grant_authorities(&app, request).await?, + )) +} + +async fn control_dynamic_inspect_ref_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::InspectRef(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => Ok(Json(response)), + _ => unreachable!("inspect_ref should return inspect response"), + } +} + +async fn control_dynamic_resolve_ref_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + if auth != DynamicCapsRequestAuth::Sidecar { + return Err(ProtocolApiError::unauthorized( + "dynamic capability ref resolution must be requested by the holder component sidecar" + .to_string(), + )); + } + Ok(Json( + execute_site_controller_dynamic_caps_resolve_ref(&app, request).await?, + )) +} + +async fn control_dynamic_revoke_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.caller_component_id).await?; + match execute_site_controller_dynamic_caps_mutate( + &app, + DynamicCapsMutateRequest::Revoke(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsMutateResponse::Revoke(response) => Ok(Json(response)), + _ => unreachable!("revoke should return revoke response"), + } +} + +async fn control_dynamic_resolve_origin_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::ResolveOrigin(request), + matches!(auth, DynamicCapsRequestAuth::RemoteController), + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(Json(response)), + _ => unreachable!("resolve_origin should return origin resolution"), + } +} diff --git a/cli/src/framework_component/state.rs b/runtime/site-controller/src/state.rs similarity index 64% rename from cli/src/framework_component/state.rs rename to runtime/site-controller/src/state.rs index bb7f044c..1bd280cc 100644 --- a/cli/src/framework_component/state.rs +++ b/runtime/site-controller/src/state.rs @@ -2,15 +2,20 @@ use super::{http::*, orchestration::*, planner::*, *}; pub(super) const CONTROL_STATE_SCHEMA: &str = "amber.framework_component.control_state"; pub(super) const CONTROL_STATE_VERSION: u32 = 1; -pub(super) const CONTROL_SERVICE_PLAN_SCHEMA: &str = - "amber.framework_component.control_service_plan"; -pub(super) const CONTROL_SERVICE_PLAN_VERSION: u32 = 1; -pub(super) const CCS_PLAN_SCHEMA: &str = "amber.framework_component.ccs_plan"; -pub(super) const CCS_PLAN_VERSION: u32 = 1; -pub(super) const CONTROL_SERVICE_PATH: &str = "/v1/control-state"; +pub(super) const SITE_CONTROLLER_PLAN_SCHEMA: &str = + "amber.framework_component.site_controller_plan"; +pub(super) const SITE_CONTROLLER_PLAN_VERSION: u32 = 1; +pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; -pub(super) const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; +pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; +pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; +pub const SITE_CONTROLLER_MESH_IDENTITY_PATH: &str = "/amber/mesh/mesh-identity.json"; + +pub fn site_controller_internal_route_id(site_id: &str) -> String { + format!("site-controller:{site_id}") +} #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct FrozenPlacementState { @@ -41,6 +46,10 @@ pub(crate) struct CapabilityInstanceRecord { pub(crate) recipient_component_moniker: String, pub(crate) recipient_peer_id: String, pub(crate) recipient_site_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub(crate) controller_site_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub(crate) managed_site_id: String, pub(crate) capability: String, pub(crate) slot: String, pub(crate) generation: u64, @@ -64,8 +73,6 @@ pub(crate) struct LiveChildRecord { #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub(crate) assignments: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) site_plans: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) overlay_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) overlays: Vec, @@ -86,52 +93,33 @@ pub(crate) struct PendingDestroyRecord { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicSitePlanRecord { - pub(crate) site_id: String, - pub(crate) kind: SiteKind, - pub(crate) router_identity_id: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) component_ids: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) assigned_components: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) artifact_files: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) desired_artifact_files: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) proxy_exports: BTreeMap, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) routed_inputs: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicProxyExportRecord { - pub(crate) component_id: usize, - pub(crate) component: String, - pub(crate) provide: String, - pub(crate) protocol: String, - pub(crate) capability_kind: String, +pub struct DynamicInputDirectRecord { + pub component: String, + pub slot: String, + pub provider_component: String, + pub protocol: String, + pub capability_kind: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) capability_profile: Option, - pub(crate) target_port: u16, + pub capability_profile: Option, + #[serde(flatten)] + pub target: DynamicInputRouteTarget, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicInputRouteRecord { - pub(crate) component: String, - pub(crate) slot: String, - pub(crate) provider_component: String, - pub(crate) protocol: String, - pub(crate) capability_kind: String, +pub struct DynamicProxyExportRecord { + pub component_id: usize, + pub component: String, + pub provide: String, + pub protocol: String, + pub capability_kind: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) capability_profile: Option, - #[serde(flatten)] - pub(crate) target: DynamicInputRouteTarget, + pub capability_profile: Option, + pub target_port: u16, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "route_kind", rename_all = "snake_case")] -pub(crate) enum DynamicInputRouteTarget { +pub enum DynamicInputRouteTarget { ComponentProvide { provide: String }, } @@ -211,7 +199,7 @@ pub(crate) struct ControlJournalEntry { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlState { +pub struct FrameworkControlState { pub(crate) schema: String, pub(crate) version: u32, pub(crate) run_id: String, @@ -225,18 +213,22 @@ pub(crate) struct FrameworkControlState { pub(crate) next_child_id: u64, #[serde(default)] pub(crate) next_tx_id: u64, + #[serde(default = "default_framework_id_stride")] + pub(crate) id_stride: u64, #[serde(default)] pub(crate) next_component_id: usize, #[serde(default)] pub(crate) capability_instances: BTreeMap, #[serde(default)] pub(crate) journal: Vec, - #[serde(default)] - pub(crate) dynamic_capability_signing_seed_b64: String, + #[serde(skip)] + pub(crate) controller_identity: Option, #[serde(default)] pub(crate) next_dynamic_capability_grant_id: u64, #[serde(default)] pub(crate) dynamic_capability_grants: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) dynamic_capability_grant_authority_sites: BTreeMap, #[serde(default)] pub(crate) dynamic_capability_journal: Vec, #[serde(default)] @@ -247,34 +239,176 @@ pub(crate) struct FrameworkControlState { pub(crate) pending_destroys: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlStateServicePlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) listen_addr: SocketAddr, - pub(crate) state_path: String, - pub(crate) run_root: String, - pub(crate) state_root: String, - pub(crate) mesh_scope: String, - pub(crate) auth_token: String, +fn default_framework_id_stride() -> u64 { + 1 } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkCcsPlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) site_id: String, - pub(crate) site_state_root: String, - pub(crate) listen_addr: SocketAddr, - pub(crate) control_state_url: String, - pub(crate) router_auth_token: String, - pub(crate) control_state_auth_token: String, +pub struct SiteControllerPlan { + pub schema: String, + pub version: u32, + pub run_id: String, + pub mesh_scope: String, + pub site_id: String, + pub kind: SiteKind, + pub listen_addr: SocketAddr, + pub authority_url: String, + pub router_identity_id: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_site_router_urls: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_router_identities: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_router_mesh_addrs: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub published_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_consumer_router_mesh_addr: Option, + pub state_path: String, + pub run_root: String, + pub state_root: String, + pub site_state_root: String, + pub artifact_dir: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_identity_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_endpoint_forward_ready_timeout_secs: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub launch_env: BTreeMap, } +#[cfg(test)] pub(crate) fn build_control_state( run_id: &str, run_plan: &RunPlan, ) -> Result { + let mut state = build_control_state_base(run_id, run_plan)?; + state.controller_identity = Some(test_controller_identity(run_id)); + Ok(state) +} + +#[cfg(test)] +pub(crate) fn test_controller_identity(run_id: &str) -> amber_mesh::MeshIdentitySecret { + amber_mesh::MeshIdentitySecret::from_identity(&amber_mesh::MeshIdentity::derive( + "/site/test/controller", + Some("test".to_string()), + run_id, + )) +} + +pub fn build_site_controller_state( + run_id: &str, + run_plan: &RunPlan, + site_id: &str, + site_index: usize, + site_count: usize, +) -> Result { + let mut state = build_control_state_base(run_id, run_plan)?; + localize_framework_control_state(&mut state, site_id)?; + let site_offset = site_index as u64; + let id_stride = site_count.max(1) as u64; + state.next_child_id = localized_last_allocated_counter( + site_offset, + id_stride, + state + .live_children + .iter() + .map(|child| child.child_id) + .chain( + state + .pending_creates + .iter() + .map(|record| record.child.child_id), + ) + .chain( + state + .pending_destroys + .iter() + .map(|record| record.child.child_id), + ), + ); + state.next_tx_id = localized_last_allocated_counter( + site_offset, + id_stride, + state + .pending_creates + .iter() + .map(|record| record.tx_id) + .chain(state.pending_destroys.iter().map(|record| record.tx_id)) + .chain(state.journal.iter().map(|entry| entry.tx_id)), + ); + state.next_dynamic_capability_grant_id = localized_next_available_counter( + site_offset, + id_stride, + state + .dynamic_capability_grants + .keys() + .filter_map(|grant_id| dynamic_capability_grant_counter(grant_id)) + .chain( + state + .dynamic_capability_journal + .iter() + .filter_map(|entry| entry.grant_id.as_deref()) + .filter_map(dynamic_capability_grant_counter), + ), + ); + state.id_stride = id_stride; + Ok(state) +} + +fn localized_last_allocated_counter( + site_offset: u64, + id_stride: u64, + used_ids: impl Iterator, +) -> u64 { + let max_used_id = used_ids.max(); + match max_used_id { + Some(max_used_id) if max_used_id > site_offset => { + site_offset + ((max_used_id - site_offset) / id_stride.max(1)) * id_stride.max(1) + } + _ => site_offset, + } +} + +fn localized_next_available_counter( + site_offset: u64, + id_stride: u64, + used_ids: impl Iterator, +) -> u64 { + let max_used_id = used_ids.max(); + match max_used_id { + Some(max_used_id) if max_used_id >= site_offset => { + site_offset + (((max_used_id - site_offset) / id_stride.max(1)) + 1) * id_stride.max(1) + } + _ => site_offset, + } +} + +fn dynamic_capability_grant_counter(grant_id: &str) -> Option { + grant_id + .strip_prefix(dynamic_caps::DYNAMIC_CAPABILITY_GRANT_ID_PREFIX) + .and_then(|suffix| u64::from_str_radix(suffix, 16).ok()) +} + +fn build_control_state_base(run_id: &str, run_plan: &RunPlan) -> Result { let scenario = Scenario::try_from(run_plan.base_scenario.clone()) .into_diagnostic() .map_err(|err| miette::miette!("failed to decode frozen base scenario: {err}"))?; @@ -304,16 +438,14 @@ pub(crate) fn build_control_state( generation: 0, next_child_id: 0, next_tx_id: 0, + id_stride: default_framework_id_stride(), next_component_id, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: amber_mesh::dynamic_caps::signing_seed_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed( - amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), dynamic_capability_journal: Vec::new(), live_children: Vec::new(), pending_creates: Vec::new(), @@ -331,6 +463,159 @@ pub(crate) fn build_control_state( Ok(state) } +pub(crate) fn localize_framework_control_state( + state: &mut FrameworkControlState, + site_id: &str, +) -> Result<()> { + let local_child_ids = state + .live_children + .iter() + .chain(state.pending_creates.iter().map(|record| &record.child)) + .chain(state.pending_destroys.iter().map(|record| &record.child)) + .filter_map(|child| { + child_authority_site_id(state, child) + .ok() + .filter(|authority_site_id| authority_site_id == site_id) + .map(|_| child.child_id) + }) + .collect::>(); + state + .live_children + .retain(|child| local_child_ids.contains(&child.child_id)); + state + .pending_creates + .retain(|record| local_child_ids.contains(&record.child.child_id)); + state + .pending_destroys + .retain(|record| local_child_ids.contains(&record.child.child_id)); + state + .journal + .retain(|entry| local_child_ids.contains(&entry.child_id)); + + let roots = dynamic_caps::derive_root_authorities(state) + .map_err(|err| miette::miette!(err.message.clone()))?; + let local_grant_authority_sites = state + .dynamic_capability_grants + .iter() + .filter_map(|(grant_id, grant)| { + let authority_site_id = roots + .get(&dynamic_caps::root_authority_key( + &grant.root_authority_selector, + )) + .and_then(|root| component_site_id(state, &root.holder_component_id).ok())?; + let holder_site_id = component_site_id(state, &grant.holder_component_id).ok()?; + (holder_site_id == site_id && authority_site_id != site_id) + .then_some((grant_id.clone(), authority_site_id)) + }) + .collect::>(); + let local_grant_ids = state + .dynamic_capability_grants + .iter() + .filter_map(|(grant_id, grant)| { + roots + .get(&dynamic_caps::root_authority_key( + &grant.root_authority_selector, + )) + .and_then(|root| component_site_id(state, &root.holder_component_id).ok()) + .filter(|root_site_id| root_site_id == site_id) + .map(|_| grant_id.clone()) + }) + .collect::>(); + state + .dynamic_capability_grants + .retain(|grant_id, _| local_grant_ids.contains(grant_id)); + state.dynamic_capability_grant_authority_sites = local_grant_authority_sites; + state.dynamic_capability_journal.retain(|entry| { + entry + .grant_id + .as_ref() + .is_none_or(|grant_id| local_grant_ids.contains(grant_id)) + }); + Ok(()) +} + +fn child_authority_site_id( + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result { + let authority_moniker = decode_live_scenario(state)? + .component(ComponentId(child.authority_realm_id)) + .moniker + .to_string(); + site_id_for_moniker( + state, + &authority_moniker, + &format!("authority realm `{authority_moniker}`"), + ) + .or_else(|_| child_runtime_site_id(child)) +} + +fn component_site_id( + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; + let moniker = moniker.to_string(); + site_id_for_moniker( + state, + &moniker, + &format!("component `{logical_component_id}`"), + ) +} + +fn site_id_for_moniker( + state: &FrameworkControlState, + moniker: &str, + subject: &str, +) -> std::result::Result { + let components = planned_component_site_map(state); + if let Some(site_id) = components.get(moniker) { + return Ok(site_id.clone()); + } + + let descendant_sites = components + .iter() + .filter(|(assigned_moniker, _)| moniker_contains(assigned_moniker, moniker)) + .map(|(_, site_id)| site_id.clone()) + .collect::>(); + if descendant_sites.len() == 1 { + return Ok(descendant_sites + .into_iter() + .next() + .expect("single descendant site should be present")); + } + + Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &if descendant_sites.is_empty() { + format!("{subject} is missing a site assignment") + } else { + format!( + "{subject} is missing a site assignment and spans multiple sites: {}", + descendant_sites.into_iter().collect::>().join(", ") + ) + }, + )) +} + +fn planned_component_site_map(state: &FrameworkControlState) -> BTreeMap { + let mut components = state.placement.placement_components.clone(); + components.extend(state.placement.assignments.clone()); + for child in visible_child_records(state) { + components.extend(child.assignments.clone()); + } + components +} + +fn moniker_contains(candidate: &str, realm: &str) -> bool { + if realm == "/" { + return true; + } + candidate + .strip_prefix(realm) + .is_some_and(|suffix| suffix.starts_with('/')) +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(super) struct FrameworkChildSnapshotRecord { pub(super) child: FrameworkChildSnapshotState, @@ -529,7 +814,6 @@ pub(super) fn restore_framework_children_from_snapshot( fragment: Some(child.fragment), input_bindings: child.input_bindings, assignments: child.assignments, - site_plans: Vec::new(), overlay_ids: Vec::new(), overlays: Vec::new(), outputs: child.outputs, @@ -776,33 +1060,16 @@ fn remap_output_handle_for_snapshot( }) } -pub(crate) fn generate_framework_auth_token(mesh_scope: &str, purpose: &str) -> String { - base64::engine::general_purpose::STANDARD.encode( - MeshIdentity::generate( - format!("/framework/{purpose}"), - Some(mesh_scope.to_string()), - ) - .public_key, - ) -} - -pub(crate) fn control_state_service_url(listen_addr: SocketAddr) -> String { - format!("http://{listen_addr}") -} - -pub(crate) fn ccs_listen_addr_for_site(kind: SiteKind, port: u16) -> SocketAddr { - host_service_bind_addr_for_consumer(kind, port) -} - -pub(crate) fn ccs_url_for_site(kind: SiteKind, port: u16) -> String { - let host = match kind { - SiteKind::Direct | SiteKind::Vm => Ipv4Addr::LOCALHOST.to_string(), - SiteKind::Compose | SiteKind::Kubernetes => host_service_host_for_consumer(kind), +pub fn authority_url_for_listen_addr(listen_addr: SocketAddr) -> String { + let dial_addr = if listen_addr.ip().is_unspecified() { + SocketAddr::from(([127, 0, 0, 1], listen_addr.port())) + } else { + listen_addr }; - format!("http://{host}:{port}") + format!("http://{dial_addr}") } -pub(crate) fn write_control_state(path: &Path, state: &FrameworkControlState) -> Result<()> { +pub fn write_control_state(path: &Path, state: &FrameworkControlState) -> Result<()> { write_json(path, state) } @@ -937,48 +1204,116 @@ pub(super) fn child_destroy_tx_id( }) } -pub(crate) fn write_control_state_service_plan( +#[allow(clippy::too_many_arguments)] +pub fn write_site_controller_plan( path: &Path, + run_id: &str, + mesh_scope: &str, + site_id: &str, + kind: SiteKind, listen_addr: SocketAddr, + authority_url: &str, + router_identity_id: &str, + peer_site_router_urls: &BTreeMap, + peer_router_identities: &BTreeMap, + peer_router_mesh_addrs: &BTreeMap, + local_router_control: Option<&str>, + published_router_mesh_addr: Option<&str>, + compose_consumer_router_mesh_addr: Option<&str>, + kubernetes_consumer_router_mesh_addr: Option<&str>, state_path: &Path, run_root: &Path, state_root: &Path, - mesh_scope: &str, - auth_token: &str, -) -> Result { - let plan = FrameworkControlStateServicePlan { - schema: CONTROL_SERVICE_PLAN_SCHEMA.to_string(), - version: CONTROL_SERVICE_PLAN_VERSION, + site_state_root: &Path, + artifact_dir: &Path, + controller_identity_path: Option<&str>, + storage_root: Option<&str>, + runtime_root: Option<&str>, + router_mesh_port: Option, + compose_project: Option<&str>, + kubernetes_namespace: Option<&str>, + context: Option<&str>, + observability_endpoint: Option<&str>, + launch_env: &BTreeMap, +) -> Result { + let plan = SiteControllerPlan { + schema: SITE_CONTROLLER_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_PLAN_VERSION, + run_id: run_id.to_string(), + mesh_scope: mesh_scope.to_string(), + site_id: site_id.to_string(), + kind, listen_addr, + authority_url: authority_url.to_string(), + router_identity_id: router_identity_id.to_string(), + peer_site_router_urls: peer_site_router_urls.clone(), + peer_router_identities: peer_router_identities.clone(), + peer_router_mesh_addrs: peer_router_mesh_addrs.clone(), + local_router_control: local_router_control.map(str::to_string), + published_router_mesh_addr: published_router_mesh_addr.map(str::to_string), + compose_consumer_router_mesh_addr: compose_consumer_router_mesh_addr.map(str::to_string), + kubernetes_consumer_router_mesh_addr: kubernetes_consumer_router_mesh_addr + .map(str::to_string), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), - mesh_scope: mesh_scope.to_string(), - auth_token: auth_token.to_string(), + site_state_root: site_state_root.display().to_string(), + artifact_dir: artifact_dir.display().to_string(), + controller_identity_path: controller_identity_path.map(str::to_string), + storage_root: storage_root.map(str::to_string), + runtime_root: runtime_root.map(str::to_string), + router_mesh_port, + compose_project: compose_project.map(str::to_string), + kubernetes_namespace: kubernetes_namespace.map(str::to_string), + context: context.map(str::to_string), + observability_endpoint: observability_endpoint.map(str::to_string), + vm_endpoint_forward_ready_timeout_secs: Some( + crate::default_runtime::vm_endpoint_forward_ready_timeout().as_secs(), + ), + launch_env: launch_env.clone(), }; write_json(path, &plan)?; Ok(plan) } -pub(crate) fn write_framework_ccs_plan( - path: &Path, - site_id: &str, - site_state_root: &Path, - listen_addr: SocketAddr, - control_state_url: &str, - router_auth_token: &str, - control_state_auth_token: &str, -) -> Result { - let plan = FrameworkCcsPlan { - schema: CCS_PLAN_SCHEMA.to_string(), - version: CCS_PLAN_VERSION, - site_id: site_id.to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - control_state_url: control_state_url.to_string(), - router_auth_token: router_auth_token.to_string(), - control_state_auth_token: control_state_auth_token.to_string(), - }; - write_json(path, &plan)?; - Ok(plan) +pub(crate) fn site_id_for_dynamic_grant( + state: &FrameworkControlState, + grant_id: &str, +) -> std::result::Result { + if let Some(grant) = state.dynamic_capability_grants.get(grant_id) { + return site_id_for_root_authority_selector(state, &grant.root_authority_selector); + } + state + .dynamic_capability_grant_authority_sites + .get(grant_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::UnknownSource, + &format!("dynamic grant `{grant_id}` is not live"), + ) + }) +} + +pub(crate) fn site_id_for_logical_component( + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + component_site_id(state, logical_component_id) +} + +pub(crate) fn site_id_for_root_authority_selector( + state: &FrameworkControlState, + selector: &RootAuthoritySelectorIr, +) -> std::result::Result { + let roots = dynamic_caps::derive_root_authorities(state)?; + let root = roots + .get(&dynamic_caps::root_authority_key(selector)) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::UnknownSource, + "dynamic capability root authority is not live", + ) + })?; + site_id_for_logical_component(state, &root.holder_component_id) } diff --git a/cli/src/framework_component/tests.rs b/runtime/site-controller/src/tests.rs similarity index 51% rename from cli/src/framework_component/tests.rs rename to runtime/site-controller/src/tests.rs index 7e6a96d7..448c5652 100644 --- a/cli/src/framework_component/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -1,6 +1,15 @@ -use std::fs; +use std::{ + fs, + future::Future, + io::{Read, Write}, + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration as StdDuration, +}; -use amber_compiler::run_plan::build_run_plan; +use amber_compiler::run_plan::{FrameworkComponentControllerMoniker, build_run_plan}; use amber_mesh::{ InboundRoute, InboundTarget, MeshConfigPublic, MeshIdentityPublic, MeshPeer, MeshProtocol, OutboundRoute, TransportConfig, @@ -10,6 +19,7 @@ use amber_mesh::{ HeldEntryKind, HeldEntryState, RootAuthoritySelectorIr, }, }; +use axum::{Router, http::HeaderMap}; use reqwest::{Client, StatusCode}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; @@ -17,6 +27,820 @@ use tempfile::TempDir; use url::Url; use super::{api::*, http::*, orchestration::*, planner::*, state::*, *}; +use crate::{ + ccs_api::FrameworkComponentInspectRequest, + runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, +}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DirectRuntimeState { + #[serde(default)] + ready: bool, + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + dynamic_caps_port_by_component: BTreeMap, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { + plan_root.join(".amber").join("direct-runtime.json") +} + +#[derive(Clone, Default)] +struct TestSiteControllerRuntime; + +fn ready_site_controller_flag() -> Arc { + Arc::new(std::sync::atomic::AtomicBool::new(true)) +} + +impl SiteControllerRuntime for TestSiteControllerRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn resolve_link_external_url<'a>( + &'a self, + _provider: &'a LaunchedSite, + _provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + _consumer_kind: SiteKind, + _run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async move { + Ok(match link.protocol { + amber_manifest::NetworkProtocol::Tcp => "tcp://127.0.0.1:1".to_string(), + _ => "http://127.0.0.1:1".to_string(), + }) + }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + if plan.kind != SiteKind::Direct { + return Err(miette::miette!( + "test runtime only implements direct site runtime metadata" + )); + } + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .ok_or_else(|| miette::miette!("direct test site is missing its runtime root"))?, + ); + let direct_plan: serde_json::Value = + read_json(&artifact_dir.join("direct-plan.json"), "direct plan")?; + let runtime_state: DirectRuntimeState = read_json( + &direct_runtime_state_path(&artifact_dir), + "direct runtime state", + )?; + let mut components = BTreeMap::new(); + for component in direct_plan["components"].as_array().into_iter().flatten() { + let Some(component_id) = component.get("id").and_then(serde_json::Value::as_u64) else { + continue; + }; + let Some(moniker) = component.get("moniker").and_then(serde_json::Value::as_str) else { + continue; + }; + let Some(port) = runtime_state + .component_mesh_port_by_id + .get(&(component_id as usize)) + .copied() + else { + continue; + }; + let Some(mesh_config_path) = component + .get("sidecar") + .and_then(|value| value.get("mesh_config_path")) + .and_then(serde_json::Value::as_str) + else { + continue; + }; + let mesh_config = read_json( + &runtime_root.join(mesh_config_path), + "component mesh config", + )?; + components.insert( + moniker.to_string(), + LiveComponentRuntimeMetadata { + moniker: moniker.to_string(), + router_reachable_mesh_addr: format!("127.0.0.1:{port}"), + component_reachable_mesh_addr: format!("127.0.0.1:{port}"), + control_endpoint: None, + mesh_config, + }, + ); + } + Ok(components) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + if plan.kind != SiteKind::Direct { + return Err(miette::miette!( + "test runtime only implements direct site router metadata" + )); + } + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .ok_or_else(|| miette::miette!("direct test site is missing its runtime root"))?, + ); + let direct_plan: serde_json::Value = + read_json(&artifact_dir.join("direct-plan.json"), "direct plan")?; + let mesh_config_path = direct_plan["router"]["mesh_config_path"] + .as_str() + .ok_or_else(|| { + miette::miette!("direct test site is missing router mesh config path") + })?; + read_json(&runtime_root.join(mesh_config_path), "router mesh config") + } + + fn router_mesh_addr_for_consumer( + &self, + _provider_kind: SiteKind, + _consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + Ok(router_mesh_addr.to_string()) + } + + fn update_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn update_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } +} + +fn test_runtime() -> SharedSiteControllerRuntime { + Arc::new(TestSiteControllerRuntime) +} + +type DestroyCalls = Arc>>; +type PublishCalls = Arc>>; +type ResolveCalls = Arc>>; + +#[derive(Clone, Default)] +struct FailingPublishRuntime { + destroy_calls: DestroyCalls, +} + +impl SiteControllerRuntime for FailingPublishRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn resolve_link_external_url<'a>( + &'a self, + _provider: &'a LaunchedSite, + _provider_output_dir: &'a Path, + _link: &'a amber_compiler::run_plan::RunLink, + _consumer_kind: SiteKind, + _run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async { Ok("http://127.0.0.1:1".to_string()) }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Err(miette::miette!("publish exploded")) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let destroy_calls = self.destroy_calls.clone(); + Box::pin(async move { + let site_id = child_runtime_site_id(&child).expect("child site id"); + destroy_calls + .lock() + .expect("destroy call log mutex should lock") + .push((child.child_id, site_id)); + Ok(()) + }) + } + + fn collect_live_component_runtime_metadata( + &self, + _plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + Ok(BTreeMap::new()) + } + + fn load_live_site_router_mesh_config( + &self, + _plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + Err(miette::miette!( + "failing publish runtime should not load live site router mesh config" + )) + } + + fn router_mesh_addr_for_consumer( + &self, + _provider_kind: SiteKind, + _consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + Ok(router_mesh_addr.to_string()) + } + + fn update_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn update_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } +} + +#[derive(Clone, Default)] +struct RecordingPublishRuntime { + publish_calls: PublishCalls, +} + +impl SiteControllerRuntime for RecordingPublishRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + TestSiteControllerRuntime.resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let publish_calls = self.publish_calls.clone(); + Box::pin(async move { + let site_id = child_runtime_site_id(&child).expect("child site id"); + publish_calls + .lock() + .expect("publish call log mutex should lock") + .push((child.child_id, site_id)); + Ok(()) + }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +#[derive(Clone, Default)] +struct PublishBeforeResolveRuntime { + publish_calls: PublishCalls, + resolve_calls: ResolveCalls, +} + +impl SiteControllerRuntime for PublishBeforeResolveRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + let publish_calls = self.publish_calls.clone(); + let resolve_calls = self.resolve_calls.clone(); + Box::pin(async move { + resolve_calls + .lock() + .expect("resolve call log mutex should lock") + .push(link.export_name.clone()); + if publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty() + { + return Err(miette::miette!( + "resolve_link_external_url ran before publish_child" + )); + } + TestSiteControllerRuntime + .resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + .await + }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let publish_calls = self.publish_calls.clone(); + Box::pin(async move { + let site_id = + child_runtime_site_id(&child).unwrap_or_else(|_| "multi-site".to_string()); + publish_calls + .lock() + .expect("publish call log mutex should lock") + .push((child.child_id, site_id)); + Ok(()) + }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +#[derive(Clone, Default)] +struct FailingRollbackRuntime; + +impl SiteControllerRuntime for FailingRollbackRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + TestSiteControllerRuntime.resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Err(miette::miette!("rollback exploded")) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + _site_id: &'a str, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +fn with_runtime(app: &ControlStateApp, runtime: SharedSiteControllerRuntime) -> ControlStateApp { + ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + controller_plan: app.controller_plan.clone(), + authority_locks: app.authority_locks.clone(), + runtime, + } +} fn write_file(path: &Path, contents: &str) { fs::write(path, contents).expect("test fixture should write"); @@ -260,7 +1084,7 @@ fn delegated_entry_for( .expect("delegated held entry should resolve") } -async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { +fn dynamic_caps_binding_scenario_ir() -> ScenarioIr { fn path_program() -> amber_scenario::Program { serde_json::from_value(serde_json::json!({ "path": "/usr/bin/env", @@ -320,7 +1144,7 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { } } - let scenario = ScenarioIr { + ScenarioIr { schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), version: amber_scenario::SCENARIO_IR_VERSION, root: 0, @@ -417,7 +1241,11 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { ], exports: Vec::new(), manifest_catalog: BTreeMap::new(), - }; + } +} + +async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { + let scenario = dynamic_caps_binding_scenario_ir(); let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -426,13 +1254,14 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { path: Some("direct_local".to_string()), ..PlacementDefaults::default() }, - components: BTreeMap::new(), + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), dynamic_capabilities: None, framework_children: None, }; @@ -442,13 +1271,16 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { fn test_live_component_runtime( moniker: &str, peer_id: &str, - host_mesh_addr: &str, + router_reachable_mesh_addr: &str, + component_reachable_mesh_addr: &str, inbound: Vec, outbound: Vec, ) -> LiveComponentRuntimeMetadata { LiveComponentRuntimeMetadata { moniker: moniker.to_string(), - host_mesh_addr: host_mesh_addr.to_string(), + router_reachable_mesh_addr: router_reachable_mesh_addr.to_string(), + component_reachable_mesh_addr: component_reachable_mesh_addr.to_string(), + control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { id: peer_id.to_string(), @@ -486,11 +1318,12 @@ fn test_live_site_router(inbound: Vec) -> MeshConfigPublic { } #[test] -fn dynamic_capability_origin_self_provide_routes_via_component_mesh() { +fn dynamic_capability_origin_self_provide_routes_via_router_reachable_mesh() { let runtime = test_live_component_runtime( "/provider", "/provider", "127.0.0.1:24001", + "10.0.2.2:24001", vec![InboundRoute { route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -503,14 +1336,13 @@ fn dynamic_capability_origin_self_provide_routes_via_component_mesh() { }], Vec::new(), ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); let site_router = test_live_site_router(Vec::new()); + let site_components = BTreeMap::new(); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::SelfProvide { component_id: "/provider".to_string(), @@ -542,6 +1374,7 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { "/consumer", "/consumer", "127.0.0.1:24002", + "10.0.2.2:24002", Vec::new(), vec![OutboundRoute { route_id: "provider-route".to_string(), @@ -561,21 +1394,18 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { let provider_runtime = test_live_component_runtime( "/provider", "/provider", - "127.0.0.1:24001", + "127.0.0.1:24099", + "10.0.2.2:24099", Vec::new(), Vec::new(), ); - let site_components = BTreeMap::from([ - (holder_runtime.moniker.clone(), holder_runtime.clone()), - (provider_runtime.moniker.clone(), provider_runtime.clone()), - ]); let site_router = test_live_site_router(Vec::new()); + let site_components = BTreeMap::from([(provider_runtime.moniker.clone(), provider_runtime)]); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &holder_runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::Binding { consumer_component_id: "components./consumer".to_string(), @@ -592,7 +1422,7 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { assert_eq!( route.target, InboundTarget::MeshForward { - peer_addr: "127.0.0.1:24001".to_string(), + peer_addr: "127.0.0.1:24099".to_string(), peer_id: "/provider".to_string(), route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -606,6 +1436,7 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { "/consumer", "/consumer", "127.0.0.1:24002", + "10.0.2.2:24002", Vec::new(), vec![OutboundRoute { route_id: "router:external:catalog_api:http".to_string(), @@ -622,7 +1453,6 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { capability: "catalog_api".to_string(), }], ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); let site_router = test_live_site_router(vec![InboundRoute { route_id: "router:external:catalog_api:http".to_string(), capability: "catalog_api".to_string(), @@ -636,12 +1466,12 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { }, allowed_issuers: vec!["/consumer".to_string()], }]); + let site_components = BTreeMap::new(); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::ExternalSlotBinding { consumer_component_id: "components./consumer".to_string(), @@ -666,62 +1496,13 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { ); } -#[test] -fn dynamic_capability_origin_binding_rewrites_linux_slirp_peer_addr_for_host_router() { - let runtime = test_live_component_runtime( - "/consumer", - "/consumer", - "127.0.0.1:24002", - Vec::new(), - vec![OutboundRoute { - route_id: "remote-route".to_string(), - rewrite_route_id: None, - slot: "provider".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - listen_port: 20000, - listen_addr: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - peer_addr: "10.0.2.2:24077".to_string(), - peer_id: "/remote".to_string(), - capability: "provider.api".to_string(), - }], - ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); - let site_router = test_live_site_router(Vec::new()); - - let (route, _, _) = dynamic_capability_origin_route_surface( - &runtime, - &site_components, - &site_router, - SiteKind::Direct, - "dynamic-origin", - &RootAuthoritySelectorIr::Binding { - consumer_component_id: "components./consumer".to_string(), - slot_name: "provider".to_string(), - provider_component_id: "components./remote".to_string(), - provider_capability_name: "provider.api".to_string(), - }, - vec!["peer-consumer".to_string()], - ) - .expect("binding origin surface should resolve"); - - let InboundTarget::MeshForward { peer_addr, .. } = route.target else { - panic!("dynamic origin route should forward through mesh"); - }; - #[cfg(target_os = "linux")] - assert_eq!(peer_addr, "127.0.0.1:24077"); - #[cfg(not(target_os = "linux"))] - assert_eq!(peer_addr, "10.0.2.2:24077"); -} - #[test] fn dynamic_capability_origin_target_mesh_peer_uses_self_identity_for_self_provide() { let runtime = test_live_component_runtime( "/provider", "/provider", "127.0.0.1:24001", + "10.0.2.2:24001", vec![InboundRoute { route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -740,7 +1521,6 @@ fn dynamic_capability_origin_target_mesh_peer_uses_self_identity_for_self_provid &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::SelfProvide { component_id: "/provider".to_string(), @@ -767,7 +1547,9 @@ fn dynamic_capability_origin_target_mesh_peer_uses_runtime_peer_catalog_for_bind }; let runtime = LiveComponentRuntimeMetadata { moniker: "/consumer".to_string(), - host_mesh_addr: "127.0.0.1:24002".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24002".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24002".to_string(), + control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { id: "/consumer".to_string(), @@ -806,7 +1588,6 @@ fn dynamic_capability_origin_target_mesh_peer_uses_runtime_peer_catalog_for_bind &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::Binding { consumer_component_id: "components./consumer".to_string(), @@ -916,6 +1697,7 @@ async fn compile_dynamic_caps_external_root_state() -> FrameworkControlState { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1015,6 +1797,7 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1030,7 +1813,6 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; execute_create_child( &app, @@ -1071,28 +1853,28 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes .iter() .find(|child| child.name == "consumer") .expect("consumer child should be recorded"); - for site_plan in &consumer.site_plans { - assert_eq!(site_plan.routed_inputs.len(), 1); - assert_eq!(site_plan.routed_inputs[0].component, "/consumer"); - assert_eq!(site_plan.routed_inputs[0].slot, "required_api"); - assert_eq!(site_plan.routed_inputs[0].provider_component, "/required"); - assert_eq!(site_plan.routed_inputs[0].protocol, "http"); - assert_eq!(site_plan.routed_inputs[0].capability_kind, "http"); - assert_eq!( - site_plan.routed_inputs[0].target, - DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string() - }, - "same-site child exports should reuse the provider component route instead of \ - inventing a synthetic dynamic-export hop", - ); - } - - for actuator in actuators { - actuator.abort(); - } -} - + let site_id = child_runtime_site_id(consumer).expect("consumer site id"); + let runtime_spec = + build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); + assert_eq!(runtime_spec.direct_inputs.len(), 1); + assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); + assert_eq!(runtime_spec.direct_inputs[0].slot, "required_api"); + assert_eq!( + runtime_spec.direct_inputs[0].provider_component, + "/required" + ); + assert_eq!(runtime_spec.direct_inputs[0].protocol, "http"); + assert_eq!(runtime_spec.direct_inputs[0].capability_kind, "http"); + assert_eq!( + runtime_spec.direct_inputs[0].target, + DynamicInputRouteTarget::ComponentProvide { + provide: "http".to_string() + }, + "same-site child exports should reuse the provider component route without routing \ + through the site router", + ); +} + #[tokio::test] async fn same_site_static_child_export_bindings_reuse_provider_component_routes() { let dir = TempDir::new().expect("temp dir"); @@ -1173,6 +1955,7 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1188,7 +1971,6 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; execute_create_child( &app, @@ -1216,63 +1998,211 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( .iter() .find(|child| child.name == "consumer") .expect("consumer child should be recorded"); - for site_plan in &consumer.site_plans { - assert_eq!(site_plan.routed_inputs.len(), 1); - assert_eq!(site_plan.routed_inputs[0].component, "/consumer"); - assert_eq!(site_plan.routed_inputs[0].slot, "upstream"); - assert_eq!(site_plan.routed_inputs[0].provider_component, "/provider"); - assert_eq!(site_plan.routed_inputs[0].protocol, "http"); - assert_eq!(site_plan.routed_inputs[0].capability_kind, "http"); - assert_eq!( - site_plan.routed_inputs[0].target, - DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string() - }, - "same-site static child exports should reuse the provider component route", - ); - } - - for actuator in actuators { - actuator.abort(); - } -} - -#[test] -fn framework_ccs_addressing_matches_site_runtime_topology() { + let site_id = child_runtime_site_id(consumer).expect("consumer site id"); + let runtime_spec = + build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); + assert_eq!(runtime_spec.direct_inputs.len(), 1); + assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); + assert_eq!(runtime_spec.direct_inputs[0].slot, "upstream"); assert_eq!( - ccs_listen_addr_for_site(SiteKind::Direct, 41000), - SocketAddr::from(([127, 0, 0, 1], 41000)) + runtime_spec.direct_inputs[0].provider_component, + "/provider" ); + assert_eq!(runtime_spec.direct_inputs[0].protocol, "http"); + assert_eq!(runtime_spec.direct_inputs[0].capability_kind, "http"); assert_eq!( - ccs_url_for_site(SiteKind::Direct, 41000), - "http://127.0.0.1:41000" + runtime_spec.direct_inputs[0].target, + DynamicInputRouteTarget::ComponentProvide { + provide: "http".to_string() + }, + "same-site static child exports should reuse the provider component route without routing \ + through the site router", ); +} + +#[test] +fn controller_authority_url_normalizes_unspecified_bind_addresses() { assert_eq!( - ccs_listen_addr_for_site(SiteKind::Vm, 42000), - SocketAddr::from(([127, 0, 0, 1], 42000)) + authority_url_for_listen_addr(SocketAddr::from(([127, 0, 0, 1], 41000))), + "http://127.0.0.1:41000" ); assert_eq!( - ccs_url_for_site(SiteKind::Vm, 42000), + authority_url_for_listen_addr(SocketAddr::from(([0, 0, 0, 0], 42000))), "http://127.0.0.1:42000" ); - assert_eq!( - ccs_listen_addr_for_site(SiteKind::Compose, 43000), - SocketAddr::from(([0, 0, 0, 0], 43000)) +} + +#[tokio::test] +async fn dynamic_grant_routes_to_holder_site_not_offered_site_order() { + let mut state = compile_dynamic_caps_binding_state().await; + state.placement.offered_sites.insert( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, ); + let share = super::dynamic_caps::share_dynamic_capability( + &mut state, + "components./provider", + &super::dynamic_caps::source_key_from_control_request( + &dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: RootAuthoritySelectorIr::SelfProvide { + component_id: "components./provider".to_string(), + provide_name: "http".to_string(), + }, + }, + ), + "components./alice", + None, + &json!({}), + ) + .expect("share should succeed"); + let grant_id = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } + | super::dynamic_caps::DynamicCapabilityShareOutcome::Deduplicated { grant_id, .. } => { + grant_id + } + super::dynamic_caps::DynamicCapabilityShareOutcome::Noop { reason } => { + panic!("share should not noop: {reason}"); + } + }; assert_eq!( - ccs_url_for_site(SiteKind::Compose, 43000), - "http://host.docker.internal:43000" + site_id_for_dynamic_grant(&state, &grant_id).expect("grant site should resolve"), + "direct_local", ); - assert_eq!( - ccs_listen_addr_for_site(SiteKind::Kubernetes, 44000), - SocketAddr::from(([0, 0, 0, 0], 44000)) +} + +#[tokio::test] +async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_remote_controller_capability() +{ + let dir = TempDir::new().expect("temp dir"); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authority_state = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + localize_framework_control_state(&mut authority_state, "direct_a") + .expect("authority site state should localize"); + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authority_state, + "components./alice", + &root_held_id_for(&authority_state, "components./alice"), + ) + .expect("alice root source should resolve"); + let dynamic_caps::DynamicCapabilitySourceKey::RootAuthority(root_authority_selector) = + alice_root + else { + panic!("alice root held id should resolve to a root authority selector"); + }; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &authority_state).expect("authority state should write"); + + let sync_requests = Arc::new(std::sync::Mutex::new(Vec::< + dynamic_caps::ControlDynamicGrantAuthoritySyncRequest, + >::new())); + let router = Router::new().route( + "/v1/controller/dynamic-caps/grant-authorities/sync", + axum::routing::post({ + let sync_requests = sync_requests.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let sync_requests = sync_requests.clone(); + async move { + sync_requests + .lock() + .expect("sync request log poisoned") + .push(request.clone()); + Json(dynamic_caps::ControlDynamicGrantAuthoritySyncResponse { + synced: request.authority_sites.len(), + }) + } + } + }), + ); + let (holder_base_url, _holder_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, authority_state, state_path); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.site_id = "direct_a".to_string(); + controller_plan.router_identity_id = "/site/direct_a/router".to_string(); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_b".to_string(), holder_base_url)]), ); + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_mutate( + &controller_app, + super::control_state_api::DynamicCapsMutateRequest::Share( + dynamic_caps::ControlDynamicShareRequest { + caller_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector, + }, + recipient_component_id: "components./bob".to_string(), + idempotency_key: None, + options: json!({}), + }, + ), + false, + ) + .await + .expect("authority site should create the share and sync it through the holder router"); + + let super::control_state_api::DynamicCapsMutateResponse::Share(response) = response else { + panic!("share should return a share response"); + }; + let grant_id = response + .grant_id + .clone() + .expect("cross-site share should create a concrete grant id"); + assert_eq!(response.outcome, "created"); + + let sync_requests = sync_requests.lock().expect("sync request log poisoned"); + assert_eq!(sync_requests.len(), 1, "expected exactly one holder sync"); assert_eq!( - ccs_url_for_site(SiteKind::Kubernetes, 44000), - format!( - "http://{}:44000", - host_service_host_for_consumer(SiteKind::Kubernetes) - ) + sync_requests[0].authority_sites, + BTreeMap::from([(grant_id, "direct_a".to_string())]), + "the holder site should learn that direct_a remains authoritative for the shared grant", ); } @@ -1288,12 +2218,105 @@ async fn compile_empty_control_state() -> (TempDir, FrameworkControlState, PathB } "#, ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); (dir, state, state_path) } +#[tokio::test] +async fn wrapper_manifest_lowers_framework_sources_to_synthetic_slots() { + let (dir, mut state, _) = compile_empty_control_state().await; + let child_path = dir.path().join("child.json5"); + state.base_scenario.manifest_catalog.insert( + "child".to_string(), + amber_scenario::ir::ManifestCatalogEntryIr { + source_ref: file_url(&child_path), + digest: amber_manifest::ManifestDigest::new([7; 32]), + manifest: amber_manifest::Manifest::empty(), + }, + ); + + let (manifest, synthetic_sources) = build_wrapper_manifest( + &state, + &ResolvedTemplateContract { + config: BTreeMap::new(), + bindings: BTreeMap::new(), + slot_decls: BTreeMap::new(), + visible_exports: Vec::new(), + }, + "child", + "child", + None, + &[ResolvedTemplateBinding { + slot_name: "creator".to_string(), + slot_decl: amber_manifest::SlotDecl::builder() + .decl( + amber_manifest::CapabilityDecl::builder() + .kind(amber_manifest::CapabilityKind::Http) + .build(), + ) + .build(), + sources: vec![ResolvedBindingSource { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: amber_scenario::ComponentId(state.base_scenario.root), + capability: amber_manifest::FrameworkCapabilityName::try_from("component") + .expect("framework capability name should parse"), + }), + weak: true, + }], + source_child_id: None, + }], + ) + .expect("wrapper manifest should build"); + + assert!( + manifest.bindings().iter().all(|binding| !matches!( + binding.binding.from, + amber_manifest::BindingSource::Framework(_) + )), + "wrapper manifest should not recreate raw framework bindings: {:?}", + manifest.bindings() + ); + assert_eq!( + manifest.slots().len(), + 1, + "framework sources should lower to synthetic wrapper slots" + ); + assert_eq!( + synthetic_sources.len(), + 1, + "framework source mapping should still be recorded" + ); + assert!(matches!( + synthetic_sources + .values() + .next() + .expect("synthetic source should exist") + .actual_source, + BindingFrom::Framework(_) + )); +} + async fn compile_exact_template_control_state() -> (TempDir, FrameworkControlState, PathBuf) { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); @@ -1331,7 +2354,26 @@ async fn compile_exact_template_control_state() -> (TempDir, FrameworkControlSta child = file_url(&child_path), ), ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); (dir, state, state_path) @@ -1365,8 +2407,8 @@ async fn compile_framework_binding_control_state() -> ( { manifest_version: "0.3.0", program: { - path: "/bin/echo", - args: ["worker"], + path: "/usr/bin/env", + args: ["python3", "-m", "http.server", "8080"], network: { endpoints: [{ name: "out", port: 8080, protocol: "http" }] } }, provides: { out: { kind: "http", endpoint: "out" } }, @@ -1399,7 +2441,26 @@ async fn compile_framework_binding_control_state() -> ( ), ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); let record = state @@ -1411,12 +2472,84 @@ async fn compile_framework_binding_control_state() -> ( (dir, state, state_path, record) } +async fn compile_cross_site_framework_authority_control_state() +-> (TempDir, FrameworkControlState, PathBuf) { + let dir = TempDir::new().expect("temp dir"); + let admin_path = dir.path().join("admin.json5"); + let authority_path = dir.path().join("authority.json5"); + let root_path = dir.path().join("root.json5"); + write_file( + &admin_path, + r#" + { + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" }, + } + "#, + ); + write_file( + &authority_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ admin: "{admin}" }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + program: {{ + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: {{ endpoints: [{{ name: "http", port: 8081, protocol: "http" }}] }} + }}, + provides: {{ api: {{ kind: "http", endpoint: "http" }} }}, + exports: {{ + api: "api", + admin_api: "#admin.api" + }}, + }} + "##, + admin = file_url(&admin_path), + ), + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ authority: "{authority}" }}, + exports: {{ + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" + }}, + }} + "##, + authority = file_url(&authority_path), + ), + ); + + let state = compile_control_state(&root_path).await; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + (dir, state, state_path) +} + fn empty_live_child( authority_realm_id: usize, name: &str, child_id: u64, state: ChildState, ) -> LiveChildRecord { + let child_moniker = format!("/{name}"); LiveChildRecord { child_id, authority_realm_id, @@ -1424,10 +2557,29 @@ fn empty_live_child( state, template_name: Some("worker".to_string()), selected_manifest_catalog_key: None, - fragment: None, + fragment: Some(LiveScenarioFragment { + root_component_id: child_id as usize + 10_000, + components: vec![ComponentIr { + id: child_id as usize + 10_000, + moniker: child_moniker.clone(), + parent: Some(authority_realm_id), + children: Vec::new(), + resolved_url: Some(format!("file:///tmp/{name}.json5")), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }], + bindings: Vec::new(), + }), input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: Vec::new(), + assignments: BTreeMap::from([(child_moniker, "direct_local".to_string())]), overlay_ids: Vec::new(), overlays: Vec::new(), outputs: BTreeMap::new(), @@ -1442,25 +2594,169 @@ fn pending_destroy(tx_id: u64, child: LiveChildRecord) -> PendingDestroyRecord { PendingDestroyRecord { tx_id, child } } -fn test_control_state_app( - dir: &TempDir, - state: FrameworkControlState, +fn test_router_control_addr() -> String { + static ROUTER_CONTROL_ADDR: OnceLock = OnceLock::new(); + ROUTER_CONTROL_ADDR + .get_or_init(|| { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .expect("mock router control listener should bind"); + let addr = listener + .local_addr() + .expect("mock router control listener addr"); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + stream + .set_read_timeout(Some(StdDuration::from_millis(100))) + .expect("mock router control read timeout should set"); + let mut request = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + match stream.read(&mut buf) { + Ok(0) => break, + Ok(read) => request.extend_from_slice(&buf[..read]), + Err(err) + if matches!( + err.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + break; + } + Err(_) => break, + } + } + let request = String::from_utf8_lossy(&request); + let response = if request.starts_with("GET /identity ") { + let body = serde_json::to_string(&json!({ + "id": "/site/test/router", + "public_key": vec![9u8; 32], + "mesh_scope": "test-mesh", + })) + .expect("mock router identity should serialize"); + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ) + } else { + "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string() + }; + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + }); + addr.to_string() + }) + .clone() +} + +fn test_control_state_app( + dir: &TempDir, + state: FrameworkControlState, state_path: PathBuf, ) -> ControlStateApp { let run_root = dir.path().join("run"); let state_root = dir.path().join("state"); fs::create_dir_all(&run_root).expect("run root should exist"); fs::create_dir_all(&state_root).expect("state root should exist"); + let offered_sites = if state.placement.offered_sites.is_empty() { + BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + )]) + } else { + state.placement.offered_sites.clone() + }; + let router_control = test_router_control_addr(); + let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + let site_id = state + .placement + .assignments + .get("/") + .cloned() + .or_else(|| state.placement.defaults.path.clone()) + .or_else(|| offered_sites.keys().next().cloned()) + .expect("offered sites should contain at least one site"); + for (site_id, site_definition) in &offered_sites { + let site_state_root = state_root.join(site_id); + let artifact_dir = dir.path().join("artifact").join(site_id); + let storage_root = dir.path().join("storage").join(site_id); + let runtime_root = dir.path().join("runtime").join(site_id); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&storage_root).expect("storage root should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &json!({ + "status": "running", + "kind": site_definition.kind, + "artifact_dir": artifact_dir.display().to_string(), + "supervisor_pid": 1u32, + "router_control": router_control.clone(), + "router_mesh_addr": "127.0.0.1:24000", + "router_identity_id": format!("/site/{site_id}/router"), + "router_public_key_b64": router_public_key_b64.clone(), + "site_controller_url": "http://127.0.0.1:0", + }), + ) + .expect("site manager state should write"); + } + let site_state_root = state_root.join(&site_id); + let artifact_dir = dir.path().join("artifact").join(&site_id); + let storage_root = dir.path().join("storage").join(&site_id); + let runtime_root = dir.path().join("runtime").join(&site_id); ControlStateApp { control_state: Arc::new(Mutex::new(state)), client: ReqwestClient::new(), - state_path, - run_root, - state_root, + state_path: state_path.clone(), + run_root: run_root.clone(), + state_root: state_root.clone(), mesh_scope: Arc::::from("test-mesh"), - control_state_auth_token: Arc::::from("test-control-state-auth"), + controller_plan: Arc::new(SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: site_id.clone(), + kind: SiteKind::Direct, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: format!("/site/{site_id}/router"), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: None, + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + state_path: state_path.display().to_string(), + run_root: run_root.display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: site_state_root.display().to_string(), + artifact_dir: artifact_dir.display().to_string(), + controller_identity_path: None, + storage_root: Some(storage_root.display().to_string()), + runtime_root: Some(runtime_root.display().to_string()), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, + launch_env: BTreeMap::new(), + }), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), - bridge_proxies: Arc::new(Mutex::new(BTreeMap::new())), + runtime: test_runtime(), } } @@ -1483,10 +2779,21 @@ fn sse_json_rpc_message(body: &str) -> Value { .unwrap_or_else(|err| panic!("parse JSON-RPC payload from SSE: {err}; {payload}")) } +fn json_rpc_message(body: &str) -> Value { + serde_json::from_str(body).unwrap_or_else(|_| sse_json_rpc_message(body)) +} + async fn spawn_test_router(router: Router) -> (String, tokio::task::JoinHandle<()>) { let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await .expect("test listener"); + spawn_test_router_on(listener, router) +} + +fn spawn_test_router_on( + listener: tokio::net::TcpListener, + router: Router, +) -> (String, tokio::task::JoinHandle<()>) { let addr = listener.local_addr().expect("test listener addr"); let handle = tokio::spawn(async move { axum::serve(listener, router.into_make_service()) @@ -1511,13 +2818,14 @@ async fn http_get_json( url: &str, headers: &[(String, String)], ) -> T { - let response = apply_headers(client.get(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send GET {url}: {err}")); + let response = with_test_timeout( + format!("GET {url}"), + apply_headers(client.get(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send GET {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read GET {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read GET {url}: {err}")); assert_eq!(status, StatusCode::OK, "GET {url} failed: {body}"); @@ -1531,14 +2839,14 @@ async fn http_post_json( headers: &[(String, String)], body: &Req, ) -> T { - let response = apply_headers(client.post(url), headers) - .json(body) - .send() - .await - .unwrap_or_else(|err| panic!("send POST {url}: {err}")); + let response = with_test_timeout( + format!("POST {url}"), + apply_headers(client.post(url), headers).json(body).send(), + ) + .await + .unwrap_or_else(|err| panic!("send POST {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read POST {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read POST {url}: {err}")); assert_eq!(status, StatusCode::OK, "POST {url} failed: {body}"); @@ -1551,13 +2859,14 @@ async fn http_post_empty_json( url: &str, headers: &[(String, String)], ) -> T { - let response = apply_headers(client.post(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send POST {url}: {err}")); + let response = with_test_timeout( + format!("POST {url}"), + apply_headers(client.post(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send POST {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read POST {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read POST {url}: {err}")); assert_eq!(status, StatusCode::OK, "POST {url} failed: {body}"); @@ -1566,13 +2875,14 @@ async fn http_post_empty_json( } async fn http_delete_empty(client: &Client, url: &str, headers: &[(String, String)]) { - let response = apply_headers(client.delete(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send DELETE {url}: {err}")); + let response = with_test_timeout( + format!("DELETE {url}"), + apply_headers(client.delete(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send DELETE {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read DELETE {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read DELETE {url}: {err}")); assert_eq!( @@ -1604,6 +2914,275 @@ fn normalize_dynamic_share_ref(value: &mut Value) { } } +const TEST_SITE_STATE_SCHEMA: &str = "amber.run.site_state"; +const TEST_SITE_STATE_VERSION: u32 = 3; +const TEST_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(15); + +async fn with_test_timeout(label: impl Into, future: impl Future) -> T { + let label = label.into(); + tokio::time::timeout(TEST_REQUEST_TIMEOUT, future) + .await + .unwrap_or_else(|_| panic!("{label} timed out after {:?}", TEST_REQUEST_TIMEOUT)) +} + +fn with_controller_endpoint( + app: &ControlStateApp, + authority_url: &str, + listen_addr: SocketAddr, +) -> ControlStateApp { + let mut controller_plan = app.controller_plan.as_ref().clone(); + controller_plan.authority_url = authority_url.to_string(); + controller_plan.listen_addr = listen_addr; + ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + controller_plan: Arc::new(controller_plan), + authority_locks: app.authority_locks.clone(), + runtime: app.runtime.clone(), + } +} + +#[tokio::test] +async fn remote_dynamic_capability_controller_post_retries_transient_unavailable() { + async fn handler( + axum::extract::State(attempts): axum::extract::State>, + ) -> axum::response::Response { + use axum::response::IntoResponse as _; + + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "code": "ControlStateUnavailable", + "message": "site controller is still recovering" + })), + ) + .into_response(); + } + (StatusCode::OK, Json(json!({"held": []}))).into_response() + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let (url, handle) = spawn_test_router( + Router::new() + .route( + "/v1/controller/dynamic-caps/held", + axum::routing::post(handler), + ) + .with_state(attempts.clone()), + ) + .await; + let (dir, state, state_path) = compile_empty_control_state().await; + let mut control = test_control_state_app(&dir, state, state_path); + install_remote_controller_peer_url_fixture( + &mut control, + &BTreeMap::from([("remote".to_string(), url)]), + ); + let app = SiteControllerApp { + control, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./caller".to_string(), + }, + ), + false, + ) + .await + .expect("request should succeed after transient controller unavailability"); + let super::control_state_api::DynamicCapsInspectResponse::HeldList(response) = response else { + panic!("held_list should return held entries"); + }; + assert!(response.held.is_empty()); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + + handle.abort(); + let _ = handle.await; +} + +async fn install_framework_site_controller_fixture( + app: &ControlStateApp, +) -> Vec> { + let plan = app.controller_plan.as_ref(); + let site_state_root = PathBuf::from(&plan.site_state_root); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(artifact_dir.join(".amber").join("control")) + .expect("artifact control dir should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json(&site_controller_plan_path(&site_state_root), plan) + .expect("site controller plan should write"); + + let overlay_router = Router::new().route( + "/overlays/{overlay_id}", + axum::routing::put(|| async { StatusCode::NO_CONTENT }) + .delete(|| async { StatusCode::NO_CONTENT }), + ); + let (router_base_url, overlay_handle) = spawn_test_router(overlay_router).await; + let router_control = router_base_url + .strip_prefix("http://") + .expect("router control URL should be absolute HTTP") + .to_string(); + + write_json( + &site_state_path(&app.state_root, &plan.site_id), + &json!({ + "schema": TEST_SITE_STATE_SCHEMA, + "version": TEST_SITE_STATE_VERSION, + "run_id": plan.run_id, + "site_id": plan.site_id, + "kind": plan.kind, + "status": "running", + "artifact_dir": plan.artifact_dir, + "supervisor_pid": 1, + "router_control": router_control, + "router_mesh_addr": format!("127.0.0.1:{}", plan.router_mesh_port.expect("router mesh port")), + "router_identity_id": plan.router_identity_id, + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([11; 32]), + "site_controller_url": plan.authority_url, + }), + ) + .expect("manager state should write"); + write_json( + &artifact_dir.join("direct-plan.json"), + &json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [], + "components": [], + "router": { + "identity_id": plan.router_identity_id, + "mesh_port": plan.router_mesh_port.expect("router mesh port"), + "control_port": 39011, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json", + }, + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { + router_mesh_port: plan.router_mesh_port, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + let mut router_mesh = test_live_site_router(Vec::new()); + router_mesh.identity.id = plan.router_identity_id.clone(); + router_mesh.mesh_listen = format!( + "127.0.0.1:{}", + plan.router_mesh_port.expect("router mesh port") + ) + .parse() + .expect("router mesh listen should parse"); + write_json(&runtime_root.join("router-mesh.json"), &router_mesh) + .expect("router mesh config should write"); + + vec![overlay_handle] +} + +fn install_remote_controller_peer_url_fixture( + app: &mut ControlStateApp, + remote_base_urls: &BTreeMap, +) { + let plan = Arc::make_mut(&mut app.controller_plan); + plan.peer_site_router_urls.extend(remote_base_urls.clone()); +} + +fn retarget_test_controller_site(app: &mut ControlStateApp, site_id: &str) { + let plan = Arc::make_mut(&mut app.controller_plan); + plan.site_id = site_id.to_string(); + plan.router_identity_id = format!("/site/{site_id}/router"); + plan.site_state_root = app.state_root.join(site_id).display().to_string(); + if let Some(parent) = Path::new(&plan.artifact_dir).parent() { + plan.artifact_dir = parent.join(site_id).display().to_string(); + } + if let Some(parent) = plan + .runtime_root + .as_deref() + .and_then(|runtime_root| Path::new(runtime_root).parent()) + { + plan.runtime_root = Some(parent.join(site_id).display().to_string()); + } + if let Some(parent) = plan + .storage_root + .as_deref() + .and_then(|storage_root| Path::new(storage_root).parent()) + { + plan.storage_root = Some(parent.join(site_id).display().to_string()); + } +} + +fn install_bob_holder_runtime_fixture(app: &ControlStateApp) { + let plan = app.controller_plan.as_ref(); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + write_json( + &artifact_dir.join("direct-plan.json"), + &json!({ + "components": [{ + "id": 3, + "moniker": "/bob", + "sidecar": { + "mesh_config_path": "bob-mesh.json", + }, + }], + "router": { + "mesh_config_path": "router-mesh.json", + }, + }), + ) + .expect("holder direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { + component_mesh_port_by_id: BTreeMap::from([(3, 24003)]), + ..Default::default() + }, + ) + .expect("holder direct runtime state should write"); + write_json( + &runtime_root.join("bob-mesh.json"), + &test_live_component_runtime( + "/bob", + "/bob", + "127.0.0.1:24003", + "127.0.0.1:24003", + Vec::new(), + Vec::new(), + ) + .mesh_config, + ) + .expect("holder mesh config should write"); + write_json( + &runtime_root.join("router-mesh.json"), + &test_live_site_router(Vec::new()), + ) + .expect("holder router mesh config should write"); +} + struct TestMcpClient { client: Client, endpoint: String, @@ -1614,31 +3193,17 @@ struct TestMcpClient { impl TestMcpClient { async fn connect(base_url: &str, client_name: &str, headers: Vec<(String, String)>) -> Self { + Self::connect_endpoint(&format!("{base_url}/mcp"), client_name, headers).await + } + + async fn connect_endpoint( + endpoint: &str, + client_name: &str, + headers: Vec<(String, String)>, + ) -> Self { let client = Client::new(); - let endpoint = format!("{base_url}/mcp"); - let initialize = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "initialize", - "params": { - "protocolVersion": "2025-06-18", - "capabilities": {}, - "clientInfo": { - "name": client_name, - "version": "0.0.0", - }, - }, - }); - let response = apply_headers(client.post(&endpoint), &headers) - .header("content-type", "application/json") - .header("accept", "application/json, text/event-stream") - .json(&initialize) - .send() - .await - .expect("send initialize request"); - let status = response.status(); - let response_headers = response.headers().clone(); - let body = response.text().await.expect("read initialize response"); + let (status, response_headers, payload, body) = + Self::initialize_response(&client, endpoint, client_name, &headers).await; assert_eq!(status, StatusCode::OK, "initialize failed: {body}"); let session_id = response_headers .get("mcp-session-id") @@ -1646,7 +3211,6 @@ impl TestMcpClient { .to_str() .expect("session ID should be valid UTF-8") .to_string(); - let payload = sse_json_rpc_message(&body); assert!( payload.get("error").is_none(), "initialize returned error: {payload:#?}" @@ -1656,28 +3220,72 @@ impl TestMcpClient { Some("2025-06-18") ); - let notification = apply_headers(client.post(&endpoint), &headers) - .header("content-type", "application/json") - .header("accept", "application/json, text/event-stream") - .header("mcp-session-id", &session_id) - .json(&json!({ - "jsonrpc": "2.0", - "method": "notifications/initialized", - })) - .send() - .await - .expect("send initialized notification"); + let notification = with_test_timeout( + format!("MCP initialized notification to {endpoint}"), + apply_headers(client.post(endpoint), &headers) + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .header("mcp-session-id", &session_id) + .json(&json!({ + "jsonrpc": "2.0", + "method": "notifications/initialized", + })) + .send(), + ) + .await + .expect("send initialized notification"); assert_eq!(notification.status(), StatusCode::ACCEPTED); Self { client, - endpoint, + endpoint: endpoint.to_string(), session_id, headers, next_id: 1, } } + async fn initialize_response( + client: &Client, + endpoint: &str, + client_name: &str, + headers: &[(String, String)], + ) -> (StatusCode, HeaderMap, Value, String) { + let initialize = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": { + "name": client_name, + "version": "0.0.0", + }, + }, + }); + let response = with_test_timeout( + format!("MCP initialize request to {endpoint}"), + apply_headers(client.post(endpoint), headers) + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .json(&initialize) + .send(), + ) + .await + .expect("send initialize request"); + let status = response.status(); + let response_headers = response.headers().clone(); + let body = with_test_timeout( + format!("read MCP initialize response from {endpoint}"), + response.text(), + ) + .await + .expect("read initialize response"); + let payload = json_rpc_message(&body); + (status, response_headers, payload, body) + } + async fn tools_list(&mut self) -> Vec { self.request("tools/list", json!({})) .await @@ -1732,7 +3340,35 @@ impl TestMcpClient { .unwrap_or_else(|err| panic!("deserialize tool result for {name}: {err}; {result:#?}")) } + async fn call_tool_error(&mut self, name: &str, arguments: Value) -> Value { + let payload = self + .request_payload( + "tools/call", + json!({ + "name": name, + "arguments": arguments, + }), + ) + .await; + payload + .get("error") + .cloned() + .unwrap_or_else(|| panic!("tool {name} unexpectedly succeeded: {payload:#?}")) + } + async fn request(&mut self, method: &str, params: Value) -> Value { + let payload = self.request_payload(method, params).await; + assert!( + payload.get("error").is_none(), + "MCP request {method} returned error: {payload:#?}" + ); + payload + .get("result") + .cloned() + .expect("MCP response should include result") + } + + async fn request_payload(&mut self, method: &str, params: Value) -> Value { let id = self.next_id; self.next_id += 1; let response = apply_headers( @@ -1749,14 +3385,20 @@ impl TestMcpClient { "method": method, "params": params, })) - .send() + .send(); + let response = with_test_timeout( + format!("MCP request {method} to {}", self.endpoint), + response, + ) .await .unwrap_or_else(|err| panic!("send MCP request {method}: {err}")); let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|err| panic!("read MCP response for {method}: {err}")); + let body = with_test_timeout( + format!("read MCP response for {method} from {}", self.endpoint), + response.text(), + ) + .await + .unwrap_or_else(|err| panic!("read MCP response for {method}: {err}")); assert_eq!( status, StatusCode::OK, @@ -1764,14 +3406,7 @@ impl TestMcpClient { ); let payload = sse_json_rpc_message(&body); assert_eq!(payload["id"].as_u64(), Some(id)); - assert!( - payload.get("error").is_none(), - "MCP request {method} returned error: {payload:#?}" - ); payload - .get("result") - .cloned() - .expect("MCP response should include result") } } @@ -1781,73 +3416,53 @@ struct FrameworkMcpHarness { base_url: String, route_id: String, peer_id: String, - auth_token: String, handles: Vec>, } impl FrameworkMcpHarness { - async fn start(with_actuators: bool) -> Self { + async fn start(with_live_site_runtime: bool) -> Self { let (dir, state, state_path, record) = compile_framework_binding_control_state().await; - let app = test_control_state_app(&dir, state, state_path); - let mut handles = if with_actuators { - install_success_site_actuator(&app).await + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("site controller listener"); + let listen_addr = listener.local_addr().expect("site controller addr"); + let base_url = format!("http://{listen_addr}"); + let app = with_controller_endpoint( + &test_control_state_app(&dir, state, state_path), + &base_url, + listen_addr, + ); + assert_eq!(app.controller_plan.site_id, record.recipient_site_id); + let mut handles = if with_live_site_runtime { + install_framework_site_controller_fixture(&app).await } else { Vec::new() }; - let control_router = Router::new() - .route(CONTROL_SERVICE_PATH, get(get_control_state)) - .route("/v1/control-state/children", post(control_create_child)) - .route( - "/v1/control-state/children/{child}/destroy", - post(control_destroy_child), - ) - .with_state(app.clone()); - let (control_state_url, control_handle) = spawn_test_router(control_router).await; - handles.push(control_handle); - - let auth_token = "test-router-auth".to_string(); - let ccs_app = CcsApp { - client: ReqwestClient::new(), - site_state_root: app.state_root.clone(), - control_state_url: Arc::::from(control_state_url), - router_auth_token: Arc::::from(auth_token.clone()), - control_state_auth_token: app.control_state_auth_token.clone(), + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), }; - let ccs_router = Router::new() - .nest_service("/mcp", mcp::service(ccs_app.clone())) - .route("/v1/templates", get(ccs_list_templates)) - .route("/v1/templates/{template}", get(ccs_describe_template)) - .route( - "/v1/templates/{template}/resolve", - post(ccs_resolve_template), - ) - .route( - "/v1/children", - get(ccs_list_children).post(ccs_create_child), - ) - .route( - "/v1/children/{child}", - get(ccs_describe_child).delete(ccs_destroy_child), + handles.push(tokio::spawn(async move { + axum::serve( + listener, + super::site_controller::site_controller_router(controller_app).into_make_service(), ) - .route("/v1/snapshot", post(ccs_snapshot)) - .with_state(ccs_app); - let (base_url, ccs_handle) = spawn_test_router(ccs_router).await; - handles.push(ccs_handle); + .await + .expect("site controller should serve"); + })); Self { _dir: dir, client: Client::new(), base_url, - route_id: record.cap_instance_id, + route_id: record.route_id, peer_id: record.recipient_peer_id, - auth_token, handles, } } fn http_headers(&self) -> Vec<(String, String)> { vec![ - (FRAMEWORK_AUTH_HEADER.to_string(), self.auth_token.clone()), (FRAMEWORK_ROUTE_ID_HEADER.to_string(), self.route_id.clone()), (FRAMEWORK_PEER_ID_HEADER.to_string(), self.peer_id.clone()), ] @@ -1943,18 +3558,23 @@ async fn framework_component_mcp_discovers_compact_surface() { help.contains("amber.v1.framework_component.inspect"), "help resource should point callers to the inspect tool" ); - assert!( - help.contains("/mcp"), - "help resource should explain the MCP endpoint path" - ); } #[tokio::test] async fn framework_component_mcp_matches_http_surface() { - let http = FrameworkMcpHarness::start(true).await; - let mcp_harness = FrameworkMcpHarness::start(true).await; - let mut mcp = mcp_harness.connect().await; - let mut same_state_mcp = http.connect().await; + let http = with_test_timeout( + "start framework HTTP harness", + FrameworkMcpHarness::start(true), + ) + .await; + let mcp_harness = with_test_timeout( + "start framework MCP harness", + FrameworkMcpHarness::start(true), + ) + .await; + let mut mcp = with_test_timeout("connect framework MCP client", mcp_harness.connect()).await; + let mut same_state_mcp = + with_test_timeout("connect same-state MCP client", http.connect()).await; let http_templates: TemplateListResponse = http.get_json("/v1/templates").await; let mcp_templates: Value = mcp @@ -2087,11 +3707,304 @@ async fn framework_component_mcp_matches_http_surface() { ); } -struct DynamicCapsMcpHarness { - _dir: TempDir, - client: Client, - base_url: String, - auth_token: String, +#[tokio::test] +async fn framework_component_rejects_stale_nonlocal_controller_delivery() { + let (_dir, state, state_path, record) = compile_framework_binding_control_state().await; + let app = test_control_state_app(&_dir, state.clone(), state_path); + let mut controller_plan = app.controller_plan.as_ref().clone(); + controller_plan.site_id = "wrong-site".to_string(); + let controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + controller_plan: Arc::new(controller_plan), + authority_locks: app.authority_locks.clone(), + runtime: app.runtime.clone(), + }, + ready: ready_site_controller_flag(), + }; + + let err = match super::site_controller::execute_site_controller_framework_inspect( + &controller_app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await + { + Ok(_) => panic!("stale controller delivery should be rejected"), + Err(err) => err, + }; + assert_eq!(err.0.code, ProtocolErrorCode::ControlStateUnavailable); + assert!( + err.0.message.contains("bound to controller site"), + "unexpected error: {}", + err.0.message + ); +} + +#[tokio::test] +async fn framework_component_rejects_requests_while_controller_recovers() { + let (_dir, state, state_path, record) = compile_framework_binding_control_state().await; + let controller_app = SiteControllerApp { + control: test_control_state_app(&_dir, state.clone(), state_path), + ready: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let err = match super::site_controller::execute_site_controller_framework_inspect( + &controller_app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await + { + Ok(_) => panic!("recovering controller should reject public framework requests"), + Err(err) => err, + }; + + assert_eq!(err.0.code, ProtocolErrorCode::ControlStateUnavailable); + assert_eq!(err.0.message, "site controller is still recovering"); +} + +#[tokio::test] +async fn refresh_capability_instances_rejects_unlowered_framework_component_bindings() { + let (_dir, mut state, _state_path, _) = compile_framework_binding_control_state().await; + let binding = state + .base_scenario + .bindings + .iter_mut() + .find(|binding| binding.to.slot == "ctl") + .expect("framework binding should exist"); + binding.from = BindingFromIr::Framework { + authority_realm: state.base_scenario.root, + capability: "component".to_string(), + }; + + let err = refresh_capability_instances(&mut state) + .expect_err("unlowered framework.component bindings must be rejected"); + assert!( + err.to_string() + .contains("must be lowered before capability refresh"), + "unexpected framework lowering error: {err}", + ); +} +#[tokio::test] +async fn recover_control_state_reconciles_framework_routes_without_live_remote_controllers() { + let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; + state.placement.offered_sites = BTreeMap::from([ + ( + "authority".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "consumer".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]); + state.placement.defaults = PlacementDefaults { + path: Some("authority".to_string()), + ..PlacementDefaults::default() + }; + state.placement.placement_components = BTreeMap::from([ + ("/".to_string(), "authority".to_string()), + ("/admin".to_string(), "consumer".to_string()), + ]); + state.placement.assignments = state.placement.placement_components.clone(); + refresh_capability_instances(&mut state).expect("framework routes should refresh"); + write_control_state(&state_path, &state).expect("state should write"); + let app = test_control_state_app(&dir, state, state_path); + + let mut authority_plan = app.controller_plan.as_ref().clone(); + authority_plan.site_id = "authority".to_string(); + authority_plan.router_identity_id = "/site/authority/router".to_string(); + authority_plan.peer_router_identities = BTreeMap::from([( + "consumer".to_string(), + MeshIdentityPublic { + id: "/site/consumer/router".to_string(), + public_key: [8u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]); + let authority_app = ControlStateApp { + controller_plan: Arc::new(authority_plan), + ..app + }; + + recover_control_state(&authority_app).await.expect( + "recovery should not require a live remote controller when peer router identities were \ + already planned", + ); +} + +#[test] +fn inject_site_controller_peer_router_routes_records_peer_router_identities() { + let temp = TempDir::new().expect("tempdir"); + let artifact_root = temp.path(); + write_json( + &artifact_root.join("mesh-provision-plan.json"), + &amber_mesh::MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![ + amber_mesh::MeshProvisionTarget { + kind: amber_mesh::MeshProvisionTargetKind::Component, + config: amber_mesh::MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/controller".to_string(), + mesh_scope: Some("test-mesh".to_string()), + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: vec![amber_mesh::InboundRoute { + route_id: amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: amber_mesh::InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/site/local/router".to_string()], + }], + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: amber_mesh::MeshProvisionOutput::Filesystem { + dir: "mesh/controller".to_string(), + }, + }, + amber_mesh::MeshProvisionTarget { + kind: amber_mesh::MeshProvisionTargetKind::Router, + config: amber_mesh::MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/router".to_string(), + mesh_scope: Some("test-mesh".to_string()), + }, + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: amber_mesh::MeshProvisionOutput::Filesystem { + dir: "mesh/router".to_string(), + }, + }, + ], + }, + ) + .expect("mesh provision plan should write"); + + inject_site_controller_peer_router_routes( + artifact_root, + "local", + &["/site/peer/router".to_string()], + &[SiteControllerPeerRouterRoute { + site_id: "peer".to_string(), + peer_router: MeshIdentityPublic { + id: "/site/peer/router".to_string(), + public_key: [5u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + peer_addr: "10.0.0.2:24000".to_string(), + listen_addr: "127.0.0.1".to_string(), + listen_port: 24123, + }], + ) + .expect("site controller peer routes should inject"); + + let plan: amber_mesh::MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + ) + .expect("mesh provision plan should read"); + assert!( + plan.existing_peer_identities + .iter() + .any(|identity| identity.id == "/site/peer/router" && identity.public_key == [5u8; 32]), + "the injected router plan should carry peer router identities so startup does not need \ + runtime discovery", + ); + let router = plan + .targets + .iter() + .find(|target| matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Router)) + .expect("router target should remain present"); + assert!( + router.config.inbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("local") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && matches!( + route.target, + amber_mesh::InboundTarget::MeshForward { + ref peer_id, + ref peer_addr, + ref route_id, + ref capability, + } if peer_id == "/site/local/controller" + && peer_addr == "127.0.0.1:23001" + && route_id + == &amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ) + && capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + ) + }), + "the injected router plan should terminate peer site-controller traffic at the lowered \ + controller component", + ); + assert!( + router + .config + .peers + .iter() + .any(|peer| peer.id == "/site/local/controller"), + "the router plan should trust the lowered local controller peer it forwards to", + ); + assert!( + router + .config + .outbound + .iter() + .any(|route| route.route_id == "site-controller:peer" + && route.peer_id == "/site/peer/router"), + "the router plan should include the site-controller forwarding route for the peer site", + ); +} + +struct DynamicCapsMcpHarness { + _dir: TempDir, + client: Client, + base_url: String, + controller_route_id: String, handles: Vec>, } @@ -2101,64 +4014,114 @@ impl DynamicCapsMcpHarness { let state = compile_dynamic_caps_binding_state().await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let mut handles = vec![install_dynamic_caps_origin_fixture(&app).await]; - let router = Router::new() - .nest_service("/mcp", control_state_mcp::service(app.clone())) - .route( - "/v1/control-state/dynamic-caps/held", - post(control_dynamic_held_list), - ) - .route( - "/v1/control-state/dynamic-caps/held/detail", - post(control_dynamic_held_detail), - ) - .route( - "/v1/control-state/dynamic-caps/share", - post(control_dynamic_share), - ) - .route( - "/v1/control-state/dynamic-caps/inspect-ref", - post(control_dynamic_inspect_ref), - ) - .route( - "/v1/control-state/dynamic-caps/revoke", - post(control_dynamic_revoke), + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("site controller listener"); + let listen_addr = listener.local_addr().expect("site controller addr"); + let base_url = format!("http://{listen_addr}"); + let app = with_controller_endpoint( + &test_control_state_app(&dir, state, state_path), + &base_url, + listen_addr, + ); + let controller_route_id = { + let state = app.control_state.lock().await; + let scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == app.controller_plan.site_id) + }) + .map(|(_, component)| component) + .expect("site controller component should be injected"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, ) - .route( - "/v1/control-state/dynamic-caps/resolve-origin", - post(control_dynamic_resolve_origin), + }; + let mut handles = vec![install_dynamic_caps_origin_fixture(&app).await]; + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + handles.push(tokio::spawn(async move { + axum::serve( + listener, + super::site_controller::site_controller_router(controller_app).into_make_service(), ) - .with_state(app); - let (base_url, handle) = spawn_test_router(router).await; - handles.push(handle); + .await + .expect("site controller should serve"); + })); Self { _dir: dir, client: Client::new(), base_url, - auth_token: "test-control-state-auth".to_string(), + controller_route_id, handles, } } - fn http_headers(&self) -> Vec<(String, String)> { - vec![(FRAMEWORK_AUTH_HEADER.to_string(), self.auth_token.clone())] + fn dynamic_caps_headers_for_component(&self, component_id: &str) -> Vec<(String, String)> { + let peer_id = dynamic_caps::moniker_from_logical_component_id(component_id) + .expect("logical component id should map to a moniker") + .to_string(); + vec![ + ( + FRAMEWORK_ROUTE_ID_HEADER.to_string(), + self.controller_route_id.clone(), + ), + (FRAMEWORK_PEER_ID_HEADER.to_string(), peer_id), + ] } - async fn connect(&self) -> TestMcpClient { - TestMcpClient::connect( - &self.base_url, + fn dynamic_caps_http_headers(&self, body: &Req) -> Vec<(String, String)> { + let body = serde_json::to_value(body).expect("dynamic caps request should serialize"); + let component_id = body + .get("holder_component_id") + .or_else(|| body.get("caller_component_id")) + .and_then(Value::as_str) + .expect("dynamic caps control requests should identify the caller component"); + self.dynamic_caps_headers_for_component(component_id) + } + + async fn connect_as(&self, component_id: &str) -> TestMcpClient { + TestMcpClient::connect_endpoint( + &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), "framework-dynamic-caps-test", - self.http_headers(), + self.dynamic_caps_headers_for_component(component_id), ) .await } + async fn connect(&self) -> TestMcpClient { + self.connect_as("components./alice").await + } + + async fn initialize_with_headers(&self, headers: Vec<(String, String)>) -> (StatusCode, Value) { + let client = Client::new(); + let (status, _response_headers, payload, body) = TestMcpClient::initialize_response( + &client, + &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), + "framework-dynamic-caps-test", + &headers, + ) + .await; + assert!( + status.is_success(), + "initialize returned transport failure {status}: {body}" + ); + (status, payload) + } + async fn post_json(&self, path: &str, body: &Req) -> T { http_post_json( &self.client, &format!("{}{}", self.base_url, path), - &self.http_headers(), + &self.dynamic_caps_http_headers(body), body, ) .await @@ -2174,205 +4137,783 @@ impl Drop for DynamicCapsMcpHarness { } async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::task::JoinHandle<()> { - let site_id = "direct_local"; - let site_state_root = Path::new(&app.state_root).join(site_id); - let artifact_dir = site_state_root.join("artifact"); - let runtime_root = site_state_root.join("runtime"); - fs::create_dir_all(artifact_dir.join(".amber")).expect("artifact root should exist"); + let plan = app.controller_plan.as_ref(); + let site_state_root = PathBuf::from(&plan.site_state_root); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(artifact_dir.join(".amber").join("control")) + .expect("artifact control dir should exist"); fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + + write_json(&site_controller_plan_path(&site_state_root), plan) + .expect("site controller plan should write"); + + let overlay_router = Router::new().route( + "/overlays/{overlay_id}", + axum::routing::put(|| async { StatusCode::NO_CONTENT }) + .delete(|| async { StatusCode::NO_CONTENT }), + ); + let (router_base_url, overlay_handle) = spawn_test_router(overlay_router).await; + let router_control = router_base_url + .strip_prefix("http://") + .expect("router control URL should be absolute HTTP") + .to_string(); + + write_json( + &site_state_path(&app.state_root, &plan.site_id), + &json!({ + "status": "running", + "kind": plan.kind, + "artifact_dir": artifact_dir.display().to_string(), + "supervisor_pid": 1, + "router_control": router_control, + "router_mesh_addr": "127.0.0.1:39001", + "router_identity_id": plan.router_identity_id, + "router_public_key_b64": "dGVzdC1yb3V0ZXIta2V5", + "site_controller_url": plan.authority_url, + }), + ) + .expect("manager state should write"); + write_json( + &site_state_root.join("site-controller-runtime-state.json"), + &json!({ + "schema": "amber.run.site_controller_runtime_state", + "version": 1, + "run_id": plan.run_id, + "site_id": plan.site_id, + "kind": plan.kind, + "children": {}, + }), + ) + .expect("site controller runtime state should write"); + write_json( + &artifact_dir.join("direct-plan.json"), + &json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [1, 2], + "components": [ + { + "id": 1, + "moniker": "/provider", + "log_name": "provider", + "sidecar": { + "log_name": "provider-sidecar", + "mesh_port": 24001, + "mesh_config_path": "provider-mesh.json", + "mesh_identity_path": "provider-identity.json", + }, + "program": { + "log_name": "provider-program", + "work_dir": ".", + "execution": { + "kind": "direct", + "entrypoint": ["/bin/true"], + }, + }, + }, + { + "id": 2, + "moniker": "/alice", + "log_name": "alice", + "sidecar": { + "log_name": "alice-sidecar", + "mesh_port": 24002, + "mesh_config_path": "alice-mesh.json", + "mesh_identity_path": "alice-identity.json", + }, + "program": { + "log_name": "alice-program", + "work_dir": ".", + "execution": { + "kind": "direct", + "entrypoint": ["/bin/true"], + }, + }, + }, + ], + "router": { + "identity_id": plan.router_identity_id, + "mesh_port": 39001, + "control_port": 39011, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json", + }, + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { + component_mesh_port_by_id: BTreeMap::from([(1, 24001), (2, 24002)]), + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + write_json( + &runtime_root.join("provider-mesh.json"), + &test_live_component_runtime( + "/provider", + "/provider", + "127.0.0.1:24001", + "10.0.2.2:24001", + Vec::new(), + Vec::new(), + ) + .mesh_config, + ) + .expect("provider mesh config should write"); + write_json( + &runtime_root.join("alice-mesh.json"), + &test_live_component_runtime( + "/alice", + "/alice", + "127.0.0.1:24002", + "10.0.2.2:24002", + Vec::new(), + vec![OutboundRoute { + route_id: "provider-route".to_string(), + rewrite_route_id: None, + slot: "upstream".to_string(), + capability_kind: Some("http".to_string()), + capability_profile: None, + listen_port: 20000, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:24001".to_string(), + peer_id: "/provider".to_string(), + capability: "http".to_string(), + }], + ) + .mesh_config, + ) + .expect("alice mesh config should write"); + write_json(&runtime_root.join("router-mesh.json"), &{ + let mut router = test_live_site_router(Vec::new()); + router.identity.id = plan.router_identity_id.clone(); + router + }) + .expect("router mesh config should write"); + + overlay_handle +} + +#[tokio::test] +async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { + let dir = TempDir::new().expect("temp dir"); + let mut state = compile_dynamic_caps_binding_state().await; + state.placement.offered_sites.insert( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ); + state.placement.offered_sites.insert( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: None, + }, + ); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let controller_app = SiteControllerApp { + control: test_control_state_app(&dir, state, state_path), + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), + }, + ), + false, + ) + .await + .expect("held list should stay local when no remote controller routes exist"); + + let super::control_state_api::DynamicCapsInspectResponse::HeldList(held) = response else { + panic!("held list request should return a held list response"); + }; + assert!( + held.held + .iter() + .any(|entry| entry.entry_kind == HeldEntryKind::RootAuthority), + "local held roots should still be returned", + ); +} + +#[tokio::test] +async fn localize_framework_control_state_tracks_remote_grant_authority_sites() { + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut state = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &state, + "components./alice", + &root_held_id_for(&state, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut state, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } = share else { + panic!("cross-site share should create a grant"); + }; + + localize_framework_control_state(&mut state, "direct_b") + .expect("recipient site state should localize"); + + assert!( + state.dynamic_capability_grants.is_empty(), + "recipient site should not retain the authoritative grant record", + ); + assert_eq!( + state + .dynamic_capability_grant_authority_sites + .get(&grant_id), + Some(&"direct_a".to_string()), + "recipient site should retain a lightweight authority index for delegated grants", + ); +} + +#[tokio::test] +async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { + let dir = TempDir::new().expect("temp dir"); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let (grant_id, shared_ref) = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, r#ref } => { + (grant_id, r#ref) + } + _ => panic!("cross-site share should create a grant"), + }; + + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new().route( + "/v1/controller/dynamic-caps/inspect-ref", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + hits.lock() + .expect("inspect-ref hit log poisoned") + .push(request.r#ref.clone()); + Json(amber_mesh::dynamic_caps::InspectRefResponse { + state: HeldEntryState::Live, + grant_id: remote_grant_id, + holder_component_id: "components./bob".to_string(), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + held_id: Some(super::dynamic_caps::held_id_for_grant("g_0000000000000000")), + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, holder_state, state_path); + retarget_test_controller_site(&mut app, "direct_b"); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::InspectRef( + dynamic_caps::ControlDynamicInspectRefRequest { + holder_component_id: "components./bob".to_string(), + r#ref: shared_ref.clone(), + }, + ), + false, + ) + .await + .expect("holder site should route inspect_ref through the authority site"); + + let super::control_state_api::DynamicCapsInspectResponse::InspectRef(response) = response + else { + panic!("inspect_ref should return inspect response"); + }; + assert_eq!(response.state, HeldEntryState::Live); + assert_eq!(response.grant_id, grant_id); + assert_eq!( + hits.lock() + .expect("inspect-ref hit log poisoned") + .as_slice(), + &[shared_ref], + "inspect_ref should route exactly once through the authority site router", + ); +} + +#[tokio::test] +async fn resolve_ref_routes_remote_grants_and_preserves_ref_suffix() { + let dir = TempDir::new().expect("temp dir"); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let (grant_id, shared_ref) = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, r#ref } => { + (grant_id, r#ref) + } + _ => panic!("cross-site share should create a grant"), + }; + let mut suffixed_ref = Url::parse(&shared_ref).expect("shared ref should be a URL"); + let token_path = suffixed_ref.path().trim_end_matches('/'); + suffixed_ref.set_path(&format!("{token_path}/nested/path")); + suffixed_ref.set_query(Some("q=1")); + suffixed_ref.set_fragment(Some("section")); + let suffixed_ref = suffixed_ref.to_string(); + + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new() + .route( + "/v1/controller/dynamic-caps/inspect-ref", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + hits.lock() + .expect("resolve-ref hit log poisoned") + .push(format!("inspect:{}", request.r#ref)); + Json(amber_mesh::dynamic_caps::InspectRefResponse { + state: HeldEntryState::Live, + grant_id: remote_grant_id.clone(), + holder_component_id: "components./bob".to_string(), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + held_id: Some(super::dynamic_caps::held_id_for_grant( + &remote_grant_id, + )), + }) + } + } + }), + ) + .route( + "/v1/internal/dynamic-caps/resolve-origin", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + let dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id, + } = request.source + else { + panic!("resolve-ref should resolve the inspected grant"); + }; + hits.lock() + .expect("resolve-ref hit log poisoned") + .push(format!("resolve:{grant_id}")); + Json(dynamic_caps::ControlDynamicResolveOriginResponse { + held_id: super::dynamic_caps::held_id_for_grant(&remote_grant_id), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + origin_route_id: "dynamic-origin".to_string(), + origin_capability: "provider.http".to_string(), + origin_protocol: "http".to_string(), + origin_peer_id: "/site/direct_a/router".to_string(), + origin_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode([9u8; 32]), + origin_peer_addr: "127.0.0.1:24000".to_string(), + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, holder_state, state_path); + retarget_test_controller_site(&mut app, "direct_b"); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); + install_bob_holder_runtime_fixture(&app); + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_resolve_ref( + &controller_app, + dynamic_caps::ControlDynamicResolveRefRequest { + holder_component_id: "components./bob".to_string(), + r#ref: suffixed_ref.clone(), + }, + ) + .await + .expect("holder site should resolve remote refs through the authority site"); + + assert_eq!( + response.origin.held_id, + super::dynamic_caps::held_id_for_grant(&grant_id) + ); + assert_eq!(response.relative_path, "/nested/path"); + assert_eq!(response.query.as_deref(), Some("q=1")); + assert_eq!(response.fragment.as_deref(), Some("section")); + assert_eq!( + hits.lock() + .expect("resolve-ref hit log poisoned") + .as_slice(), + &[ + format!("inspect:{suffixed_ref}"), + format!("resolve:{grant_id}") + ], + "resolve_ref should validate the raw ref and then resolve the inspected grant", + ); +} + +#[tokio::test] +async fn held_list_aggregates_remote_grants_via_remote_controller_capability() { + let dir = TempDir::new().expect("temp dir"); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let grant_id = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } => grant_id, + _ => panic!("cross-site share should create a grant"), + }; - let control_state_auth_token = app.control_state_auth_token.to_string(); - let publish_router = Router::new().route( - "/v1/internal/dynamic-caps/origins/publish", - post({ - let expected = control_state_auth_token.clone(); - move |headers: HeaderMap, - Json(request): Json| { - let expected = expected.clone(); + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new().route( + "/v1/controller/dynamic-caps/held", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); async move { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(expected.as_str()), - "origin publish should authenticate with the control-state token", - ); - Json(dynamic_caps::PublishDynamicCapabilityOriginResponse { - route_id: request.route_id, - capability: "provider.http".to_string(), - protocol: "http".to_string(), + hits.lock() + .expect("held-list hit log poisoned") + .push(request.holder_component_id.clone()); + Json(amber_mesh::dynamic_caps::HeldListResponse { + held: vec![HeldEntrySummary { + held_id: super::dynamic_caps::held_id_for_grant(&remote_grant_id), + entry_kind: HeldEntryKind::DelegatedGrant, + grant_id: Some(remote_grant_id), + root_authority_selector: None, + state: HeldEntryState::Live, + from_component: Some("components./alice".to_string()), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + materializations: Vec::new(), + }], }) } } }), ); - let (publish_base_url, publish_handle) = spawn_test_router(publish_router).await; - let publish_addr = publish_base_url - .strip_prefix("http://") - .expect("publish base URL should be absolute HTTP") - .parse() - .expect("publish base URL should parse as socket address"); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.to_string(), - kind: SiteKind::Direct, - router_identity_id: "/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: "127.0.0.1:0".parse().expect("listen addr"), - storage_root: None, - runtime_root: Some(runtime_root.display().to_string()), - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); - write_json( - &site_state_root.join("framework-ccs-plan.json"), - &FrameworkCcsPlan { - schema: CCS_PLAN_SCHEMA.to_string(), - version: CCS_PLAN_VERSION, - site_id: site_id.to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: publish_addr, - control_state_url: "http://127.0.0.1:0".to_string(), - router_auth_token: "unused".to_string(), - control_state_auth_token: control_state_auth_token.clone(), - }, - ) - .expect("framework ccs plan should write"); - write_json( - &site_state_root.join("manager-state.json"), - &json!({ - "status": "running", - "kind": SiteKind::Direct, - "artifact_dir": artifact_dir.display().to_string(), - "supervisor_pid": 1, - "router_mesh_addr": "127.0.0.1:39001", - "router_identity_id": "/router", - "router_public_key_b64": "dGVzdC1yb3V0ZXIta2V5", - }), - ) - .expect("manager state should write"); - write_json( - &site_state_root.join("site-actuator-state.json"), - &json!({ - "schema": "amber.site_actuator_state", - "version": 1, - "run_id": "test-run", - "site_id": site_id, - "kind": SiteKind::Direct, - "children": {}, - }), - ) - .expect("site actuator state should write"); - write_json( - &artifact_dir.join("direct-plan.json"), - &json!({ - "version": "3", - "mesh_provision_plan": "mesh-provision-plan.json", - "startup_order": [1, 2], - "components": [ - { - "id": 1, - "moniker": "/provider", - "log_name": "provider", - "sidecar": { - "log_name": "provider-sidecar", - "mesh_port": 24001, - "mesh_config_path": "provider-mesh.json", - "mesh_identity_path": "provider-identity.json", - }, - "program": { - "log_name": "provider-program", - "work_dir": ".", - "execution": { - "kind": "direct", - "entrypoint": ["/bin/true"], - }, - }, - }, - { - "id": 2, - "moniker": "/alice", - "log_name": "alice", - "sidecar": { - "log_name": "alice-sidecar", - "mesh_port": 24002, - "mesh_config_path": "alice-mesh.json", - "mesh_identity_path": "alice-identity.json", - }, - "program": { - "log_name": "alice-program", - "work_dir": ".", - "execution": { - "kind": "direct", - "entrypoint": ["/bin/true"], - }, - }, - }, - ], - "router": { - "identity_id": "/router", - "mesh_port": 39001, - "control_port": 39011, - "control_socket_path": "router.sock", - "mesh_config_path": "router-mesh.json", - "mesh_identity_path": "router-identity.json", + let mut app = test_control_state_app(&dir, holder_state, state_path); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.site_id = "direct_b".to_string(); + controller_plan.router_identity_id = "/site/direct_b/router".to_string(); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./bob".to_string(), }, - }), - ) - .expect("direct plan should write"); - write_json( - &crate::direct_runtime::direct_runtime_state_path(&artifact_dir), - &crate::direct_runtime::DirectRuntimeState { - component_mesh_port_by_id: BTreeMap::from([(1, 24001), (2, 24002)]), - ..Default::default() - }, - ) - .expect("direct runtime state should write"); - write_json( - &runtime_root.join("provider-mesh.json"), - &test_live_component_runtime( - "/provider", - "/provider", - "127.0.0.1:24001", - Vec::new(), - Vec::new(), - ) - .mesh_config, - ) - .expect("provider mesh config should write"); - write_json( - &runtime_root.join("alice-mesh.json"), - &test_live_component_runtime( - "/alice", - "/alice", - "127.0.0.1:24002", - Vec::new(), - Vec::new(), - ) - .mesh_config, - ) - .expect("alice mesh config should write"); - write_json( - &runtime_root.join("router-mesh.json"), - &test_live_site_router(Vec::new()), + ), + false, ) - .expect("router mesh config should write"); + .await + .expect("holder site should aggregate remote grants through the authority site router"); - publish_handle + let super::control_state_api::DynamicCapsInspectResponse::HeldList(response) = response else { + panic!("held_list should return a held list response"); + }; + assert!( + response + .held + .iter() + .any(|entry| entry.entry_kind == HeldEntryKind::RootAuthority), + "holder site should keep its local root authority entries", + ); + assert!( + response.held.iter().any(|entry| { + entry.entry_kind == HeldEntryKind::DelegatedGrant + && entry.grant_id.as_deref() == Some(grant_id.as_str()) + }), + "holder site should merge delegated grants from the authority site", + ); + assert_eq!( + hits.lock().expect("held-list hit log poisoned").as_slice(), + &["components./bob".to_string()], + "held_list should route exactly once through the authority site router", + ); } #[tokio::test] @@ -2412,9 +4953,56 @@ async fn dynamic_caps_mcp_discovers_compact_surface() { help.contains("amber.v1.framework_dynamic_caps.inspect"), "help resource should point callers to the inspect tool" ); +} + +#[tokio::test] +async fn dynamic_caps_mcp_rejects_control_state_auth_without_route_identity() { + let harness = DynamicCapsMcpHarness::start().await; + let (_status, payload) = harness + .initialize_with_headers(vec![( + "x-amber-control-state-auth".to_string(), + "test-control-state-auth".to_string(), + )]) + .await; + + let error = payload + .get("error") + .unwrap_or_else(|| panic!("initialize unexpectedly succeeded: {payload:#?}")); + assert!( + error.to_string().contains(FRAMEWORK_ROUTE_ID_HEADER), + "dynamic-caps MCP must require router-injected route identity, got {error:#?}" + ); +} + +#[tokio::test] +async fn dynamic_caps_mcp_rejects_tool_calls_for_a_different_component_identity() { + let harness = DynamicCapsMcpHarness::start().await; + let mut alice_mcp = harness.connect_as("components./alice").await; + + let error = alice_mcp + .call_tool_error( + "amber.v1.framework_dynamic_caps.inspect", + json!({ + "op": "held_list", + "holder_component_id": "components./carol", + }), + ) + .await; + + assert_eq!( + error + .get("data") + .and_then(|data| data.get("code")) + .and_then(Value::as_str), + Some("unauthorized"), + "dynamic-caps MCP must reject cross-component tool arguments, got {error:#?}" + ); assert!( - help.contains("/v1/control-state/dynamic-caps"), - "help resource should explain the HTTP endpoint family", + error + .to_string() + .contains("unknown framework capability instance") + || error.to_string().contains("components./carol"), + "dynamic-caps MCP should fail before executing a cross-component operation, got {error:#?}" ); } @@ -2422,15 +5010,16 @@ async fn dynamic_caps_mcp_discovers_compact_surface() { async fn dynamic_caps_mcp_matches_http_surface() { let http = DynamicCapsMcpHarness::start().await; let mcp_harness = DynamicCapsMcpHarness::start().await; - let mut mcp = mcp_harness.connect().await; + let mut alice_mcp = mcp_harness.connect_as("components./alice").await; + let mut carol_mcp = mcp_harness.connect_as("components./carol").await; let held_list_request = dynamic_caps::ControlDynamicHeldListRequest { holder_component_id: "components./alice".to_string(), }; let http_held: amber_mesh::dynamic_caps::HeldListResponse = http - .post_json("/v1/control-state/dynamic-caps/held", &held_list_request) + .post_json("/v1/controller/dynamic-caps/held", &held_list_request) .await; - let mcp_held: Value = mcp + let mcp_held: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2456,11 +5045,11 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_detail: HeldEntryDetail = http .post_json( - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &held_detail_request, ) .await; - let mcp_detail: Value = mcp + let mcp_detail: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2490,9 +5079,9 @@ async fn dynamic_caps_mcp_matches_http_surface() { options: Value::Null, }; let http_share: amber_mesh::dynamic_caps::ShareResponse = http - .post_json("/v1/control-state/dynamic-caps/share", &share_request) + .post_json("/v1/controller/dynamic-caps/share", &share_request) .await; - let mcp_share: Value = mcp + let mcp_share: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.mutate", json!({ @@ -2531,9 +5120,9 @@ async fn dynamic_caps_mcp_matches_http_surface() { holder_component_id: "components./carol".to_string(), }; let http_carol_held: amber_mesh::dynamic_caps::HeldListResponse = http - .post_json("/v1/control-state/dynamic-caps/held", &carol_held_request) + .post_json("/v1/controller/dynamic-caps/held", &carol_held_request) .await; - let mcp_carol_held: Value = mcp + let mcp_carol_held: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2553,11 +5142,11 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_inspect_ref: amber_mesh::dynamic_caps::InspectRefResponse = http .post_json( - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &inspect_ref_request, ) .await; - let mcp_inspect_ref: Value = mcp + let mcp_inspect_ref: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2580,11 +5169,11 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = http .post_json( - "/v1/control-state/dynamic-caps/resolve-origin", + "/v1/controller/dynamic-caps/resolve-origin", &resolve_origin_request, ) .await; - let mcp_resolve_origin: Value = mcp + let mcp_resolve_origin: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2610,9 +5199,9 @@ async fn dynamic_caps_mcp_matches_http_surface() { }, }; let http_revoke: amber_mesh::dynamic_caps::RevokeResponse = http - .post_json("/v1/control-state/dynamic-caps/revoke", &revoke_request) + .post_json("/v1/controller/dynamic-caps/revoke", &revoke_request) .await; - let mcp_revoke: Value = mcp + let mcp_revoke: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.mutate", json!({ @@ -2636,11 +5225,11 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_revoked_detail: HeldEntryDetail = http .post_json( - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &revoked_detail_request, ) .await; - let mcp_revoked_detail: Value = mcp + let mcp_revoked_detail: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -2656,318 +5245,129 @@ async fn dynamic_caps_mcp_matches_http_surface() { ); } -async fn install_success_site_actuator(app: &ControlStateApp) -> Vec> { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), +#[tokio::test] +async fn dynamic_caps_resolve_origin_tolerates_missing_static_runtime_state_file() { + let harness = DynamicCapsMcpHarness::start().await; + + let held: amber_mesh::dynamic_caps::HeldListResponse = harness + .post_json( + "/v1/controller/dynamic-caps/held", + &dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), }, ) - .expect("site actuator plan should write"); - let app = Router::new() - .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/rollback", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); - } - handles -} + .await; + let root_held_id = held + .held + .iter() + .find(|entry| entry.entry_kind == HeldEntryKind::RootAuthority) + .map(|entry| entry.held_id.clone()) + .expect("alice should have a root authority"); + let detail: HeldEntryDetail = harness + .post_json( + "/v1/controller/dynamic-caps/held/detail", + &dynamic_caps::ControlDynamicHeldDetailRequest { + holder_component_id: "components./alice".to_string(), + held_id: root_held_id, + }, + ) + .await; + let selector = detail + .summary + .root_authority_selector + .clone() + .expect("root held detail should include a root authority selector"); + + fs::remove_file( + harness + ._dir + .path() + .join("state") + .join("direct_local") + .join("site-controller-runtime-state.json"), + ) + .expect("fixture runtime state should be removable"); -async fn install_failing_rollback_site_actuator( - app: &ControlStateApp, -) -> Vec> { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), + let resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = harness + .post_json( + "/v1/controller/dynamic-caps/resolve-origin", + &dynamic_caps::ControlDynamicResolveOriginRequest { + holder_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: selector, + }, }, ) - .expect("site actuator plan should write"); - let app = Router::new() - .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/rollback", - post(|| async { StatusCode::INTERNAL_SERVER_ERROR }), - ) - .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); - } - handles + .await; + + assert!( + !resolve_origin.origin_peer_addr.is_empty(), + "static components should still resolve a live origin even when the dynamic runtime state \ + file is absent", + ); } -async fn install_barrier_destroy_site_actuator( - app: &ControlStateApp, -) -> ( - Vec>, - tokio::sync::mpsc::UnboundedReceiver, - Arc, -) { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), +#[tokio::test] +async fn dynamic_caps_resolve_origin_prefers_published_router_mesh_addr() { + let harness = DynamicCapsMcpHarness::start().await; + + write_json( + &harness + ._dir + .path() + .join("state") + .join("direct_local") + .join("manager-state.json"), + &json!({ + "schema": TEST_SITE_STATE_SCHEMA, + "version": TEST_SITE_STATE_VERSION, + "run_id": "test-run", + "site_id": "direct_local", + "kind": "direct", + "status": "running", + "artifact_dir": harness._dir.path().join("artifact/direct_local").display().to_string(), + "supervisor_pid": 1u32, + "router_control": test_router_control_addr(), + "router_mesh_addr": "127.0.0.1:1", + "router_identity_id": "/site/direct_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([7u8; 32]), + "site_controller_url": harness.base_url.clone(), + }), + ) + .expect("stale manager state should write"); + + let held: amber_mesh::dynamic_caps::HeldListResponse = harness + .post_json( + "/v1/controller/dynamic-caps/held", + &dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), }, ) - .expect("site actuator plan should write"); - let start_tx = tx.clone(); - let destroy_barrier = barrier.clone(); - let site_id_for_destroy = site_id.clone(); - let app = Router::new() - .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/destroy", - post(move || { - let start_tx = start_tx.clone(); - let destroy_barrier = destroy_barrier.clone(); - let site_id = site_id_for_destroy.clone(); - async move { - start_tx - .send(site_id) - .expect("destroy start notification should send"); - destroy_barrier.wait().await; - StatusCode::NO_CONTENT - } - }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); - } - (handles, rx, barrier) -} + .await; + let root_selector = held + .held + .iter() + .find(|entry| entry.entry_kind == HeldEntryKind::RootAuthority) + .and_then(|entry| entry.root_authority_selector.clone()) + .expect("alice should have a root authority selector"); -async fn install_barrier_publish_site_actuator( - app: &ControlStateApp, -) -> ( - Vec>, - tokio::sync::mpsc::UnboundedReceiver, - Arc, -) { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), + let resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = harness + .post_json( + "/v1/controller/dynamic-caps/resolve-origin", + &dynamic_caps::ControlDynamicResolveOriginRequest { + holder_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: root_selector, + }, }, ) - .expect("site actuator plan should write"); - let start_tx = tx.clone(); - let publish_barrier = barrier.clone(); - let site_id_for_publish = site_id.clone(); - let app = Router::new() - .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/publish", - post(move || { - let start_tx = start_tx.clone(); - let publish_barrier = publish_barrier.clone(); - let site_id = site_id_for_publish.clone(); - async move { - start_tx - .send(site_id) - .expect("publish start notification should send"); - publish_barrier.wait().await; - StatusCode::NO_CONTENT - } - }), - ) - .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); - } - (handles, rx, barrier) + .await; + + assert_eq!( + resolve_origin.origin_peer_addr, "127.0.0.1:24000", + "resolve-origin should use the controller plan's published router mesh address instead of \ + stale manager-state data", + ); } #[tokio::test] @@ -3018,6 +5418,7 @@ async fn create_snapshot_and_destroy_exact_child() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -3025,6 +5426,7 @@ async fn create_snapshot_and_destroy_exact_child() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -3338,6 +5740,41 @@ async fn open_template_replay_uses_admitted_manifest_after_source_mutation() { fs::remove_file(&beta_path).expect("beta source should be removable before replay"); fs::remove_file(&beta_leaf_path).expect("beta leaf source should be removable before replay"); + let replay_scenario_ir: ScenarioIr = serde_json::from_value(snapshot_response.scenario.clone()) + .expect("snapshot scenario should decode for replay run plan"); + let replay_placement = placement_from_snapshot(&snapshot_response); + let replay_compiled = + CompiledScenario::from_ir(replay_scenario_ir).expect("snapshot replay should compile"); + let replay_run_plan = + build_run_plan(&replay_compiled, Some(&replay_placement)).expect("replay run plan"); + let mut localized_replay = + build_site_controller_state("replay-run", &replay_run_plan, "direct_local", 0, 1) + .expect("site-local replay state should build"); + assert!( + localized_replay + .live_children + .iter() + .any(|child| child.name == "job-open" && child.state == ChildState::Live), + "site-local replay should preserve live children even when the root authority has no \ + explicit assignment" + ); + let replayed_live_child_id = localized_replay + .live_children + .iter() + .map(|child| child.child_id) + .max() + .expect("site-local replay should keep the restored live child id"); + assert_eq!( + localized_replay.next_child_id, replayed_live_child_id, + "site-local replay must preserve the child id allocator after restoring live children" + ); + let replay_next_child_id = allocate_child_id(&mut localized_replay); + assert!( + replay_next_child_id > replayed_live_child_id, + "site-local replay must allocate a fresh child id instead of colliding with restored \ + children" + ); + let mut replayed = compile_control_state_from_snapshot(&snapshot_response).await; let replay_state_path = dir.path().join("replay-control-state.json"); write_control_state(&replay_state_path, &replayed).expect("replay state should write"); @@ -3350,8 +5787,13 @@ async fn open_template_replay_uses_admitted_manifest_after_source_mutation() { "replay should restore the child fragment as authoritative semantic state", ); assert!( - !replayed.live_children[0].site_plans.is_empty(), - "replay should rebuild derived site plans from the restored child fragment", + build_local_child_runtime_spec( + &replayed, + &replayed.live_children[0], + &child_runtime_site_id(&replayed.live_children[0]).expect("replayed child site id"), + ) + .is_ok(), + "replay should rebuild local runtime realization from the restored child fragment", ); assert!( list_children(&replayed, replay_root_authority) @@ -3497,56 +5939,243 @@ async fn open_template_admission_uses_canonical_manifest_url_and_freezes_redirec create_child( &mut state, - root_authority, + root_authority, + CreateChildRequest { + template: "worker".to_string(), + name: "job-open".to_string(), + manifest: Some(requested_url.parse().expect("manifest ref")), + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + &state_path, + ) + .await + .expect("redirected open-template create should succeed"); + server.join().expect("manifest server should stop cleanly"); + + assert_eq!( + state.live_children[0] + .selected_manifest_catalog_key + .as_deref(), + Some(canonical_root_url.as_str()) + ); + assert!( + state + .base_scenario + .manifest_catalog + .contains_key(canonical_root_url.as_str()), + "admitted runtime manifests should be keyed by the resolver's final URL" + ); + assert!( + state + .base_scenario + .manifest_catalog + .contains_key(canonical_leaf_url.as_str()), + "admitting an open template should freeze transitive redirected dependencies" + ); + + let snapshot_response = + snapshot(&state, root_authority).expect("snapshot should succeed after redirected create"); + let scenario_ir: ScenarioIr = serde_json::from_value(snapshot_response.scenario) + .expect("snapshot scenario should decode"); + assert!( + scenario_ir + .components + .iter() + .any(|component| component.moniker == "/job-open/leaf"), + "snapshot should contain the redirected transitive child component" + ); +} + +#[tokio::test] +async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_origin_realm() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let parent_path = dir.path().join("parent.json5"); + let worker_path = dir.path().join("worker.json5"); + let root_worker_path = dir.path().join("root-worker.json5"); + write_file( + &worker_path, + r#" + { + manifest_version: "0.3.0", + slots: { + realm: { kind: "component", optional: true } + }, + program: { + path: "/bin/echo", + args: ["worker", "${slots.realm.url}"], + network: { endpoints: [{ name: "http", port: 8080 }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" }, + } + "#, + ); + write_file( + &root_worker_path, + r#" + { + manifest_version: "0.3.0", + program: { + path: "/bin/echo", + args: ["root-worker"], + network: { endpoints: [{ name: "http", port: 8082 }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" }, + } + "#, + ); + write_file( + &parent_path, + &format!( + r#" + {{ + manifest_version: "0.3.0", + slots: {{ + realm: {{ kind: "component", optional: true }} + }}, + program: {{ + path: "/bin/echo", + args: ["parent", "${{slots.realm.url}}"], + network: {{ endpoints: [{{ name: "http", port: 8081 }}] }} + }}, + provides: {{ http: {{ kind: "http", endpoint: "http" }} }}, + exports: {{ http: "provides.http" }}, + child_templates: {{ + worker: {{ + manifest: "{worker}", + bindings: {{ + realm: "slots.realm" + }} + }} + }}, + }} + "#, + worker = file_url(&worker_path), + ), + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + program: {{ path: "/bin/echo", args: ["root"] }}, + slots: {{ + realm: {{ kind: "component", optional: true }} + }}, + child_templates: {{ + root_worker: {{ + manifest: "{root_worker}" + }} + }}, + components: {{ + parent: "{parent}" + }}, + bindings: [ + {{ to: "#parent.realm", from: "framework.component" }} + ], + exports: {{ + parent_http: "#parent.http" + }}, + }} + "##, + root_worker = file_url(&root_worker_path), + parent = file_url(&parent_path), + ), + ); + + let mut state = compile_control_state(&root_path).await; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let base = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let parent_id = base + .components_iter() + .find(|(_, component)| component.moniker.as_str() == "/parent") + .map(|(id, _)| id.0) + .expect("parent component should exist"); + let static_parent_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent") + .cloned() + .expect("static parent should have a realm capability instance"); + assert_eq!(static_parent_record.authority_realm_moniker, "/"); + + create_child( + &mut state, + parent_id, + CreateChildRequest { + template: "worker".to_string(), + name: "delegate".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + &state_path, + ) + .await + .expect("delegate child should be created"); + + let dynamic_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent/delegate") + .cloned() + .expect("dynamic child should receive its own realm capability instance"); + let root_authority = state.base_scenario.root; + assert_eq!(dynamic_record.authority_realm_id, root_authority); + assert_eq!(dynamic_record.authority_realm_moniker, "/"); + let authorized = + authorize_capability_instance(&state, &dynamic_record.cap_instance_id, "/parent/delegate") + .expect("dynamic child capability instance should authorize for its own peer"); + let delegated_authority_realm_id = authorized.authority_realm_id; + assert_eq!(delegated_authority_realm_id, root_authority); + + create_child( + &mut state, + delegated_authority_realm_id, CreateChildRequest { - template: "worker".to_string(), - name: "job-open".to_string(), - manifest: Some(requested_url.parse().expect("manifest ref")), + template: "root_worker".to_string(), + name: "sibling".to_string(), + manifest: None, config: BTreeMap::new(), bindings: BTreeMap::new(), }, &state_path, ) .await - .expect("redirected open-template create should succeed"); - server.join().expect("manifest server should stop cleanly"); + .expect("forwarded realm authority should create a sibling in the parent realm"); - assert_eq!( - state.live_children[0] - .selected_manifest_catalog_key - .as_deref(), - Some(canonical_root_url.as_str()) - ); + let live_scenario = live_scenario_ir(&state).expect("live scenario should materialize"); + let live = Scenario::try_from(live_scenario).expect("live scenario should decode"); assert!( - state - .base_scenario - .manifest_catalog - .contains_key(canonical_root_url.as_str()), - "admitted runtime manifests should be keyed by the resolver's final URL" + live.components_iter() + .any(|(_, component)| component.moniker.as_str() == "/parent/delegate"), + "delegate should live under the parent realm" ); assert!( - state - .base_scenario - .manifest_catalog - .contains_key(canonical_leaf_url.as_str()), - "admitting an open template should freeze transitive redirected dependencies" + live.components_iter() + .any(|(_, component)| component.moniker.as_str() == "/sibling"), + "forwarded realm capability should create in the origin realm, not under the caller" ); - let snapshot_response = - snapshot(&state, root_authority).expect("snapshot should succeed after redirected create"); - let scenario_ir: ScenarioIr = serde_json::from_value(snapshot_response.scenario) - .expect("snapshot scenario should decode"); + destroy_child(&mut state, parent_id, "delegate", &state_path) + .await + .expect("destroy should succeed"); assert!( - scenario_ir - .components - .iter() - .any(|component| component.moniker == "/job-open/leaf"), - "snapshot should contain the redirected transitive child component" + !state + .capability_instances + .values() + .any(|record| record.recipient_component_moniker == "/parent/delegate"), + "destroy should revoke dynamic capability instances owned by the removed child" ); } #[tokio::test] -async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_origin_realm() { +async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authority_site() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let parent_path = dir.path().join("parent.json5"); @@ -3596,11 +6225,8 @@ async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_or }}, program: {{ path: "/bin/echo", - args: ["parent", "${{slots.realm.url}}"], - network: {{ endpoints: [{{ name: "http", port: 8081 }}] }} + args: ["parent", "${{slots.realm.url}}"] }}, - provides: {{ http: {{ kind: "http", endpoint: "http" }} }}, - exports: {{ http: "provides.http" }}, child_templates: {{ worker: {{ manifest: "{worker}", @@ -3620,7 +6246,6 @@ async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_or r##" {{ manifest_version: "0.3.0", - program: {{ path: "/bin/echo", args: ["root"] }}, slots: {{ realm: {{ kind: "component", optional: true }} }}, @@ -3635,35 +6260,58 @@ async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_or bindings: [ {{ to: "#parent.realm", from: "framework.component" }} ], - exports: {{ - parent_http: "#parent.http" - }}, }} "##, root_worker = file_url(&root_worker_path), parent = file_url(&parent_path), ), ); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/parent".to_string(), "compose_local".to_string()), + ("/parent/delegate".to_string(), "direct_local".to_string()), + ("/sibling".to_string(), "compose_local".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; - let mut state = compile_control_state(&root_path).await; - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let base = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let mut authoritative = + compile_control_state_with_placement(&root_path, Some(&placement)).await; + let authoritative_state_path = dir.path().join("authoritative-state.json"); + write_control_state(&authoritative_state_path, &authoritative).expect("state should write"); + let base = Scenario::try_from(authoritative.base_scenario.clone()).expect("base scenario"); let parent_id = base .components_iter() .find(|(_, component)| component.moniker.as_str() == "/parent") .map(|(id, _)| id.0) .expect("parent component should exist"); - let static_parent_record = state - .capability_instances - .values() - .find(|record| record.recipient_component_moniker == "/parent") - .cloned() - .expect("static parent should have a realm capability instance"); - assert_eq!(static_parent_record.authority_realm_moniker, "/"); - create_child( - &mut state, + &mut authoritative, parent_id, CreateChildRequest { template: "worker".to_string(), @@ -3672,63 +6320,168 @@ async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_or config: BTreeMap::new(), bindings: BTreeMap::new(), }, - &state_path, + &authoritative_state_path, ) .await .expect("delegate child should be created"); - let dynamic_record = state + let mut direct_state = authoritative.clone(); + localize_framework_control_state(&mut direct_state, "direct_local") + .expect("direct-local state should localize"); + direct_state.placement.assignments = + BTreeMap::from([("/parent/delegate".to_string(), "direct_local".to_string())]); + let direct_record = direct_state .capability_instances .values() .find(|record| record.recipient_component_moniker == "/parent/delegate") .cloned() - .expect("dynamic child should receive its own realm capability instance"); - let root_authority = state.base_scenario.root; - assert_eq!(dynamic_record.authority_realm_id, root_authority); - assert_eq!(dynamic_record.authority_realm_moniker, "/"); - let authorized = - authorize_capability_instance(&state, &dynamic_record.cap_instance_id, "/parent/delegate") - .expect("dynamic child capability instance should authorize for its own peer"); - let delegated_authority_realm_id = authorized.authority_realm_id; - assert_eq!(delegated_authority_realm_id, root_authority); - - create_child( - &mut state, - delegated_authority_realm_id, - CreateChildRequest { + .expect("delegate should have a forwarded framework capability instance"); + assert_eq!( + direct_record.controller_site_id, "compose_local", + "delegated root authority should stay anchored at the forwarded realm site", + ); + let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([11u8; 32]); + let peer_router_attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let peer_router = Router::new().route( + "/v1/internal/router-identity", + axum::routing::post({ + let router_public_key_b64 = router_public_key_b64.clone(); + let peer_router_attempts = peer_router_attempts.clone(); + move |Json(request): Json| { + let router_public_key_b64 = router_public_key_b64.clone(); + let peer_router_attempts = peer_router_attempts.clone(); + async move { + assert_eq!(request.site_id, "direct_local"); + let mut attempts = peer_router_attempts + .lock() + .expect("peer router attempt log poisoned"); + *attempts += 1; + if *attempts == 1 { + return Err(StatusCode::BAD_GATEWAY); + } + Ok(Json(RouterIdentityResponse { + id: "/site/direct_local/router".to_string(), + public_key_b64: router_public_key_b64, + })) + } + } + }), + ); + let (peer_router_base_url, _peer_router_handle) = spawn_test_router(peer_router).await; + let direct_state_path = dir.path().join("direct-state.json"); + write_control_state(&direct_state_path, &direct_state).expect("direct state should write"); + let mut direct_plan = test_control_state_app(&dir, direct_state.clone(), direct_state_path) + .controller_plan + .as_ref() + .clone(); + direct_plan.site_id = "direct_local".to_string(); + direct_plan.kind = SiteKind::Direct; + direct_plan.peer_site_router_urls = + BTreeMap::from([("compose_local".to_string(), peer_router_base_url.clone())]); + direct_plan.peer_router_mesh_addrs = + BTreeMap::from([("compose_local".to_string(), "127.0.0.1:25000".to_string())]); + let direct_controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: Arc::new(Mutex::new(direct_state.clone())), + controller_plan: Arc::new(direct_plan), + ..test_control_state_app( + &dir, + direct_state.clone(), + dir.path().join("direct-state.json"), + ) + }, + ready: ready_site_controller_flag(), + }; + let stale_err = match super::site_controller::execute_site_controller_framework_mutate( + &direct_controller_app, + &direct_record, + &direct_state, + super::ccs_api::FrameworkComponentMutateRequest::CreateChild(CreateChildRequest { template: "root_worker".to_string(), name: "sibling".to_string(), manifest: None, config: BTreeMap::new(), bindings: BTreeMap::new(), - }, - &state_path, + }), ) .await - .expect("forwarded realm authority should create a sibling in the parent realm"); - - let live_scenario = live_scenario_ir(&state).expect("live scenario should materialize"); - let live = Scenario::try_from(live_scenario).expect("live scenario should decode"); - assert!( - live.components_iter() - .any(|(_, component)| component.moniker.as_str() == "/parent/delegate"), - "delegate should live under the parent realm" - ); + { + Ok(_) => panic!("the delegated caller site should reject stale local delivery"), + Err(err) => err, + }; + assert_eq!(stale_err.0.code, ProtocolErrorCode::ControlStateUnavailable); assert!( - live.components_iter() - .any(|(_, component)| component.moniker.as_str() == "/sibling"), - "forwarded realm capability should create in the origin realm, not under the caller" + stale_err.0.message.contains("bound to controller site"), + "unexpected stale-delivery error: {}", + stale_err.0.message ); - destroy_child(&mut state, parent_id, "delegate", &state_path) + let mut compose_state = authoritative; + localize_framework_control_state(&mut compose_state, "compose_local") + .expect("compose-local state should localize"); + let compose_record = compose_state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent/delegate") + .cloned() + .expect("compose-local controller should retain the delegated capability instance"); + let compose_state_path = dir.path().join("compose-state.json"); + write_control_state(&compose_state_path, &compose_state).expect("compose state should write"); + let mut compose_plan = + test_control_state_app(&dir, compose_state.clone(), compose_state_path.clone()) + .controller_plan + .as_ref() + .clone(); + compose_plan.site_id = "compose_local".to_string(); + compose_plan.kind = SiteKind::Direct; + compose_plan.peer_site_router_urls = + BTreeMap::from([("direct_local".to_string(), peer_router_base_url)]); + compose_plan.peer_router_mesh_addrs = + BTreeMap::from([("direct_local".to_string(), "127.0.0.1:24000".to_string())]); + let compose_controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: Arc::new(Mutex::new(compose_state.clone())), + controller_plan: Arc::new(compose_plan), + ..test_control_state_app(&dir, compose_state.clone(), compose_state_path) + }, + ready: ready_site_controller_flag(), + }; + let response = super::site_controller::execute_site_controller_framework_mutate( + &compose_controller_app, + &compose_record, + &compose_state, + super::ccs_api::FrameworkComponentMutateRequest::CreateChild(CreateChildRequest { + template: "root_worker".to_string(), + name: "sibling".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }), + ) + .await + .expect("forwarded realm create should succeed on the authority site controller"); + let super::ccs_api::FrameworkComponentMutateResponse::CreateChild(response) = response else { + panic!("framework mutate response should be a create result"); + }; + assert_eq!( + response.child.selector, "children.sibling", + "the authority controller should create the child under the forwarded root realm", + ); + let persisted = compose_controller_app + .control + .control_state + .lock() .await - .expect("destroy should succeed"); + .clone(); assert!( - !state - .capability_instances - .values() - .any(|record| record.recipient_component_moniker == "/parent/delegate"), - "destroy should revoke dynamic capability instances owned by the removed child" + persisted + .live_children + .iter() + .any(|child| child.name == "sibling" + && child_runtime_site_id(child) + .as_deref() + .is_ok_and(|site_id| site_id == "compose_local")), + "the forwarded root child should be owned by the compose-local controller state" ); } @@ -3790,21 +6543,354 @@ async fn capability_instance_auth_and_snapshot_scope_are_enforced() { .find(|record| record.recipient_component_moniker == "/parent") .expect("parent should have a realm capability instance"); - let wrong_peer = authorize_capability_instance(&state, &record.cap_instance_id, "/root") - .expect_err("peer mismatch should be rejected"); - assert_eq!(wrong_peer.code, ProtocolErrorCode::Unauthorized); + let wrong_peer = authorize_capability_instance(&state, &record.cap_instance_id, "/root") + .expect_err("peer mismatch should be rejected"); + assert_eq!(wrong_peer.code, ProtocolErrorCode::Unauthorized); + + let unknown = authorize_capability_instance(&state, "cap.missing", "/parent") + .expect_err("unknown capability instance should be rejected"); + assert_eq!(unknown.code, ProtocolErrorCode::Unauthorized); + + let snapshot_err = + snapshot(&state, parent_id).expect_err("non-root authority should not be able to snapshot"); + assert_eq!(snapshot_err.code, ProtocolErrorCode::ScopeNotAllowed); +} + +#[tokio::test] +async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + write_file( + &admin_path, + r#" + { + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { path: "/bin/echo", args: ["admin", "${slots.ctl.url}"] } + } + "#, + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + program: {{ path: "/bin/echo", args: ["root"] }}, + components: {{ admin: "{admin}" }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + }} + "##, + admin = file_url(&admin_path), + ), + ); + + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/job-1/consumer".to_string(), "compose_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; + for execution_site in ["direct_local", "compose_local"] { + if state.base_scenario.components.iter().any(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == execution_site) + }) { + continue; + } + let mut scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let root_id = scenario.root; + let controller_id = amber_scenario::ComponentId(scenario.components.len()); + scenario.components.push(Some(amber_scenario::Component { + id: controller_id, + parent: Some(root_id), + moniker: FrameworkComponentControllerMoniker::for_site(execution_site) + .into_string() + .into(), + digest: scenario.component(root_id).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value( + amber_compiler::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: execution_site.to_string(), + grants: BTreeMap::new(), + }, + ) + .expect("controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario.component_mut(root_id).children.push(controller_id); + scenario.normalize_order(); + scenario.assert_invariants(); + state.base_scenario = amber_scenario::ScenarioIr::from(&scenario); + } + let state_path = dir.path().join("control-state.json"); + let root_authority = state.base_scenario.root; + let child_root_id = 10_001usize; + let provider_id = 10_002usize; + let consumer_id = 10_003usize; + let link = RunLink { + provider_site: "direct_local".to_string(), + consumer_site: "compose_local".to_string(), + provider_component: "/job-1/provider".to_string(), + provide: "http".to_string(), + consumer_component: "/job-1/consumer".to_string(), + slot: "upstream".to_string(), + weak: false, + protocol: amber_manifest::NetworkProtocol::Http, + export_name: "amber_export_test".to_string(), + external_slot_name: "amber_link_test".to_string(), + }; + let child = LiveChildRecord { + child_id: 1, + authority_realm_id: root_authority, + name: "job-1".to_string(), + state: ChildState::CreateCommittedHidden, + template_name: Some("worker".to_string()), + selected_manifest_catalog_key: None, + fragment: Some(LiveScenarioFragment { + root_component_id: child_root_id, + components: vec![ + ComponentIr { + id: child_root_id, + moniker: "/job-1".to_string(), + parent: Some(root_authority), + children: vec![provider_id, consumer_id], + resolved_url: Some("file:///tmp/job-1.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ComponentIr { + id: provider_id, + moniker: "/job-1/provider".to_string(), + parent: Some(child_root_id), + children: Vec::new(), + resolved_url: Some("file:///tmp/job-1-provider.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ComponentIr { + id: consumer_id, + moniker: "/job-1/consumer".to_string(), + parent: Some(child_root_id), + children: Vec::new(), + resolved_url: Some("file:///tmp/job-1-consumer.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ], + bindings: Vec::new(), + }), + input_bindings: Vec::new(), + assignments: BTreeMap::from([ + ("/job-1".to_string(), "direct_local".to_string()), + ("/job-1/provider".to_string(), "direct_local".to_string()), + ("/job-1/consumer".to_string(), "compose_local".to_string()), + ]), + overlay_ids: vec![ + "child:1:consumer:compose_local:amber_link_test".to_string(), + "child:1:provider:direct_local:amber_export_test".to_string(), + ], + overlays: vec![ + DynamicOverlayRecord { + overlay_id: "child:1:consumer:compose_local:amber_link_test".to_string(), + site_id: "compose_local".to_string(), + action: DynamicOverlayAction::ExternalSlot { link: link.clone() }, + }, + DynamicOverlayRecord { + overlay_id: "child:1:provider:direct_local:amber_export_test".to_string(), + site_id: "direct_local".to_string(), + action: DynamicOverlayAction::ExportPeer { link: link.clone() }, + }, + ], + outputs: BTreeMap::new(), + }; + state.pending_creates.push(pending_create(1, child.clone())); + write_control_state(&state_path, &state).expect("state should write"); + + let peer_router = Router::new() + .route( + "/v1/internal/router-identity", + axum::routing::post(|Json(request): Json| async move { + assert_eq!(request.site_id, "compose_local"); + Json(RouterIdentityResponse { + id: "/site/compose_local/router".to_string(), + public_key_b64: base64::engine::general_purpose::STANDARD.encode([13u8; 32]), + }) + }), + ) + .route( + "/v1/internal/children/prepare", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/publish", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/destroy", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/rollback", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/link-overlays/external-slot/publish", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/link-overlays/external-slot/clear", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ); + let (peer_router_url, _peer_router_handle) = spawn_test_router(peer_router).await; - let unknown = authorize_capability_instance(&state, "cap.missing", "/parent") - .expect_err("unknown capability instance should be rejected"); - assert_eq!(unknown.code, ProtocolErrorCode::Unauthorized); + let publish_calls = Arc::new(std::sync::Mutex::new(Vec::new())); + let resolve_calls = Arc::new(std::sync::Mutex::new(Vec::new())); + let runtime = Arc::new(PublishBeforeResolveRuntime { + publish_calls: publish_calls.clone(), + resolve_calls: resolve_calls.clone(), + }); + let base_app = test_control_state_app(&dir, state, state_path); + let mut plan = base_app.controller_plan.as_ref().clone(); + plan.site_id = "direct_local".to_string(); + plan.kind = SiteKind::Direct; + plan.router_identity_id = "/site/direct_local/router".to_string(); + plan.site_state_root = dir + .path() + .join("state") + .join("direct_local") + .display() + .to_string(); + plan.artifact_dir = dir + .path() + .join("artifact") + .join("direct_local") + .display() + .to_string(); + plan.storage_root = Some( + dir.path() + .join("storage") + .join("direct_local") + .display() + .to_string(), + ); + plan.runtime_root = Some( + dir.path() + .join("runtime") + .join("direct_local") + .display() + .to_string(), + ); + let mut app = ControlStateApp { + controller_plan: Arc::new(plan), + runtime, + ..base_app + }; + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("compose_local".to_string(), peer_router_url.clone())]), + ); + assert_eq!( + app.controller_plan + .peer_site_router_urls + .get("compose_local") + .map(String::as_str), + Some(peer_router_url.as_str()), + "controller runtime should use the router-local peer URL for compose-local controller \ + traffic", + ); + continue_create_committed_hidden(&app, child.child_id) + .await + .expect("create should publish the child before resolving cross-site overlays"); - let snapshot_err = - snapshot(&state, parent_id).expect_err("non-root authority should not be able to snapshot"); - assert_eq!(snapshot_err.code, ProtocolErrorCode::ScopeNotAllowed); + assert!( + !publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty(), + "local child publication should happen before overlay resolution", + ); + assert!( + !resolve_calls + .lock() + .expect("resolve call log mutex should lock") + .is_empty(), + "cross-site overlay resolution should still occur for the compose consumer", + ); + let live_state = app.control_state.lock().await.clone(); + assert!( + live_state + .live_children + .iter() + .any(|live_child| live_child.child_id == child.child_id), + "the committed-hidden child should become live after publication", + ); } #[tokio::test] -async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id() { +async fn destroy_and_recreate_same_child_name_revokes_then_restores_stable_framework_route() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let parent_path = dir.path().join("parent.json5"); @@ -3885,6 +6971,13 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( .map(|(id, _)| id.0) .expect("parent component should exist"); + let parent_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent") + .cloned() + .expect("parent should have its own framework capability instance"); + create_child( &mut state, parent_id, @@ -3899,12 +6992,30 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( ) .await .expect("first delegate create should succeed"); - let first_cap_instance_id = state + authorize_capability_instance(&state, &parent_record.cap_instance_id, "/parent").expect( + "parent authorization should survive when a child receives the same controller route", + ); + let (first_cap_instance_id, first_route_id, first_peer_id) = state .capability_instances .values() .find(|record| record.recipient_component_moniker == "/parent/delegate") - .map(|record| record.cap_instance_id.clone()) + .map(|record| { + ( + record.cap_instance_id.clone(), + record.route_id.clone(), + record.recipient_peer_id.clone(), + ) + }) .expect("first delegate capability instance should exist"); + assert_ne!( + first_cap_instance_id, parent_record.cap_instance_id, + "distinct recipients must retain distinct framework grant identities even when they share \ + a controller route", + ); + assert_eq!( + first_route_id, parent_record.route_id, + "distinct recipients in the same authority should share the same stable controller route", + ); destroy_child(&mut state, parent_id, "delegate", &state_path) .await @@ -3916,6 +7027,10 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( .any(|record| record.recipient_component_moniker == "/parent/delegate"), "destroy should revoke the first child lifetime's capability instance", ); + let destroyed = authorize_capability_instance(&state, &first_cap_instance_id, &first_peer_id) + .expect_err("destroyed child should no longer authorize"); + assert_eq!(destroyed.code, ProtocolErrorCode::Unauthorized); + assert_eq!(destroyed.message, "unknown framework capability instance"); create_child( &mut state, @@ -3931,40 +7046,425 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( ) .await .expect("second delegate create should succeed"); - let second_cap_instance_id = state + let (second_cap_instance_id, second_route_id, second_peer_id) = state .capability_instances .values() .find(|record| record.recipient_component_moniker == "/parent/delegate") - .map(|record| record.cap_instance_id.clone()) + .map(|record| { + ( + record.cap_instance_id.clone(), + record.route_id.clone(), + record.recipient_peer_id.clone(), + ) + }) .expect("second delegate capability instance should exist"); assert_ne!( - first_cap_instance_id, second_cap_instance_id, - "recreating the same child name must mint a new framework capability instance id", + second_cap_instance_id, first_cap_instance_id, + "recreating the same child name must mint a fresh framework grant identity", + ); + assert_eq!( + second_route_id, first_route_id, + "ordinary framework.component bindings should reuse the stable controller route for the \ + same authority and child path", ); + assert_eq!( + second_peer_id, first_peer_id, + "recreating the same child name should restore the same peer identity path", + ); + authorize_capability_instance(&state, &second_cap_instance_id, &second_peer_id) + .expect("recreated child should regain framework authorization"); } -#[test] -fn framework_auth_header_must_match_expected_token() { +#[tokio::test] +async fn local_controller_requests_require_internal_route_headers_not_framework_auth() { + let (dir, state, state_path, _) = compile_framework_binding_control_state().await; + let app = SiteControllerApp { + control: test_control_state_app(&dir, state, state_path), + ready: ready_site_controller_flag(), + }; + let controller_route_id = { + let state = app.control.control_state.lock().await.clone(); + let scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some() + }) + .map(|(_, component)| component) + .expect("site controller component should be injected"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; + let mut headers = HeaderMap::new(); + headers.insert( + super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, + "1".parse().expect("header should parse"), + ); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + controller_route_id + .parse() + .expect("route header should parse"), + ); + headers.insert( + FRAMEWORK_PEER_ID_HEADER, + app.control + .controller_plan + .router_identity_id + .parse() + .expect("peer header should parse"), + ); + + super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect( + "router-local controller requests should authorize without a control-state auth header", + ); + + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + "component:/wrong:__amber_internal_site_controller:http" + .parse() + .expect("header should parse"), + ); + let err = super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect_err("wrong internal route should be rejected"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + +#[tokio::test] +async fn local_controller_requests_select_the_controller_for_the_current_site() { + let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; + let mut scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let root_id = scenario.root; + let compose_controller_id = amber_scenario::ComponentId(scenario.components.len()); + scenario.components.push(Some(amber_scenario::Component { + id: compose_controller_id, + parent: Some(root_id), + moniker: FrameworkComponentControllerMoniker::for_site("compose_local") + .into_string() + .into(), + digest: scenario.component(root_id).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value( + amber_compiler::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: "compose_local".to_string(), + grants: BTreeMap::new(), + }, + ) + .expect("controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario + .component_mut(root_id) + .children + .push(compose_controller_id); + scenario.normalize_order(); + scenario.assert_invariants(); + state.base_scenario = amber_scenario::ScenarioIr::from(&scenario); + + let route_id_for_site = |scenario: &Scenario, execution_site: &str| { + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == execution_site) + }) + .map(|(_, component)| component) + .expect("site controller component should be injected for the site"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; + + let direct_route_id = route_id_for_site(&scenario, "direct_local"); + let compose_route_id = route_id_for_site(&scenario, "compose_local"); + + let control = test_control_state_app(&dir, state, state_path.clone()); + let mut controller_plan = control.controller_plan.as_ref().clone(); + controller_plan.site_id = "direct_local".to_string(); + controller_plan.kind = SiteKind::Direct; + controller_plan.router_identity_id = "/site/direct_local/router".to_string(); + let app = SiteControllerApp { + control: ControlStateApp { + controller_plan: Arc::new(controller_plan), + ..control + }, + ready: ready_site_controller_flag(), + }; + let mut headers = HeaderMap::new(); - let missing = authorize_framework_auth_header(&headers, "expected") - .expect_err("missing auth header should be rejected"); - assert_eq!(missing.0.code, ProtocolErrorCode::Unauthorized); + headers.insert( + super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, + "1".parse().expect("header should parse"), + ); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + direct_route_id.parse().expect("route header should parse"), + ); + headers.insert( + FRAMEWORK_PEER_ID_HEADER, + "/site/direct_local/router" + .parse() + .expect("peer header should parse"), + ); + super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect("the direct-local controller should accept its own lowered internal route"); headers.insert( - FRAMEWORK_AUTH_HEADER, - "wrong".parse().expect("header should parse"), + FRAMEWORK_ROUTE_ID_HEADER, + compose_route_id.parse().expect("route header should parse"), + ); + let err = super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect_err("a different site's controller route should be rejected"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + +#[test] +fn dynamic_caps_sidecar_requests_require_internal_route_headers() { + let headers = HeaderMap::new(); + + let err = super::site_controller::authorize_dynamic_caps_sidecar_request( + &headers, + "components./alice", + ) + .expect_err("dynamic-caps sidecar auth must require the router-injected internal route"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + +#[tokio::test] +async fn cross_site_framework_capability_records_bind_to_provider_router_peers() { + let (_dir, state, _state_path) = compile_cross_site_framework_authority_control_state().await; + let expected_provider_router = "/site/compose_local/router"; + + let admin_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/authority/admin") + .cloned() + .expect("cross-site admin should receive a framework capability record"); + assert_eq!(admin_record.controller_site_id, "compose_local"); + assert_eq!(admin_record.recipient_site_id, "direct_local"); + assert_eq!(admin_record.recipient_peer_id, expected_provider_router); + authorize_capability_instance( + &state, + &admin_record.cap_instance_id, + expected_provider_router, + ) + .expect("cross-site framework grants should authorize through the provider site router"); + let admin_err = + authorize_capability_instance(&state, &admin_record.cap_instance_id, "/authority/admin") + .expect_err( + "cross-site framework grants must not authorize the remote component directly", + ); + assert_eq!(admin_err.code, ProtocolErrorCode::Unauthorized); + + let remote_record = state + .capability_instances + .values() + .find(|record| { + record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + && record.controller_site_id == "compose_local" + && record.recipient_site_id == "direct_local" + }) + .cloned() + .expect("compose-local controller should expose the remote controller internal capability"); + assert_eq!(remote_record.recipient_peer_id, expected_provider_router); + authorize_capability_instance( + &state, + &remote_record.cap_instance_id, + expected_provider_router, + ) + .expect( + "cross-site remote controller capabilities should authorize through the provider site \ + router", + ); + let peer_err = authorize_capability_instance( + &state, + &remote_record.cap_instance_id, + remote_record.recipient_component_moniker.as_str(), + ) + .expect_err( + "cross-site remote controller capabilities must not authorize the remote controller \ + component directly", ); - let wrong = authorize_framework_auth_header(&headers, "expected") - .expect_err("mismatched auth header should be rejected"); - assert_eq!(wrong.0.code, ProtocolErrorCode::Unauthorized); + assert_eq!(peer_err.code, ProtocolErrorCode::Unauthorized); +} +#[tokio::test] +async fn dynamic_caps_remote_controller_requests_authorize_over_public_remote_controller_capability() + { + let (dir, state, state_path) = compile_cross_site_framework_authority_control_state().await; + let app = SiteControllerApp { + control: test_control_state_app(&dir, state.clone(), state_path), + ready: ready_site_controller_flag(), + }; + let remote_record = state + .capability_instances + .values() + .find(|record| { + record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + && record.controller_site_id == "compose_local" + && record.recipient_site_id == "direct_local" + }) + .cloned() + .expect("compose-local controller should expose the remote controller internal capability"); + let mut headers = HeaderMap::new(); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + remote_record + .route_id + .parse() + .expect("route header should parse"), + ); headers.insert( - FRAMEWORK_AUTH_HEADER, - "expected".parse().expect("header should parse"), + FRAMEWORK_PEER_ID_HEADER, + remote_record + .recipient_peer_id + .parse() + .expect("peer header should parse"), + ); + + let auth = super::site_controller::authorize_dynamic_caps_request( + &app, + &headers, + "components./remote-holder", + ) + .await + .expect( + "remote dynamic capability requests should authorize through the router-local controller \ + route", + ); + assert_eq!( + auth, + super::site_controller::DynamicCapsRequestAuth::RemoteController + ); +} + +#[tokio::test] +async fn router_identity_lookup_requests_target_site_from_remote_controller_site() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + write_file( + &root_path, + r#" + { + manifest_version: "0.3.0", + program: { path: "/bin/echo", args: ["root"] }, + } + "#, + ); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("compose_local".to_string()), + vm: Some("vm_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; + let state_path = dir.path().join("state.json"); + write_control_state(&state_path, &state).expect("state should write"); + + let requested_sites = Arc::new(std::sync::Mutex::new(Vec::::new())); + let vm_public_key = [23u8; 32]; + let peer_router = Router::new().route( + "/v1/internal/router-identity", + axum::routing::post({ + let requested_sites = requested_sites.clone(); + move |Json(request): Json| { + let requested_sites = requested_sites.clone(); + async move { + requested_sites + .lock() + .expect("requested site log should lock") + .push(request.site_id.clone()); + Json(RouterIdentityResponse { + id: format!("/site/{}/router", request.site_id), + public_key_b64: base64::engine::general_purpose::STANDARD + .encode(vm_public_key), + }) + } + } + }), + ); + let (direct_controller_url, _direct_controller_handle) = spawn_test_router(peer_router).await; + + let mut app = test_control_state_app(&dir, state, state_path); + let plan = Arc::make_mut(&mut app.controller_plan); + plan.site_id = "compose_local".to_string(); + plan.kind = SiteKind::Compose; + plan.peer_site_router_urls = BTreeMap::from([( + "direct_local".to_string(), + direct_controller_url.to_string(), + )]); + + let identity = router_identity_for_overlay(&app, "vm_local") + .await + .expect("remote controller site should return the target VM router identity"); + assert_eq!(identity.id, "/site/vm_local/router"); + assert_eq!(identity.public_key, vm_public_key); + assert_eq!( + requested_sites + .lock() + .expect("requested site log should lock") + .clone(), + vec!["vm_local".to_string()], + "the remote controller request must name the target site, not the controller site", ); - authorize_framework_auth_header(&headers, "expected") - .expect("matching auth header should succeed"); } #[tokio::test] @@ -4185,14 +7685,12 @@ fn shared_cross_site_link_is_retained_while_another_child_still_needs_it() { generation: 0, next_child_id: 2, next_tx_id: 0, + id_stride: 1, next_component_id: 0, - dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( - &mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), dynamic_capability_journal: Vec::new(), capability_instances: BTreeMap::new(), journal: Vec::new(), @@ -4289,18 +7787,82 @@ async fn create_rejects_duplicate_names_and_destroy_is_idempotent() { &state_path, ) .await - .expect_err("duplicate child name should be rejected"); - assert_eq!(duplicate.code, ProtocolErrorCode::NameConflict); + .expect_err("duplicate child name should be rejected"); + assert_eq!(duplicate.code, ProtocolErrorCode::NameConflict); + + destroy_child(&mut state, root_authority, "job", &state_path) + .await + .expect("first destroy should succeed"); + destroy_child(&mut state, root_authority, "job", &state_path) + .await + .expect("destroy should be idempotent once the child is gone"); + assert!( + state.live_children.is_empty(), + "destroy should remove the child" + ); +} + +#[tokio::test] +async fn create_aborts_and_destroys_partially_published_child_when_publish_fails() { + let (dir, state, state_path) = compile_exact_template_control_state().await; + let authority_realm_id = state.base_scenario.root; + let base_app = test_control_state_app(&dir, state, state_path); + let runtime = Arc::new(FailingPublishRuntime::default()); + let app = with_runtime(&base_app, runtime.clone()); + + let err = execute_create_child( + &app, + authority_realm_id, + CreateChildRequest { + template: "worker".to_string(), + name: "broken".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + ) + .await + .expect_err("publish failure should abort the create"); + assert_eq!(err.0.code, ProtocolErrorCode::PublishFailed); + assert!( + err.0.message.contains("publish exploded"), + "publish failure should preserve the underlying detail, got: {}", + err.0.message + ); - destroy_child(&mut state, root_authority, "job", &state_path) - .await - .expect("first destroy should succeed"); - destroy_child(&mut state, root_authority, "job", &state_path) - .await - .expect("destroy should be idempotent once the child is gone"); + let state = app.control_state.lock().await.clone(); assert!( - state.live_children.is_empty(), - "destroy should remove the child" + state.pending_creates.is_empty(), + "failed publish cleanup must remove the pending create" + ); + assert!( + state + .live_children + .iter() + .all(|child| child.name != "broken"), + "failed publish cleanup must not leave a live child behind" + ); + assert_eq!( + state.journal.last().map(|entry| entry.state), + Some(ChildState::CreateAborted), + "failed publish cleanup must record an aborted transaction" + ); + + let destroy_calls = runtime + .destroy_calls + .lock() + .expect("destroy call log mutex should lock") + .clone(); + assert_eq!( + destroy_calls.len(), + 1, + "failed publish cleanup must destroy the prepared site-local child" + ); + assert_eq!(destroy_calls[0].0, 1); + assert_eq!( + Some(destroy_calls[0].1.as_str()), + Some("direct_local"), + "failed publish cleanup must reconcile the child site back to the desired plan", ); } @@ -4399,6 +7961,7 @@ async fn snapshot_is_stable_across_dynamic_create_order() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -4406,6 +7969,7 @@ async fn snapshot_is_stable_across_dynamic_create_order() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -4538,6 +8102,7 @@ async fn create_rejects_unoffered_backend_without_committing_child_state() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -4562,7 +8127,6 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { let (dir, state, state_path) = compile_exact_template_control_state().await; let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; let request = CreateChildRequest { template: "worker".to_string(), name: "job".to_string(), @@ -4611,303 +8175,6 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { 1, "snapshot should remain clean after the same-name race", ); - for actuator in actuators { - actuator.abort(); - } -} - -#[tokio::test] -async fn destroy_retracted_tears_down_sites_concurrently() { - let dir = TempDir::new().expect("temp dir"); - let root_path = dir.path().join("root.json5"); - write_file( - &root_path, - r#" - { - manifest_version: "0.3.0", - program: { - path: "/bin/sh", - args: ["-c", "sleep 1"] - } - } - "#, - ); - - let placement = PlacementFile { - schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), - version: amber_compiler::run_plan::PLACEMENT_VERSION, - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - SiteDefinition { - kind: SiteKind::Compose, - context: None, - }, - ), - ( - "direct_local".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]), - defaults: PlacementDefaults { - path: Some("direct_local".to_string()), - image: Some("compose_local".to_string()), - ..PlacementDefaults::default() - }, - components: BTreeMap::new(), - dynamic_capabilities: None, - framework_children: None, - }; - - let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let root_authority = state.base_scenario.root; - let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut destroy_starts, barrier) = - install_barrier_destroy_site_actuator(&app).await; - { - let mut state = app.control_state.lock().await; - state.pending_destroys.push(pending_destroy( - 1, - LiveChildRecord { - child_id: 7, - authority_realm_id: root_authority, - name: "job-compose".to_string(), - state: ChildState::DestroyRetracted, - template_name: Some("fixture".to_string()), - selected_manifest_catalog_key: None, - fragment: None, - input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: vec![ - DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - ], - overlay_ids: Vec::new(), - overlays: Vec::new(), - outputs: BTreeMap::new(), - }, - )); - } - - let destroy = tokio::spawn({ - let app = app.clone(); - async move { continue_destroy_retracted(&app, 7).await } - }); - - let first = tokio::time::timeout(Duration::from_secs(5), destroy_starts.recv()) - .await - .expect("first destroy should start in time") - .expect("first destroy notification should arrive"); - let second = tokio::time::timeout(Duration::from_secs(5), destroy_starts.recv()) - .await - .expect("second destroy should start in time") - .expect("second destroy notification should arrive"); - assert_ne!( - first, second, - "destroy should reach both site actuators before either completes" - ); - - barrier.wait().await; - destroy - .await - .expect("destroy task should join") - .expect("destroy should succeed"); - - let recovered = app.control_state.lock().await.clone(); - assert!( - recovered.live_children.is_empty(), - "successful destroy should remove the child after concurrent site teardown", - ); - for actuator in actuators { - actuator.abort(); - } -} - -#[tokio::test] -async fn create_committed_hidden_publishes_independent_sites_concurrently() { - let dir = TempDir::new().expect("temp dir"); - let root_path = dir.path().join("root.json5"); - let child_path = dir.path().join("child.json5"); - write_file( - &child_path, - r#" - { - manifest_version: "0.3.0", - program: { - path: "/bin/echo", - args: ["child"], - network: { endpoints: [{ name: "out", port: 8080, protocol: "http" }] } - }, - provides: { out: { kind: "http", endpoint: "out" } }, - exports: { out: "provides.out" }, - } - "#, - ); - write_file( - &root_path, - &format!( - r#" - {{ - manifest_version: "0.3.0", - program: {{ - path: "/bin/sh", - args: ["-c", "sleep 1"] - }}, - slots: {{ - realm: {{ kind: "component", optional: true }} - }}, - child_templates: {{ - fixture: {{ manifest: "{child}" }} - }} - }} - "#, - child = file_url(&child_path), - ), - ); - - let placement = PlacementFile { - schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), - version: amber_compiler::run_plan::PLACEMENT_VERSION, - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - SiteDefinition { - kind: SiteKind::Compose, - context: None, - }, - ), - ( - "direct_local".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]), - defaults: PlacementDefaults { - path: Some("direct_local".to_string()), - image: Some("compose_local".to_string()), - ..PlacementDefaults::default() - }, - components: BTreeMap::new(), - dynamic_capabilities: None, - framework_children: None, - }; - - let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; - let state_path = dir.path().join("control-state.json"); - let root_authority = state.base_scenario.root; - let mut child = prepare_child_record( - &mut state, - root_authority, - &CreateChildRequest { - template: "fixture".to_string(), - name: "job-compose".to_string(), - manifest: None, - config: BTreeMap::new(), - bindings: BTreeMap::new(), - }, - ) - .await - .expect("child should plan successfully"); - let child_id = child.child_id; - child.state = ChildState::CreateCommittedHidden; - child.site_plans = vec![ - DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - ]; - write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut publish_starts, barrier) = - install_barrier_publish_site_actuator(&app).await; - { - let mut state = app.control_state.lock().await; - state.pending_creates.push(pending_create(1, child)); - } - - let publish = tokio::spawn({ - let app = app.clone(); - async move { continue_create_committed_hidden(&app, child_id).await } - }); - - let first = tokio::time::timeout(Duration::from_secs(5), publish_starts.recv()) - .await - .expect("first publish should start in time") - .expect("first publish notification should arrive"); - let second = tokio::time::timeout(Duration::from_secs(5), publish_starts.recv()) - .await - .expect("second publish should start in time") - .expect("second publish notification should arrive"); - assert_ne!( - first, second, - "create should reach both independent site actuators before either completes" - ); - - barrier.wait().await; - publish - .await - .expect("publish task should join") - .expect("publish should succeed"); - - let recovered = app.control_state.lock().await.clone(); - let child = recovered - .live_children - .iter() - .find(|child| child.child_id == child_id) - .expect("child should remain present"); - assert_eq!( - child.state, - ChildState::Live, - "successful concurrent site publication should promote the child to live", - ); - for actuator in actuators { - actuator.abort(); - } } #[tokio::test] @@ -4955,6 +8222,7 @@ async fn concurrent_distinct_creates_commit_both_children() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -4970,7 +8238,6 @@ async fn concurrent_distinct_creates_commit_both_children() { write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; let (left, right) = tokio::join!( execute_create_child( @@ -5031,9 +8298,6 @@ async fn concurrent_distinct_creates_commit_both_children() { .any(|component| component.moniker == "/job-b"), "snapshot should contain the second child", ); - for actuator in actuators { - actuator.abort(); - } } #[tokio::test] @@ -5083,6 +8347,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -5090,6 +8355,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { SiteDefinition { kind: SiteKind::Kubernetes, context: None, + controller_site: None, }, ), ]), @@ -5126,7 +8392,44 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { } #[tokio::test] -async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { +async fn prepare_child_record_grants_internal_controller_route_only_to_dynamic_child() { + let (_dir, mut state, _state_path, _admin_record) = + compile_framework_binding_control_state().await; + let root_authority = state.base_scenario.root; + let child = prepare_child_record( + &mut state, + root_authority, + &CreateChildRequest { + template: "worker".to_string(), + name: "job".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + ) + .await + .expect("dynamic child should plan successfully"); + let spec = build_local_child_runtime_spec(&state, &child, "direct_local") + .expect("dynamic child should build a direct runtime spec"); + let internal_routes = spec + .controller_routes + .iter() + .filter(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + .collect::>(); + + assert_eq!(internal_routes.len(), 1); + assert_eq!( + internal_routes[0].allowed_issuers, + vec!["/job".to_string()], + "dynamic child sidecars need the local controller route for dynamic-capability control, \ + but the overlay must not grant access to pre-existing components or unrelated children", + ); +} + +#[tokio::test] +async fn prepare_child_record_supports_cross_site_dynamic_fragments() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let child_path = dir.path().join("child-compose.json5"); @@ -5287,6 +8590,7 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -5294,6 +8598,7 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -5301,6 +8606,7 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { SiteDefinition { kind: SiteKind::Kubernetes, context: None, + controller_site: None, }, ), ( @@ -5308,6 +8614,7 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { SiteDefinition { kind: SiteKind::Vm, context: None, + controller_site: None, }, ), ]), @@ -5346,88 +8653,80 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { }, ) .await - .expect("matrix child should plan successfully"); - + .expect("cross-site dynamic children should retain their multi-site placement"); assert_eq!( - child - .assignments - .get("/job-compose/root") - .map(String::as_str), - Some("compose_local"), + child_runtime_site_ids(&child).expect("child site ids"), + vec![ + "compose_local".to_string(), + "direct_local".to_string(), + "kind_local".to_string(), + "vm_local".to_string() + ] ); + let compose_spec = + build_local_child_runtime_spec(&state, &child, "compose_local").expect("compose spec"); + assert_eq!(compose_spec.assigned_components, vec!["/job-compose/root"]); + assert!( + compose_spec.direct_inputs.is_empty(), + "cross-site child bindings should not be modeled as same-site direct inputs" + ); + let direct_spec = + build_local_child_runtime_spec(&state, &child, "direct_local").expect("direct spec"); assert_eq!( - child - .assignments - .get("/job-compose/kind_helper") - .map(String::as_str), - Some("kind_local"), + direct_spec.assigned_components, + vec!["/job-compose/direct_helper"] + ); + assert!( + direct_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" ); + let kind_spec = + build_local_child_runtime_spec(&state, &child, "kind_local").expect("kind spec"); assert_eq!( - child - .assignments - .get("/job-compose/direct_helper") - .map(String::as_str), - Some("direct_local"), + kind_spec.assigned_components, + vec!["/job-compose/kind_helper"] + ); + assert!( + kind_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" + ); + let vm_spec = build_local_child_runtime_spec(&state, &child, "vm_local").expect("vm spec"); + assert_eq!( + vm_spec.assigned_components, + vec!["/job-compose/vm_helper/root"] + ); + assert!( + vm_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" ); assert_eq!( - child - .assignments - .get("/job-compose/vm_helper/root") - .map(String::as_str), - Some("vm_local"), + child.overlays.len(), + 6, + "three cross-site child bindings should lower to three consumer external-slot overlays and three provider export-peer overlays" ); assert_eq!( child - .site_plans + .overlays .iter() - .map(|site_plan| site_plan.site_id.as_str()) - .collect::>(), - BTreeSet::from(["compose_local", "direct_local", "kind_local", "vm_local"]), - "cross-backend child planning should retain all expected site slices", + .filter(|overlay| { + overlay.site_id == "compose_local" + && matches!(overlay.action, DynamicOverlayAction::ExternalSlot { .. }) + }) + .count(), + 3, + "the child root site should receive one external-slot overlay per cross-site binding" ); - let proxy_exports_by_site = child - .site_plans - .iter() - .map(|site_plan| { - ( - site_plan.site_id.as_str(), - site_plan - .proxy_exports - .keys() - .map(String::as_str) - .collect::>(), - ) - }) - .collect::>(); assert_eq!( - proxy_exports_by_site.get("compose_local"), - Some(&BTreeSet::from(["http"])), - "compose site should own the dynamic child root export", - ); - for (site_id, public_export) in [ - ("kind_local", "kind_http"), - ("direct_local", "direct_http"), - ("vm_local", "vm_http"), - ] { - let exports = proxy_exports_by_site - .get(site_id) - .unwrap_or_else(|| panic!("missing proxy export set for {site_id}")); - assert!( - exports.contains(public_export), - "{site_id} should keep its public helper export", - ); - assert!( - exports.iter().any(|name| name.starts_with("amber_export_")), - "{site_id} should also publish its internal routed link export", - ); - } - assert!( child - .site_plans + .overlays .iter() - .all(|site_plan| site_plan.routed_inputs.is_empty()), - "bindings that stay inside the created fragment must remain intra-fragment wiring, not \ - site-router routed inputs", + .filter(|overlay| { + overlay.site_id != "compose_local" + && matches!(overlay.action, DynamicOverlayAction::ExportPeer { .. }) + }) + .count(), + 3, + "each remote provider site should expose one export-peer overlay for the child binding" ); } @@ -6017,6 +9316,7 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -6030,35 +9330,13 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); let root_authority = state.base_scenario.root; - let app = test_control_state_app(&dir, state.clone(), state_path.clone()); - let actuators = install_failing_rollback_site_actuator(&app).await; + let app = with_runtime( + &test_control_state_app(&dir, state.clone(), state_path.clone()), + Arc::new(FailingRollbackRuntime), + ); state.pending_creates.push(pending_create( 1, - LiveChildRecord { - child_id: 1, - authority_realm_id: root_authority, - name: "prepared".to_string(), - state: ChildState::CreatePrepared, - template_name: Some("worker".to_string()), - selected_manifest_catalog_key: None, - fragment: None, - input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: vec![DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }], - overlay_ids: Vec::new(), - overlays: Vec::new(), - outputs: BTreeMap::new(), - }, + empty_live_child(root_authority, "prepared", 1, ChildState::CreatePrepared), )); write_control_state(&state_path, &state).expect("state should write"); *app.control_state.lock().await = state; @@ -6082,9 +9360,6 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { recovered.journal.is_empty(), "failed rollback must not pretend the child was aborted" ); - for actuator in actuators { - actuator.abort(); - } } #[tokio::test] @@ -6105,7 +9380,6 @@ async fn recover_control_state_promotes_create_committed_hidden_children_to_live .await .expect("child should plan successfully"); child.state = ChildState::CreateCommittedHidden; - child.site_plans.clear(); state.pending_creates.push(pending_create(1, child)); write_control_state(&state_path, &state).expect("state should write"); let app = test_control_state_app(&dir, state, state_path); @@ -6154,6 +9428,7 @@ async fn recover_control_state_does_not_republish_live_children() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -6177,34 +9452,27 @@ async fn recover_control_state_does_not_republish_live_children() { fragment: None, input_bindings: Vec::new(), assignments: BTreeMap::new(), - site_plans: vec![DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }], overlay_ids: Vec::new(), overlays: Vec::new(), outputs: BTreeMap::new(), }); write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut publish_starts, _barrier) = - install_barrier_publish_site_actuator(&app).await; + let runtime = Arc::new(RecordingPublishRuntime::default()); + let app = with_runtime( + &test_control_state_app(&dir, state, state_path), + runtime.clone(), + ); recover_control_state(&app) .await .expect("recovery should leave live children alone"); assert!( - tokio::time::timeout(Duration::from_millis(200), publish_starts.recv()) - .await - .is_err(), + runtime + .publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty(), "live recovery should not call publish again", ); let recovered = app.control_state.lock().await.clone(); @@ -6215,9 +9483,6 @@ async fn recover_control_state_does_not_republish_live_children() { recovered.journal.is_empty(), "live recovery should not append synthetic journal entries", ); - for actuator in actuators { - actuator.abort(); - } } #[tokio::test] @@ -6838,13 +10103,18 @@ async fn dynamic_capabilities_inspect_ref_rejects_unsupported_token_versions() { super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } => grant_id, other => panic!("unexpected share outcome: {other:?}"), }; - let signing_key = - mesh_dynamic_caps::signing_key_from_seed_b64(&state.dynamic_capability_signing_seed_b64) - .expect("test signing key should decode"); + let identity = state + .controller_identity + .as_ref() + .expect("test controller identity should be loaded"); + let signing_key = identity + .signing_key() + .expect("test controller identity should be valid"); let unsupported_ref = mesh_dynamic_caps::build_dynamic_capability_ref_url( DynamicCapabilityRefClaims { version: mesh_dynamic_caps::DYNAMIC_CAPS_REF_VERSION + 1, run_id: state.run_id.clone(), + issuer_id: identity.id.clone(), grant_id, holder_component_id: "components./carol".to_string(), descriptor_hint: Some("provider.http".to_string()), @@ -6862,3 +10132,756 @@ async fn dynamic_capabilities_inspect_ref_rejects_unsupported_token_versions() { assert_eq!(err.code, ProtocolErrorCode::MalformedRef); assert!(err.message.contains("unsupported")); } + +#[test] +fn compose_site_controller_is_injected_as_a_compose_service() { + let temp = TempDir::new().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("compose-site"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + fs::write( + artifact_root.join("compose.yaml"), + r#" +services: + amber-router: + image: ghcr.io/rdi-foundation/amber-router:test + volumes: + - amber-router-control:/amber/control + amber-router-control-init: + image: busybox + amber-provisioner: + image: ghcr.io/rdi-foundation/amber-provisioner:test + amber-site-controller: + image: __amber_internal/site-controller + command: + - --plan + - /amber/site/state/site-controller-plan.json + environment: + - EXISTING_ENV=kept +networks: + amber_mesh: {} +volumes: + amber-router-control: {} +"#, + ) + .expect("compose yaml should write"); + + let controller_port = 32111; + let plan = write_site_controller_plan( + &site_controller_plan_path(&site_state_root), + "test-run", + "test-mesh", + "compose-site", + SiteKind::Compose, + SocketAddr::from(([0, 0, 0, 0], controller_port)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{controller_port}"), + "/site/compose-site/router", + &BTreeMap::new(), + &BTreeMap::new(), + &BTreeMap::new(), + Some("unix:///amber/control/router-control.sock"), + Some("127.0.0.1:24000"), + Some("host.docker.internal:24000"), + Some("192.168.65.254:24000"), + &site_state_root.join("site-controller-state.json"), + temp.path(), + &temp.path().join("state"), + &site_state_root, + &artifact_root, + Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), + None, + None, + Some(24000), + Some("amber_test_compose"), + None, + None, + None, + &BTreeMap::new(), + ) + .expect("compose site controller plan should write"); + + inject_compose_site_controller( + &artifact_root, + &plan, + &site_controller_plan_path(&site_state_root), + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose controller should inject"); + + let document: serde_yaml::Value = serde_yaml::from_str( + &fs::read_to_string(artifact_root.join("compose.yaml")).expect("compose yaml should read"), + ) + .expect("compose yaml should parse"); + let service = document["services"][SITE_CONTROLLER_SERVICE_NAME] + .as_mapping() + .expect("controller service should exist"); + assert_eq!( + service + .get(serde_yaml::Value::String("image".to_string())) + .and_then(serde_yaml::Value::as_str), + Some("ghcr.io/rdi-foundation/amber-site-controller:test") + ); + let command = service + .get(serde_yaml::Value::String("command".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should have a command"); + assert_eq!(command[0].as_str(), Some("--plan")); + assert_eq!( + command[1].as_str(), + Some("/amber/site/state/site-controller-plan.json") + ); + #[cfg(unix)] + { + let expected_user = format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }); + assert_eq!( + service + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + Some(expected_user.as_str()), + "compose site controller should run as the launching user so its host bind-mounted \ + state directory can stay private" + ); + } + #[cfg(not(unix))] + assert_eq!( + service + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + Some("0:0") + ); + let volumes = service + .get(serde_yaml::Value::String("volumes".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should mount volumes"); + let extra_hosts = service + .get(serde_yaml::Value::String("extra_hosts".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should set extra_hosts"); + let healthcheck = service + .get(serde_yaml::Value::String("healthcheck".to_string())) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller service should define a healthcheck"); + assert!(volumes.iter().any(|value| { + value.as_str() + == Some(&format!( + "{}:{}", + temp.path().display(), + temp.path().display() + )) + })); + let environment = service + .get(serde_yaml::Value::String("environment".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should preserve environment"); + assert!( + environment + .iter() + .any(|value| value.as_str() == Some("EXISTING_ENV=kept")) + ); + assert!( + volumes + .iter() + .any(|value| value.as_str() == Some("amber-router-control:/amber/control")) + ); + assert!( + volumes + .iter() + .any(|value| value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock")) + ); + let group_add = service + .get(serde_yaml::Value::String("group_add".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should join the router runtime group"); + assert!( + group_add + .iter() + .any(|value| value.as_str() == Some("65532")), + "compose site controller runs as the launching user, so it needs the router runtime group \ + to access the router-control socket" + ); + assert!( + extra_hosts + .iter() + .any(|value| value.as_str() == Some("host.docker.internal:host-gateway")), + "compose site controller should resolve host.docker.internal inside the site network", + ); + let healthcheck_test = healthcheck + .get(serde_yaml::Value::String("test".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller healthcheck should define a test command"); + assert!( + healthcheck_test.iter().any(|value| value + .as_str() + .is_some_and(|value| value.contains("/healthz"))), + "compose site controller healthcheck should wait for the controller readiness endpoint", + ); +} + +#[test] +fn kubernetes_site_controller_resources_are_injected_into_the_artifact() { + let temp = TempDir::new().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("kube-site"); + fs::create_dir_all(artifact_root.join("05-networkpolicies")) + .expect("network policies dir should create"); + fs::create_dir_all(artifact_root.join("04-services")).expect("services dir should create"); + fs::create_dir_all(site_state_root.clone()).expect("site state root should create"); + fs::write( + artifact_root.join("kustomization.yaml"), + "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ + 04-services/amber-site-controller.yaml\n - \ + 05-networkpolicies/amber-router-netpol.yaml\n - \ + 05-networkpolicies/amber-site-controller-netpol.yaml\n", + ) + .expect("kustomization should write"); + fs::write( + artifact_root.join("04-services/amber-router.yaml"), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-router +spec: + ports: + - name: mesh + port: 24000 + targetPort: 24000 + protocol: TCP + - name: control + port: 24100 + targetPort: 24100 + protocol: TCP +"#, + ) + .expect("router service should write"); + fs::write( + artifact_root.join("05-networkpolicies/amber-router-netpol.yaml"), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-router +spec: + ingress: [] +"#, + ) + .expect("router netpol should write"); + fs::write( + artifact_root.join("05-networkpolicies/amber-site-controller-netpol.yaml"), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-site-controller-netpol +spec: + egress: + - to: + - podSelector: + matchLabels: + amber.io/component: amber-router + ports: + - protocol: TCP + port: 24100 +"#, + ) + .expect("site controller netpol should write"); + fs::write( + site_state_root.join("site-controller-state.json"), + "{\"schema\":\"amber.test\",\"version\":1}", + ) + .expect("controller state should write"); + fs::write( + site_state_root.join("desired-links.json"), + "{\"schema\":\"amber.test\",\"version\":1}", + ) + .expect("desired links should write"); + let peer_site_router_urls = BTreeMap::from([( + "peer-site".to_string(), + "http://amber-router:37046".to_string(), + )]); + + fs::create_dir_all(artifact_root.join("03-deployments")) + .expect("deployments dir should create"); + fs::write( + artifact_root.join("03-deployments/amber-site-controller.yaml"), + r#" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: amber-site-controller +spec: + template: + spec: + containers: + - name: main + image: __amber_internal/site-controller + args: + - --plan + - /amber/site/state/site-controller-plan.json + env: + - name: EXISTING_ENV + value: kept + - name: sidecar + image: ghcr.io/rdi-foundation/amber-router:test +"#, + ) + .expect("controller deployment should write"); + fs::write( + artifact_root.join("04-services/amber-site-controller.yaml"), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-site-controller +spec: + ports: + - name: framework-component + port: 32123 + targetPort: 32123 + protocol: TCP +"#, + ) + .expect("controller service should write"); + let controller_port = 32123; + let plan = write_site_controller_plan( + &site_controller_plan_path(&site_state_root), + "test-run", + "test-mesh", + "kube-site", + SiteKind::Kubernetes, + SocketAddr::from(([0, 0, 0, 0], controller_port)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{controller_port}"), + "/site/kube-site/router", + &peer_site_router_urls, + &BTreeMap::new(), + &BTreeMap::new(), + Some("amber-router:24100"), + Some("127.0.0.1:24000"), + Some("host.docker.internal:24000"), + Some("192.168.65.254:24000"), + &site_state_root.join("site-controller-state.json"), + temp.path(), + &temp.path().join("state"), + &site_state_root, + &artifact_root, + Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), + None, + None, + Some(24000), + None, + Some("amber-test-kube-site"), + None, + None, + &BTreeMap::new(), + ) + .expect("kubernetes site controller plan should write"); + + inject_kubernetes_site_controller( + &artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes controller should inject"); + + let kustomization = fs::read_to_string(artifact_root.join("kustomization.yaml")) + .expect("kustomization should read"); + assert!(kustomization.contains("01-secrets/amber-site-controller-seed.yaml")); + assert!(kustomization.contains("03-persistentvolumeclaims/amber-site-controller-state.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-dynamic-role.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-dynamic-rolebinding.yaml")); + assert!(kustomization.contains("03-deployments/amber-site-controller.yaml")); + assert!(kustomization.contains("04-services/amber-site-controller.yaml")); + + fn rule_contains(rule: &serde_yaml::Value, key: &str, expected: &str) -> bool { + rule.as_mapping() + .and_then(|mapping| mapping.get(serde_yaml::Value::String(key.to_string()))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|values| values.iter().any(|value| value.as_str() == Some(expected))) + } + + let controller_role_raw = + fs::read_to_string(artifact_root.join("02-rbac/amber-site-controller-role.yaml")) + .expect("controller role should read"); + let controller_role: serde_yaml::Value = + serde_yaml::from_str(&controller_role_raw).expect("controller role should parse"); + let controller_role_rules = controller_role["rules"] + .as_sequence() + .expect("controller role should include rules"); + assert!( + controller_role_rules.iter().all(|rule| { + !rule_contains(rule, "apiGroups", "rbac.authorization.k8s.io") + && !rule_contains(rule, "resources", "serviceaccounts") + && !rule_contains(rule, "resources", "roles") + && !rule_contains(rule, "resources", "rolebindings") + }), + "site-controller Role should not be able to create or update Kubernetes RBAC objects" + ); + + let dynamic_provisioner_role_raw = + fs::read_to_string(artifact_root.join("02-rbac/amber-provisioner-dynamic-role.yaml")) + .expect("dynamic provisioner role should read"); + let dynamic_provisioner_role: serde_yaml::Value = + serde_yaml::from_str(&dynamic_provisioner_role_raw) + .expect("dynamic provisioner role should parse"); + let provisioner_rules = dynamic_provisioner_role["rules"] + .as_sequence() + .expect("dynamic provisioner role should include rules"); + assert_eq!(provisioner_rules.len(), 1); + let rule = &provisioner_rules[0]; + assert!(rule_contains(rule, "resources", "secrets")); + assert!(rule_contains(rule, "verbs", "create")); + assert!(rule_contains(rule, "verbs", "get")); + assert!(rule_contains(rule, "verbs", "update")); + + let deployment_path = artifact_root.join("03-deployments/amber-site-controller.yaml"); + let deployment_raw = fs::read_to_string(&deployment_path).expect("deployment should read"); + assert!(deployment_raw.contains("amber-site-controller")); + assert!(deployment_raw.contains("ghcr.io/rdi-foundation/amber-site-controller:test")); + assert!(deployment_raw.contains("/amber/site/state/site-controller-plan.json")); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment should parse"); + let init_container = deployment["spec"]["template"]["spec"]["initContainers"] + .as_sequence() + .and_then(|containers| containers.first()) + .and_then(serde_yaml::Value::as_mapping) + .expect("deployment should contain a seed init container"); + assert_eq!( + init_container + .get(serde_yaml::Value::String("image".to_string())) + .and_then(serde_yaml::Value::as_str), + Some("ghcr.io/rdi-foundation/amber-site-controller:test"), + "kubernetes site controller seeding should reuse the site-controller image so tests and \ + CI do not depend on an extra public init image pull", + ); + let containers = deployment["spec"]["template"]["spec"]["containers"] + .as_sequence() + .expect("deployment should contain site-controller containers"); + assert_eq!( + containers.len(), + 2, + "kubernetes bootstrap should patch the rendered controller deployment instead of \ + replacing it" + ); + let container = containers + .iter() + .find(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(serde_yaml::Value::String("name".to_string()))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("deployment should keep the main site-controller container"); + assert!( + !container.contains_key(serde_yaml::Value::String("command".to_string())), + "kubernetes site controller should keep the image entrypoint and pass only args" + ); + let args = container + .get(serde_yaml::Value::String("args".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("deployment should pass site-controller args"); + assert_eq!(args[0].as_str(), Some("--plan")); + assert_eq!( + args[1].as_str(), + Some("/amber/site/state/site-controller-plan.json") + ); + + let seed = fs::read_to_string(artifact_root.join("01-secrets/amber-site-controller-seed.yaml")) + .expect("seed secret should read"); + assert!(seed.contains("kind: Secret")); + assert!(seed.contains("stringData:")); + assert!(seed.contains("site-controller-plan.json")); + assert!( + !seed.contains("artifact.tar.b64"), + "kubernetes controller bootstrap should seed ordinary files instead of a tarball" + ); + assert!( + deployment_raw.contains("cp -R /amber/seed/artifact/. /amber/site/artifact"), + "kubernetes controller bootstrap should copy seeded artifact files directly" + ); + assert!( + deployment_raw.contains("seed_if_missing"), + "kubernetes controller bootstrap should not overwrite mutable state on pod recreation" + ); + assert!( + deployment_raw.contains("persistentVolumeClaim"), + "kubernetes controller state should be backed by a PVC" + ); + assert!(seed.contains(&format!("http://amber-site-controller:{controller_port}"))); + assert!(deployment_raw.contains("EXISTING_ENV")); + + let router_netpol = + fs::read_to_string(artifact_root.join("05-networkpolicies/amber-router-netpol.yaml")) + .expect("router netpol should read"); + assert!(router_netpol.contains("amber-site-controller")); + assert!(router_netpol.contains("24000")); + assert!(router_netpol.contains("24100")); + assert!(router_netpol.contains("37046")); + let router_service = fs::read_to_string(artifact_root.join("04-services/amber-router.yaml")) + .expect("router service should read"); + assert!(router_service.contains("controller-route-37046")); + assert!(router_service.contains("port: 37046")); + let controller_netpol = fs::read_to_string( + artifact_root.join("05-networkpolicies/amber-site-controller-netpol.yaml"), + ) + .expect("site controller netpol should read"); + assert!( + controller_netpol.contains("24000"), + "site controller needs router mesh egress for in-cluster router readiness probes" + ); + assert!(controller_netpol.contains("24100")); + assert!(controller_netpol.contains("37046")); +} + +#[test] +fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { + let temp = TempDir::new().expect("tempdir should create"); + let state_root = temp.path().join("state"); + let site_state_root = state_root.join("compose-site"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([( + "compose-site".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + )]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["compose-site".to_string()], + dynamic_enabled_sites: vec!["compose-site".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + controller_identity: None, + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + let state_path = site_state_root.join("site-controller-state.json"); + write_json(&state_path, &state).expect("state should write"); + let mut app = test_control_state_app(&temp, state, state_path); + fs::remove_file(site_state_path(&temp.path().join("state"), "compose-site")) + .expect("host manager state should be removed for fallback test"); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.kind = SiteKind::Compose; + controller_plan.compose_project = Some("amber_test_compose".to_string()); + controller_plan.local_router_control = + Some("unix:///amber/control/router-control.sock".to_string()); + controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); + controller_plan.authority_url = "http://amber-site-controller:32111".to_string(); + + let state = load_site_manager_state(&app, "compose-site") + .expect("local controller should synthesize site metadata from its own plan"); + assert_eq!(state.status, "running"); + assert_eq!( + state.router_control.as_deref(), + Some("unix:///amber/control/router-control.sock") + ); + assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); + let authority_url = "http://amber-site-controller:32111".to_string(); + assert_eq!( + state.site_controller_url.as_deref(), + Some(authority_url.as_str()) + ); +} + +#[test] +fn site_execution_site_routes_vm_sites_to_their_direct_controller() { + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["direct_local".to_string(), "vm_local".to_string()], + dynamic_enabled_sites: vec!["direct_local".to_string(), "vm_local".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + controller_identity: None, + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + + assert_eq!( + site_execution_site_from_state(&state, "direct_local").expect("direct site execution"), + "direct_local" + ); + assert_eq!( + site_execution_site_from_state(&state, "vm_local").expect("vm site execution"), + "direct_local" + ); +} + +#[test] +fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() { + let temp = TempDir::new().expect("temp dir"); + let state_root = temp.path().join("state"); + let site_state_root = state_root.join("compose-site"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([( + "compose-site".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + )]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["compose-site".to_string()], + dynamic_enabled_sites: vec!["compose-site".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + controller_identity: None, + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + let state_path = site_state_root.join("site-controller-state.json"); + write_json(&state_path, &state).expect("state should write"); + let mut app = test_control_state_app(&temp, state, state_path); + fs::write( + site_state_path(&state_root, "compose-site"), + serde_json::to_vec(&serde_json::json!({ + "status": "stopped", + "kind": "compose", + "artifact_dir": "/tmp/stale-artifact", + "supervisor_pid": 1, + "compose_project": "stale-project", + "router_control": "unix:///tmp/stale-router.sock", + "router_mesh_addr": "127.0.0.1:1", + "site_controller_url": "http://stale-controller" + })) + .expect("stale manager state should serialize"), + ) + .expect("stale manager state should write"); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.kind = SiteKind::Compose; + controller_plan.compose_project = Some("amber_test_compose".to_string()); + controller_plan.local_router_control = + Some("unix:///amber/control/router-control.sock".to_string()); + controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); + controller_plan.authority_url = "http://amber-site-controller:32111".to_string(); + + let state = load_site_manager_state(&app, "compose-site") + .expect("local controller should not trust stale host manager state"); + assert_eq!(state.status, "running"); + assert_eq!( + state.router_control.as_deref(), + Some("unix:///amber/control/router-control.sock") + ); + assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); + let authority_url = "http://amber-site-controller:32111".to_string(); + assert_eq!( + state.site_controller_url.as_deref(), + Some(authority_url.as_str()) + ); +}