diff --git a/.dockerignore b/.dockerignore index b39af6cd0..aa5a9e733 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,8 @@ !application/single_app/** !application/v2_ui/ !application/v2_ui/** +!application/map_server/ +!application/map_server/** !docker-customization/ !docker-customization/** !LICENSE diff --git a/application/map_server/Dockerfile b/application/map_server/Dockerfile new file mode 100644 index 000000000..b1afd8366 --- /dev/null +++ b/application/map_server/Dockerfile @@ -0,0 +1,59 @@ +# Dockerfile (map server) +# Build from the repository root: docker build -f application/map_server/Dockerfile . +ARG UID=65532 +ARG GID=65532 + +FROM mcr.microsoft.com/azurelinux/base/python:3.12 AS builder + +ARG UID +ARG GID + +COPY docker-customization/pip.conf /etc/pip.conf +COPY docker-customization/custom-ca-certificates/ /etc/pki/ca-trust/source/anchors +RUN update-ca-trust enable \ + && update-ca-trust extract + +ENV PYTHONUNBUFFERED=1 + +RUN set -eux; \ + echo "nonroot:x:${GID}:" >> /etc/group; \ + echo "nonroot:x:${UID}:${GID}:nonroot:/home/nonroot:/bin/bash" >> /etc/passwd; \ + mkdir -p /home/nonroot /app; \ + chown ${UID}:${GID} /home/nonroot /app; \ + chmod 744 /app + +WORKDIR /app +COPY application/map_server/requirements.txt /tmp/requirements.txt +RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt + +FROM mcr.microsoft.com/azurelinux/distroless/python:3.12 + +ARG UID +ARG GID + +COPY --from=builder /etc/pki /etc/pki +COPY --from=builder /etc/ssl/certs /etc/ssl/certs +COPY --from=builder /home/nonroot /home/nonroot +COPY --from=builder /etc/passwd /etc/passwd +COPY --from=builder /etc/group /etc/group +COPY --from=builder /usr/lib/python3.12 /usr/lib/python3.12 + +USER ${UID}:${GID} + +COPY --from=builder --chown=${UID}:${GID} /app /app + +ENV HOME=/home/nonroot \ + PYTHONIOENCODING=utf-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt \ + SSL_CERT_DIR=/etc/ssl/certs \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt + +WORKDIR /app +COPY --chown=${UID}:${GID} application/map_server/mapserver ./mapserver + +EXPOSE 8080 +ENTRYPOINT ["python3", "-m", "uvicorn", "mapserver.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--no-server-header"] diff --git a/application/map_server/deploy/Deploy-MapServer.ps1 b/application/map_server/deploy/Deploy-MapServer.ps1 new file mode 100644 index 000000000..a6c5e64f3 --- /dev/null +++ b/application/map_server/deploy/Deploy-MapServer.ps1 @@ -0,0 +1,270 @@ +# Deploy-MapServer.ps1 +#Requires -Version 7.0 +<# +.SYNOPSIS + Builds and deploys the SimpleChat map server to Azure Container Apps for a proof of concept. + +.DESCRIPTION + Uses the signed-in Azure CLI account. Safe to run again: existing resources are reused. + 1. Builds the image in Azure Container Registry from a staged context (only the map server and docker-customization). + 2. Creates the Cosmos DB database "mapserver" with containers "maps" (/map_id) and "map_links" (/scope_key). + 3. Creates a user-assigned managed identity and grants it AcrPull, Cosmos DB data access and, when a Maps + account is given, Azure Maps Data Reader. + 4. Registers the map server API with the MapServer.ActOnBehalf app role and assigns that role to SimpleChat's + managed identity. If the role can't be assigned, SimpleChat's identity is allowlisted by object ID instead. + 5. Creates or updates the container app and prints the values SimpleChat needs. + +.EXAMPLE + ./Deploy-MapServer.ps1 -ResourceGroup my-rg -RegistryName myacr -ContainerAppsEnvironment my-env ` + -CosmosAccountName my-cosmos -SimpleChatPrincipalId 00000000-0000-0000-0000-000000000000 -MapsAccountName my-maps +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $RegistryName, + [Parameter(Mandatory)] [string] $ContainerAppsEnvironment, + [Parameter(Mandatory)] [string] $CosmosAccountName, + [Parameter(Mandatory)] [ValidatePattern('^[0-9a-fA-F-]{36}$')] [string] $SimpleChatPrincipalId, + [string] $RegistryResourceGroup = $ResourceGroup, + [string] $CosmosResourceGroup = $ResourceGroup, + [string] $MapsAccountName = '', + [string] $MapsResourceGroup = $ResourceGroup, + [ValidatePattern('^[a-z][a-z0-9-]{1,30}[a-z0-9]$')] [string] $ContainerAppName = 'simplechat-map-server', + [string] $ImageTag = (Get-Date -Format 'yyyyMMdd-HHmmss'), + [string] $ApiAppId = '', + [ValidateSet('external', 'internal')] [string] $Ingress = 'external', + [switch] $SkipBuild, + [string] $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path +) + +$ErrorActionPreference = 'Stop' +$RoleValue = 'MapServer.ActOnBehalf' +$CosmosDataContributorRole = '00000000-0000-0000-0000-000000000002' +$ImageName = 'simplechat-map-server' +$AzErrorFile = New-TemporaryFile + +function Invoke-AzCommand { + param([Parameter(Mandatory)] [string[]] $Arguments, [switch] $AllowFailure) + $output = & az @Arguments 2>$AzErrorFile.FullName + if ($LASTEXITCODE -ne 0) { + if ($AllowFailure) { return $null } + $details = (Get-Content -Raw -Path $AzErrorFile.FullName -ErrorAction SilentlyContinue) + throw "az $($Arguments[0]) $($Arguments[1]) failed: $details" + } + return (($output | Out-String).Trim()) +} + +function Invoke-AzJson { + param([Parameter(Mandatory)] [string[]] $Arguments, [switch] $AllowFailure) + # az is a .cmd on Windows, so JMESPath filters with parentheses get mangled; filter JSON in PowerShell instead. + $raw = Invoke-AzCommand -Arguments ($Arguments + @('-o', 'json')) -AllowFailure:$AllowFailure + if (-not $raw) { return $null } + return ($raw | ConvertFrom-Json) +} + +function Write-Step { + param([string] $Message) + Write-Host "==> $Message" -ForegroundColor Cyan +} + +function Invoke-ImageBuild { + param([string] $Image, [string] $SourceRoot) + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "map-server-build-$([guid]::NewGuid().ToString('N'))" + try { + New-Item -ItemType Directory -Path (Join-Path $staging 'application') | Out-Null + Copy-Item -Recurse -Path (Join-Path $SourceRoot 'application/map_server') -Destination (Join-Path $staging 'application/map_server') + Copy-Item -Recurse -Path (Join-Path $SourceRoot 'docker-customization') -Destination (Join-Path $staging 'docker-customization') + Get-ChildItem -Path $staging -Recurse -Directory -Filter '__pycache__' | Remove-Item -Recurse -Force + + $queued = & az acr build --registry $RegistryName --resource-group $RegistryResourceGroup --image $Image ` + --file (Join-Path $staging 'application/map_server/Dockerfile') --no-wait $staging 2>&1 | Out-String + if ($queued -notmatch 'Queued a build with ID:\s*(\S+)') { + throw "The image build wasn't queued: $queued" + } + $runId = $Matches[1] + Write-Host " build $runId queued" + + $deadline = (Get-Date).AddMinutes(30) + $unreadable = 0 + while ($true) { + Start-Sleep -Seconds 10 + $status = Invoke-AzCommand -AllowFailure -Arguments @( + 'acr', 'task', 'show-run', '--registry', $RegistryName, '--resource-group', $RegistryResourceGroup, + '--run-id', $runId, '--query', 'status', '-o', 'tsv') + if (-not $status) { + $unreadable++ + if ($unreadable -gt 15) { throw "Couldn't read the status of build $runId." } + } + elseif ($status -eq 'Succeeded') { break } + elseif ($status -in @('Failed', 'Canceled', 'Error', 'Timeout')) { + throw "Build $runId ended with status $status. See: az acr task logs --registry $RegistryName --run-id $runId" + } + if ((Get-Date) -gt $deadline) { throw "Build $runId didn't finish within 30 minutes." } + } + } + finally { + Remove-Item -Recurse -Force -Path $staging -ErrorAction SilentlyContinue + } +} + +function Grant-AzureRole { + param([string] $PrincipalId, [string] $Role, [string] $Scope) + # Match by object ID: --assignee looks the principal up in Entra ID, which lags for a new identity. + $existing = @(Invoke-AzJson -Arguments @('role', 'assignment', 'list', '--role', $Role, '--scope', $Scope) | + Where-Object { $_.principalId -eq $PrincipalId }) + if ($existing.Count -gt 0) { return $false } + Invoke-AzCommand -Arguments @( + 'role', 'assignment', 'create', '--assignee-object-id', $PrincipalId, '--assignee-principal-type', 'ServicePrincipal', + '--role', $Role, '--scope', $Scope) | Out-Null + return $true +} + +try { + Write-Step 'Checking the Azure CLI sign-in' + $tenantId = Invoke-AzCommand -Arguments @('account', 'show', '--query', 'tenantId', '-o', 'tsv') + if (-not (Invoke-AzCommand -AllowFailure -Arguments @('extension', 'show', '--name', 'containerapp', '--query', 'name', '-o', 'tsv'))) { + Invoke-AzCommand -Arguments @('extension', 'add', '--name', 'containerapp', '--only-show-errors') | Out-Null + } + $loginServer = Invoke-AzCommand -Arguments @('acr', 'show', '-n', $RegistryName, '-g', $RegistryResourceGroup, '--query', 'loginServer', '-o', 'tsv') + $registryId = Invoke-AzCommand -Arguments @('acr', 'show', '-n', $RegistryName, '-g', $RegistryResourceGroup, '--query', 'id', '-o', 'tsv') + $image = "$ImageName`:$ImageTag" + + if ($SkipBuild) { + Write-Step "Skipping the image build; using $loginServer/$image" + } + else { + Write-Step "Building $image in $RegistryName" + Invoke-ImageBuild -Image $image -SourceRoot $RepoRoot + } + + Write-Step 'Preparing Cosmos DB' + $cosmosAccount = Invoke-AzJson -Arguments @('cosmosdb', 'show', '-n', $CosmosAccountName, '-g', $CosmosResourceGroup) + $cosmosEndpoint = $cosmosAccount.documentEndpoint + $serverless = @($cosmosAccount.capabilities | Where-Object { $_.name -eq 'EnableServerless' }).Count -gt 0 + if (-not (Invoke-AzCommand -AllowFailure -Arguments @('cosmosdb', 'sql', 'database', 'show', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup, '-n', 'mapserver', '--query', 'name', '-o', 'tsv'))) { + Invoke-AzCommand -Arguments @('cosmosdb', 'sql', 'database', 'create', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup, '-n', 'mapserver') | Out-Null + } + foreach ($container in @(@{ Name = 'maps'; Key = '/map_id' }, @{ Name = 'map_links'; Key = '/scope_key' })) { + $exists = Invoke-AzCommand -AllowFailure -Arguments @( + 'cosmosdb', 'sql', 'container', 'show', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup, '-d', 'mapserver', + '-n', $container.Name, '--query', 'name', '-o', 'tsv') + if (-not $exists) { + $arguments = @('cosmosdb', 'sql', 'container', 'create', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup, + '-d', 'mapserver', '-n', $container.Name, '-p', $container.Key) + if (-not $serverless) { $arguments += @('--max-throughput', '1000') } + Invoke-AzCommand -Arguments $arguments | Out-Null + } + } + + Write-Step 'Preparing the managed identity' + $identityName = "$ContainerAppName-id" + $identityJson = Invoke-AzCommand -AllowFailure -Arguments @('identity', 'show', '-n', $identityName, '-g', $ResourceGroup, '-o', 'json') + if (-not $identityJson) { + $identityJson = Invoke-AzCommand -Arguments @('identity', 'create', '-n', $identityName, '-g', $ResourceGroup, '-o', 'json') + } + $identity = $identityJson | ConvertFrom-Json + $rolesGranted = Grant-AzureRole -PrincipalId $identity.principalId -Role 'AcrPull' -Scope $registryId + + $cosmosAssignments = @(Invoke-AzJson -Arguments @('cosmosdb', 'sql', 'role', 'assignment', 'list', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup)) + $cosmosAssigned = @($cosmosAssignments | Where-Object { + $_.principalId -eq $identity.principalId -and $_.roleDefinitionId.EndsWith($CosmosDataContributorRole) + }).Count -gt 0 + if (-not $cosmosAssigned) { + Invoke-AzCommand -Arguments @( + 'cosmosdb', 'sql', 'role', 'assignment', 'create', '-a', $CosmosAccountName, '-g', $CosmosResourceGroup, + '--role-definition-id', $CosmosDataContributorRole, '--principal-id', $identity.principalId, '--scope', '/dbs/mapserver') | Out-Null + } + + $mapsClientId = '' + if ($MapsAccountName) { + Write-Step 'Granting Azure Maps access' + $mapsAccount = Invoke-AzCommand -Arguments @('maps', 'account', 'show', '-n', $MapsAccountName, '-g', $MapsResourceGroup, '-o', 'json') | ConvertFrom-Json + $mapsClientId = $mapsAccount.properties.uniqueId + $rolesGranted = (Grant-AzureRole -PrincipalId $identity.principalId -Role 'Azure Maps Data Reader' -Scope $mapsAccount.id) -or $rolesGranted + } + else { + Write-Host ' no Maps account given; the map viewer will show a black background' + } + + Write-Step 'Registering the map server API' + if (-not $ApiAppId) { + $roleFile = New-TemporaryFile + @(@{ + allowedMemberTypes = @('Application') + description = 'Act on behalf of people when reading and writing maps.' + displayName = 'Act on behalf of people' + id = [guid]::NewGuid().ToString() + isEnabled = $true + value = $RoleValue + }) | ConvertTo-Json -Depth 4 -AsArray | Set-Content -Path $roleFile.FullName -Encoding utf8 + $ApiAppId = Invoke-AzCommand -Arguments @( + 'ad', 'app', 'create', '--display-name', "SimpleChat map server ($ContainerAppName)", '--sign-in-audience', 'AzureADMyOrg', + '--app-roles', "@$($roleFile.FullName)", '--query', 'appId', '-o', 'tsv') + Remove-Item -Path $roleFile.FullName -Force + Invoke-AzCommand -Arguments @('ad', 'app', 'update', '--id', $ApiAppId, '--identifier-uris', "api://$ApiAppId") | Out-Null + } + if (-not (Invoke-AzCommand -AllowFailure -Arguments @('ad', 'sp', 'show', '--id', $ApiAppId, '--query', 'id', '-o', 'tsv'))) { + Invoke-AzCommand -Arguments @('ad', 'sp', 'create', '--id', $ApiAppId) | Out-Null + } + $apiPrincipal = Invoke-AzJson -Arguments @('ad', 'sp', 'show', '--id', $ApiAppId) + $apiPrincipalId = $apiPrincipal.id + $roleId = @($apiPrincipal.appRoles | Where-Object { $_.value -eq $RoleValue })[0].id + if (-not $roleId) { throw "The map server API has no $RoleValue app role." } + + $assignmentsUri = "https://graph.microsoft.com/v1.0/servicePrincipals/$SimpleChatPrincipalId/appRoleAssignments" + $existingGrants = Invoke-AzJson -AllowFailure -Arguments @('rest', '--method', 'GET', '--uri', $assignmentsUri) + $assigned = if (@($existingGrants.value | Where-Object { $_.resourceId -eq $apiPrincipalId -and $_.appRoleId -eq $roleId }).Count -gt 0) { '1' } else { '0' } + if ($assigned -ne '1') { + $bodyFile = New-TemporaryFile + @{ principalId = $SimpleChatPrincipalId; resourceId = $apiPrincipalId; appRoleId = $roleId } | + ConvertTo-Json | Set-Content -Path $bodyFile.FullName -Encoding utf8 + $granted = Invoke-AzCommand -AllowFailure -Arguments @( + 'rest', '--method', 'POST', '--uri', $assignmentsUri, '--headers', 'Content-Type=application/json', '--body', "@$($bodyFile.FullName)") + Remove-Item -Path $bodyFile.FullName -Force + $assigned = if ($null -ne $granted) { '1' } else { '0' } + } + $allowedCallers = '' + if ($assigned -ne '1') { + Write-Warning "Couldn't assign $RoleValue to SimpleChat's identity. Allowlisting its object ID instead." + $allowedCallers = $SimpleChatPrincipalId + } + + Write-Step "Deploying the container app ($Ingress ingress)" + $envVars = @( + "MAP_SERVER_TENANT_ID=$tenantId", + "MAP_SERVER_AUDIENCES=api://$ApiAppId,$ApiAppId", + "MAP_SERVER_REQUIRED_ROLE=$RoleValue", + "MAP_SERVER_STORE=cosmos", + "MAP_SERVER_COSMOS_ENDPOINT=$cosmosEndpoint", + "AZURE_CLIENT_ID=$($identity.clientId)" + ) + if ($allowedCallers) { $envVars += "MAP_SERVER_ALLOWED_CALLER_IDS=$allowedCallers" } + if ($mapsClientId) { $envVars += "AZURE_MAPS_CLIENT_ID=$mapsClientId" } + + $appExists = Invoke-AzCommand -AllowFailure -Arguments @('containerapp', 'show', '-n', $ContainerAppName, '-g', $ResourceGroup, '--query', 'name', '-o', 'tsv') + if ($rolesGranted -and -not $appExists) { + Write-Host ' waiting 60 seconds for the new role assignments to take effect' + Start-Sleep -Seconds 60 + } + if ($appExists) { + Invoke-AzCommand -Arguments (@('containerapp', 'update', '-n', $ContainerAppName, '-g', $ResourceGroup, + '--image', "$loginServer/$image", '--set-env-vars') + $envVars) | Out-Null + } + else { + Invoke-AzCommand -Arguments (@('containerapp', 'create', '-n', $ContainerAppName, '-g', $ResourceGroup, + '--environment', $ContainerAppsEnvironment, '--image', "$loginServer/$image", + '--registry-server', $loginServer, '--registry-identity', $identity.id, '--user-assigned', $identity.id, + '--target-port', '8080', '--ingress', $Ingress, '--min-replicas', '1', '--max-replicas', '2', + '--cpu', '0.5', '--memory', '1.0Gi', '--env-vars') + $envVars) | Out-Null + } + $fqdn = Invoke-AzCommand -Arguments @('containerapp', 'show', '-n', $ContainerAppName, '-g', $ResourceGroup, '--query', 'properties.configuration.ingress.fqdn', '-o', 'tsv') + + Write-Step 'Done' + Write-Host " Map server URL: https://$fqdn" + Write-Host " Token audience: api://$ApiAppId" + Write-Host " Caller check: $(if ($allowedCallers) { 'object ID allowlist' } else { "app role $RoleValue" })" + Write-Host " Health check: https://$fqdn/healthz" +} +finally { + Remove-Item -Path $AzErrorFile.FullName -Force -ErrorAction SilentlyContinue +} diff --git a/application/map_server/mapserver/__init__.py b/application/map_server/mapserver/__init__.py new file mode 100644 index 000000000..27f957dbd --- /dev/null +++ b/application/map_server/mapserver/__init__.py @@ -0,0 +1,4 @@ +# __init__.py +"""SimpleChat map server: persistent, phase-tagged maps that agents build up over a conversation.""" + +__version__ = "0.1.0" diff --git a/application/map_server/mapserver/__main__.py b/application/map_server/mapserver/__main__.py new file mode 100644 index 000000000..f4ff1e887 --- /dev/null +++ b/application/map_server/mapserver/__main__.py @@ -0,0 +1,20 @@ +# __main__.py +"""Run the map server locally: python -m mapserver (from application/map_server).""" + +import os + +import uvicorn + + +def main() -> None: + uvicorn.run( + "mapserver.app:create_app", + factory=True, + host=os.environ.get("MAP_SERVER_HOST", "127.0.0.1"), + port=int(os.environ.get("MAP_SERVER_PORT", "8080")), + server_header=False, + ) + + +if __name__ == "__main__": + main() diff --git a/application/map_server/mapserver/app.py b/application/map_server/mapserver/app.py new file mode 100644 index 000000000..0c8102473 --- /dev/null +++ b/application/map_server/mapserver/app.py @@ -0,0 +1,355 @@ +# app.py +"""HTTP API for the map server. Every route but the health check needs a caller holding the map server role.""" + +import asyncio +import json +import logging +import sys +import time +from contextlib import asynccontextmanager +from typing import Any, AsyncIterator, Dict, Optional + +from azure.identity.aio import DefaultAzureCredential +from fastapi import Depends, FastAPI, Header, Query, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.datastructures import MutableHeaders + +from . import __version__ +from .auth import ACTOR_HEADER, Actor, Caller, TokenValidator, parse_actor +from .cosmos_store import CosmosMapStore +from .engine import MapEngine +from .errors import MapServerError +from .models import ( + AddFeaturesRequest, + CreateMapRequest, + LinkConversationRequest, + RetractFeatureRequest, + StartPhaseRequest, + UpdateFeatureRequest, + UpdateMapRequest, + UpdatePhaseRequest, +) +from .settings import STORE_COSMOS, STORE_MEMORY, Settings +from .store import InMemoryMapStore +from .tiles import TileService + +LOGGER = logging.getLogger("mapserver") + +MAX_BODY_BYTES = 1_048_576 +MAX_VALIDATION_DETAILS = 20 +EVENT_RETRY_MILLISECONDS = 5000 + + +def _configure_logging() -> None: + logger = logging.getLogger("mapserver") + if not logger.handlers: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False + + +class RequestGuard: + """Caps request bodies and adds security headers, without buffering streamed responses.""" + + def __init__(self, app: Any): + self.app = app + + async def __call__(self, scope: Dict[str, Any], receive: Any, send: Any) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + length = dict(scope.get("headers") or []).get(b"content-length", b"") + if length.isdigit() and int(length) > MAX_BODY_BYTES: + response = JSONResponse( + status_code=413, content={"error": "request_too_large", "message": "The request body is too large."}, + ) + await response(scope, receive, send) + return + + async def send_with_headers(message: Dict[str, Any]) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(scope=message) + headers.setdefault("x-content-type-options", "nosniff") + if "cache-control" not in headers: + headers["cache-control"] = "no-store" + await send(message) + + await self.app(scope, receive, send_with_headers) + + +def _sse(event: str, data: Dict[str, Any], event_id: str = "") -> str: + lines = [f"id: {event_id}"] if event_id else [] + lines.append(f"event: {event}") + lines.append(f"data: {json.dumps(data, separators=(',', ':'), ensure_ascii=False)}") + return "\n".join(lines) + "\n\n" + + +async def _event_stream( + request: Request, engine: MapEngine, actor: Actor, map_id: str, start_version: int, settings: Settings +) -> AsyncIterator[str]: + """One event per map change. Access is checked again on every poll.""" + last_version = start_version + started = last_sent = time.monotonic() + yield f"retry: {EVENT_RETRY_MILLISECONDS}\n\n" + while True: + try: + if await engine.current_version(actor, map_id) > last_version: + page = await engine.list_changes(actor, map_id, since_version=last_version, limit=100) + for change in page["items"]: + yield _sse("change", change, event_id=str(change["version"])) + last_version = change["version"] + last_sent = time.monotonic() + except MapServerError as exc: + yield _sse("error", {"error": exc.code, "message": exc.message}) + return + now = time.monotonic() + if now - started >= settings.events_max_seconds: + yield _sse("end", {"version": last_version}) + return + if now - last_sent >= settings.events_keepalive_seconds: + yield ": keepalive\n\n" + last_sent = now + if await request.is_disconnected(): + return + await asyncio.sleep(settings.events_poll_seconds) + + +def create_app( + settings: Optional[Settings] = None, + *, + store: Optional[Any] = None, + token_validator: Optional[TokenValidator] = None, + tile_service: Optional[TileService] = None, + credential: Optional[Any] = None, +) -> FastAPI: + settings = settings or Settings.from_env() + _configure_logging() + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + owned_credential = None + app_credential = credential + needs_credential = (settings.store == STORE_COSMOS and not settings.cosmos_key) or ( + settings.azure_maps_client_id and not settings.azure_maps_key + ) + if app_credential is None and needs_credential: + app_credential = DefaultAzureCredential(managed_identity_client_id=settings.managed_identity_client_id or None) + owned_credential = app_credential + + map_store = store + if map_store is None: + map_store = InMemoryMapStore() if settings.store == STORE_MEMORY else CosmosMapStore.from_settings(settings, app_credential) + tiles = tile_service or TileService(settings, credential=app_credential) + app.state.settings = settings + app.state.engine = MapEngine(map_store, settings) + app.state.tiles = tiles + app.state.validator = token_validator or TokenValidator(settings) + if settings.local_dev_key: + LOGGER.warning("[MAP_SERVER] Running with the local developer key and the in-memory store. Don't expose this server.") + LOGGER.info(f"[MAP_SERVER] Map server {__version__} started with the {settings.store} store.") + try: + yield + finally: + if store is None: + await map_store.close() + if tile_service is None: + await tiles.close() + if owned_credential is not None: + await owned_credential.close() + + app = FastAPI( + title="SimpleChat map server", + version=__version__, + docs_url=None, + redoc_url=None, + swagger_ui_oauth2_redirect_url=None, + lifespan=lifespan, + ) + app.add_middleware(RequestGuard) + + @app.exception_handler(MapServerError) + async def handle_map_error(request: Request, exc: MapServerError) -> JSONResponse: + body: Dict[str, Any] = {"error": exc.code, "message": exc.message} + if exc.details: + body["details"] = exc.details + return JSONResponse(status_code=exc.status_code, content=body) + + @app.exception_handler(RequestValidationError) + async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: + details = [ + {"location": ".".join(str(part) for part in error.get("loc", ())), "problem": str(error.get("msg", ""))} + for error in exc.errors()[:MAX_VALIDATION_DETAILS] + ] + return JSONResponse( + status_code=400, content={"error": "invalid_request", "message": "The request isn't valid.", "details": details}, + ) + + @app.exception_handler(Exception) + async def handle_unexpected(request: Request, exc: Exception) -> JSONResponse: + LOGGER.error(f"[MAP_SERVER] Unexpected error ({type(exc).__name__}).") + return JSONResponse(status_code=500, content={"error": "internal_error", "message": "Unexpected error."}) + + def get_caller(request: Request) -> Caller: + return request.app.state.validator.validate(request.headers.get("authorization")) + + def get_actor(request: Request, caller: Caller = Depends(get_caller)) -> Actor: + return parse_actor(request.headers.get(ACTOR_HEADER)) + + def get_engine(request: Request) -> MapEngine: + return request.app.state.engine + + @app.get("/healthz", include_in_schema=False) + async def health() -> Dict[str, str]: + return {"status": "ok", "version": __version__} + + @app.post("/v1/maps", status_code=201) + async def create_map(body: CreateMapRequest, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine)): + return await engine.create_map(actor, body) + + @app.get("/v1/maps") + async def list_maps( + conversation_id: str = Query(default="", max_length=128), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.list_maps(actor, conversation_id) + + @app.get("/v1/maps/{map_id}") + async def get_map(map_id: str, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine)): + return await engine.get_map(actor, map_id) + + @app.patch("/v1/maps/{map_id}") + async def update_map( + map_id: str, + body: UpdateMapRequest, + if_match: str = Header(default="", alias="If-Match", max_length=128), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.update_map(actor, map_id, body, if_match) + + @app.post("/v1/maps/{map_id}/links") + async def link_conversation( + map_id: str, body: LinkConversationRequest, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine), + ): + return await engine.link_conversation(actor, map_id, body.conversation_id) + + @app.delete("/v1/maps/{map_id}/links/{conversation_id}") + async def unlink_conversation( + map_id: str, conversation_id: str, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine), + ): + return await engine.unlink_conversation(actor, map_id, conversation_id) + + @app.post("/v1/maps/{map_id}/phases", status_code=201) + async def start_phase( + map_id: str, body: StartPhaseRequest, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine), + ): + return await engine.start_phase(actor, map_id, body) + + @app.patch("/v1/maps/{map_id}/phases/{phase_id}") + async def update_phase( + map_id: str, + phase_id: str, + body: UpdatePhaseRequest, + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.update_phase(actor, map_id, phase_id, body) + + @app.post("/v1/maps/{map_id}/features:batch") + async def add_features( + map_id: str, body: AddFeaturesRequest, actor: Actor = Depends(get_actor), engine: MapEngine = Depends(get_engine), + ): + return await engine.add_features(actor, map_id, body) + + @app.patch("/v1/maps/{map_id}/features/{feature_id}") + async def update_feature( + map_id: str, + feature_id: str, + body: UpdateFeatureRequest, + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.update_feature(actor, map_id, feature_id, body) + + @app.post("/v1/maps/{map_id}/features/{feature_id}:retract") + async def retract_feature( + map_id: str, + feature_id: str, + body: RetractFeatureRequest, + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.retract_feature(actor, map_id, feature_id, body) + + @app.get("/v1/maps/{map_id}/features") + async def list_features( + map_id: str, + phase_id: str = Query(default="", max_length=8), + category: str = Query(default="", max_length=40), + kind: str = Query(default="", max_length=8), + status: str = Query(default="active", max_length=10), + since_version: Optional[int] = Query(default=None, ge=0), + bbox: str = Query(default="", max_length=128), + limit: int = Query(default=200, ge=1, le=500), + cursor: str = Query(default="", max_length=16), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.list_features( + actor, map_id, phase_id=phase_id, category=category, kind=kind, status=status, + since_version=since_version, bbox=bbox, limit=limit, cursor=cursor, + ) + + @app.get("/v1/maps/{map_id}/snapshot") + async def snapshot( + map_id: str, + as_of_version: Optional[int] = Query(default=None, ge=1), + phases: str = Query(default="", max_length=400), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + phase_ids = [phase.strip() for phase in phases.split(",") if phase.strip()] + return await engine.snapshot(actor, map_id, as_of_version=as_of_version, phase_ids=phase_ids) + + @app.get("/v1/maps/{map_id}/changes") + async def list_changes( + map_id: str, + since_version: int = Query(default=0, ge=0), + limit: int = Query(default=100, ge=1, le=500), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + return await engine.list_changes(actor, map_id, since_version=since_version, limit=limit) + + @app.get("/v1/maps/{map_id}/events") + async def map_events( + map_id: str, + request: Request, + since_version: Optional[int] = Query(default=None, ge=0), + last_event_id: str = Header(default="", alias="Last-Event-ID", max_length=16), + actor: Actor = Depends(get_actor), + engine: MapEngine = Depends(get_engine), + ): + current = await engine.current_version(actor, map_id) + if last_event_id.isdigit(): + start = int(last_event_id) + elif since_version is not None: + start = since_version + else: + start = current + return StreamingResponse( + _event_stream(request, engine, actor, map_id, min(start, current), settings), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @app.get("/v1/tiles/{tileset}/{z}/{x}/{y}") + async def tile(tileset: str, z: int, x: int, y: int, request: Request, caller: Caller = Depends(get_caller)): + content, media_type = await request.app.state.tiles.get_tile(tileset, z, x, y) + return Response(content=content, media_type=media_type, headers={"Cache-Control": "private, max-age=3600"}) + + return app diff --git a/application/map_server/mapserver/auth.py b/application/map_server/mapserver/auth.py new file mode 100644 index 000000000..1314ff561 --- /dev/null +++ b/application/map_server/mapserver/auth.py @@ -0,0 +1,142 @@ +# auth.py +"""Who is calling: a trusted caller's Entra token, and the person that caller acts for.""" + +import base64 +import binascii +import hmac +import json +import logging +from dataclasses import dataclass +from typing import Any, Dict, FrozenSet, Literal, Optional + +import jwt +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from .errors import MapServerError, forbidden, unauthorized +from .settings import Settings + +LOGGER = logging.getLogger("mapserver.auth") + +ACTOR_HEADER = "X-Map-Actor" +MAX_ACTOR_HEADER_LENGTH = 4096 +TOKEN_ALGORITHMS = ["RS256"] +TOKEN_LEEWAY_SECONDS = 60 +LOCAL_DEV_CALLER_ID = "local-dev" +ID_PATTERN = r"^[A-Za-z0-9._:@-]{1,128}$" +OPTIONAL_ID_PATTERN = r"^[A-Za-z0-9._:@-]{0,128}$" +SCOPE_PATTERN = r"^(user|group):[A-Za-z0-9._-]{1,128}$" + + +@dataclass(frozen=True) +class Caller: + """The application that presented the token, such as SimpleChat's managed identity.""" + + object_id: str + app_id: str + roles: FrozenSet[str] + + +class ActorAgent(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + id: str = Field(default="", pattern=OPTIONAL_ID_PATTERN) + name: str = Field(default="", max_length=120) + + +class Actor(BaseModel): + """The person a trusted caller acts for, the scope it vouches for, and where the request came from.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + user_id: str = Field(pattern=ID_PATTERN) + display_name: str = Field(default="", max_length=120) + scope: str = Field(pattern=SCOPE_PATTERN) + access: Literal["read", "write"] + conversation_id: str = Field(default="", pattern=OPTIONAL_ID_PATTERN) + message_id: str = Field(default="", pattern=OPTIONAL_ID_PATTERN) + run_id: str = Field(default="", pattern=OPTIONAL_ID_PATTERN) + agent: Optional[ActorAgent] = None + + @property + def can_write(self) -> bool: + return self.access == "write" + + def summary(self) -> Dict[str, Any]: + """What the map records about who made a change.""" + summary: Dict[str, Any] = {"user_id": self.user_id} + for name in ("display_name", "conversation_id", "message_id", "run_id"): + value = getattr(self, name) + if value: + summary[name] = value + if self.agent and (self.agent.id or self.agent.name): + summary["agent"] = {key: value for key, value in self.agent.model_dump().items() if value} + return summary + + +def encode_actor(actor: Dict[str, Any]) -> str: + """Encode an actor context for the X-Map-Actor header.""" + raw = json.dumps(actor, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def parse_actor(raw_header: Optional[str]) -> Actor: + if not raw_header: + raise MapServerError(400, "actor_required", f"The {ACTOR_HEADER} header is required.") + if len(raw_header) > MAX_ACTOR_HEADER_LENGTH: + raise MapServerError(400, "invalid_actor", "The actor context is invalid.") + try: + padded = raw_header.strip() + "=" * (-len(raw_header.strip()) % 4) + payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + return Actor.model_validate(payload) + except (UnicodeError, binascii.Error, ValueError, ValidationError) as exc: + LOGGER.info(f"[MAP_SERVER_AUTH] Rejected an invalid actor context ({type(exc).__name__}).") + raise MapServerError(400, "invalid_actor", "The actor context is invalid.") from exc + + +def _bearer_token(authorization: Optional[str]) -> str: + scheme, _, token = str(authorization or "").partition(" ") + if scheme.lower() != "bearer" or not token.strip(): + raise unauthorized() + return token.strip() + + +class TokenValidator: + """Checks that a request carries an Entra token from a caller allowed to act for people.""" + + def __init__(self, settings: Settings, jwk_client: Optional[Any] = None): + self._settings = settings + self._jwk_client = jwk_client + if self._jwk_client is None and settings.jwks_url and not settings.local_dev_key: + self._jwk_client = jwt.PyJWKClient(settings.jwks_url, cache_keys=True, lifespan=3600, timeout=10) + + def validate(self, authorization: Optional[str]) -> Caller: + token = _bearer_token(authorization) + if self._settings.local_dev_key: + if hmac.compare_digest(token.encode("utf-8"), self._settings.local_dev_key.encode("utf-8")): + return Caller(LOCAL_DEV_CALLER_ID, LOCAL_DEV_CALLER_ID, frozenset({self._settings.required_role})) + raise unauthorized() + + try: + signing_key = self._jwk_client.get_signing_key_from_jwt(token) + claims = jwt.decode( + token, + signing_key.key, + algorithms=TOKEN_ALGORITHMS, + audience=list(self._settings.audiences), + issuer=list(self._settings.issuers), + leeway=TOKEN_LEEWAY_SECONDS, + options={"require": ["exp", "iss", "aud"]}, + ) + except jwt.PyJWTError as exc: + LOGGER.info(f"[MAP_SERVER_AUTH] Rejected a caller token ({type(exc).__name__}).") + raise unauthorized() from exc + + object_id = str(claims.get("oid") or "") + app_id = str(claims.get("azp") or claims.get("appid") or "") + roles = frozenset(role for role in claims.get("roles") or [] if isinstance(role, str)) + has_role = bool(self._settings.required_role) and self._settings.required_role in roles + if has_role or (object_id and object_id in self._settings.allowed_caller_ids): + return Caller(object_id, app_id, roles) + + LOGGER.warning(f"[MAP_SERVER_AUTH] Refused a caller without the map server role (caller {object_id or 'unknown'}).") + raise forbidden("caller_not_allowed", "This caller isn't allowed to use the map server.") diff --git a/application/map_server/mapserver/cosmos_store.py b/application/map_server/mapserver/cosmos_store.py new file mode 100644 index 000000000..48f18bce9 --- /dev/null +++ b/application/map_server/mapserver/cosmos_store.py @@ -0,0 +1,143 @@ +# cosmos_store.py +"""Cosmos DB storage: one partition per map, written with transactional batches, plus a scope and conversation index.""" + +import logging +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from azure.cosmos import exceptions as cosmos_exceptions +from azure.cosmos.aio import CosmosClient + +from .settings import Settings +from .store import MAX_BATCH_OPERATIONS, ConflictError, StoreError + +LOGGER = logging.getLogger("mapserver.store") + +SYSTEM_PROPERTIES = ("_rid", "_self", "_attachments", "_ts", "_etag", "_lsn") +CONFLICT_STATUS_CODES = (409, 412) + + +def _clean(item: Dict[str, Any]) -> Dict[str, Any]: + return {key: value for key, value in item.items() if key not in SYSTEM_PROPERTIES} + + +class CosmosMapStore: + def __init__(self, client: CosmosClient, database_name: str, maps_container_name: str, links_container_name: str): + database = client.get_database_client(database_name) + self._client = client + self._maps = database.get_container_client(maps_container_name) + self._links = database.get_container_client(links_container_name) + + @classmethod + def from_settings(cls, settings: Settings, credential: Optional[Any] = None) -> "CosmosMapStore": + client = CosmosClient(settings.cosmos_endpoint, credential=settings.cosmos_key or credential) + return cls(client, settings.cosmos_database, settings.maps_container, settings.links_container) + + async def _query(self, container: Any, query: str, parameters: List[Dict[str, Any]], partition_key: str) -> List[Dict[str, Any]]: + try: + return [ + _clean(item) + async for item in container.query_items(query=query, parameters=parameters, partition_key=partition_key) + ] + except cosmos_exceptions.CosmosHttpResponseError as exc: + raise StoreError(f"Query failed with status {exc.status_code}.") from exc + + async def read_map(self, map_id: str) -> Optional[Tuple[Dict[str, Any], str]]: + item = await self.read_item(map_id, map_id, keep_etag=True) + if item is None or item.get("type") != "map": + return None + etag = str(item.get("_etag") or "") + return _clean(item), etag + + async def read_item(self, map_id: str, item_id: str, keep_etag: bool = False) -> Optional[Dict[str, Any]]: + try: + item = await self._maps.read_item(item=item_id, partition_key=map_id) + except cosmos_exceptions.CosmosResourceNotFoundError: + return None + except cosmos_exceptions.CosmosHttpResponseError as exc: + raise StoreError(f"Read failed with status {exc.status_code}.") from exc + return item if keep_etag else _clean(item) + + async def query_features(self, map_id: str, *, dup_keys: Optional[Sequence[str]] = None) -> List[Dict[str, Any]]: + if dup_keys is None: + return await self._query(self._maps, "SELECT * FROM c WHERE c.type = 'feature'", [], map_id) + if not dup_keys: + return [] + return await self._query( + self._maps, + "SELECT * FROM c WHERE c.type = 'feature' AND c.status = 'active' AND ARRAY_CONTAINS(@keys, c.dup_key)", + [{"name": "@keys", "value": list(dup_keys)}], + map_id, + ) + + async def query_feature_revisions( + self, map_id: str, feature_ids: Sequence[str], as_of_version: int + ) -> List[Dict[str, Any]]: + if not feature_ids: + return [] + return await self._query( + self._maps, + "SELECT * FROM c WHERE c.type = 'feature_rev' AND ARRAY_CONTAINS(@ids, c.feature_id) " + "AND c.valid_from_version <= @version AND c.valid_to_version >= @version", + [{"name": "@ids", "value": list(feature_ids)}, {"name": "@version", "value": int(as_of_version)}], + map_id, + ) + + async def query_changes(self, map_id: str, since_version: int, limit: int) -> List[Dict[str, Any]]: + # Cosmos needs TOP as a literal; limit is an int the engine already bounded. + top = max(1, min(1000, int(limit))) + return await self._query( + self._maps, + f"SELECT TOP {top} * FROM c WHERE c.type = 'change' AND c.version > @since ORDER BY c.version ASC", + [{"name": "@since", "value": int(since_version)}], + map_id, + ) + + async def commit( + self, + map_id: str, + map_doc: Dict[str, Any], + expected_etag: Optional[str], + creates: Sequence[Dict[str, Any]], + replaces: Sequence[Dict[str, Any]], + ) -> str: + operations: List[Tuple[Any, ...]] = [] + if expected_etag is None: + operations.append(("create", (map_doc,))) + else: + operations.append(("replace", (map_id, map_doc), {"if_match_etag": expected_etag})) + operations.extend(("create", (item,)) for item in creates) + operations.extend(("replace", (item["id"], item)) for item in replaces) + if len(operations) > MAX_BATCH_OPERATIONS: + raise StoreError("Too many operations for one batch.") + try: + results = await self._maps.execute_item_batch(batch_operations=operations, partition_key=map_id) + except cosmos_exceptions.CosmosBatchOperationError as exc: + if exc.status_code in CONFLICT_STATUS_CODES: + raise ConflictError() from exc + raise StoreError(f"Batch failed with status {exc.status_code}.") from exc + except cosmos_exceptions.CosmosHttpResponseError as exc: + if exc.status_code in CONFLICT_STATUS_CODES: + raise ConflictError() from exc + raise StoreError(f"Batch failed with status {exc.status_code}.") from exc + first = results[0] if results else {} + return str(first.get("eTag") or (first.get("resourceBody") or {}).get("_etag") or "") + + async def upsert_link(self, row: Dict[str, Any]) -> None: + try: + await self._links.upsert_item(body=row) + except cosmos_exceptions.CosmosHttpResponseError as exc: + raise StoreError(f"Index write failed with status {exc.status_code}.") from exc + + async def delete_link(self, scope_key: str, map_id: str) -> None: + try: + await self._links.delete_item(item=map_id, partition_key=scope_key) + except cosmos_exceptions.CosmosResourceNotFoundError: + return + except cosmos_exceptions.CosmosHttpResponseError as exc: + raise StoreError(f"Index delete failed with status {exc.status_code}.") from exc + + async def list_links(self, scope_key: str) -> List[Dict[str, Any]]: + return await self._query(self._links, "SELECT * FROM c", [], scope_key) + + async def close(self) -> None: + await self._client.close() diff --git a/application/map_server/mapserver/engine.py b/application/map_server/mapserver/engine.py new file mode 100644 index 000000000..221fb13be --- /dev/null +++ b/application/map_server/mapserver/engine.py @@ -0,0 +1,925 @@ +# engine.py +"""Map operations: who may do what, and how every change becomes one new, logged map version.""" + +import asyncio +import copy +import logging +import random +import secrets +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +from . import validation +from .auth import Actor +from .errors import conflict, forbidden, invalid, not_found, precondition_failed, store_unavailable +from .models import ( + AddFeaturesRequest, + CreateMapRequest, + RetractFeatureRequest, + StartPhaseRequest, + UpdateFeatureRequest, + UpdateMapRequest, + UpdatePhaseRequest, +) +from .settings import Settings +from .store import MAX_BATCH_OPERATIONS, ConflictError, MapStore, StoreError + +LOGGER = logging.getLogger("mapserver.engine") + +SCHEMA_VERSION = 1 +MAX_WRITE_ATTEMPTS = 6 +MAX_PHASES = 50 +MAX_LINKS = 50 +MAX_LISTED_MAPS = 200 +MAX_CHANGE_LABELS = 5 +DEFAULT_PAGE_SIZE = 200 +MAX_PAGE_SIZE = 500 +MAX_CHANGES_PAGE = 500 +MAX_CURSOR = 10_000_000 +DEFAULT_LINE_WIDTH = 4 +TILE_ATTRIBUTION = "© Microsoft Corporation © OpenStreetMap contributors" +BACKGROUND_COLOR = "#000000" +PHASE_COLORS = ("#0d6efd", "#fd7e14", "#20c997", "#d63384", "#6f42c1", "#ffc107", "#198754", "#dc3545", "#0dcaf0", "#6610f2") +ZERO_COUNTS = {"active": 0, "retracted": 0, "point": 0, "path": 0, "area": 0} +PHASE_OUTPUT_FIELDS = ( + "id", "order", "name", "description", "color", "status", + "started_at", "started_by", "started_version", "closed_at", "closed_version", +) +FEATURE_OUTPUT_FIELDS = ( + "id", "kind", "geometry", "label", "category", "description", "observed_at", "source", "media", "fields", "style", + "status", "phase_id", "last_phase_id", "revision", "created_version", "retracted_version", "retract_reason", + "created_by", "created_at", "updated_by", "updated_at", +) +CHANGE_OUTPUT_FIELDS = ( + "id", "version", "action", "phase_id", "added", "updated", "retracted", "skipped", "labels", "note", "actor", "at", +) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +@dataclass +class WritePlan: + """What one write changes, besides the map document itself.""" + + phase_id: str = "" + creates: List[Dict[str, Any]] = field(default_factory=list) + replaces: List[Dict[str, Any]] = field(default_factory=list) + added: List[str] = field(default_factory=list) + updated: List[str] = field(default_factory=list) + retracted: List[str] = field(default_factory=list) + skipped: int = 0 + labels: List[str] = field(default_factory=list) + note: str = "" + no_op: bool = False + result: Dict[str, Any] = field(default_factory=dict) + + +Build = Callable[[Dict[str, Any], int, str, str], Awaitable[WritePlan]] + + +def _plural(count: int, noun: str) -> str: + return f"{count} {noun}{'' if count == 1 else 's'}" + + +def _translucent(color: str) -> str: + hex_digits = color.lstrip("#") if color.startswith("#") else "" + if len(hex_digits) == 3: + hex_digits = "".join(digit * 2 for digit in hex_digits) + if len(hex_digits) in (6, 8): + red, green, blue = (int(hex_digits[index:index + 2], 16) for index in (0, 2, 4)) + return f"rgba({red}, {green}, {blue}, 0.20)" + return "rgba(13, 110, 253, 0.20)" + + +def _who(summary: Optional[Dict[str, Any]]) -> str: + summary = summary or {} + return str((summary.get("agent") or {}).get("name") or summary.get("display_name") or "") + + +class MapEngine: + def __init__(self, store: MapStore, settings: Settings, *, clock: Callable[[], str] = utc_now): + self._store = store + self._settings = settings + self._clock = clock + + # ------------------------------------------------------------------------------------------ + # Access and writes + # ------------------------------------------------------------------------------------------ + + async def _guard(self, awaitable: Awaitable[Any]) -> Any: + try: + return await awaitable + except StoreError as exc: + LOGGER.error(f"[MAP_SERVER_STORE] A store call failed ({type(exc).__name__}).") + raise store_unavailable() from exc + + async def _load(self, actor: Actor, map_id: str, *, write: bool = False) -> Tuple[Dict[str, Any], str]: + """The map, if the actor's scope owns it. Other scopes get the same answer as a missing map.""" + validation.require_map_id(map_id) + loaded = await self._guard(self._store.read_map(map_id)) + if loaded is None or loaded[0].get("owner_scope") != actor.scope: + raise not_found() + if write and not actor.can_write: + raise forbidden("read_only", "This request can only read the map.") + return loaded + + async def _commit( + self, + map_id: str, + map_doc: Dict[str, Any], + etag: Optional[str], + creates: Sequence[Dict[str, Any]], + replaces: Sequence[Dict[str, Any]], + ) -> str: + try: + return await self._store.commit(map_id, map_doc, etag, creates, replaces) + except StoreError as exc: + LOGGER.error(f"[MAP_SERVER_STORE] A map write failed ({type(exc).__name__}).") + raise store_unavailable() from exc + + async def _write(self, actor: Actor, map_id: str, action: str, build: Build) -> Dict[str, Any]: + """Apply one change as the next map version, retrying from a fresh read when another write lands first.""" + for attempt in range(MAX_WRITE_ATTEMPTS): + map_doc, etag = await self._load(actor, map_id, write=True) + version = int(map_doc.get("version") or 0) + 1 + now = self._clock() + plan = await build(map_doc, version, now, etag) + if plan.no_op: + return {"map_id": map_id, "version": map_doc["version"], "change_id": None, "unchanged": True, **plan.result} + change = self._change_doc(map_id, version, action, actor, now, plan, map_doc) + map_doc["version"] = version + map_doc["updated_at"] = now + map_doc["updated_by"] = actor.summary() + try: + await self._commit(map_id, map_doc, etag, [*plan.creates, change], plan.replaces) + except ConflictError: + await asyncio.sleep(random.uniform(0.005, 0.02) * (attempt + 1)) + continue + return {"map_id": map_id, "version": version, "change_id": change["id"], **plan.result} + LOGGER.warning(f"[MAP_SERVER] A write kept conflicting and gave up (map {map_id}, action {action}).") + raise conflict("map_busy", "The map kept changing while saving. Try again.") + + def _change_doc( + self, map_id: str, version: int, action: str, actor: Actor, now: str, plan: WritePlan, map_doc: Dict[str, Any] + ) -> Dict[str, Any]: + return { + "id": f"C-{version:06d}", + "map_id": map_id, + "type": "change", + "schema_version": SCHEMA_VERSION, + "version": version, + "action": action, + "phase_id": plan.phase_id or map_doc.get("current_phase_id", ""), + "added": plan.added, + "updated": plan.updated, + "retracted": plan.retracted, + "skipped": plan.skipped, + "labels": plan.labels[:MAX_CHANGE_LABELS], + "note": plan.note, + "actor": actor.summary(), + "at": now, + } + + async def _index(self, map_id: str, scope_key: str, now: str) -> None: + # The index is written before the map, so a failed write leaves a row that reads ignore, never a hidden map. + await self._guard(self._store.upsert_link({"id": map_id, "scope_key": scope_key, "map_id": map_id, "at": now})) + + # ------------------------------------------------------------------------------------------ + # Shapes returned to callers + # ------------------------------------------------------------------------------------------ + + @staticmethod + def _phase(map_doc: Dict[str, Any], phase_id: Optional[str]) -> Optional[Dict[str, Any]]: + return next((phase for phase in map_doc.get("phases", []) if phase["id"] == phase_id), None) + + @staticmethod + def _public_phase(phase: Dict[str, Any]) -> Dict[str, Any]: + return {name: phase.get(name) for name in PHASE_OUTPUT_FIELDS} + + @staticmethod + def _linked_ids(map_doc: Dict[str, Any]) -> List[str]: + return [link["conversation_id"] for link in map_doc.get("links", [])] + + def _map_view(self, map_doc: Dict[str, Any], etag: str) -> Dict[str, Any]: + return { + "map_id": map_doc["map_id"], + "title": map_doc["title"], + "description": map_doc.get("description", ""), + "owner_scope": map_doc["owner_scope"], + "basemap": map_doc["basemap"], + "version": map_doc["version"], + "current_phase_id": map_doc["current_phase_id"], + "phases": [self._public_phase(phase) for phase in map_doc.get("phases", [])], + "counts": dict(ZERO_COUNTS, **map_doc.get("counts", {})), + "links": self._linked_ids(map_doc), + "created_at": map_doc.get("created_at"), + "created_by": map_doc.get("created_by"), + "updated_at": map_doc.get("updated_at"), + "updated_by": map_doc.get("updated_by"), + "etag": etag, + } + + def _map_summary(self, map_doc: Dict[str, Any]) -> Dict[str, Any]: + current = self._phase(map_doc, map_doc.get("current_phase_id")) or {} + return { + "map_id": map_doc["map_id"], + "title": map_doc["title"], + "version": map_doc["version"], + "counts": dict(ZERO_COUNTS, **map_doc.get("counts", {})), + "current_phase": {"id": current.get("id"), "name": current.get("name"), "color": current.get("color")}, + "updated_at": map_doc.get("updated_at"), + } + + @staticmethod + def _public_feature(feature: Dict[str, Any], phases_by_id: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + output = {name: copy.deepcopy(feature.get(name)) for name in FEATURE_OUTPUT_FIELDS} + output["updated_version"] = feature.get("version_start") + output["phase_name"] = (phases_by_id.get(feature.get("phase_id")) or {}).get("name", "") + return output + + @staticmethod + def _public_change(change: Dict[str, Any]) -> Dict[str, Any]: + return {name: copy.deepcopy(change.get(name)) for name in CHANGE_OUTPUT_FIELDS} + + @staticmethod + def _brief(feature: Dict[str, Any]) -> Dict[str, Any]: + return {"id": feature["id"], "label": feature["label"], "kind": feature["kind"]} + + @staticmethod + def _content(content: Dict[str, Any]) -> Dict[str, Any]: + return {name: copy.deepcopy(content.get(name)) for name in validation.CONTENT_FIELDS} + + @staticmethod + def _new_phase(order: int, name: str, description: str, color: str, actor: Actor, now: str, version: int) -> Dict[str, Any]: + return { + "id": f"P{order}", + "order": order, + "name": name, + "description": description, + "color": color or PHASE_COLORS[(order - 1) % len(PHASE_COLORS)], + "status": "open", + "started_by": actor.summary(), + "started_at": now, + "started_version": version, + "closed_at": None, + "closed_version": None, + } + + def _writable_phase(self, map_doc: Dict[str, Any], phase_id: str) -> Dict[str, Any]: + if not phase_id: + return self._phase(map_doc, map_doc["current_phase_id"]) + validation.require_phase_id(phase_id) + phase = self._phase(map_doc, phase_id) + if phase is None: + raise not_found("phase_not_found", "Phase not found.") + if phase["status"] != "open": + raise conflict("phase_closed", f"Phase {phase['id']} is closed. Use the current phase or start a new one.") + return phase + + @staticmethod + def _revision_doc(feature: Dict[str, Any], version: int) -> Dict[str, Any]: + revision = copy.deepcopy(feature) + revision.update({ + "id": f"{feature['id']}@r{feature['revision']}", + "type": "feature_rev", + "feature_id": feature["id"], + "valid_from_version": feature["version_start"], + "valid_to_version": version - 1, + }) + revision.pop("dup_key", None) + return revision + + async def _read_feature(self, map_id: str, feature_id: str) -> Dict[str, Any]: + item = await self._guard(self._store.read_item(map_id, feature_id)) + if not item or item.get("type") != "feature": + raise not_found("feature_not_found", "Feature not found.") + return item + + # ------------------------------------------------------------------------------------------ + # Maps and links + # ------------------------------------------------------------------------------------------ + + async def create_map(self, actor: Actor, request: CreateMapRequest) -> Dict[str, Any]: + if not actor.can_write: + raise forbidden("read_only", "This request can only read maps.") + title = validation.require_text(request.title, validation.TITLE_MAX, "A map needs a title.") + description = validation.normalize_text(request.description, validation.MAP_DESCRIPTION_MAX, multiline=True) + basemap = validation.normalize_tileset(request.basemap or self._settings.default_basemap) + phase_name = validation.normalize_text(request.first_phase_name, validation.PHASE_NAME_MAX) or "Phase 1" + phase_description = validation.normalize_text( + request.first_phase_description, validation.PHASE_DESCRIPTION_MAX, multiline=True, + ) + conversation_id = validation.normalize_conversation_id(request.conversation_id) + + now = self._clock() + map_id = f"map-{secrets.token_hex(10)}" + phase = self._new_phase(1, phase_name, phase_description, "", actor, now, 1) + links = [{"conversation_id": conversation_id, "linked_at": now, "linked_by": actor.summary()}] if conversation_id else [] + map_doc = { + "id": map_id, + "map_id": map_id, + "type": "map", + "schema_version": SCHEMA_VERSION, + "title": title, + "description": description, + "owner_scope": actor.scope, + "basemap": basemap, + "version": 1, + "phases": [phase], + "current_phase_id": phase["id"], + "phase_seq": 1, + "feature_seq": 0, + "counts": dict(ZERO_COUNTS), + "links": links, + "created_by": actor.summary(), + "created_at": now, + "updated_by": actor.summary(), + "updated_at": now, + } + change = self._change_doc(map_id, 1, "create_map", actor, now, WritePlan(phase_id=phase["id"], note=title), map_doc) + await self._index(map_id, actor.scope, now) + if conversation_id: + await self._index(map_id, f"conversation:{conversation_id}", now) + try: + etag = await self._commit(map_id, map_doc, None, [change], []) + except ConflictError as exc: + raise conflict("map_exists", "Couldn't create the map. Try again.") from exc + return self._map_view(map_doc, etag) + + async def list_maps(self, actor: Actor, conversation_id: str = "") -> Dict[str, Any]: + conversation_id = validation.normalize_conversation_id(conversation_id) + scope_key = f"conversation:{conversation_id}" if conversation_id else actor.scope + rows = await self._guard(self._store.list_links(scope_key)) + items = [] + for row in rows[:MAX_LISTED_MAPS]: + map_id = str(row.get("map_id") or "") + if not validation.MAP_ID_PATTERN.fullmatch(map_id): + continue + loaded = await self._guard(self._store.read_map(map_id)) + if loaded is None or loaded[0].get("owner_scope") != actor.scope: + continue + if conversation_id and conversation_id not in self._linked_ids(loaded[0]): + continue + items.append(self._map_summary(loaded[0])) + items.sort(key=lambda item: item.get("updated_at") or "", reverse=True) + return {"items": items} + + async def get_map(self, actor: Actor, map_id: str) -> Dict[str, Any]: + map_doc, etag = await self._load(actor, map_id) + return self._map_view(map_doc, etag) + + async def update_map(self, actor: Actor, map_id: str, request: UpdateMapRequest, if_match: str = "") -> Dict[str, Any]: + changes: Dict[str, Any] = {} + if request.title is not None: + changes["title"] = validation.require_text(request.title, validation.TITLE_MAX, "A map needs a title.") + if request.description is not None: + changes["description"] = validation.normalize_text( + request.description, validation.MAP_DESCRIPTION_MAX, multiline=True, + ) + if request.basemap is not None: + changes["basemap"] = validation.normalize_tileset(request.basemap) + if not changes: + raise invalid("Send a title, description or basemap to change.") + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + if if_match and if_match != etag: + raise precondition_failed("map_changed", "The map changed since you read it.") + if all(map_doc.get(name) == value for name, value in changes.items()): + return WritePlan(no_op=True) + map_doc.update(changes) + return WritePlan(note=", ".join(sorted(changes))) + + await self._write(actor, map_id, "update_map", build) + return await self.get_map(actor, map_id) + + async def link_conversation(self, actor: Actor, map_id: str, conversation_id: str) -> Dict[str, Any]: + conversation_id = validation.normalize_conversation_id(conversation_id) + if not conversation_id: + raise invalid("conversation_id is required.") + for attempt in range(MAX_WRITE_ATTEMPTS): + map_doc, etag = await self._load(actor, map_id, write=True) + if conversation_id in self._linked_ids(map_doc): + return {"map_id": map_id, "conversation_id": conversation_id, "links": self._linked_ids(map_doc)} + if len(map_doc.get("links", [])) >= MAX_LINKS: + raise conflict("too_many_links", f"A map can be linked to up to {MAX_LINKS} conversations.") + now = self._clock() + await self._index(map_id, f"conversation:{conversation_id}", now) + map_doc.setdefault("links", []).append( + {"conversation_id": conversation_id, "linked_at": now, "linked_by": actor.summary()} + ) + try: + await self._commit(map_id, map_doc, etag, [], []) + except ConflictError: + await asyncio.sleep(random.uniform(0.005, 0.02) * (attempt + 1)) + continue + return {"map_id": map_id, "conversation_id": conversation_id, "links": self._linked_ids(map_doc)} + raise conflict("map_busy", "The map kept changing while saving. Try again.") + + async def unlink_conversation(self, actor: Actor, map_id: str, conversation_id: str) -> Dict[str, Any]: + conversation_id = validation.normalize_conversation_id(conversation_id) + if not conversation_id: + raise invalid("conversation_id is required.") + for attempt in range(MAX_WRITE_ATTEMPTS): + map_doc, etag = await self._load(actor, map_id, write=True) + remaining = [link for link in map_doc.get("links", []) if link["conversation_id"] != conversation_id] + if len(remaining) == len(map_doc.get("links", [])): + break + map_doc["links"] = remaining + try: + await self._commit(map_id, map_doc, etag, [], []) + except ConflictError: + await asyncio.sleep(random.uniform(0.005, 0.02) * (attempt + 1)) + continue + break + else: + raise conflict("map_busy", "The map kept changing while saving. Try again.") + await self._guard(self._store.delete_link(f"conversation:{conversation_id}", map_id)) + return {"map_id": map_id, "conversation_id": conversation_id, "links": self._linked_ids(map_doc)} + + # ------------------------------------------------------------------------------------------ + # Phases + # ------------------------------------------------------------------------------------------ + + async def start_phase(self, actor: Actor, map_id: str, request: StartPhaseRequest) -> Dict[str, Any]: + name = validation.require_text(request.name, validation.PHASE_NAME_MAX, "A phase needs a name.") + description = validation.normalize_text(request.description, validation.PHASE_DESCRIPTION_MAX, multiline=True) + color = validation.normalize_color(request.color) + if request.color and not color: + raise invalid("color must be a hex or rgba color, such as #0d6efd.") + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + phases = map_doc.setdefault("phases", []) + if len(phases) >= MAX_PHASES: + raise conflict("too_many_phases", f"A map can have up to {MAX_PHASES} phases.") + current = self._phase(map_doc, map_doc.get("current_phase_id")) + if current and current["status"] == "open": + current.update({"status": "closed", "closed_at": now, "closed_version": version}) + order = int(map_doc.get("phase_seq") or len(phases)) + 1 + map_doc["phase_seq"] = order + phase = self._new_phase(order, name, description, color, actor, now, version) + phases.append(phase) + map_doc["current_phase_id"] = phase["id"] + return WritePlan(phase_id=phase["id"], note=name, result={"phase": self._public_phase(phase)}) + + return await self._write(actor, map_id, "start_phase", build) + + async def update_phase(self, actor: Actor, map_id: str, phase_id: str, request: UpdatePhaseRequest) -> Dict[str, Any]: + validation.require_phase_id(phase_id) + changes: Dict[str, Any] = {} + if request.name is not None: + changes["name"] = validation.require_text(request.name, validation.PHASE_NAME_MAX, "A phase needs a name.") + if request.description is not None: + changes["description"] = validation.normalize_text( + request.description, validation.PHASE_DESCRIPTION_MAX, multiline=True, + ) + if request.color is not None: + changes["color"] = validation.normalize_color(request.color) + if not changes["color"]: + raise invalid("color must be a hex or rgba color, such as #0d6efd.") + if not changes: + raise invalid("Send a name, description or color to change.") + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + phase = self._phase(map_doc, phase_id) + if phase is None: + raise not_found("phase_not_found", "Phase not found.") + if all(phase.get(name) == value for name, value in changes.items()): + return WritePlan(no_op=True, result={"phase": self._public_phase(phase)}) + phase.update(changes) + return WritePlan(phase_id=phase_id, note=phase["name"], result={"phase": self._public_phase(phase)}) + + return await self._write(actor, map_id, "update_phase", build) + + # ------------------------------------------------------------------------------------------ + # Features + # ------------------------------------------------------------------------------------------ + + async def add_features(self, actor: Actor, map_id: str, request: AddFeaturesRequest) -> Dict[str, Any]: + limit = self._settings.max_features_per_call + if len(request.features) > limit: + raise invalid(f"Send up to {limit} features per call, and split larger sets.", code="too_many_features") + normalized = [validation.normalize_feature(raw, index) for index, raw in enumerate(request.features)] + contents = [content for content, _ in normalized] + images_dropped = sum(1 for _, dropped in normalized if dropped) + update_duplicates = request.on_duplicate == "update" + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + phase = self._writable_phase(map_doc, request.phase_id) + keys = sorted({content["dup_key"] for content in contents if content["dup_key"]}) + existing: Dict[str, Dict[str, Any]] = {} + if keys: + for feature in await self._guard(self._store.query_features(map_id, dup_keys=keys)): + existing.setdefault(feature["dup_key"], feature) + + counts = dict(ZERO_COUNTS, **map_doc.get("counts", {})) + created: Dict[str, Dict[str, Any]] = {} + new_docs: List[Dict[str, Any]] = [] + updates: Dict[str, Tuple[Dict[str, Any], Dict[str, Any]]] = {} + skipped: Dict[str, Dict[str, Any]] = {} + for content in contents: + key = content["dup_key"] + if key and key in created: + if update_duplicates: + created[key].update(self._content(content)) + else: + skipped[created[key]["id"]] = created[key] + continue + if key and key in existing: + original = existing[key] + current = updates[original["id"]][1] if original["id"] in updates else copy.deepcopy(original) + if update_duplicates and not validation.content_equal(current, content): + current.update(self._content(content)) + updates[original["id"]] = (original, current) + elif original["id"] not in updates: + skipped[original["id"]] = original + continue + if counts["active"] >= self._settings.max_active_features: + raise conflict( + "map_full", + f"A map holds up to {self._settings.max_active_features} active features. Retract ones that no longer matter.", + ) + sequence = int(map_doc.get("feature_seq") or 0) + 1 + map_doc["feature_seq"] = sequence + doc = { + "id": f"F-{sequence:04d}", + "map_id": map_id, + "type": "feature", + "schema_version": SCHEMA_VERSION, + "kind": content["kind"], + **self._content(content), + "phase_id": phase["id"], + "last_phase_id": phase["id"], + "status": "active", + "created_version": version, + "version_start": version, + "retracted_version": None, + "retract_reason": None, + "revision": 1, + "created_by": actor.summary(), + "created_at": now, + "updated_by": actor.summary(), + "updated_at": now, + } + new_docs.append(doc) + counts["active"] += 1 + counts[content["kind"]] += 1 + if key: + created[key] = doc + + plan = WritePlan(phase_id=phase["id"]) + for original, current in updates.values(): + plan.creates.append(self._revision_doc(original, version)) + current.update({ + "revision": original["revision"] + 1, + "version_start": version, + "last_phase_id": phase["id"], + "updated_by": actor.summary(), + "updated_at": now, + }) + plan.replaces.append(current) + plan.creates.extend(new_docs) + if 2 + len(plan.creates) + len(plan.replaces) > MAX_BATCH_OPERATIONS: + raise invalid("Too many changes for one call. Send fewer features.", code="too_many_changes") + + updated_docs = [current for _, current in updates.values()] + plan.no_op = not new_docs and not updated_docs + map_doc["counts"] = counts + plan.added = [doc["id"] for doc in new_docs] + plan.updated = [doc["id"] for doc in updated_docs] + plan.skipped = len(skipped) + plan.labels = [doc["label"] for doc in [*new_docs, *updated_docs]] + plan.result = { + "phase": self._public_phase(phase), + "added": [self._brief(doc) for doc in new_docs], + "updated": [self._brief(doc) for doc in updated_docs], + "skipped": [self._brief(doc) for doc in skipped.values()], + "images_dropped": images_dropped, + "counts": counts, + } + return plan + + return await self._write(actor, map_id, "add_features", build) + + async def update_feature( + self, actor: Actor, map_id: str, feature_id: str, request: UpdateFeatureRequest + ) -> Dict[str, Any]: + validation.require_feature_id(feature_id) + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + phase = self._writable_phase(map_doc, request.phase_id) + phases_by_id = {item["id"]: item for item in map_doc.get("phases", [])} + current = await self._read_feature(map_id, feature_id) + if current["status"] != "active": + raise conflict("feature_retracted", "This feature was retracted and can't be changed.") + if request.expected_revision is not None and request.expected_revision != current["revision"]: + raise precondition_failed("revision_mismatch", "The feature changed since you read it.") + content, image_dropped = validation.apply_changes(current, request.changes) + if validation.content_equal(current, content): + return WritePlan(no_op=True, result={"updated": [], "feature": self._public_feature(current, phases_by_id)}) + if content["dup_key"] and content["dup_key"] != current.get("dup_key"): + clashes = await self._guard(self._store.query_features(map_id, dup_keys=[content["dup_key"]])) + if any(item["id"] != feature_id for item in clashes): + raise conflict("duplicate_source", "Another feature on this map already has that source record.") + + updated = copy.deepcopy(current) + updated.update(self._content(content)) + updated.update({ + "revision": current["revision"] + 1, + "version_start": version, + "last_phase_id": phase["id"], + "updated_by": actor.summary(), + "updated_at": now, + }) + return WritePlan( + phase_id=phase["id"], + creates=[self._revision_doc(current, version)], + replaces=[updated], + updated=[feature_id], + labels=[updated["label"]], + result={ + "phase": self._public_phase(phase), + "updated": [self._brief(updated)], + "feature": self._public_feature(updated, phases_by_id), + "images_dropped": int(image_dropped), + }, + ) + + return await self._write(actor, map_id, "update_feature", build) + + async def retract_feature( + self, actor: Actor, map_id: str, feature_id: str, request: RetractFeatureRequest + ) -> Dict[str, Any]: + validation.require_feature_id(feature_id) + reason = validation.require_text(request.reason, validation.REASON_MAX, "Say why the feature is being retracted.") + + async def build(map_doc: Dict[str, Any], version: int, now: str, etag: str) -> WritePlan: + phase = self._writable_phase(map_doc, request.phase_id) + current = await self._read_feature(map_id, feature_id) + if current["status"] != "active": + return WritePlan(no_op=True, result={"retracted": []}) + updated = copy.deepcopy(current) + updated.update({ + "status": "retracted", + "retracted_version": version, + "retract_reason": reason, + "last_phase_id": phase["id"], + "updated_by": actor.summary(), + "updated_at": now, + }) + counts = dict(ZERO_COUNTS, **map_doc.get("counts", {})) + counts["active"] = max(0, counts["active"] - 1) + counts[current["kind"]] = max(0, counts[current["kind"]] - 1) + counts["retracted"] += 1 + map_doc["counts"] = counts + return WritePlan( + phase_id=phase["id"], + replaces=[updated], + retracted=[feature_id], + labels=[current["label"]], + note=reason, + result={"phase": self._public_phase(phase), "retracted": [self._brief(updated)], "counts": counts}, + ) + + return await self._write(actor, map_id, "retract_feature", build) + + # ------------------------------------------------------------------------------------------ + # Reads + # ------------------------------------------------------------------------------------------ + + @staticmethod + def _cursor_offset(cursor: str) -> int: + if not cursor: + return 0 + if not cursor.isdigit() or int(cursor) > MAX_CURSOR: + raise invalid("cursor isn't valid.") + return int(cursor) + + @staticmethod + def _intersects(geometry: Dict[str, Any], box: Tuple[float, float, float, float]) -> bool: + points = validation.feature_points(geometry) + if not points: + return False + longitudes = [point[0] for point in points] + latitudes = [point[1] for point in points] + west, south, east, north = box + return min(longitudes) <= east and max(longitudes) >= west and min(latitudes) <= north and max(latitudes) >= south + + async def list_features( + self, + actor: Actor, + map_id: str, + *, + phase_id: str = "", + category: str = "", + kind: str = "", + status: str = "active", + since_version: Optional[int] = None, + bbox: str = "", + limit: int = DEFAULT_PAGE_SIZE, + cursor: str = "", + ) -> Dict[str, Any]: + map_doc, _ = await self._load(actor, map_id) + if status not in ("active", "retracted", "all"): + raise invalid("status must be active, retracted or all.") + if kind and kind not in validation.FEATURE_KINDS: + raise invalid("kind must be point, path or area.") + if phase_id and not validation.PHASE_ID_PATTERN.fullmatch(phase_id): + raise invalid("phase_id must be a phase ID such as P1.") + wanted_category = validation.normalize_category(category) if category else "" + box = validation.parse_bbox(bbox) + page_size = max(1, min(MAX_PAGE_SIZE, int(limit))) + offset = self._cursor_offset(cursor) + + selected = [] + for feature in await self._guard(self._store.query_features(map_id)): + if status != "all" and feature["status"] != status: + continue + if phase_id and feature["phase_id"] != phase_id: + continue + if wanted_category and feature["category"] != wanted_category: + continue + if kind and feature["kind"] != kind: + continue + if since_version is not None and max(feature["version_start"], feature.get("retracted_version") or 0) <= since_version: + continue + if box and not self._intersects(feature["geometry"], box): + continue + selected.append(feature) + selected.sort(key=lambda feature: (feature["created_version"], feature["id"])) + page = selected[offset:offset + page_size] + phases_by_id = {phase["id"]: phase for phase in map_doc.get("phases", [])} + return { + "items": [self._public_feature(feature, phases_by_id) for feature in page], + "next_cursor": str(offset + page_size) if offset + page_size < len(selected) else None, + "total": len(selected), + "version": map_doc["version"], + } + + @staticmethod + def _as_of(feature: Dict[str, Any], revision: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if revision is None: + return feature + restored = dict(feature) + for name in (*validation.CONTENT_FIELDS, "revision", "last_phase_id", "updated_by", "updated_at"): + restored[name] = revision.get(name) + restored["version_start"] = revision["valid_from_version"] + return restored + + def _render_entry(self, feature: Dict[str, Any], phase: Dict[str, Any]) -> Tuple[str, Dict[str, Any]]: + """One feature in the marker, path and area shapes the chat's inline map renderers already draw.""" + style = feature.get("style") or {} + color = style.get("color") or phase.get("color") or PHASE_COLORS[0] + entry: Dict[str, Any] = { + "id": feature["id"], + "feature_id": feature["id"], + "label": feature["label"], + "description": feature.get("description") or "", + "category": feature.get("category"), + "phase_id": feature["phase_id"], + "phase_name": phase.get("name", ""), + "observed_at": feature.get("observed_at"), + "source": feature.get("source"), + "added_by": _who(feature.get("created_by")), + "created_version": feature["created_version"], + "updated_version": feature["version_start"], + } + if feature.get("fields"): + entry["fields"] = feature["fields"] + geometry = feature["geometry"] + if feature["kind"] == "point": + longitude, latitude = geometry["coordinates"] + entry.update({"latitude": latitude, "longitude": longitude, "color": color, "icon_name": ""}) + media = feature.get("media") or {} + if media.get("image_url"): + entry["image_url"] = media["image_url"] + if media.get("caption"): + entry["image_caption"] = media["caption"] + return "markers", entry + if feature["kind"] == "path": + entry.update({ + "coordinates": geometry["coordinates"], + "stroke_color": color, + "line_width": style.get("line_width") or DEFAULT_LINE_WIDTH, + }) + return "paths", entry + entry.update({ + "coordinates": geometry["coordinates"][0], + "stroke_color": color, + "fill_color": style.get("fill_color") or _translucent(color), + }) + return "areas", entry + + @staticmethod + def _view(markers: List[Dict[str, Any]], paths: List[Dict[str, Any]], areas: List[Dict[str, Any]]) -> Dict[str, Any]: + points: List[Tuple[float, float]] = [(marker["longitude"], marker["latitude"]) for marker in markers] + for shape in [*paths, *areas]: + points.extend((point[0], point[1]) for point in shape["coordinates"]) + if not points: + return {"center": [0.0, 20.0], "zoom": 2, "max_zoom": 15, "fit_to_features": False} + center = [round(sum(point[0] for point in points) / len(points), 6), round(sum(point[1] for point in points) / len(points), 6)] + zoom = 14 if len(markers) == 1 and not paths and not areas else 10 + return {"center": center, "zoom": zoom, "max_zoom": 15, "fit_to_features": True} + + @staticmethod + def _summary_text(markers: Sequence[Any], paths: Sequence[Any], areas: Sequence[Any], phase_count: int, target: int, current: int) -> str: + parts = [ + _plural(len(items), noun) + for items, noun in ((markers, "place"), (paths, "route"), (areas, "area")) + if items + ] + if not parts: + text = "No features yet" + elif len(parts) == 1: + text = parts[0] + else: + text = f"{', '.join(parts[:-1])} and {parts[-1]}" + text = f"{text} in {_plural(phase_count, 'phase')}" + if target != current: + text = f"{text}, as of version {target}" + return f"{text}." + + async def snapshot( + self, actor: Actor, map_id: str, *, as_of_version: Optional[int] = None, phase_ids: Iterable[str] = () + ) -> Dict[str, Any]: + """The map as it stood at a version, in the shape today's inline map renderers draw.""" + map_doc, _ = await self._load(actor, map_id) + current_version = int(map_doc["version"]) + target = current_version if as_of_version is None else int(as_of_version) + if not 1 <= target <= current_version: + raise invalid(f"as_of_version must be between 1 and {current_version}.") + wanted = {phase_id for phase_id in phase_ids if phase_id} + for phase_id in wanted: + if not validation.PHASE_ID_PATTERN.fullmatch(phase_id): + raise invalid("phases must be phase IDs such as P1,P2.") + + features = await self._guard(self._store.query_features(map_id)) + visible = [ + feature for feature in features + if feature["created_version"] <= target + and (feature.get("retracted_version") is None or feature["retracted_version"] > target) + ] + stale_ids = [feature["id"] for feature in visible if feature["version_start"] > target] + if stale_ids: + revisions = await self._guard(self._store.query_feature_revisions(map_id, stale_ids, target)) + by_feature = {revision["feature_id"]: revision for revision in revisions} + visible = [self._as_of(feature, by_feature.get(feature["id"])) for feature in visible] + if wanted: + visible = [feature for feature in visible if feature["phase_id"] in wanted] + visible.sort(key=lambda feature: (feature["created_version"], feature["id"])) + + phases = [phase for phase in map_doc.get("phases", []) if phase["started_version"] <= target] + phases_by_id = {phase["id"]: phase for phase in phases} + shapes: Dict[str, List[Dict[str, Any]]] = {"markers": [], "paths": [], "areas": []} + phase_counts: Dict[str, int] = {} + for feature in visible: + bucket, entry = self._render_entry(feature, phases_by_id.get(feature["phase_id"]) or {}) + shapes[bucket].append(entry) + phase_counts[feature["phase_id"]] = phase_counts.get(feature["phase_id"], 0) + 1 + + public_phases = [] + for phase in phases: + public = self._public_phase(phase) + closed = phase.get("closed_version") is not None and phase["closed_version"] <= target + public["status"] = "closed" if closed else "open" + public["feature_count"] = phase_counts.get(phase["id"], 0) + public_phases.append(public) + + return { + "map_id": map_id, + "title": map_doc["title"], + "summary": self._summary_text( + shapes["markers"], shapes["paths"], shapes["areas"], len(phases), target, current_version, + ), + "map_provider": "azure_maps", + "map_library": "openlayers", + "tileset_id": map_doc["basemap"], + "tile_attribution": TILE_ATTRIBUTION, + "background_color": BACKGROUND_COLOR, + "view": self._view(shapes["markers"], shapes["paths"], shapes["areas"]), + "markers": shapes["markers"], + "paths": shapes["paths"], + "areas": shapes["areas"], + "phases": public_phases, + "version": current_version, + "as_of_version": target, + } + + async def list_changes(self, actor: Actor, map_id: str, *, since_version: int = 0, limit: int = 100) -> Dict[str, Any]: + map_doc, _ = await self._load(actor, map_id) + since = max(0, int(since_version)) + page_size = max(1, min(MAX_CHANGES_PAGE, int(limit))) + changes = await self._guard(self._store.query_changes(map_id, since, page_size + 1)) + more = len(changes) > page_size + changes = changes[:page_size] + return { + "items": [self._public_change(change) for change in changes], + "version": map_doc["version"], + "next_since_version": changes[-1]["version"] if more else None, + } + + async def current_version(self, actor: Actor, map_id: str) -> int: + map_doc, _ = await self._load(actor, map_id) + return int(map_doc["version"]) diff --git a/application/map_server/mapserver/errors.py b/application/map_server/mapserver/errors.py new file mode 100644 index 000000000..1405d738d --- /dev/null +++ b/application/map_server/mapserver/errors.py @@ -0,0 +1,43 @@ +# errors.py +"""Errors the map server returns to callers, each with a stable code and a message that's safe to show.""" + +from typing import Any, Optional + + +class MapServerError(Exception): + """A request failed in a way the caller can act on.""" + + def __init__(self, status_code: int, code: str, message: str, details: Optional[Any] = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.details = details + + +def invalid(message: str, code: str = "invalid_request") -> MapServerError: + return MapServerError(400, code, message) + + +def unauthorized(message: str = "A valid access token is required.") -> MapServerError: + return MapServerError(401, "unauthorized", message) + + +def forbidden(code: str, message: str) -> MapServerError: + return MapServerError(403, code, message) + + +def not_found(code: str = "map_not_found", message: str = "Map not found.") -> MapServerError: + return MapServerError(404, code, message) + + +def conflict(code: str, message: str) -> MapServerError: + return MapServerError(409, code, message) + + +def precondition_failed(code: str, message: str) -> MapServerError: + return MapServerError(412, code, message) + + +def store_unavailable() -> MapServerError: + return MapServerError(503, "store_unavailable", "The map store is unavailable. Try again shortly.") diff --git a/application/map_server/mapserver/models.py b/application/map_server/mapserver/models.py new file mode 100644 index 000000000..be89e829f --- /dev/null +++ b/application/map_server/mapserver/models.py @@ -0,0 +1,58 @@ +# models.py +"""Request bodies the map server accepts. Text is trimmed to its stored limits by validation.py.""" + +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class _Request(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class CreateMapRequest(_Request): + title: str = Field(max_length=1000) + description: str = Field(default="", max_length=10000) + basemap: str = Field(default="", max_length=64) + first_phase_name: str = Field(default="", max_length=1000) + first_phase_description: str = Field(default="", max_length=10000) + conversation_id: str = Field(default="", max_length=128) + + +class UpdateMapRequest(_Request): + title: Optional[str] = Field(default=None, max_length=1000) + description: Optional[str] = Field(default=None, max_length=10000) + basemap: Optional[str] = Field(default=None, max_length=64) + + +class LinkConversationRequest(_Request): + conversation_id: str = Field(min_length=1, max_length=128) + + +class StartPhaseRequest(_Request): + name: str = Field(max_length=1000) + description: str = Field(default="", max_length=10000) + color: str = Field(default="", max_length=64) + + +class UpdatePhaseRequest(_Request): + name: Optional[str] = Field(default=None, max_length=1000) + description: Optional[str] = Field(default=None, max_length=10000) + color: Optional[str] = Field(default=None, max_length=64) + + +class AddFeaturesRequest(_Request): + features: List[Dict[str, Any]] = Field(min_length=1, max_length=200) + phase_id: str = Field(default="", max_length=8) + on_duplicate: Literal["skip", "update"] = "skip" + + +class UpdateFeatureRequest(_Request): + changes: Dict[str, Any] + expected_revision: Optional[int] = Field(default=None, ge=1) + phase_id: str = Field(default="", max_length=8) + + +class RetractFeatureRequest(_Request): + reason: str = Field(max_length=10000) + phase_id: str = Field(default="", max_length=8) diff --git a/application/map_server/mapserver/settings.py b/application/map_server/mapserver/settings.py new file mode 100644 index 000000000..e32f50b86 --- /dev/null +++ b/application/map_server/mapserver/settings.py @@ -0,0 +1,126 @@ +# settings.py +"""Map server configuration, read once from environment variables and checked at startup.""" + +import os +from dataclasses import dataclass +from typing import FrozenSet, Mapping, Optional, Tuple + +DEFAULT_REQUIRED_ROLE = "MapServer.ActOnBehalf" +DEFAULT_AUTHORITY_HOST = "login.microsoftonline.com" +DEFAULT_AZURE_MAPS_ENDPOINT = "https://atlas.microsoft.com" +DEFAULT_BASEMAP = "microsoft.base.road" +STORE_COSMOS = "cosmos" +STORE_MEMORY = "memory" +LOCAL_DEV_KEY_MIN_LENGTH = 32 +# A transactional batch holds 100 operations; an update costs two (revision + replace) plus the map and change. +MAX_FEATURES_PER_CALL_LIMIT = 49 + + +class SettingsError(ValueError): + """The map server is misconfigured.""" + + +def _split_list(raw: Optional[str]) -> Tuple[str, ...]: + return tuple(item.strip() for item in (raw or "").split(",") if item.strip()) + + +def _read_number(env: Mapping[str, str], name: str, default: float, minimum: float, maximum: float) -> float: + raw = str(env.get(name) or "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError as exc: + raise SettingsError(f"{name} must be a number.") from exc + if not minimum <= value <= maximum: + raise SettingsError(f"{name} must be between {minimum:g} and {maximum:g}.") + return value + + +@dataclass(frozen=True) +class Settings: + tenant_id: str = "" + audiences: Tuple[str, ...] = () + issuers: Tuple[str, ...] = () + jwks_url: str = "" + required_role: str = DEFAULT_REQUIRED_ROLE + allowed_caller_ids: FrozenSet[str] = frozenset() + local_dev_key: str = "" + store: str = STORE_COSMOS + cosmos_endpoint: str = "" + cosmos_key: str = "" + cosmos_database: str = "mapserver" + maps_container: str = "maps" + links_container: str = "map_links" + managed_identity_client_id: str = "" + azure_maps_endpoint: str = DEFAULT_AZURE_MAPS_ENDPOINT + azure_maps_key: str = "" + azure_maps_client_id: str = "" + default_basemap: str = DEFAULT_BASEMAP + tile_cache_entries: int = 1024 + events_poll_seconds: float = 2.0 + events_keepalive_seconds: float = 15.0 + events_max_seconds: float = 600.0 + max_features_per_call: int = 40 + max_active_features: int = 10000 + + @classmethod + def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "Settings": + env = os.environ if env is None else env + tenant_id = str(env.get("MAP_SERVER_TENANT_ID") or "").strip() + authority_host = str(env.get("MAP_SERVER_AUTHORITY_HOST") or DEFAULT_AUTHORITY_HOST).strip() + issuers = _split_list(env.get("MAP_SERVER_ISSUERS")) + if not issuers and tenant_id: + # Managed identity tokens carry the v1 issuer unless the API app requests v2 tokens. + issuers = (f"https://sts.windows.net/{tenant_id}/", f"https://{authority_host}/{tenant_id}/v2.0") + jwks_url = str(env.get("MAP_SERVER_JWKS_URL") or "").strip() + if not jwks_url and tenant_id: + jwks_url = f"https://{authority_host}/{tenant_id}/discovery/v2.0/keys" + + settings = cls( + tenant_id=tenant_id, + audiences=_split_list(env.get("MAP_SERVER_AUDIENCES")), + issuers=issuers, + jwks_url=jwks_url, + required_role=str(env.get("MAP_SERVER_REQUIRED_ROLE") or DEFAULT_REQUIRED_ROLE).strip(), + allowed_caller_ids=frozenset(_split_list(env.get("MAP_SERVER_ALLOWED_CALLER_IDS"))), + local_dev_key=str(env.get("MAP_SERVER_LOCAL_DEV_KEY") or "").strip(), + store=str(env.get("MAP_SERVER_STORE") or STORE_COSMOS).strip().lower(), + cosmos_endpoint=str(env.get("MAP_SERVER_COSMOS_ENDPOINT") or "").strip(), + cosmos_key=str(env.get("MAP_SERVER_COSMOS_KEY") or "").strip(), + cosmos_database=str(env.get("MAP_SERVER_COSMOS_DATABASE") or "mapserver").strip(), + maps_container=str(env.get("MAP_SERVER_MAPS_CONTAINER") or "maps").strip(), + links_container=str(env.get("MAP_SERVER_LINKS_CONTAINER") or "map_links").strip(), + managed_identity_client_id=str(env.get("AZURE_CLIENT_ID") or "").strip(), + azure_maps_endpoint=str(env.get("AZURE_MAPS_ENDPOINT") or DEFAULT_AZURE_MAPS_ENDPOINT).strip().rstrip("/"), + azure_maps_key=str(env.get("AZURE_MAPS_KEY") or "").strip(), + azure_maps_client_id=str(env.get("AZURE_MAPS_CLIENT_ID") or "").strip(), + default_basemap=str(env.get("MAP_SERVER_DEFAULT_BASEMAP") or DEFAULT_BASEMAP).strip(), + tile_cache_entries=int(_read_number(env, "MAP_SERVER_TILE_CACHE_ENTRIES", 1024, 0, 100000)), + events_poll_seconds=_read_number(env, "MAP_SERVER_EVENTS_POLL_SECONDS", 2.0, 0.05, 60), + events_keepalive_seconds=_read_number(env, "MAP_SERVER_EVENTS_KEEPALIVE_SECONDS", 15.0, 1, 300), + events_max_seconds=_read_number(env, "MAP_SERVER_EVENTS_MAX_SECONDS", 600.0, 1, 3600), + max_features_per_call=int(_read_number( + env, "MAP_SERVER_MAX_FEATURES_PER_CALL", 40, 1, MAX_FEATURES_PER_CALL_LIMIT, + )), + max_active_features=int(_read_number(env, "MAP_SERVER_MAX_ACTIVE_FEATURES", 10000, 1, 100000)), + ) + settings.validate() + return settings + + def validate(self) -> None: + if self.store not in (STORE_COSMOS, STORE_MEMORY): + raise SettingsError("MAP_SERVER_STORE must be 'cosmos' or 'memory'.") + if self.local_dev_key: + if self.store != STORE_MEMORY: + raise SettingsError("MAP_SERVER_LOCAL_DEV_KEY only works with MAP_SERVER_STORE=memory.") + if len(self.local_dev_key) < LOCAL_DEV_KEY_MIN_LENGTH: + raise SettingsError(f"MAP_SERVER_LOCAL_DEV_KEY must be at least {LOCAL_DEV_KEY_MIN_LENGTH} characters.") + elif not (self.tenant_id and self.audiences and self.issuers and self.jwks_url): + raise SettingsError("MAP_SERVER_TENANT_ID and MAP_SERVER_AUDIENCES are required.") + if not self.local_dev_key and not (self.required_role or self.allowed_caller_ids): + raise SettingsError("Set MAP_SERVER_REQUIRED_ROLE or MAP_SERVER_ALLOWED_CALLER_IDS.") + if self.store == STORE_COSMOS and not self.cosmos_endpoint: + raise SettingsError("MAP_SERVER_COSMOS_ENDPOINT is required with the Cosmos store.") + if not 1 <= self.max_features_per_call <= MAX_FEATURES_PER_CALL_LIMIT: + raise SettingsError(f"max_features_per_call must be between 1 and {MAX_FEATURES_PER_CALL_LIMIT}.") diff --git a/application/map_server/mapserver/store.py b/application/map_server/mapserver/store.py new file mode 100644 index 000000000..daa5d66a0 --- /dev/null +++ b/application/map_server/mapserver/store.py @@ -0,0 +1,161 @@ +# store.py +"""Storage for maps: the interface the engine relies on, and an in-memory store for tests and local runs.""" + +import copy +import itertools +import threading +from typing import Any, Dict, List, Optional, Protocol, Sequence, Tuple + +MAX_BATCH_OPERATIONS = 100 + + +class ConflictError(Exception): + """The map changed after it was read, so the write must be retried.""" + + +class StoreError(Exception): + """The store failed for a reason other than a conflict.""" + + +class MapStore(Protocol): + """Every item of a map lives in one partition keyed by its map_id; links live in a separate index.""" + + async def read_map(self, map_id: str) -> Optional[Tuple[Dict[str, Any], str]]: + """The map document and its ETag, or None.""" + + async def read_item(self, map_id: str, item_id: str) -> Optional[Dict[str, Any]]: + """Any item in the map's partition, or None.""" + + async def query_features(self, map_id: str, *, dup_keys: Optional[Sequence[str]] = None) -> List[Dict[str, Any]]: + """Every feature, or only active features whose dup_key is in dup_keys.""" + + async def query_feature_revisions( + self, map_id: str, feature_ids: Sequence[str], as_of_version: int + ) -> List[Dict[str, Any]]: + """Earlier states of the given features that were current at as_of_version.""" + + async def query_changes(self, map_id: str, since_version: int, limit: int) -> List[Dict[str, Any]]: + """Changes after since_version, oldest first.""" + + async def commit( + self, + map_id: str, + map_doc: Dict[str, Any], + expected_etag: Optional[str], + creates: Sequence[Dict[str, Any]], + replaces: Sequence[Dict[str, Any]], + ) -> str: + """Atomically write the map document and items. A None ETag creates the map. Returns the new ETag.""" + + async def upsert_link(self, row: Dict[str, Any]) -> None: + """Add or refresh an index row (scope_key, map_id).""" + + async def delete_link(self, scope_key: str, map_id: str) -> None: + """Remove an index row if it exists.""" + + async def list_links(self, scope_key: str) -> List[Dict[str, Any]]: + """Index rows for a scope or conversation.""" + + async def close(self) -> None: + """Release connections.""" + + +class InMemoryMapStore: + """A store that keeps everything in memory, with the same all-or-nothing commit as a Cosmos batch.""" + + def __init__(self): + self._partitions: Dict[str, Dict[str, Dict[str, Any]]] = {} + self._etags: Dict[str, str] = {} + self._links: Dict[str, Dict[str, Dict[str, Any]]] = {} + self._lock = threading.Lock() + self._etag_counter = itertools.count(1) + + async def read_map(self, map_id: str) -> Optional[Tuple[Dict[str, Any], str]]: + with self._lock: + doc = self._partitions.get(map_id, {}).get(map_id) + if doc is None: + return None + return copy.deepcopy(doc), self._etags[map_id] + + async def read_item(self, map_id: str, item_id: str) -> Optional[Dict[str, Any]]: + with self._lock: + doc = self._partitions.get(map_id, {}).get(item_id) + return copy.deepcopy(doc) if doc is not None else None + + async def query_features(self, map_id: str, *, dup_keys: Optional[Sequence[str]] = None) -> List[Dict[str, Any]]: + wanted = set(dup_keys) if dup_keys is not None else None + with self._lock: + items = list(self._partitions.get(map_id, {}).values()) + features = [item for item in items if item.get("type") == "feature"] + if wanted is not None: + features = [item for item in features if item.get("status") == "active" and item.get("dup_key") in wanted] + return copy.deepcopy(features) + + async def query_feature_revisions( + self, map_id: str, feature_ids: Sequence[str], as_of_version: int + ) -> List[Dict[str, Any]]: + wanted = set(feature_ids) + with self._lock: + items = list(self._partitions.get(map_id, {}).values()) + return copy.deepcopy([ + item for item in items + if item.get("type") == "feature_rev" + and item.get("feature_id") in wanted + and item.get("valid_from_version", 0) <= as_of_version <= item.get("valid_to_version", -1) + ]) + + async def query_changes(self, map_id: str, since_version: int, limit: int) -> List[Dict[str, Any]]: + with self._lock: + items = list(self._partitions.get(map_id, {}).values()) + changes = sorted( + (item for item in items if item.get("type") == "change" and item.get("version", 0) > since_version), + key=lambda item: item["version"], + ) + return copy.deepcopy(changes[:limit]) + + async def commit( + self, + map_id: str, + map_doc: Dict[str, Any], + expected_etag: Optional[str], + creates: Sequence[Dict[str, Any]], + replaces: Sequence[Dict[str, Any]], + ) -> str: + if 1 + len(creates) + len(replaces) > MAX_BATCH_OPERATIONS: + raise StoreError("Too many operations for one batch.") + with self._lock: + partition = self._partitions.get(map_id, {}) + if expected_etag is None: + if map_id in partition: + raise ConflictError() + elif self._etags.get(map_id) != expected_etag: + raise ConflictError() + create_ids = [doc["id"] for doc in creates] + if len(set(create_ids)) != len(create_ids) or any(item_id in partition for item_id in create_ids): + raise ConflictError() + if any(doc["id"] not in partition for doc in replaces): + raise StoreError("A replaced item doesn't exist.") + + updated = dict(partition) + updated[map_id] = copy.deepcopy(map_doc) + for doc in list(creates) + list(replaces): + updated[doc["id"]] = copy.deepcopy(doc) + self._partitions[map_id] = updated + etag = f'"{next(self._etag_counter)}"' + self._etags[map_id] = etag + return etag + + async def upsert_link(self, row: Dict[str, Any]) -> None: + with self._lock: + self._links.setdefault(row["scope_key"], {})[row["id"]] = copy.deepcopy(row) + + async def delete_link(self, scope_key: str, map_id: str) -> None: + with self._lock: + self._links.get(scope_key, {}).pop(map_id, None) + + async def list_links(self, scope_key: str) -> List[Dict[str, Any]]: + with self._lock: + return copy.deepcopy(list(self._links.get(scope_key, {}).values())) + + async def close(self) -> None: + return None diff --git a/application/map_server/mapserver/tiles.py b/application/map_server/mapserver/tiles.py new file mode 100644 index 000000000..c4bd11be1 --- /dev/null +++ b/application/map_server/mapserver/tiles.py @@ -0,0 +1,112 @@ +# tiles.py +"""Azure Maps raster tiles for the map viewer, fetched with the server's own credentials and cached in memory.""" + +import collections +import logging +import re +import threading +import time +from typing import Any, Dict, Optional, Tuple + +import httpx +from azure.core.exceptions import AzureError + +from .errors import MapServerError, invalid +from .settings import Settings + +LOGGER = logging.getLogger("mapserver.tiles") + +TILE_API_VERSION = "2024-04-01" +TILE_LANGUAGE = "en-US" +TILE_VIEW = "Auto" +TILE_SIZE = 256 +MAX_ZOOM = 22 +TILE_TIMEOUT_SECONDS = 15 +TOKEN_REFRESH_MARGIN_SECONDS = 300 +MAPS_TOKEN_SCOPE = "https://atlas.microsoft.com/.default" +TILESET_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") + + +class TileService: + def __init__(self, settings: Settings, *, http_client: Optional[httpx.AsyncClient] = None, credential: Optional[Any] = None): + self._settings = settings + self._http = http_client + self._owns_http = http_client is None + self._credential = credential + self._cache: "collections.OrderedDict[str, Tuple[bytes, str]]" = collections.OrderedDict() + self._cache_lock = threading.Lock() + self._token = "" + self._token_expires_on = 0.0 + + @property + def configured(self) -> bool: + return bool(self._settings.azure_maps_key or (self._settings.azure_maps_client_id and self._credential)) + + async def get_tile(self, tileset: str, z: int, x: int, y: int) -> Tuple[bytes, str]: + if not TILESET_PATTERN.fullmatch(tileset or ""): + raise invalid("The tileset isn't valid.", code="invalid_tile") + if not (0 <= z <= MAX_ZOOM and 0 <= x < 2 ** z and 0 <= y < 2 ** z): + raise invalid("The tile coordinates are out of range.", code="invalid_tile") + if not self.configured: + raise MapServerError(404, "tiles_unavailable", "Map tiles aren't configured, so the map shows a black background.") + + cache_key = f"{tileset}/{z}/{x}/{y}" + with self._cache_lock: + cached = self._cache.get(cache_key) + if cached is not None: + self._cache.move_to_end(cache_key) + return cached + + params = { + "api-version": TILE_API_VERSION, + "tilesetId": tileset, + "zoom": z, + "x": x, + "y": y, + "tileSize": TILE_SIZE, + "language": TILE_LANGUAGE, + "view": TILE_VIEW, + } + headers = await self._auth_headers() + try: + response = await self._client().get( + f"{self._settings.azure_maps_endpoint}/map/tile", params=params, headers=headers, timeout=TILE_TIMEOUT_SECONDS, + ) + except httpx.HTTPError as exc: + LOGGER.warning(f"[MAP_SERVER_TILES] The tile request failed ({type(exc).__name__}).") + raise MapServerError(502, "tile_upstream_error", "The map tile service didn't respond.") from exc + + media_type = response.headers.get("content-type", "").split(";")[0].strip().lower() + if response.status_code != 200 or not media_type.startswith("image/"): + LOGGER.warning(f"[MAP_SERVER_TILES] The tile service answered {response.status_code} ({media_type or 'no type'}).") + raise MapServerError(502, "tile_upstream_error", "The map tile service returned an error.") + + tile = (response.content, media_type) + if self._settings.tile_cache_entries: + with self._cache_lock: + self._cache[cache_key] = tile + while len(self._cache) > self._settings.tile_cache_entries: + self._cache.popitem(last=False) + return tile + + def _client(self) -> httpx.AsyncClient: + if self._http is None: + self._http = httpx.AsyncClient() + return self._http + + async def _auth_headers(self) -> Dict[str, str]: + if self._settings.azure_maps_key: + return {"subscription-key": self._settings.azure_maps_key} + if time.time() > self._token_expires_on - TOKEN_REFRESH_MARGIN_SECONDS: + try: + access = await self._credential.get_token(MAPS_TOKEN_SCOPE) + except AzureError as exc: + LOGGER.error(f"[MAP_SERVER_TILES] Couldn't get an Azure Maps token ({type(exc).__name__}).") + raise MapServerError(502, "tile_auth_failed", "The map server couldn't sign in to Azure Maps.") from exc + self._token, self._token_expires_on = access.token, float(access.expires_on) + return {"Authorization": f"Bearer {self._token}", "x-ms-client-id": self._settings.azure_maps_client_id} + + async def close(self) -> None: + if self._owns_http and self._http is not None: + await self._http.aclose() + self._http = None diff --git a/application/map_server/mapserver/validation.py b/application/map_server/mapserver/validation.py new file mode 100644 index 000000000..49f514f44 --- /dev/null +++ b/application/map_server/mapserver/validation.py @@ -0,0 +1,366 @@ +# validation.py +"""Turns what a caller sends into the map's stored shapes, enforcing limits and safe links.""" + +import math +import re +from datetime import datetime +from typing import Any, Dict, List, Optional, Sequence, Tuple +from urllib.parse import urlparse + +from .errors import invalid, not_found + +TITLE_MAX = 160 +MAP_DESCRIPTION_MAX = 1000 +PHASE_NAME_MAX = 80 +PHASE_DESCRIPTION_MAX = 500 +LABEL_MAX = 160 +DESCRIPTION_MAX = 1000 +CATEGORY_MAX = 40 +FIELD_LIMIT = 12 +FIELD_LABEL_MAX = 60 +FIELD_VALUE_MAX = 300 +IMAGE_URL_MAX = 2048 +CAPTION_MAX = 200 +SOURCE_SYSTEM_MAX = 80 +SOURCE_RECORD_MAX = 120 +REASON_MAX = 500 +OBSERVED_AT_MAX = 64 +MAX_VERTICES = 2000 +MIN_LINE_WIDTH = 1 +MAX_LINE_WIDTH = 12 +DEFAULT_CATEGORY = "general" +FEATURE_KINDS = ("point", "path", "area") +GEOMETRY_KINDS = {"Point": "point", "LineString": "path", "Polygon": "area"} +GEOMETRY_KEYS = ("geometry", "latitude", "longitude", "lat", "lon", "lng", "coordinates") +CONTENT_FIELDS = ("geometry", "label", "category", "description", "observed_at", "source", "media", "fields", "style", "dup_key") + +UNSAFE_URL_CHARACTERS = re.compile(r"[\s\"'<>\\]") +CATEGORY_UNSAFE = re.compile(r"[^a-z0-9_-]+") +COLOR_PATTERN = re.compile( + r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$" + r"|^rgba?\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*(?:,\s*(?:0|1|0?\.\d+)\s*)?\)$" +) +TILESET_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +MAP_ID_PATTERN = re.compile(r"^map-[0-9a-f]{20}$") +FEATURE_ID_PATTERN = re.compile(r"^F-\d{4,7}$") +PHASE_ID_PATTERN = re.compile(r"^P\d{1,3}$") +CONVERSATION_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]{1,128}$") + + +def normalize_text(value: Any, max_length: int, *, multiline: bool = False) -> str: + if value is None or isinstance(value, (dict, list)): + return "" + text = str(value).strip() if multiline else " ".join(str(value).split()) + return text[:max_length] + + +def require_text(value: Any, max_length: int, message: str) -> str: + text = normalize_text(value, max_length) + if not text: + raise invalid(message) + return text + + +def require_map_id(map_id: str) -> str: + if not MAP_ID_PATTERN.fullmatch(str(map_id or "")): + raise not_found() + return map_id + + +def require_feature_id(feature_id: str) -> str: + if not FEATURE_ID_PATTERN.fullmatch(str(feature_id or "")): + raise not_found("feature_not_found", "Feature not found.") + return feature_id + + +def require_phase_id(phase_id: str) -> str: + if not PHASE_ID_PATTERN.fullmatch(str(phase_id or "")): + raise not_found("phase_not_found", "Phase not found.") + return phase_id + + +def normalize_conversation_id(value: Any) -> str: + conversation_id = str(value or "").strip() + if conversation_id and not CONVERSATION_ID_PATTERN.fullmatch(conversation_id): + raise invalid("conversation_id isn't valid.") + return conversation_id + + +def normalize_tileset(value: Any) -> str: + tileset = str(value or "").strip() + if not TILESET_PATTERN.fullmatch(tileset): + raise invalid("The basemap must be an Azure Maps tileset ID such as microsoft.base.road.") + return tileset + + +def normalize_color(value: Any) -> str: + color = str(value or "").strip() + return color if color and COLOR_PATTERN.fullmatch(color) else "" + + +def normalize_category(value: Any) -> str: + category = CATEGORY_UNSAFE.sub("_", str(value or "").strip().lower()).strip("_")[:CATEGORY_MAX] + return category or DEFAULT_CATEGORY + + +def normalize_https_url(value: Any) -> str: + """An https link the viewer can load, or an empty string.""" + candidate = str(value or "").strip() + if not candidate or len(candidate) > IMAGE_URL_MAX or UNSAFE_URL_CHARACTERS.search(candidate): + return "" + try: + parsed = urlparse(candidate) + host = parsed.hostname + except ValueError: + return "" + if parsed.scheme.lower() != "https" or not host: + return "" + return candidate + + +def normalize_fields(raw_fields: Any) -> List[Dict[str, str]]: + if isinstance(raw_fields, dict): + pairs = list(raw_fields.items()) + elif isinstance(raw_fields, list): + pairs = [(item.get("label") or item.get("name"), item.get("value")) for item in raw_fields if isinstance(item, dict)] + else: + return [] + + fields: List[Dict[str, str]] = [] + for raw_label, raw_value in pairs: + if isinstance(raw_value, bool): + raw_value = "Yes" if raw_value else "No" + if not isinstance(raw_value, (str, int, float)): + continue + label = normalize_text(raw_label, FIELD_LABEL_MAX) + value = normalize_text(raw_value, FIELD_VALUE_MAX) + if label and value: + fields.append({"label": label, "value": value}) + if len(fields) == FIELD_LIMIT: + break + return fields + + +def normalize_observed_at(value: Any, where: str) -> Optional[str]: + if value in (None, ""): + return None + text = str(value).strip() + if len(text) > OBSERVED_AT_MAX: + raise invalid(f"{where}.observed_at must be an ISO 8601 date and time.") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00").replace("z", "+00:00")) + except ValueError as exc: + raise invalid(f"{where}.observed_at must be an ISO 8601 date and time.") from exc + return parsed.isoformat(timespec="seconds") + + +def _finite(value: Any, where: str) -> float: + if isinstance(value, bool): + raise invalid(f"{where} must be a number.") + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise invalid(f"{where} must be a number.") from exc + if not math.isfinite(number): + raise invalid(f"{where} must be a finite number.") + return number + + +def _coordinate(point: Any, where: str) -> List[float]: + if not isinstance(point, (list, tuple)) or len(point) < 2: + raise invalid(f"{where} must be a [longitude, latitude] pair.") + longitude = _finite(point[0], f"{where}[0]") + latitude = _finite(point[1], f"{where}[1]") + if not (-180 <= longitude <= 180 and -90 <= latitude <= 90): + raise invalid(f"{where} is out of range: longitude must be -180 to 180 and latitude -90 to 90.") + return [round(longitude, 6), round(latitude, 6)] + + +def _coordinate_list(coordinates: Any, where: str) -> List[List[float]]: + if not isinstance(coordinates, (list, tuple)) or not coordinates: + raise invalid(f"{where}.coordinates must be a list of [longitude, latitude] pairs.") + first = coordinates[0] + if isinstance(first, (list, tuple)) and first and isinstance(first[0], (list, tuple)): + coordinates = first + if len(coordinates) > MAX_VERTICES + 1: + raise invalid(f"{where} has more than {MAX_VERTICES} points.") + return [_coordinate(point, f"{where}.coordinates[{index}]") for index, point in enumerate(coordinates)] + + +def normalize_geometry(raw: Dict[str, Any], where: str) -> Tuple[str, Dict[str, Any]]: + """The feature kind and its GeoJSON geometry, from GeoJSON, latitude and longitude, or coordinates.""" + kind = str(raw.get("kind") or "").strip().lower() + if kind and kind not in FEATURE_KINDS: + raise invalid(f"{where}.kind must be point, path or area.") + + geometry = raw.get("geometry") + latitude = raw.get("latitude", raw.get("lat")) + longitude = raw.get("longitude", raw.get("lon", raw.get("lng"))) + if isinstance(geometry, dict): + geometry_kind = GEOMETRY_KINDS.get(str(geometry.get("type") or "")) + if not geometry_kind: + raise invalid(f"{where}.geometry.type must be Point, LineString or Polygon.") + if kind and kind != geometry_kind: + raise invalid(f"{where}.kind doesn't match its geometry.") + kind, coordinates = geometry_kind, geometry.get("coordinates") + elif latitude is not None or longitude is not None: + if kind and kind != "point": + raise invalid(f"{where} uses latitude and longitude, which only a point can have.") + kind, coordinates = "point", [longitude, latitude] + elif raw.get("coordinates") is not None: + if kind not in ("path", "area"): + raise invalid(f"{where} needs kind 'path' or 'area' to go with its coordinates.") + coordinates = raw.get("coordinates") + else: + raise invalid(f"{where} needs a geometry, latitude and longitude, or coordinates.") + + if kind == "point": + return "point", {"type": "Point", "coordinates": _coordinate(coordinates, f"{where}.coordinates")} + points = _coordinate_list(coordinates, where) + if kind == "path": + if len(points) < 2: + raise invalid(f"{where} needs at least two points.") + return "path", {"type": "LineString", "coordinates": points} + if points[0] != points[-1]: + points.append(list(points[0])) + if len({tuple(point) for point in points}) < 3: + raise invalid(f"{where} needs at least three distinct points.") + return "area", {"type": "Polygon", "coordinates": [points]} + + +def normalize_source(raw: Dict[str, Any]) -> Optional[Dict[str, str]]: + raw_source = raw.get("source") if isinstance(raw.get("source"), dict) else {} + system = normalize_text(raw_source.get("system") or raw.get("source_system"), SOURCE_SYSTEM_MAX) + record_id = normalize_text(raw_source.get("record_id") or raw.get("source_record_id"), SOURCE_RECORD_MAX) + url = normalize_https_url(raw_source.get("url") or raw.get("source_url")) + source = {key: value for key, value in (("system", system), ("record_id", record_id), ("url", url)) if value} + return source or None + + +def normalize_media(raw: Dict[str, Any]) -> Tuple[Optional[Dict[str, str]], bool]: + """The photo for a feature, and whether a photo link was left out because it wasn't a safe https link.""" + raw_media = raw.get("media") if isinstance(raw.get("media"), dict) else {} + raw_url = raw_media.get("image_url") or raw.get("image_url") + if not raw_url: + return None, False + image_url = normalize_https_url(raw_url) + if not image_url: + return None, True + media = {"image_url": image_url} + caption = normalize_text(raw_media.get("caption") or raw.get("image_caption"), CAPTION_MAX) + if caption: + media["caption"] = caption + return media, False + + +def normalize_style(raw: Dict[str, Any], kind: str) -> Dict[str, Any]: + raw_style = raw.get("style") if isinstance(raw.get("style"), dict) else {} + style: Dict[str, Any] = {} + color = normalize_color(raw_style.get("color") or raw.get("color") or raw.get("stroke_color")) + if color: + style["color"] = color + if kind == "path": + raw_width = raw_style.get("line_width", raw.get("line_width")) + if raw_width not in (None, "") and not isinstance(raw_width, bool): + try: + style["line_width"] = max(MIN_LINE_WIDTH, min(MAX_LINE_WIDTH, int(float(raw_width)))) + except (TypeError, ValueError): + pass + if kind == "area": + fill = normalize_color(raw_style.get("fill_color") or raw.get("fill_color")) + if fill: + style["fill_color"] = fill + return style + + +def duplicate_key(kind: str, source: Optional[Dict[str, str]]) -> Optional[str]: + if not source or not source.get("record_id"): + return None + return f"{kind}|{source.get('system', '').lower()}|{source['record_id']}" + + +def normalize_feature(raw: Any, index: int, where: Optional[str] = None) -> Tuple[Dict[str, Any], bool]: + """A feature's stored content, and whether its photo link was left out.""" + where = where or f"features[{index}]" + if not isinstance(raw, dict): + raise invalid(f"{where} must be an object.") + kind, geometry = normalize_geometry(raw, where) + label = require_text(raw.get("label") or raw.get("title") or raw.get("name"), LABEL_MAX, f"{where} needs a label.") + source = normalize_source(raw) + media, image_dropped = normalize_media(raw) + content = { + "kind": kind, + "geometry": geometry, + "label": label, + "category": normalize_category(raw.get("category")), + "description": normalize_text(raw.get("description"), DESCRIPTION_MAX, multiline=True), + "observed_at": normalize_observed_at(raw.get("observed_at"), where), + "source": source, + "media": media, + "fields": normalize_fields(raw.get("fields")), + "style": normalize_style(raw, kind), + "dup_key": duplicate_key(kind, source), + } + return content, image_dropped + + +def feature_to_input(feature: Dict[str, Any]) -> Dict[str, Any]: + """A stored feature in the shape normalize_feature accepts, so a change can be validated in full.""" + raw = { + "kind": feature["kind"], + "geometry": feature["geometry"], + "label": feature.get("label"), + "category": feature.get("category"), + "description": feature.get("description"), + "observed_at": feature.get("observed_at"), + "source": feature.get("source") or {}, + "fields": feature.get("fields") or [], + "style": feature.get("style") or {}, + } + if feature.get("media"): + raw["media"] = dict(feature["media"]) + return raw + + +def apply_changes(feature: Dict[str, Any], changes: Dict[str, Any]) -> Tuple[Dict[str, Any], bool]: + """A feature's content with the requested changes applied and validated.""" + if not isinstance(changes, dict) or not changes: + raise invalid("changes must name at least one field to change.") + if "kind" in changes and str(changes.get("kind") or "").strip().lower() != feature["kind"]: + raise invalid("A feature can't change kind. Retract it and add a new one.") + merged = feature_to_input(feature) + if any(key in changes for key in GEOMETRY_KEYS): + merged.pop("geometry", None) + if "media" in changes and not changes.get("media"): + merged.pop("media", None) + changes = {key: value for key, value in changes.items() if key != "media"} + if any(key in changes for key in ("image_url", "image_caption")): + merged.pop("media", None) + merged.update(changes) + merged["kind"] = feature["kind"] + return normalize_feature(merged, 0, where="changes") + + +def content_equal(left: Dict[str, Any], right: Dict[str, Any]) -> bool: + return all(left.get(name) == right.get(name) for name in CONTENT_FIELDS) + + +def parse_bbox(value: Optional[str]) -> Optional[Tuple[float, float, float, float]]: + if not value: + return None + parts = [part.strip() for part in str(value).split(",")] + if len(parts) != 4: + raise invalid("bbox must be min_longitude,min_latitude,max_longitude,max_latitude.") + west, south, east, north = (_finite(part, "bbox") for part in parts) + if west > east or south > north: + raise invalid("bbox must be min_longitude,min_latitude,max_longitude,max_latitude.") + return west, south, east, north + + +def feature_points(geometry: Dict[str, Any]) -> Sequence[Sequence[float]]: + coordinates = geometry.get("coordinates") or [] + if geometry.get("type") == "Point": + return [coordinates] + if geometry.get("type") == "Polygon": + return coordinates[0] if coordinates else [] + return coordinates diff --git a/application/map_server/requirements.txt b/application/map_server/requirements.txt new file mode 100644 index 000000000..6b538b258 --- /dev/null +++ b/application/map_server/requirements.txt @@ -0,0 +1,10 @@ +# requirements.txt (map server) +fastapi==0.141.1 +starlette==1.3.1 +uvicorn==0.50.2 +pydantic==2.11.10 +httpx==0.28.1 +pyjwt[crypto]==2.13.0 +azure-cosmos==4.9.0 +azure-identity==1.24.0 +aiohttp==3.14.3 diff --git a/application/single_app/config.py b/application/single_app/config.py index 17903e612..db6cdd541 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.252" +VERSION = "0.261.254" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/docs/explanation/features/SHARED_MAP_SERVER_DESIGN.md b/docs/explanation/features/SHARED_MAP_SERVER_DESIGN.md new file mode 100644 index 000000000..1053178d0 --- /dev/null +++ b/docs/explanation/features/SHARED_MAP_SERVER_DESIGN.md @@ -0,0 +1,504 @@ +# Shared map server design + +Prepared: **2026-10-04**. Repository: **microsoft/simplechat**. + +Planned against version: **0.261.233** (`application\single_app\config.py`). This design is documentation only, so it +doesn't change the version. Each build phase records the version it ships in. + +Planning branch: `paullizer-map-server`. Status: **Phase 1 (map server core) built in 0.261.254**, scoped as a proof +of concept. No GitHub issue. + +Dependencies: the Azure Maps action (`semantic_kernel_plugins/azure_maps_openlayers_plugin.py`, +`functions_azure_maps.py`), the Simple Chat action (`semantic_kernel_plugins/simplechat_plugin.py`, +`functions_simplechat_operations.py`), the workflow visualization mirror (`functions_workflow_runner.py`), and the V2 +UI (`application\v2_ui`). + +## Why + +Agents draw maps today with `create_map_visualization`. Each call returns a self-contained snapshot: a `map_payload` +holding every marker, path and area, a view, and a tile URL that wraps the action's Azure Maps key in an encrypted, +expiring token. The snapshot is stored in that message's `agent_citations`, and each message draws its own map. + +That works for a one-off map. It doesn't work when a team builds up a picture over a long conversation, such as an +investigation, an incident response, a field survey or a logistics plan: + +- Nothing connects one message's map to the next. To show the whole picture, the model has to send every point again, + which costs tokens and lets points drift, drop out or change between messages. +- Nobody can tell what was learned when. A map shows the current points, not which stage of the work found them. +- People read a long thread of separate maps instead of one current picture. +- A workflow's map reaches its created conversation as a copy, not as something the team keeps adding to. + +The shared map server keeps **one persistent map per piece of work**, a common operating picture that agents add to over +time. Every addition is tagged with the **phase** of the work that produced it. Each message shows what it changed, and +one live map shows everything, filterable by phase and replayable in order. + +## Decisions + +Made 2026-10-04: + +1. **Standalone service.** The map server is its own deployable service with a REST API. SimpleChat reaches it through + a new action and never calls Azure Maps directly for shared maps. +2. **Ownership.** A map belongs to a person or a group, and can be linked to any number of conversations in that scope. + A workflow run and the team's conversation can build one map, and the map outlives any one conversation. +3. **Messages show a change card; the conversation has one live map.** A message no longer embeds a full map. It shows + what it added, updated or retracted and in which phase, and opens the live map at that point. +4. **Design first.** This document is reviewed before any code is written. + +Made 2026-10-04 in review. This is a proof of concept, so version 1 stays small: + +5. **SimpleChat only.** SimpleChat is the map server's only client. Version 1 has no MCP endpoint and no other clients. +6. **Only agents write.** Agents, including workflow runs, create maps, start phases, link conversations, and add, + update or retract features. People view the map, and the panel has no editing. +7. **No sharing.** A map stays in its owner's scope. It isn't shared across groups or with other users, and it links + only to conversations in its own scope. +8. **No deletion rules yet.** Retention, archiving and deleting maps are out of scope for now. +9. **Black background without Azure Maps.** When map tiles aren't available, the map draws its features on a plain + black background. No other basemaps are offered. +10. **Agents use the shared map instead of the Azure Maps action.** An agent that maps its work gets the Shared map + action in place of the Azure Maps action (`create_map_visualization`). The Azure Maps action stays in SimpleChat + for agents that don't use shared maps. + +## Concepts + +| Concept | What it is | +|---|---| +| Map | A titled, persistent picture owned by a user or a group. It has phases, features, a change log, a monotonically increasing `version`, a basemap, and links to conversations. | +| Phase | A named, ordered, colored stage of the work, such as "Initial enrichment" or "Follow-up". One phase is current; writes without a phase go to it. A phase records who started it and when, and can be closed. | +| Feature | One thing on the map: a point, a path or an area, with a label, category, time observed, source record, optional photo and labelled fields. It records the phase that added it and who added it. | +| Change | One entry in the map's append-only log: what was added, updated or retracted, by whom, in which phase, from which conversation, message or workflow run, and the map version after the change. | +| Link | A map shown in a conversation. Linking lets the conversation's participants view the map; it doesn't change who can edit it. A map links only to conversations in its own scope. | + +Features are never deleted by agents. They're **retracted**, with a reason, so the history stays explainable and an +earlier version of the map can still be shown. + +## Architecture + +```mermaid +flowchart LR + subgraph Browser + V2[V2 chat: change cards and live map panel] + CL[Classic chat: change cards and map page] + end + subgraph SimpleChat["SimpleChat (Flask)"] + ACT[Shared map action] + RTS[Map routes: snapshot, events, tiles] + end + subgraph MapServer["Map server (container app)"] + API[REST API /v1] + TIL[Tile proxy] + end + DB[(Cosmos DB: maps, map_links)] + AZM[Azure Maps] + + V2 --> RTS + CL --> RTS + ACT -->|managed identity + actor context| API + RTS -->|managed identity + actor context| API + RTS --> TIL + API --> DB + TIL -->|managed identity| AZM +``` + +- **Service.** Python with FastAPI: async, with a generated OpenAPI document. An MCP endpoint can be mounted in the same + app later. Source in `application/map_server/`, its own container image. +- **Hosting.** Azure Container Apps or an App Service container, with a managed identity. Ingress is internal when + SimpleChat can reach the environment's network, and external otherwise. Either way every request needs an Entra token + from an allowed caller. +- **Browser traffic goes through SimpleChat.** The browser talks only to SimpleChat, which checks access and proxies + map reads, live events and tiles to the map server. This keeps the map server private, needs no CORS or CSP change, + and works in deployments that use private endpoints. A direct browser-to-server option is listed under alternatives. + +## Authentication and authorization + +### SimpleChat to the map server + +SimpleChat calls the map server with its managed identity and an app role on the map server's app registration +(proposed name `MapServer.ActOnBehalf`). Each call carries an **actor context**: the user ID and display name, the scope +it acts in (`user:` or `group:`), and the conversation, message or workflow run it came from. + +The map server accepts an actor context only from callers holding that role. SimpleChat owns users, groups and +conversations, so it checks membership and roles before it calls. The map server then checks that the asserted scope +matches the map's owner scope. + +On-behalf-of token exchange isn't enough by itself: scheduled workflow runs have no user session to exchange, and group +membership lives in SimpleChat. It can be added later for interactive calls. + +### Why the generic MCP action isn't enough + +SimpleChat's MCP action sends one fixed credential per action: a bearer token, key, basic credentials or the managed +identity. The server can't tell which user or group is calling, so it can't enforce per-user or per-group access, and +every write would be attributed to SimpleChat. MCP tool results also pass through the action's result text policy, so a +structured change card can't be returned reliably. + +### Access rules + +- **Personal map.** Its owner can view it, and agents write to it for the owner. +- **Group map.** Group members can view it. Agents write for the person who invoked them, so a write needs that person's + group role to allow contributing, the same roles that can add content to the group workspace. +- **Linked conversation.** Everyone who can open a linked conversation can view the map while the link exists. A + personal map links only to its owner's conversations, and a group map only to that group's conversations. +- **Scope follows the conversation.** A map opened for a group conversation is a group map, and one opened for a + personal conversation is a personal map. +- **Workflow runs.** A run acts as the workflow's Run-as user. A run that creates a group conversation opens the map for + that conversation, so it writes to the group's map, and the Run-as user needs a contributing role in that group. + +### Tiles + +The map server fetches Azure Maps tiles with its managed identity, caches them, and SimpleChat proxies them to the +browser. No Azure Maps key reaches the browser or a token. Today's tile URL carries the key in an encrypted, expiring +token that's refreshed whenever messages load; shared maps don't need that. + +The map viewer's background is black. When Azure Maps isn't configured or a tile can't be fetched, the features still +draw, on black. + +## Data model + +Cosmos DB for NoSQL, in its own database (`mapserver`), so deployments reuse an account type they already run. + +**Container `maps`, partition key `/map_id`.** Every item for one map shares a partition, so reads and writes for a map +never cross partitions. Items carry a `type` discriminator and a `schema_version`: + +- `map`: title, description, `owner_scope`, basemap (an Azure Maps tileset), `current_phase_id`, `version`, counts, + created and updated stamps, and two short lists kept on the map so they change under its ETag: + - `phases`: name, description, color, order, status, started by and at, started and closed versions. Up to 50. + - `links`: the conversations the map is linked to, with who linked them and when. Up to 50. +- `feature`: the current state of one feature (example below). +- `feature_rev`: an earlier state of an updated feature, valid from one version to another, so the map can be shown as + of an earlier version. +- `change`: one log entry, with `version`, action, phase, actor, origin (conversation, message, run), the IDs it + added, updated or retracted, and up to five labels. + +Features are separate items, not embedded in the map document, so a large map never nears the 2 MB item limit and each +feature updates on its own. At about 2 KB a feature, a map with 10,000 features and its log is well under the 20 GB +logical partition limit. + +**Container `map_links`, partition key `/scope_key`.** Small rows that answer "which maps can I see here" in one +partition: `user:` and `group:` rows for ownership, and `conversation:` rows for links. Each row holds the +`map_id`. A row is written before the map changes and every read checks the map itself, so a failed write can leave an +extra row but never hides a map or shows one outside its scope. + +```json +{ + "type": "feature", + "schema_version": 1, + "id": "F-0007", + "map_id": "map-3f6c1a9e2b7d4c8e0a15", + "kind": "point", + "geometry": { "type": "Point", "coordinates": [-73.9368, 40.7393] }, + "label": "Storage facility gate", + "category": "location", + "description": "Entry recorded by the facility's gate log.", + "observed_at": "2026-10-04T06:36:00-04:00", + "phase_id": "P2", + "last_phase_id": "P2", + "source": { "system": "records", "record_id": "R-004512" }, + "dup_key": "point|records|R-004512", + "media": { "image_url": "https://example.org/still.png", "caption": "Gate camera" }, + "fields": [{ "label": "Unit", "value": "214" }], + "style": {}, + "status": "active", + "created_version": 7, + "version_start": 7, + "retracted_version": null, + "retract_reason": null, + "revision": 1, + "created_by": { "user_id": "user-123", "agent": { "name": "Field analyst" }, "conversation_id": "conv-1" }, + "created_at": "2026-10-04T10:22:41Z" +} +``` + +### Versions and concurrency + +Each write is one Cosmos transactional batch in the map's partition. It replaces the map document with the next +`version` on the condition that its ETag hasn't changed, writes the features and any `feature_rev` items, and creates the +`change`. When another write lands first the ETag check fails, and the write starts again from a fresh read, up to six +times, so versions never skip or repeat. Linking or unlinking a conversation also goes through the ETag check but +doesn't add a version, because the map's content doesn't change. + +Showing the map **as of version N** means: features with `created_version <= N` that weren't retracted at N, each in +the state it had at N. A feature updated after N is shown from the `feature_rev` that was current at N. + +### Duplicates + +A feature is a duplicate when it repeats the `kind`, `source.system` and `source.record_id` of an active feature on the +same map. The caller chooses whether a duplicate is skipped or updates the existing feature. Features without a source +record are never treated as duplicates. + +### Limits + +These are checked on the server: + +- Up to 40 features per call, configurable up to 49. A Cosmos transactional batch holds 100 operations, and updating a + feature takes two: its revision and its new state. +- Up to 10,000 active features, 50 phases and 50 linked conversations per map. +- Labels up to 160 characters and descriptions up to 1,000. +- Up to 12 labelled fields, with 60-character labels and 300-character values. +- Photos must be https links of up to 2,048 characters, with captions up to 200. This matches the current map action. +- Up to 2,000 vertices per path or area. +- Latitude between -90 and 90, longitude between -180 and 180. + +## Map server API (version 1) + +| Method and path | Purpose | +|---|---| +| `POST /v1/maps` | Create a map (title, owner scope, basemap, optional first phase, optional conversation to link). | +| `GET /v1/maps?scope=` | List maps for a scope, read from `map_links`. | +| `GET /v1/maps/{id}` | Map, phases, counts and current version. | +| `PATCH /v1/maps/{id}` | Title, description or basemap, with `If-Match`. | +| `POST /v1/maps/{id}/links`, `DELETE /v1/maps/{id}/links/{conversation_id}` | Link or unlink a conversation. | +| `POST /v1/maps/{id}/phases` | Start a phase and make it current. | +| `PATCH /v1/maps/{id}/phases/{phase_id}` | Rename a phase or change its description or color. A phase closes when the next one starts. | +| `POST /v1/maps/{id}/features:batch` | Add features to a phase, with a duplicate rule. Returns the new version and what changed. | +| `PATCH /v1/maps/{id}/features/{feature_id}` | Update a feature, with `If-Match`. | +| `POST /v1/maps/{id}/features/{feature_id}:retract` | Retract a feature with a reason. | +| `GET /v1/maps/{id}/features` | Features, filtered by phase, category, `since_version`, `as_of_version` or bounding box. Paged. | +| `GET /v1/maps/{id}/snapshot` | A render-ready payload, as of a version and for chosen phases, in the shape today's `map_payload` uses, so existing renderers can draw it. | +| `GET /v1/maps/{id}/changes` | The change log since a version. Paged. | +| `GET /v1/maps/{id}/events` | Server-sent events, one per change: version, phase and counts. | +| `GET /v1/tiles/{tileset}/{z}/{x}/{y}` | Cached Azure Maps tiles. Answers 404 when Azure Maps isn't configured. | + +## SimpleChat integration + +### New action type: Shared map + +A native action type, `shared_map`, with a definition schema, the admin and workspace action forms, and a reference page +at `docs/reference/actions/shared-map.md`. It's configured with the map server URL, the audience for its managed +identity token, a default basemap and limits. + +The action resolves the user, group and conversation from the invocation, the same way the Simple Chat action's +`*_for_current_user` operations do. The model never supplies a user or group. It can only name a map or a conversation +the acting user can open, and the action checks that before every call. + +| Function | What it does | +|---|---| +| `open_map(title, conversation_id, create_if_missing)` | Use the map linked to a conversation: this one by default, or one the run created. Otherwise find a map with that title in the conversation's scope, or create one, and link it. | +| `start_phase(name, description)` | Start the next phase. | +| `add_to_map(features_json, phase, on_duplicate)` | Add points, paths and areas. `features_json` accepts today's `locations_json`, `paths_json` and `areas_json` shapes, so agent instructions carry over. | +| `update_map_feature(feature_id, changes_json)` | Correct a feature. | +| `retract_map_feature(feature_id, reason)` | Retract a feature. | +| `get_map_summary()` | Phases, counts, recent changes and categories, so an agent can see what's already known before adding. | +| `show_map(as_of, phases)` | Point the conversation at the live map, at a version or for chosen phases. | + +The function descriptions teach the working pattern: + +- Start a phase when the work moves to a new stage. +- Add each place once, with its source record. +- Update a feature instead of adding it again. +- Retract a feature instead of deleting it. + +Each write returns a small result with `render_type: "shared_map_change"`: map ID and title, version, phase (name and +color), counts added, updated and retracted, a few labels, and the change ID. It carries no feature data, so it stays +small in `agent_citations`. Because it has a `render_type`, the workflow visualization mirror +(`_is_visualization_citation`) already copies it into conversations a workflow creates. + +`create_map_visualization` stays as it is for agents that don't use shared maps (decision 10). + +### SimpleChat routes + +These are new Flask routes. Each one carries `@swagger_route(security=get_auth_security())`, requires sign-in, checks +access, and is added to the route policy tests. + +- `GET /api/conversations//maps` lists the maps linked to a conversation the user can open. +- `GET /api/maps//snapshot` proxies a snapshot after checking scope membership or a conversation link. +- `GET /api/maps//events` proxies the event stream, using the existing streaming response pattern. +- `GET /api/maps//tiles////` proxies tiles. + +The routes are read-only. Agents link maps to conversations through the action. + +### V2 UI + +- **Change card** (`SharedMapChangeCard.tsx`). It shows the map title and a phase chip in the phase's color, with + counts such as "+12 places, 1 route updated, 2 retracted" and a few labels. **Open map at this point** opens the panel + at that version. It replaces the full inline map in each message. +- **Live map panel** (`SharedMapPanel.tsx`). A docked panel, with a full-screen operations view, showing the + conversation's linked map. + - It renders with the vendored OpenLayers that `loadOpenLayers` already loads. + - Each phase is a layer with its own toggle and color. You can show one phase, every phase up to one, or what one + phase added compared with the one before. + - A legend lists the categories. + - A timeline replays the map by version or by time observed. + - A feature popup shows its photo, fields, source, phase and who added it. + - Changes made since you last looked are highlighted. + - Live updates arrive as server-sent events, the same way `collaborationEvents.ts` follows collaboration conversations. +- **Conversation header.** A map button, with a badge when the map changes. + +The panel is view-only. All feature text is rendered as text, never as HTML. + +### Classic UI + +The change card renders as a compact card that opens a full-page viewer at `/maps/`. The docked panel is V2 +only. + +### Workflows + +A run writes as its Run-as user. When it creates a conversation through the Simple Chat action, it opens the map for +that conversation, so the map takes the conversation's scope. The run's change cards, mirrored into that conversation, +open the same live map. A scheduled workflow can start a phase per run, named by the agent or after the run time. + +## Phases + +Phases are what turn a series of updates into an operating picture you can explain. + +- **Ordered and named.** An agent starts a new phase when the work changes stage. Phases keep their order, so later + work never hides earlier work. +- **Attributed.** Every feature records the phase that added it, and every change records the phase it happened in. An + update in phase 3 to something first found in phase 1 shows up in both places. +- **Viewable.** You can show each phase as a layer, show the map as it stood at the end of a phase, or compare what one + phase added with the one before it. + +## Security + +- No Azure Maps key in the browser, in tokens or in messages. +- Inputs are validated on the server: coordinates, https-only photos with the current action's unsafe-character check, + and length and count limits. +- Access is checked on every read and write in SimpleChat and again in the map server. Map and feature IDs aren't + secrets. +- The change log records the actor, the user it acted for, and the conversation or run, for audit. +- Ingress is internal where SimpleChat can reach it, and external otherwise. Every request except the health check needs + an Entra token from an allowed caller. +- Errors return a stable code and a safe message, never exception text, and request bodies are capped at 1 MB. +- The map server logs to standard output with `[MAP_SERVER]`, `[MAP_SERVER_AUTH]`, `[MAP_SERVER_STORE]` and + `[MAP_SERVER_TILES]` tags. It doesn't use SimpleChat's `log_event`, which needs SimpleChat's own configuration. + +## Deployment + +- A new image, `simplechat-map-server`, in the same environment as SimpleChat. +- The map server's managed identity needs data access to the `mapserver` database and read access to Azure Maps. +- SimpleChat gets settings for the map server URL and token audience, and its identity gets the map server's app role. +- The admin setting `enable_shared_maps` is off by default. It's documented on the relevant admin settings page and + claimed in `docs/_data/features.yml`, per the documentation coverage rules. +- For the proof of concept, a script deploys the map server next to SimpleChat. Deployer modules for Bicep, Terraform + and azd come later. + +## Build plan + +Each phase is its own change and PR, and records the version it ships in. + +| Phase | Scope | Depends on | +|---|---|---| +| 0 | This design, reviewed and agreed. **Done.** | None | +| 1 | Map server core: data model, REST API, actor-context and role checks, Cosmos store, tile proxy, OpenAPI, local run, deploy script, tests. **Built in 0.261.254.** | 0 | +| 2 | Shared map action, read-only SimpleChat routes, change cards in V2 and classic, workflow linking, admin setting, docs. | 1 | +| 3 | V2 live map panel: phase layers, timeline replay, live updates. | 2 | + +### Later + +Out of scope for the proof of concept: + +- An MCP endpoint (Streamable HTTP at `/mcp`) for clients outside SimpleChat, with tools that mirror the REST API. +- Editing the map from the panel. +- Sharing maps across groups or with other users. +- Retention, archiving and deletion. +- Rate limiting per caller and per person. +- Other basemaps, including for clouds without Azure Maps. +- GeoJSON and KML export, a static map image for documents, geocoding and routing. +- Deployer modules and private networking. + +## Phase 1 as built + +Phase 1 shipped in version **0.261.254**. The map server reports its own version, starting at `0.1.0`. + +### Code layout + +All under `application/map_server/`: + +| Path | What it holds | +|---|---| +| `mapserver/app.py` | FastAPI app factory (`create_app`), routes, error handlers, the request guard and the event stream. | +| `mapserver/auth.py` | Entra token validation (`TokenValidator`) and the actor context (`Actor`, `X-Map-Actor`). | +| `mapserver/engine.py` | Map operations, access checks, versioned writes and as-of reads. | +| `mapserver/validation.py` | Normalizes features, enforces limits, and keeps photos and source links to safe https URLs. | +| `mapserver/store.py`, `mapserver/cosmos_store.py` | The storage interface, an in-memory store and the Cosmos DB store. | +| `mapserver/tiles.py` | The Azure Maps tile proxy and its cache. | +| `mapserver/settings.py`, `mapserver/models.py`, `mapserver/errors.py` | Configuration, request bodies and error codes. | +| `Dockerfile`, `requirements.txt` | The image, built from the repository root with the same Azure Linux pattern as SimpleChat. | +| `deploy/Deploy-MapServer.ps1` | The proof-of-concept deployment to Azure Container Apps. | + +### Calling the map server + +Every request except `GET /healthz` sends: + +- `Authorization: Bearer `: an Entra token for the map server's audience, from a caller holding the + `MapServer.ActOnBehalf` app role, or from an object ID in `MAP_SERVER_ALLOWED_CALLER_IDS`. +- `X-Map-Actor: `: the person the caller acts for, as `user_id`, `display_name`, `scope` (`user:` or + `group:`), `access` (`read` or `write`), and optionally `conversation_id`, `message_id`, `run_id` and + `agent` (`id`, `name`). The map server reads it only after the token passes, and refuses unknown fields. + +Tiles need only the token. + +### Configuration + +| Variable | Purpose | +|---|---| +| `MAP_SERVER_TENANT_ID`, `MAP_SERVER_AUDIENCES` | Required. The tenant and the accepted token audiences, such as `api://,`. | +| `MAP_SERVER_REQUIRED_ROLE` | The app role callers need. Defaults to `MapServer.ActOnBehalf`. | +| `MAP_SERVER_ALLOWED_CALLER_IDS` | Optional object IDs allowed without the role, for tenants where the role can't be assigned. | +| `MAP_SERVER_ISSUERS`, `MAP_SERVER_AUTHORITY_HOST`, `MAP_SERVER_JWKS_URL` | Optional overrides. By default the v1 and v2 issuers of the tenant are accepted. | +| `MAP_SERVER_STORE` | `cosmos` (default) or `memory`. | +| `MAP_SERVER_COSMOS_ENDPOINT` | Required for the Cosmos store. The managed identity signs in unless `MAP_SERVER_COSMOS_KEY` is set. | +| `MAP_SERVER_COSMOS_DATABASE`, `MAP_SERVER_MAPS_CONTAINER`, `MAP_SERVER_LINKS_CONTAINER` | Default to `mapserver`, `maps` and `map_links`. | +| `AZURE_CLIENT_ID` | The user-assigned managed identity's client ID. | +| `AZURE_MAPS_CLIENT_ID` or `AZURE_MAPS_KEY` | Azure Maps with the managed identity, or with a key. Without either, tiles answer 404 and the viewer stays black. | +| `MAP_SERVER_DEFAULT_BASEMAP` | Defaults to `microsoft.base.road`. | +| `MAP_SERVER_MAX_FEATURES_PER_CALL`, `MAP_SERVER_MAX_ACTIVE_FEATURES`, `MAP_SERVER_TILE_CACHE_ENTRIES` | Limits: 40, 10,000 and 1,024 by default. | +| `MAP_SERVER_EVENTS_POLL_SECONDS`, `MAP_SERVER_EVENTS_KEEPALIVE_SECONDS`, `MAP_SERVER_EVENTS_MAX_SECONDS` | Event stream timing: 2, 15 and 600 seconds by default. | +| `MAP_SERVER_LOCAL_DEV_KEY` | Local runs only. A shared key of at least 32 characters, accepted only with `MAP_SERVER_STORE=memory`. | + +### Running it locally + +```powershell +cd application\map_server +python -m pip install -r requirements.txt +$env:MAP_SERVER_STORE = 'memory' +$env:MAP_SERVER_LOCAL_DEV_KEY = [Convert]::ToBase64String((1..32 | ForEach-Object { Get-Random -Maximum 256 })) +python -m mapserver +``` + +The server listens on `http://127.0.0.1:8080`. Send the developer key as the bearer token. The API description is at +`/openapi.json`. There's no `/docs` page, because FastAPI's page loads its scripts from a CDN. + +### Deploying the proof of concept + +```powershell +./application/map_server/deploy/Deploy-MapServer.ps1 -ResourceGroup -RegistryName ` + -ContainerAppsEnvironment -CosmosAccountName -SimpleChatPrincipalId ` + -MapsAccountName +``` + +The script uses your Azure CLI sign-in and can be run again safely. It builds the image in the registry from a staged +context, creates the `mapserver` database and containers, and creates a managed identity with AcrPull, Cosmos DB data +access and Azure Maps Data Reader. It then registers the API with its app role, assigns the role to SimpleChat's +identity, deploys the container app, and prints the URL and token audience SimpleChat needs in Phase 2. If you can't +assign app roles in the directory, it allowlists SimpleChat's object ID instead. + +## Testing and validation + +- **Map server.** `functional_tests/test_map_server_api.py` and `functional_tests/test_map_server_storage_tiles_events.py` + cover validation, duplicates, versioning, as-of reconstruction, access checks, the REST contract, the Cosmos batch and + query shapes, tiles and live events. A concurrency test shows that several writers on one map never skip or repeat a + version. +- **SimpleChat.** Functional tests for the action against a fake map server, route policy tests for the new routes, and + UI tests for the change card and the live panel. +- **Docs.** The documentation coverage and site quality tests. + +## Alternatives considered + +- **Built into SimpleChat.** Simpler to secure and deploy. Not chosen (decision 1): a separate service can serve other + clients later without moving the data. +- **Only the generic MCP action.** It can't carry the user or group, and can't return a structured change card. Not + chosen. +- **Browser talks to the map server directly.** It would remove a proxy hop, but needs public ingress or private DNS for + every user, CORS, a CSP change (`frame-src` is `'self' blob:` today), and a second token flow. Not chosen. +- **PostGIS for storage.** It has richer spatial queries, but adds a database type that deployments don't run today. + Cosmos supports the bounding-box and point queries needed, with spatial indexing if they're needed later. + +## Open questions + +None. Decisions 5 to 10 settled the review questions. + +## Related + +- [Azure Maps OpenLayers action reference](../../reference/actions/azure-maps-openlayers.md) +- [MCP action reference](../../reference/actions/mcp.md) +- [Simple Chat action reference](../../reference/actions/simplechat.md) +- [V2 inline media and agent-posted messages](V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md) diff --git a/docs/reference/logging-tags.md b/docs/reference/logging-tags.md index d9a5e18e5..e0288f007 100644 --- a/docs/reference/logging-tags.md +++ b/docs/reference/logging-tags.md @@ -1,6 +1,6 @@ # SimpleChat Logging Tags -This reference lists the bracketed logging tags currently used in Python logging-style calls under `application/single_app`. +This reference lists the bracketed logging tags currently used in Python logging-style calls under `application/single_app` and `application/map_server`. Logging tags must use the normalized `[UPPERCASE_WITH_UNDERSCORES]` format. Prefer static tags and move dynamic values into the message body or `extra` metadata. @@ -155,6 +155,10 @@ Last inventoried: 2026-08-10 - `[MASK]` - `[MASK_API_ERROR]` - `[MASK_MESSAGE]` +- `[MAP_SERVER]` +- `[MAP_SERVER_AUTH]` +- `[MAP_SERVER_STORE]` +- `[MAP_SERVER_TILES]` - `[MCP_DESTINATION_POLICY]` - `[MCP_DISCOVERY]` - `[MCP_OUTBOUND]` diff --git a/functional_tests/test_logging_tag_standardization.py b/functional_tests/test_logging_tag_standardization.py index 4b0043d29..f7ac492f4 100644 --- a/functional_tests/test_logging_tag_standardization.py +++ b/functional_tests/test_logging_tag_standardization.py @@ -2,11 +2,12 @@ # test_logging_tag_standardization.py """ Functional test for logging tag standardization. -Version: 0.250.125 +Version: 0.261.254 Implemented in: 0.250.125 This test ensures Python logging prefixes use `[UPPERCASE_WITH_UNDERSCORES]` -and that the logging tag reference document stays synchronized with source. +and that the logging tag reference document stays synchronized with source +in SimpleChat and the map server. """ import ast @@ -16,7 +17,10 @@ REPO_ROOT = Path(__file__).resolve().parents[1] -APP_ROOT = REPO_ROOT / "application" / "single_app" +APP_ROOTS = ( + REPO_ROOT / "application" / "single_app", + REPO_ROOT / "application" / "map_server", +) LOGGING_TAG_DOC = REPO_ROOT / "docs" / "reference" / "logging-tags.md" TAG_RE = re.compile(r"^\s*\[([^\]\n]{1,100})\]") @@ -81,7 +85,7 @@ def _get_logging_message_node(self, node): def _collect_source_tags(): source_tags = {} - for path in APP_ROOT.rglob("*.py"): + for path in (path for app_root in APP_ROOTS for path in app_root.rglob("*.py")): tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) scanner = LoggingTagScanner() scanner.visit(tree) diff --git a/functional_tests/test_map_server_api.py b/functional_tests/test_map_server_api.py new file mode 100644 index 000000000..ca4fafe15 --- /dev/null +++ b/functional_tests/test_map_server_api.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +# test_map_server_api.py +""" +Functional test for the shared map server API. +Version: 0.261.254 +Implemented in: 0.261.254 + +This test ensures the map server only serves callers holding the map server role, keeps every map inside the +scope that owns it, records each change as one new version tagged with its phase, and can show any earlier version. +""" + +import asyncio +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from cryptography.hazmat.primitives.asymmetric import rsa + +from test_support.map_server_harness import ( + ISSUER_V2, + actor_headers, + create_map, + make_client, + make_settings, + make_token, +) +from test_support.versioning import assert_app_version_at_least + +from fastapi.testclient import TestClient +from mapserver.app import create_app +from mapserver.auth import Actor, encode_actor +from mapserver.engine import MapEngine +from mapserver.models import AddFeaturesRequest, CreateMapRequest +from mapserver.settings import Settings, SettingsError +from mapserver.store import ConflictError, InMemoryMapStore, StoreError + +POINT = { + "latitude": 40.7393, + "longitude": -73.9368, + "label": "Storage facility gate", + "category": "Location", + "description": "Entry recorded by the gate log.", + "observed_at": "2026-10-04T06:36:00-04:00", + "source": {"system": "records", "record_id": "R-1"}, + "image_url": "https://example.org/still.png", + "image_caption": "Gate camera", + "fields": {"Unit": "214"}, +} +PATH = {"kind": "path", "coordinates": [[-73.95, 40.73], [-73.94, 40.74]], "label": "Route", "line_width": 30} +AREA = { + "geometry": {"type": "Polygon", "coordinates": [[[-73.9, 40.7], [-73.8, 40.7], [-73.8, 40.8]]]}, + "label": "Search area", +} + + +def add(client, map_id, headers, features, **body): + return client.post(f"/v1/maps/{map_id}/features:batch", json={"features": features, **body}, headers=headers) + + +def features_of(client, map_id, headers, **params): + response = client.get(f"/v1/maps/{map_id}/features", params=params, headers=headers) + assert response.status_code == 200, response.text + return response.json() + + +def test_app_version_includes_the_map_server(): + assert_app_version_at_least("0.261.254") + return True + + +def test_callers_need_a_valid_token_with_the_map_server_role(): + print("Checking caller tokens...") + with make_client() as client: + assert client.get("/healthz").status_code == 200 + actor_only = {"X-Map-Actor": actor_headers()["X-Map-Actor"]} + assert client.get("/v1/maps", headers=actor_only).status_code == 401 + rejected_tokens = [ + make_token(audience="api://another-api"), + make_token(issuer="https://sts.windows.net/another-tenant/"), + make_token(expires_in=-3600), + make_token(key=rsa.generate_private_key(public_exponent=65537, key_size=2048)), + ] + for token in rejected_tokens: + response = client.get("/v1/maps", headers=actor_headers(token=token)) + assert response.status_code == 401, response.text + assert response.json()["error"] == "unauthorized" + no_role = client.get("/v1/maps", headers=actor_headers(token=make_token(roles=None))) + assert no_role.status_code == 403 and no_role.json()["error"] == "caller_not_allowed" + assert client.get("/v1/maps", headers=actor_headers(token=make_token(issuer=ISSUER_V2))).status_code == 200 + + allowlisted_id = "22222222-2222-2222-2222-222222222222" + with make_client(make_settings(allowed_caller_ids=frozenset({allowlisted_id}))) as client: + token = make_token(roles=None, object_id=allowlisted_id) + assert client.get("/v1/maps", headers=actor_headers(token=token)).status_code == 200 + print(" Caller tokens passed.") + return True + + +def test_actor_context_is_required_and_validated(): + print("Checking the actor context...") + with make_client() as client: + token_only = {"Authorization": f"Bearer {make_token()}"} + response = client.get("/v1/maps", headers=token_only) + assert response.status_code == 400 and response.json()["error"] == "actor_required" + for headers in ( + {**token_only, "X-Map-Actor": "not base64!"}, + actor_headers(scope="tenant:everything"), + actor_headers(access="admin"), + actor_headers(role="admin"), + actor_headers(user_id="has spaces"), + ): + response = client.get("/v1/maps", headers=headers) + assert response.status_code == 400 and response.json()["error"] == "invalid_actor", response.text + # The actor context is only read after the token is checked. + assert client.get("/v1/maps", headers={"X-Map-Actor": "garbage"}).status_code == 401 + print(" Actor context passed.") + return True + + +def test_maps_stay_inside_the_scope_that_owns_them(): + print("Checking scope isolation...") + with make_client() as client: + writer = actor_headers(scope="group:team-1") + map_id = create_map(client, writer, conversation_id="conv-1") + view = client.get(f"/v1/maps/{map_id}", headers=writer).json() + assert view["owner_scope"] == "group:team-1" and view["version"] == 1 + assert view["current_phase_id"] == "P1" and view["links"] == ["conv-1"] + + other_scope = actor_headers(scope="group:team-2") + for path in ("", "/snapshot", "/features", "/changes"): + response = client.get(f"/v1/maps/{map_id}{path}", headers=other_scope) + assert response.status_code == 404 and response.json()["error"] == "map_not_found", path + foreign_write = add(client, map_id, other_scope, [POINT]) + assert foreign_write.status_code == 404 + + reader = actor_headers(scope="group:team-1", access="read") + assert client.get(f"/v1/maps/{map_id}", headers=reader).status_code == 200 + refused = add(client, map_id, reader, [POINT]) + assert refused.status_code == 403 and refused.json()["error"] == "read_only" + refused_create = client.post("/v1/maps", json={"title": "Not allowed"}, headers=reader) + assert refused_create.status_code == 403 + + other_map = create_map(client, other_scope, conversation_id="conv-1") + assert [item["map_id"] for item in client.get("/v1/maps", headers=writer).json()["items"]] == [map_id] + by_conversation = client.get("/v1/maps", params={"conversation_id": "conv-1"}, headers=writer).json()["items"] + assert [item["map_id"] for item in by_conversation] == [map_id] + other_listing = client.get("/v1/maps", params={"conversation_id": "conv-1"}, headers=other_scope).json()["items"] + assert [item["map_id"] for item in other_listing] == [other_map] + assert client.get("/v1/maps/not-a-map", headers=writer).status_code == 404 + print(" Scope isolation passed.") + return True + + +def test_agents_add_points_paths_and_areas_with_limits_enforced(): + print("Checking feature validation...") + with make_client() as client: + headers = actor_headers(agent={"id": "agent-1", "name": "Field analyst"}, conversation_id="conv-1", message_id="msg-1") + map_id = create_map(client, headers) + insecure = {**POINT, "label": "Insecure photo", "source": {"record_id": "R-2"}, "image_url": "http://example.org/x.png"} + response = add(client, map_id, headers, [POINT, PATH, AREA, insecure]) + assert response.status_code == 200, response.text + result = response.json() + assert result["version"] == 2 and result["change_id"] == "C-000002" + assert [item["id"] for item in result["added"]] == ["F-0001", "F-0002", "F-0003", "F-0004"] + assert result["images_dropped"] == 1 + assert result["counts"] == {"active": 4, "retracted": 0, "point": 2, "path": 1, "area": 1} + + point, path, area, no_photo = features_of(client, map_id, headers)["items"] + assert point["geometry"] == {"type": "Point", "coordinates": [-73.9368, 40.7393]} + assert point["category"] == "location" + assert point["observed_at"] == "2026-10-04T06:36:00-04:00" + assert point["media"] == {"image_url": "https://example.org/still.png", "caption": "Gate camera"} + assert point["fields"] == [{"label": "Unit", "value": "214"}] + assert point["phase_id"] == "P1" and point["phase_name"] == "Phase 1" + assert point["created_by"]["agent"]["name"] == "Field analyst" and point["created_by"]["message_id"] == "msg-1" + assert path["style"]["line_width"] == 12 + ring = area["geometry"]["coordinates"][0] + assert ring[0] == ring[-1] and len(ring) == 4 + assert no_photo["media"] is None + + for bad, expected in ( + ({"latitude": 95, "longitude": 0, "label": "x"}, "out of range"), + ({"latitude": 1, "longitude": 1}, "needs a label"), + ({"coordinates": [[0, 0], [1, 1]], "label": "x"}, "kind"), + ({"kind": "path", "coordinates": [[0, 0]], "label": "x"}, "at least two"), + ({"latitude": "nan", "longitude": 1, "label": "x"}, "finite"), + ({"latitude": 1, "longitude": 1, "label": "x", "observed_at": "yesterday"}, "ISO 8601"), + ({"geometry": {"type": "MultiPoint", "coordinates": []}, "label": "x"}, "Point, LineString or Polygon"), + ): + response = add(client, map_id, headers, [bad]) + assert response.status_code == 400 and expected in response.json()["message"], (bad, response.text) + assert client.get(f"/v1/maps/{map_id}", headers=headers).json()["version"] == 2 + + too_many = [{"latitude": 1, "longitude": 1 + index * 0.001, "label": f"Point {index}"} for index in range(41)] + response = add(client, map_id, headers, too_many) + assert response.status_code == 400 and response.json()["error"] == "too_many_features" + print(" Feature validation passed.") + return True + + +def test_duplicate_source_records_are_skipped_or_updated(): + print("Checking duplicates...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers) + first = add(client, map_id, headers, [POINT]).json() + assert first["added"][0]["id"] == "F-0001" + + again = add(client, map_id, headers, [POINT]).json() + assert again["unchanged"] is True and again["version"] == 2 + assert [item["id"] for item in again["skipped"]] == ["F-0001"] + + moved = add(client, map_id, headers, [{**POINT, "latitude": 40.75}], on_duplicate="update").json() + assert moved["version"] == 3 and [item["id"] for item in moved["updated"]] == ["F-0001"] + feature = features_of(client, map_id, headers)["items"][0] + assert feature["geometry"]["coordinates"] == [-73.9368, 40.75] + assert feature["revision"] == 2 and feature["created_version"] == 2 and feature["updated_version"] == 3 + + same_record = [{**POINT, "label": "First", "source": {"record_id": "R-9"}}, {**POINT, "label": "Second", "source": {"record_id": "R-9"}}] + merged = add(client, map_id, headers, same_record, on_duplicate="update").json() + assert len(merged["added"]) == 1 and merged["added"][0]["label"] == "Second" + assert features_of(client, map_id, headers)["total"] == 2 + print(" Duplicates passed.") + return True + + +def test_phases_order_the_work_and_tag_every_feature(): + print("Checking phases...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers, first_phase_name="Initial enrichment") + first = add(client, map_id, headers, [POINT]).json() + assert first["phase"]["id"] == "P1" + + started = client.post(f"/v1/maps/{map_id}/phases", json={"name": "Follow-up", "description": "Second pass"}, headers=headers) + assert started.status_code == 201, started.text + assert started.json()["phase"]["id"] == "P2" and started.json()["phase"]["color"] == "#fd7e14" + second = add(client, map_id, headers, [PATH]).json() + assert second["phase"]["id"] == "P2" + + closed = add(client, map_id, headers, [AREA], phase_id="P1") + assert closed.status_code == 409 and closed.json()["error"] == "phase_closed" + unknown = add(client, map_id, headers, [AREA], phase_id="P9") + assert unknown.status_code == 404 + renamed = client.patch(f"/v1/maps/{map_id}/phases/P1", json={"name": "Initial sweep"}, headers=headers) + assert renamed.status_code == 200 and renamed.json()["phase"]["name"] == "Initial sweep" + bad_color = client.post(f"/v1/maps/{map_id}/phases", json={"name": "Bad", "color": "javascript:red"}, headers=headers) + assert bad_color.status_code == 400 + + view = client.get(f"/v1/maps/{map_id}", headers=headers).json() + assert view["current_phase_id"] == "P2" + first, second = view["phases"] + assert first["status"] == "closed" and first["closed_version"] == 3 and second["status"] == "open" + point, path = features_of(client, map_id, headers)["items"] + assert point["phase_id"] == "P1" and point["phase_name"] == "Initial sweep" + assert path["phase_id"] == "P2" + assert [item["id"] for item in features_of(client, map_id, headers, phase_id="P2")["items"]] == ["F-0002"] + print(" Phases passed.") + return True + + +def test_updates_and_retractions_keep_history_and_earlier_versions_can_be_shown(): + print("Checking history...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers) + add(client, map_id, headers, [POINT, PATH]) + + changes = {"label": "Gate (confirmed)", "latitude": 40.74, "longitude": -73.93} + updated = client.patch(f"/v1/maps/{map_id}/features/F-0001", json={"changes": changes}, headers=headers) + assert updated.status_code == 200, updated.text + assert updated.json()["version"] == 3 and updated.json()["feature"]["revision"] == 2 + stale = client.patch(f"/v1/maps/{map_id}/features/F-0001", json={"changes": {"label": "x"}, "expected_revision": 1}, headers=headers) + assert stale.status_code == 412 and stale.json()["error"] == "revision_mismatch" + kind_change = client.patch(f"/v1/maps/{map_id}/features/F-0001", json={"changes": {"kind": "area"}}, headers=headers) + assert kind_change.status_code == 400 + + retracted = client.post(f"/v1/maps/{map_id}/features/F-0002:retract", json={"reason": "Wrong vehicle"}, headers=headers) + assert retracted.status_code == 200 and retracted.json()["version"] == 4 + blank_reason = client.post(f"/v1/maps/{map_id}/features/F-0001:retract", json={"reason": " "}, headers=headers) + assert blank_reason.status_code == 400 + frozen = client.patch(f"/v1/maps/{map_id}/features/F-0002", json={"changes": {"label": "y"}}, headers=headers) + assert frozen.status_code == 409 and frozen.json()["error"] == "feature_retracted" + + now = client.get(f"/v1/maps/{map_id}/snapshot", headers=headers).json() + assert [(marker["label"], marker["latitude"]) for marker in now["markers"]] == [("Gate (confirmed)", 40.74)] + assert now["paths"] == [] + before = client.get(f"/v1/maps/{map_id}/snapshot", params={"as_of_version": 2}, headers=headers).json() + assert [(marker["label"], marker["latitude"]) for marker in before["markers"]] == [("Storage facility gate", 40.7393)] + assert [path["id"] for path in before["paths"]] == ["F-0002"] + assert before["as_of_version"] == 2 and before["version"] == 4 and "as of version 2" in before["summary"] + middle = client.get(f"/v1/maps/{map_id}/snapshot", params={"as_of_version": 3}, headers=headers).json() + assert middle["markers"][0]["label"] == "Gate (confirmed)" and len(middle["paths"]) == 1 + assert client.get(f"/v1/maps/{map_id}/snapshot", params={"as_of_version": 9}, headers=headers).status_code == 400 + + gone = features_of(client, map_id, headers, status="retracted")["items"] + assert [item["id"] for item in gone] == ["F-0002"] and gone[0]["retract_reason"] == "Wrong vehicle" + assert features_of(client, map_id, headers, status="all")["total"] == 2 + same = client.patch(f"/v1/maps/{map_id}/features/F-0001", json={"changes": {"label": "Gate (confirmed)"}}, headers=headers) + assert same.json()["unchanged"] is True and same.json()["version"] == 4 + print(" History passed.") + return True + + +def test_snapshot_matches_the_inline_map_payload_the_chat_already_renders(): + print("Checking the snapshot shape...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers, basemap="microsoft.base.darkgrey") + add(client, map_id, headers, [POINT, PATH, AREA]) + snapshot = client.get(f"/v1/maps/{map_id}/snapshot", headers=headers).json() + for key in ("title", "summary", "tile_attribution", "markers", "paths", "areas", "view", "tileset_id"): + assert key in snapshot, key + assert snapshot["tileset_id"] == "microsoft.base.darkgrey" and snapshot["background_color"] == "#000000" + marker = snapshot["markers"][0] + for key in ("latitude", "longitude", "label", "description", "color", "image_url", "image_caption", "fields"): + assert key in marker, key + assert marker["color"] == "#0d6efd" and marker["phase_name"] == "Phase 1" + assert {"coordinates", "stroke_color", "line_width", "label"} <= set(snapshot["paths"][0]) + area = snapshot["areas"][0] + assert area["coordinates"][0] == area["coordinates"][-1] + assert area["fill_color"] == "rgba(13, 110, 253, 0.20)" + assert snapshot["view"]["fit_to_features"] is True and len(snapshot["view"]["center"]) == 2 + assert snapshot["summary"] == "1 place, 1 route and 1 area in 1 phase." + assert snapshot["phases"][0]["feature_count"] == 3 + + client.post(f"/v1/maps/{map_id}/phases", json={"name": "Follow-up"}, headers=headers) + add(client, map_id, headers, [{"latitude": 40.8, "longitude": -73.9, "label": "Later sighting"}]) + only_follow_up = client.get(f"/v1/maps/{map_id}/snapshot", params={"phases": "P2"}, headers=headers).json() + assert [item["label"] for item in only_follow_up["markers"]] == ["Later sighting"] + assert only_follow_up["markers"][0]["color"] == "#fd7e14" and only_follow_up["paths"] == [] + assert client.get(f"/v1/maps/{map_id}/snapshot", params={"phases": "P2,"}, headers=headers).status_code == 400 + + empty_id = create_map(client, headers) + empty = client.get(f"/v1/maps/{empty_id}/snapshot", headers=headers).json() + assert empty["summary"] == "No features yet in 1 phase." and empty["view"]["zoom"] == 2 + print(" Snapshot shape passed.") + return True + + +def test_feature_list_filters_and_pages(): + print("Checking feature filters...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers) + points = [ + {"latitude": 40 + index * 0.01, "longitude": -74, "label": f"Point {index}", "category": "camera" if index % 2 else "plate"} + for index in range(30) + ] + add(client, map_id, headers, points) + first = features_of(client, map_id, headers, limit=10) + assert len(first["items"]) == 10 and first["next_cursor"] == "10" and first["total"] == 30 + second = features_of(client, map_id, headers, limit=10, cursor=first["next_cursor"]) + assert [item["label"] for item in second["items"]][0] == "Point 10" + assert features_of(client, map_id, headers, category="camera")["total"] == 15 + assert features_of(client, map_id, headers, bbox="-74.1,40.0,-73.9,40.05")["total"] == 6 + add(client, map_id, headers, [{"latitude": 41, "longitude": -74, "label": "Newest"}]) + assert [item["label"] for item in features_of(client, map_id, headers, since_version=2)["items"]] == ["Newest"] + for params in ({"cursor": "abc"}, {"bbox": "1,2,3"}, {"status": "bogus"}, {"kind": "circle"}): + response = client.get(f"/v1/maps/{map_id}/features", params=params, headers=headers) + assert response.status_code == 400, params + print(" Feature filters passed.") + return True + + +def test_change_log_records_each_version_with_its_phase_and_actor(): + print("Checking the change log...") + with make_client() as client: + headers = actor_headers(agent={"name": "Field analyst"}, run_id="run-7") + map_id = create_map(client, headers) + add(client, map_id, headers, [POINT]) + client.post(f"/v1/maps/{map_id}/phases", json={"name": "Follow-up"}, headers=headers) + client.post(f"/v1/maps/{map_id}/features/F-0001:retract", json={"reason": "Duplicate"}, headers=headers) + + log = client.get(f"/v1/maps/{map_id}/changes", headers=headers).json() + assert [item["action"] for item in log["items"]] == ["create_map", "add_features", "start_phase", "retract_feature"] + assert [item["version"] for item in log["items"]] == [1, 2, 3, 4] + added = log["items"][1] + assert added["added"] == ["F-0001"] and added["labels"] == ["Storage facility gate"] and added["phase_id"] == "P1" + assert added["actor"]["agent"]["name"] == "Field analyst" and added["actor"]["run_id"] == "run-7" + assert log["items"][2]["phase_id"] == "P2" and log["items"][2]["note"] == "Follow-up" + assert log["items"][3]["retracted"] == ["F-0001"] and log["items"][3]["note"] == "Duplicate" + + page = client.get(f"/v1/maps/{map_id}/changes", params={"since_version": 1, "limit": 2}, headers=headers).json() + assert [item["version"] for item in page["items"]] == [2, 3] and page["next_since_version"] == 3 + print(" Change log passed.") + return True + + +def test_links_connect_a_map_to_conversations_in_its_scope(): + print("Checking conversation links...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers) + linked = client.post(f"/v1/maps/{map_id}/links", json={"conversation_id": "conv-9"}, headers=headers) + assert linked.status_code == 200 and linked.json()["links"] == ["conv-9"] + again = client.post(f"/v1/maps/{map_id}/links", json={"conversation_id": "conv-9"}, headers=headers) + assert again.json()["links"] == ["conv-9"] + listed = client.get("/v1/maps", params={"conversation_id": "conv-9"}, headers=headers).json()["items"] + assert [item["map_id"] for item in listed] == [map_id] + reader = actor_headers(access="read") + reader_link = client.post(f"/v1/maps/{map_id}/links", json={"conversation_id": "conv-10"}, headers=reader) + assert reader_link.status_code == 403 + bad_link = client.post(f"/v1/maps/{map_id}/links", json={"conversation_id": "conv 9"}, headers=headers) + assert bad_link.status_code == 400 + + unlinked = client.delete(f"/v1/maps/{map_id}/links/conv-9", headers=headers) + assert unlinked.status_code == 200 and unlinked.json()["links"] == [] + assert client.get("/v1/maps", params={"conversation_id": "conv-9"}, headers=headers).json()["items"] == [] + assert client.get(f"/v1/maps/{map_id}", headers=headers).json()["version"] == 1 + print(" Conversation links passed.") + return True + + +class YieldingStore(InMemoryMapStore): + """Yields to the event loop on every read and commit, so concurrent writers really interleave.""" + + async def read_map(self, map_id): + await asyncio.sleep(0) + return await super().read_map(map_id) + + async def commit(self, *args, **kwargs): + await asyncio.sleep(0) + return await super().commit(*args, **kwargs) + + +class FlakyStore(InMemoryMapStore): + def __init__(self, failures): + super().__init__() + self.failures = failures + + async def commit(self, map_id, map_doc, expected_etag, creates, replaces): + if expected_etag is not None and self.failures > 0: + self.failures -= 1 + raise ConflictError() + return await super().commit(map_id, map_doc, expected_etag, creates, replaces) + + +def test_concurrent_writers_get_consecutive_versions(): + print("Checking concurrent writes...") + actor = Actor(user_id="user-1", scope="group:team-1", access="write") + + async def scenario(store, writers): + engine = MapEngine(store, make_settings()) + map_id = (await engine.create_map(actor, CreateMapRequest(title="Busy map")))["map_id"] + + async def write(index): + request = AddFeaturesRequest(features=[{"latitude": 1, "longitude": 1 + index * 0.01, "label": f"Point {index}"}]) + return await engine.add_features(actor, map_id, request) + + return await asyncio.gather(*(write(index) for index in range(writers)), return_exceptions=True) + + results = asyncio.run(scenario(YieldingStore(), 5)) + assert all(isinstance(result, dict) for result in results), results + assert sorted(result["version"] for result in results) == [2, 3, 4, 5, 6] + assert len({result["added"][0]["id"] for result in results}) == 5 + + retried = asyncio.run(scenario(FlakyStore(failures=2), 1))[0] + assert retried["version"] == 2 + gave_up = asyncio.run(scenario(FlakyStore(failures=50), 1))[0] + assert getattr(gave_up, "code", "") == "map_busy" and gave_up.status_code == 409 + print(" Concurrent writes passed.") + return True + + +class BrokenStore(InMemoryMapStore): + async def read_map(self, map_id): + raise StoreError("AccountKey=secret-value") + + +class ExplodingStore(InMemoryMapStore): + async def list_links(self, scope_key): + raise RuntimeError("secret-value") + + +def test_store_failures_never_reach_the_caller(): + print("Checking error responses...") + with make_client(store=BrokenStore()) as client: + response = client.get("/v1/maps/map-0123456789abcdef0123", headers=actor_headers()) + assert response.status_code == 503 and response.json()["error"] == "store_unavailable" + assert "secret" not in response.text + with make_client(store=ExplodingStore(), raise_server_exceptions=False) as client: + response = client.get("/v1/maps", headers=actor_headers()) + assert response.status_code == 500 and response.json() == {"error": "internal_error", "message": "Unexpected error."} + print(" Error responses passed.") + return True + + +def test_openapi_is_served_but_cdn_backed_docs_pages_are_not(): + print("Checking the API surface...") + with make_client() as client: + assert client.get("/openapi.json").status_code == 200 + assert client.get("/docs").status_code == 404 and client.get("/redoc").status_code == 404 + health = client.get("/healthz") + assert health.headers["x-content-type-options"] == "nosniff" and health.headers["cache-control"] == "no-store" + too_big = client.post( + "/v1/maps", content=b"x" * 1_048_577, headers={**actor_headers(), "Content-Type": "application/json"}, + ) + assert too_big.status_code == 413 + invalid_body = client.post("/v1/maps", json={"title": "x", "owner_scope": "group:other"}, headers=actor_headers()) + assert invalid_body.status_code == 400 and invalid_body.json()["error"] == "invalid_request" + print(" API surface passed.") + return True + + +def test_settings_and_the_local_developer_key(): + print("Checking settings...") + for settings in ( + Settings(local_dev_key="k" * 32, store="cosmos", cosmos_endpoint="https://example.documents.azure.com"), + Settings(local_dev_key="short", store="memory"), + Settings(store="memory"), + ): + try: + settings.validate() + except SettingsError: + continue + raise AssertionError(f"Settings should have been refused: {settings}") + try: + Settings.from_env({"MAP_SERVER_STORE": "memory", "MAP_SERVER_TENANT_ID": "tenant"}) + raise AssertionError("Missing audiences should be refused.") + except SettingsError: + pass + from_env = Settings.from_env({"MAP_SERVER_STORE": "memory", "MAP_SERVER_TENANT_ID": "tenant", "MAP_SERVER_AUDIENCES": "api://a, b"}) + assert from_env.audiences == ("api://a", "b") + assert from_env.issuers == ("https://sts.windows.net/tenant/", "https://login.microsoftonline.com/tenant/v2.0") + + dev_settings = Settings(local_dev_key="k" * 32, store="memory") + dev_settings.validate() + actor = encode_actor({"user_id": "dev", "scope": "user:dev", "access": "write"}) + with TestClient(create_app(dev_settings, store=InMemoryMapStore())) as client: + assert client.get("/v1/maps", headers={"Authorization": f"Bearer {'k' * 32}", "X-Map-Actor": actor}).status_code == 200 + assert client.get("/v1/maps", headers={"Authorization": "Bearer wrong", "X-Map-Actor": actor}).status_code == 401 + assert client.get("/v1/maps", headers={"Authorization": f"Bearer {make_token()}", "X-Map-Actor": actor}).status_code == 401 + print(" Settings passed.") + return True + + +if __name__ == "__main__": + tests = [ + test_app_version_includes_the_map_server, + test_callers_need_a_valid_token_with_the_map_server_role, + test_actor_context_is_required_and_validated, + test_maps_stay_inside_the_scope_that_owns_them, + test_agents_add_points_paths_and_areas_with_limits_enforced, + test_duplicate_source_records_are_skipped_or_updated, + test_phases_order_the_work_and_tag_every_feature, + test_updates_and_retractions_keep_history_and_earlier_versions_can_be_shown, + test_snapshot_matches_the_inline_map_payload_the_chat_already_renders, + test_feature_list_filters_and_pages, + test_change_log_records_each_version_with_its_phase_and_actor, + test_links_connect_a_map_to_conversations_in_its_scope, + test_concurrent_writers_get_consecutive_versions, + test_store_failures_never_reach_the_caller, + test_openapi_is_served_but_cdn_backed_docs_pages_are_not, + test_settings_and_the_local_developer_key, + ] + results = [] + for test in tests: + try: + results.append(bool(test())) + except Exception as exc: + import traceback + + print(f"Test {test.__name__} failed: {exc}") + traceback.print_exc() + results.append(False) + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_map_server_storage_tiles_events.py b/functional_tests/test_map_server_storage_tiles_events.py new file mode 100644 index 000000000..c56a56315 --- /dev/null +++ b/functional_tests/test_map_server_storage_tiles_events.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# test_map_server_storage_tiles_events.py +""" +Functional test for the map server's Cosmos DB store, tile proxy and live change events. +Version: 0.261.254 +Implemented in: 0.261.254 + +This test ensures each map write reaches Cosmos DB as one ETag-conditioned transactional batch with parameterized +queries, that map tiles are fetched only with the server's own credentials and cached, and that the event stream +replays changes in order to callers that can read the map. +""" + +import asyncio +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +import httpx +from azure.core.credentials import AccessToken +from azure.cosmos import exceptions as cosmos_exceptions + +from test_support.map_server_harness import actor_headers, create_map, make_client, make_settings + +from mapserver.cosmos_store import CosmosMapStore +from mapserver.store import ConflictError, StoreError +from mapserver.tiles import TileService + +PNG_BYTES = b"\x89PNG\r\n\x1a\nfake-tile" + + +class FakeContainer: + def __init__(self): + self.batches = [] + self.queries = [] + self.items = {} + self.query_results = [] + self.batch_error = None + self.upserts = [] + self.deletes = [] + + async def execute_item_batch(self, batch_operations, partition_key): + self.batches.append((list(batch_operations), partition_key)) + if self.batch_error is not None: + raise self.batch_error + return [{"statusCode": 200, "eTag": '"etag-2"', "resourceBody": {"_etag": '"etag-2"'}}] + + async def read_item(self, item, partition_key): + if (item, partition_key) not in self.items: + raise cosmos_exceptions.CosmosResourceNotFoundError(status_code=404, message="Not found") + return dict(self.items[(item, partition_key)]) + + def query_items(self, query, parameters, partition_key): + self.queries.append((query, parameters, partition_key)) + results = list(self.query_results) + + async def iterate(): + for result in results: + yield result + + return iterate() + + async def upsert_item(self, body): + self.upserts.append(body) + + async def delete_item(self, item, partition_key): + self.deletes.append((item, partition_key)) + raise cosmos_exceptions.CosmosResourceNotFoundError(status_code=404, message="Not found") + + +class FakeDatabase: + def __init__(self, containers): + self.containers = containers + + def get_container_client(self, name): + return self.containers[name] + + +class FakeCosmosClient: + def __init__(self): + self.containers = {"maps": FakeContainer(), "map_links": FakeContainer()} + self.closed = False + + def get_database_client(self, name): + assert name == "mapserver" + return FakeDatabase(self.containers) + + async def close(self): + self.closed = True + + +def make_store(): + client = FakeCosmosClient() + return CosmosMapStore(client, "mapserver", "maps", "map_links"), client.containers["maps"], client.containers["map_links"], client + + +def test_cosmos_commit_is_one_etag_conditioned_batch(): + print("Checking Cosmos batches...") + store, maps, _, client = make_store() + map_id = "map-0123456789abcdef0123" + map_doc = {"id": map_id, "map_id": map_id, "type": "map", "version": 3} + feature = {"id": "F-0001", "map_id": map_id, "type": "feature"} + revision = {"id": "F-0002@r1", "map_id": map_id, "type": "feature_rev"} + replaced = {"id": "F-0002", "map_id": map_id, "type": "feature"} + + etag = asyncio.run(store.commit(map_id, map_doc, '"etag-1"', [revision, feature], [replaced])) + operations, partition_key = maps.batches[-1] + assert etag == '"etag-2"' and partition_key == map_id + assert operations[0] == ("replace", (map_id, map_doc), {"if_match_etag": '"etag-1"'}) + assert operations[1:] == [("create", (revision,)), ("create", (feature,)), ("replace", ("F-0002", replaced))] + + asyncio.run(store.commit(map_id, map_doc, None, [], [])) + assert maps.batches[-1][0] == [("create", (map_doc,))] + + for status, expected in ((412, ConflictError), (409, ConflictError), (500, StoreError)): + maps.batch_error = cosmos_exceptions.CosmosBatchOperationError( + error_index=0, headers={}, status_code=status, message="failed", + ) + try: + asyncio.run(store.commit(map_id, map_doc, '"etag-1"', [], [])) + raise AssertionError(f"Status {status} should have raised.") + except expected: + pass + maps.batch_error = None + + calls = len(maps.batches) + try: + asyncio.run(store.commit(map_id, map_doc, '"etag-1"', [feature] * 100, [])) + raise AssertionError("An oversized batch should be refused.") + except StoreError: + pass + assert len(maps.batches) == calls + + asyncio.run(store.close()) + assert client.closed + print(" Cosmos batches passed.") + return True + + +def test_cosmos_reads_and_queries_are_partitioned_and_parameterized(): + print("Checking Cosmos reads...") + store, maps, links, _ = make_store() + map_id = "map-0123456789abcdef0123" + maps.items[(map_id, map_id)] = {"id": map_id, "type": "map", "_etag": '"e1"', "_rid": "r", "_ts": 1, "_self": "s"} + doc, etag = asyncio.run(store.read_map(map_id)) + assert etag == '"e1"' and doc == {"id": map_id, "type": "map"} + assert asyncio.run(store.read_map("map-ffffffffffffffffffff")) is None + maps.items[("map-1", "map-1")] = {"id": "map-1", "type": "feature", "_etag": '"e"'} + assert asyncio.run(store.read_map("map-1")) is None + + hostile_key = "point|records|R-1' OR 1=1 --" + asyncio.run(store.query_features(map_id, dup_keys=[hostile_key])) + query, parameters, partition_key = maps.queries[-1] + assert hostile_key not in query and parameters == [{"name": "@keys", "value": [hostile_key]}] and partition_key == map_id + assert asyncio.run(store.query_features(map_id, dup_keys=[])) == [] + + asyncio.run(store.query_changes(map_id, 5, 50)) + query, parameters, _ = maps.queries[-1] + assert "SELECT TOP 50 " in query and parameters == [{"name": "@since", "value": 5}] + + asyncio.run(store.query_feature_revisions(map_id, ["F-0001"], 4)) + _, parameters, _ = maps.queries[-1] + assert {"name": "@version", "value": 4} in parameters + + asyncio.run(store.delete_link("conversation:c1", map_id)) + assert links.deletes == [(map_id, "conversation:c1")] + asyncio.run(store.list_links("group:g1")) + assert links.queries[-1][2] == "group:g1" + print(" Cosmos reads passed.") + return True + + +class TileUpstream: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + def handler(self, request): + self.requests.append(request) + result = self.responses.pop(0) if len(self.responses) > 1 else self.responses[0] + if isinstance(result, Exception): + raise result + return result + + +class FakeCredential: + def __init__(self): + self.calls = 0 + + async def get_token(self, *scopes): + self.calls += 1 + assert scopes == ("https://atlas.microsoft.com/.default",) + return AccessToken("maps-token", 4102444800) + + +def tile_client(settings, upstream, credential=None): + tiles = TileService(settings, http_client=httpx.AsyncClient(transport=httpx.MockTransport(upstream.handler)), credential=credential) + return make_client(settings, tile_service=tiles) + + +def test_tiles_use_the_servers_own_credentials_and_are_cached(): + print("Checking tiles...") + headers = {"Authorization": actor_headers()["Authorization"]} + with make_client() as client: + unconfigured = client.get("/v1/tiles/microsoft.base.road/1/0/0", headers=headers) + assert unconfigured.status_code == 404 and unconfigured.json()["error"] == "tiles_unavailable" + assert client.get("/v1/tiles/microsoft.base.road/1/0/0").status_code == 401 + + image = httpx.Response(200, content=PNG_BYTES, headers={"content-type": "image/png"}) + upstream = TileUpstream([image]) + with tile_client(make_settings(azure_maps_key="maps-key"), upstream) as client: + first = client.get("/v1/tiles/microsoft.base.road/3/2/1", headers=headers) + second = client.get("/v1/tiles/microsoft.base.road/3/2/1", headers=headers) + assert first.status_code == 200 and first.content == PNG_BYTES and second.content == PNG_BYTES + assert first.headers["content-type"] == "image/png" and first.headers["cache-control"] == "private, max-age=3600" + assert len(upstream.requests) == 1 + request = upstream.requests[0] + assert request.headers["subscription-key"] == "maps-key" + assert request.url.path == "/map/tile" + assert request.url.params["tilesetId"] == "microsoft.base.road" and request.url.params["zoom"] == "3" + for path in ("/v1/tiles/microsoft.base.road/1/2/0", "/v1/tiles/bad$tileset/1/0/0", "/v1/tiles/microsoft.base.road/23/0/0"): + response = client.get(path, headers=headers) + assert response.status_code == 400 and response.json()["error"] == "invalid_tile", path + + credential = FakeCredential() + upstream = TileUpstream([image]) + with tile_client(make_settings(azure_maps_client_id="maps-account-id"), upstream, credential) as client: + first = client.get("/v1/tiles/microsoft.base.road/1/0/0", headers=headers) + second = client.get("/v1/tiles/microsoft.base.road/1/1/0", headers=headers) + assert first.status_code == 200 and second.status_code == 200 + assert credential.calls == 1 + assert upstream.requests[0].headers["authorization"] == "Bearer maps-token" + assert upstream.requests[0].headers["x-ms-client-id"] == "maps-account-id" + + for failure in ( + httpx.Response(200, content=b"", headers={"content-type": "text/html"}), + httpx.Response(403, content=b"denied"), + httpx.ConnectError("unreachable"), + ): + with tile_client(make_settings(azure_maps_key="maps-key"), TileUpstream([failure])) as client: + response = client.get("/v1/tiles/microsoft.base.road/1/0/0", headers=headers) + assert response.status_code == 502 and response.json()["error"] == "tile_upstream_error" + assert "maps-key" not in response.text + print(" Tiles passed.") + return True + + +def read_events(client, map_id, headers, **params): + with client.stream("GET", f"/v1/maps/{map_id}/events", params=params, headers=headers) as response: + assert response.status_code == 200, response.read() + assert response.headers["content-type"].startswith("text/event-stream") + return "".join(response.iter_text()) + + +def test_event_stream_replays_changes_in_order_and_checks_access(): + print("Checking live events...") + with make_client() as client: + headers = actor_headers() + map_id = create_map(client, headers) + client.post(f"/v1/maps/{map_id}/features:batch", json={"features": [{"latitude": 1, "longitude": 1, "label": "A"}]}, headers=headers) + client.post(f"/v1/maps/{map_id}/phases", json={"name": "Follow-up"}, headers=headers) + + body = read_events(client, map_id, headers, since_version=1) + assert body.startswith("retry: 5000") + assert "id: 2\nevent: change\n" in body and "id: 3\nevent: change\n" in body + assert body.index("id: 2\n") < body.index("id: 3\n") + assert '"action":"add_features"' in body and '"action":"start_phase"' in body + assert "event: end" in body + + resumed = read_events(client, map_id, {**headers, "Last-Event-ID": "2"}) + assert "id: 2\n" not in resumed and "id: 3\n" in resumed + caught_up = read_events(client, map_id, headers) + assert "event: change" not in caught_up + + reader = actor_headers(access="read") + assert "id: 3\n" in read_events(client, map_id, reader, since_version=2) + assert client.get(f"/v1/maps/{map_id}/events", headers=actor_headers(scope="group:team-2")).status_code == 404 + print(" Live events passed.") + return True + + +if __name__ == "__main__": + tests = [ + test_cosmos_commit_is_one_etag_conditioned_batch, + test_cosmos_reads_and_queries_are_partitioned_and_parameterized, + test_tiles_use_the_servers_own_credentials_and_are_cached, + test_event_stream_replays_changes_in_order_and_checks_access, + ] + results = [] + for test in tests: + try: + results.append(bool(test())) + except Exception as exc: + import traceback + + print(f"Test {test.__name__} failed: {exc}") + traceback.print_exc() + results.append(False) + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_support/map_server_harness.py b/functional_tests/test_support/map_server_harness.py new file mode 100644 index 000000000..bba6d5040 --- /dev/null +++ b/functional_tests/test_support/map_server_harness.py @@ -0,0 +1,100 @@ +# map_server_harness.py +"""Shared helpers for the map server functional tests: signed test tokens, actor headers and a test client.""" + +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MAP_SERVER_ROOT = REPO_ROOT / "application" / "map_server" +if str(MAP_SERVER_ROOT) not in sys.path: + sys.path.insert(0, str(MAP_SERVER_ROOT)) + +# The map server package lives outside the test tree, so these imports follow the path setup above. +import jwt # noqa: E402 +from cryptography.hazmat.primitives.asymmetric import rsa # noqa: E402 +from fastapi.testclient import TestClient # noqa: E402 + +from mapserver.app import create_app # noqa: E402 +from mapserver.auth import TokenValidator, encode_actor # noqa: E402 +from mapserver.settings import Settings # noqa: E402 +from mapserver.store import InMemoryMapStore # noqa: E402 + +TENANT_ID = "00000000-0000-0000-0000-00000000aaaa" +AUDIENCE = "api://map-server-test" +ISSUER_V1 = f"https://sts.windows.net/{TENANT_ID}/" +ISSUER_V2 = f"https://login.microsoftonline.com/{TENANT_ID}/v2.0" +CALLER_OBJECT_ID = "11111111-1111-1111-1111-111111111111" +MAP_SERVER_ROLE = "MapServer.ActOnBehalf" + +SIGNING_KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +class _SigningKey: + def __init__(self, key): + self.key = key + + +class StaticJwkClient: + """Stands in for PyJWKClient: every token is checked against the test public key.""" + + def get_signing_key_from_jwt(self, token): + return _SigningKey(SIGNING_KEY.public_key()) + + +def make_settings(**overrides) -> Settings: + values = dict( + tenant_id=TENANT_ID, + audiences=(AUDIENCE,), + issuers=(ISSUER_V1, ISSUER_V2), + jwks_url="https://keys.invalid/discovery/v2.0/keys", + store="memory", + events_poll_seconds=0.05, + events_keepalive_seconds=1.0, + events_max_seconds=1.0, + ) + values.update(overrides) + settings = Settings(**values) + settings.validate() + return settings + + +def make_token( + *, roles=(MAP_SERVER_ROLE,), audience=AUDIENCE, issuer=ISSUER_V1, object_id=CALLER_OBJECT_ID, expires_in=3600, key=None +) -> str: + now = int(time.time()) + claims = { + "aud": audience, + "iss": issuer, + "oid": object_id, + "azp": "simplechat-test-app", + "tid": TENANT_ID, + "iat": now, + "nbf": now, + "exp": now + expires_in, + } + if roles is not None: + claims["roles"] = list(roles) + return jwt.encode(claims, key or SIGNING_KEY, algorithm="RS256") + + +def actor_headers(*, user_id="user-1", scope="group:team-1", access="write", token=None, **extra) -> dict: + actor = {"user_id": user_id, "display_name": "Test user", "scope": scope, "access": access, **extra} + return {"Authorization": f"Bearer {token or make_token()}", "X-Map-Actor": encode_actor(actor)} + + +def make_client(settings=None, *, store=None, tile_service=None, raise_server_exceptions=True) -> TestClient: + settings = settings or make_settings() + app = create_app( + settings, + store=store or InMemoryMapStore(), + token_validator=TokenValidator(settings, jwk_client=StaticJwkClient()), + tile_service=tile_service, + ) + return TestClient(app, raise_server_exceptions=raise_server_exceptions) + + +def create_map(client: TestClient, headers: dict, **body) -> str: + response = client.post("/v1/maps", json={"title": "Response map", **body}, headers=headers) + assert response.status_code == 201, response.text + return response.json()["map_id"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 6cc40b558..54e5e3c1d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,5 @@ azure-mgmt-playwright==1.0.0 mcp==1.28.1 starlette==1.3.1 uvicorn==0.50.2 +fastapi==0.141.1 +httpx==0.28.1