diff --git a/CHANGELOG.md b/CHANGELOG.md index ed5de420..48ba5122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,50 @@ +## Unreleased + +### Features + +- Add SnapStart support. The adapter notifies your web application at the SnapStart + boundary via two opt-in HTTP hooks — `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` + (before checkpoint) and `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` (after restore) — + so it can drain and re-establish connections. Each hook call is bounded by a + 60-second timeout. After restore the adapter refreshes its own HTTP client and + re-runs the readiness check before admitting traffic, and it rejects external + traffic to the hook paths with 403. + - The crate now stops publishing to crates.io (`publish = false`): Lambda Web + Adapter ships as the `lambda-adapter` binary (a Lambda layer / copied + extension), not as a library, so the `lib` target has no external + API-stability contract. The internal changes SnapStart required — the + `tower::Service` impl's `Response` is now `Response>` + (was `Response`) and `check_init_health` now returns `Result` — + therefore do not affect any published API. `Bytes` and `BoxBody` are + re-exported for convenience of in-repo `Service` users. Existing crates.io + consumers keep the last published release (`1.0.0-rc1`) unchanged. +- Add `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` to configure the idle keep-alive (in + whole seconds) of the adapter's HTTP connection to your app. Default: 4 seconds. +- Add `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` to bound the readiness check + (fractional seconds allowed, e.g. `0.5`), applied to both the initial cold-start + readiness wait and the + post-SnapStart-restore readiness check. When set and the app does not become + ready within it, the adapter **refuses to serve**: cold-start init fails (the + runtime never starts) and a restore fails, rather than admitting traffic to an + app that never reported ready. When unset (the default) the wait is + **unbounded**, matching the previous behavior, so existing slow-cold-start apps + are unaffected unless they opt in. The `async_init` initial-readiness path keeps + its own fixed ~9.8s bound (non-fatal) and is not affected by this variable. + +### Bug Fixes + +- Fix `AWS_LWA_REMOVE_BASE_PATH` stripping to remove exactly one leading occurrence + on a path-segment boundary. Previously it used `trim_start_matches`, which stripped + the prefix repeatedly and byte-wise: with `AWS_LWA_REMOVE_BASE_PATH=/api`, + `/api/api/order` became `/order` (both copies removed) and `/apiorder` became + `/order` (a partial segment stripped). Now `/api/api/order` → `/api/order` and + `/apiorder` is passed through unchanged, and a configured trailing slash (`/api/`) + is normalized so it behaves like `/api`. **Upgrade note:** this changes the path + forwarded to your app for those inputs — deployments that relied on the old + repeated/partial stripping should verify their routes. + +--- + ## v1.0.1 - 2026-05-28 ### Bug Fixes diff --git a/Cargo.lock b/Cargo.lock index ed6302ce..44a3512e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "aws_lambda_events" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "087b1b9233c7fc56623d72bb2f0b1fe915b19a0606aa3c09bcd1b902d9803e6c" +checksum = "d02c123e89527e7b424f74f52d11be0d17ef2887819323a42dcae1c7630ca53d" dependencies = [ "base64", "bytes", @@ -1245,9 +1245,9 @@ dependencies = [ [[package]] name = "lambda_http" -version = "1.1.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f0300091919bd7c3d953bd0fb6a7a170d24f333486e4129421c0f6aa1164ac" +checksum = "e69eb3117d123f471f7d2b5d02b5afe150f54f9f7cff3264e578c9dd03675ad9" dependencies = [ "aws_lambda_events", "bytes", @@ -1270,9 +1270,9 @@ dependencies = [ [[package]] name = "lambda_runtime" -version = "1.1.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23583c918bee8de7bc005ba2ad8ebd84529894bfd9a863cf24226bb6c7787690" +checksum = "484647f147899f866a7b5db6aaa1671e943f89f4be76e2a383c96b1b27058294" dependencies = [ "async-stream", "base64", @@ -1295,9 +1295,9 @@ dependencies = [ [[package]] name = "lambda_runtime_api_client" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4873061514cb57ffb6a599b77c46c65d6d783efe9bad8fd56b7cba7f0459ef" +checksum = "a92e6500e47d17c1ffd3e6ad3ca224bb86382fc8b63414f6f20b1b2d98dfb7cf" dependencies = [ "bytes", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index c9a1ff49..b9448a8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,10 +11,18 @@ keywords = ["AWS", "Lambda", "APIGateway", "ALB", "API"] license = "Apache-2.0" homepage = "https://github.com/aws/aws-lambda-web-adapter" repository = "https://github.com/aws/aws-lambda-web-adapter" -documentation = "https://docs.rs/lambda_web_adapter" categories = ["web-programming::http-server"] readme = "README.md" exclude = ["examples"] +# Lambda Web Adapter ships as the `lambda-adapter` binary (packaged as a Lambda +# layer / container-copied extension), not as a library. This stops publishing the +# crate to crates.io: the `lib` target is an internal implementation detail of the +# binary and tests, with no external API-stability contract. +# +# Note for crates.io consumers: earlier `0.x` / `1.0.0-rc1` releases remain +# available and unchanged; they are simply the last published versions. New +# development ships only as the binary/layer. +publish = false [dependencies] bytes = "1.9.0" @@ -23,7 +31,7 @@ http-body = "1.0.1" http-body-util = "0.1.0" hyper = { version = "1.5.2", features = ["client"] } hyper-util = "0.1.10" -lambda_http = { version = "1.1.1", default-features = false, features = [ +lambda_http = { version = "1.3.0", default-features = false, features = [ "apigw_http", "apigw_rest", "alb", diff --git a/README.md b/README.md index f4388586..9078213c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ The same docker image can run on AWS Lambda, Amazon EC2, AWS Fargate, and local - Supports Amazon API Gateway Rest API and Http API endpoints, Lambda Function URLs, and Application Load Balancer - Supports Lambda managed runtimes, custom runtimes and docker OCI images - Supports Lambda Managed Instances for multi-concurrent request handling +- Supports Lambda SnapStart with before-checkpoint and after-restore hooks - Supports any web frameworks and languages, no new code dependency to include - Automatic encode binary response - Enables graceful shutdown @@ -59,13 +60,17 @@ The readiness check port/path and traffic port can be configured using environme | AWS_LWA_READINESS_CHECK_PROTOCOL | readiness check protocol: "http" or "tcp" | "http" | | AWS_LWA_READINESS_CHECK_HEALTHY_STATUS | HTTP status codes considered healthy (e.g., "200-399") | "100-499" | | AWS_LWA_ASYNC_INIT | enable asynchronous initialization for long initialization functions | "false" | -| AWS_LWA_REMOVE_BASE_PATH | the base path to be removed from request path | None | +| AWS_LWA_REMOVE_BASE_PATH | base path to remove from the request path; strips exactly one leading occurrence on a segment boundary (with `/api`: `/api/api/order`->`/api/order`, `/apiorder` unchanged; trailing slash normalized) | None | | AWS_LWA_ENABLE_COMPRESSION | enable gzip/br compression for response body (buffered mode only) | "false" | | AWS_LWA_INVOKE_MODE | Lambda function invoke mode: "buffered" or "response_stream" | "buffered" | | AWS_LWA_PASS_THROUGH_PATH | the path for receiving event payloads from non-http triggers | "/events" | | AWS_LWA_AUTHORIZATION_SOURCE | a header name to be replaced to `Authorization` | None | | AWS_LWA_ERROR_STATUS_CODES | HTTP status codes that will cause Lambda invocations to fail (e.g. "500,502-504") | None | | AWS_LWA_LAMBDA_RUNTIME_API_PROXY | overwrites `AWS_LAMBDA_RUNTIME_API` to allow proxying request | None | +| AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH | inner-app path the adapter POSTs to before a SnapStart snapshot (drain resources) | None | +| AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH | inner-app path the adapter POSTs to after a SnapStart restore (reconnect/reseed) | None | +| AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS | idle keep-alive (seconds) for the adapter's connection to your app | "4" | +| AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS | seconds (fractional allowed, e.g. 0.5) to wait for the app to report ready (cold-start init and after a SnapStart restore); on expiry the adapter FAILS (init fails and the runtime never starts; a restore fails) rather than serving. Unset, 0, or negative all mean wait indefinitely (a set-but-<=0 or malformed value is ignored with a warning). async_init keeps its own ~9.8s bound | unset / <=0 (unbounded) | > **Deprecation Notice:** The following non-namespaced environment variables are deprecated and will be removed in version 2.0: > `HOST`, `READINESS_CHECK_PORT`, `READINESS_CHECK_PATH`, `READINESS_CHECK_PROTOCOL`, `REMOVE_BASE_PATH`, `ASYNC_INIT`. @@ -75,6 +80,45 @@ The readiness check port/path and traffic port can be configured using environme 👉 [Detailed configuration docs](https://aws.github.io/aws-lambda-web-adapter/configuration/environment-variables.html) +### SnapStart support + +When your function uses [Lambda SnapStart](https://docs.aws.amazon.com/lambda/latest/dg/snapstart.html), +the adapter can notify your web application at the snapshot boundary so it can +drain and re-establish state (database connections, cached DNS, PRNG seeds, +unique identifiers). Both hooks are opt-in and independent. + +| Variable | When the adapter calls it | Use it to | +|---|---|---| +| `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` | Before the snapshot is taken | Drain/close resources that won't survive the snapshot | +| `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` | After restore, before serving traffic | Reconnect, refresh credentials, reseed randomness, regenerate unique IDs | + +Each hook is an empty `POST`; your application must respond with a `2xx` status. +A non-`2xx` response, a connection failure, or taking longer than 60 seconds to +respond fails the SnapStart phase (initialization for the before-checkpoint hook, +restore for the after-restore hook) instead of serving traffic against an +improperly prepared application. + +After restore, the adapter also automatically refreshes its own HTTP connection +to your application, so it never reuses a connection captured in the snapshot, and +then re-runs the readiness check before admitting traffic. By default this wait is +unbounded; set `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` (fractional seconds allowed) +to bound it, in which case a restore whose application does not report ready within +that timeout fails. + +> These hook paths are control-plane operations. External requests (via API +> Gateway or ALB) that target a configured hook path receive `403 Forbidden` and +> are never forwarded to your application, so choose paths your normal traffic +> does not use. +> +> **Warning:** that guard exists only while the adapter is in the request path — +> that is, when your application runs on Lambda behind the adapter. The hook routes +> are ordinary application routes that mutate state, so if you run the same image +> or application **without** the adapter (Amazon ECS, Amazon EKS, a local Docker +> host), they are reachable and unauthenticated. Don't expose them publicly in +> those deployments, or protect them yourself. + +See the [FastAPI with SnapStart example](examples/fastapi-snapstart-zip) for a complete, deployable application. + ## Examples - [FastAPI](examples/fastapi) @@ -84,6 +128,8 @@ The readiness check port/path and traffic port can be configured using environme - [FastAPI with Response Streaming in Zip](examples/fastapi-response-streaming-zip) - [FastAPI with Response Streaming on Lambda Managed Instances](examples/fastapi-response-streaming-lmi) - [FastAPI Response Streaming Backend with IAM Auth](examples/fastapi-backend-only-response-streaming/) +- [FastAPI with SnapStart](examples/fastapi-snapstart) +- [FastAPI with SnapStart in Zip](examples/fastapi-snapstart-zip) - [Flask](examples/flask) - [Flask in Zip](examples/flask-zip) - [Serverless Django](https://github.com/aws-hebrew-book/serverless-django) by [@efi-mk](https://github.com/efi-mk) diff --git a/docs/guide/src/SUMMARY.md b/docs/guide/src/SUMMARY.md index bd47c0ff..93ff43d9 100644 --- a/docs/guide/src/SUMMARY.md +++ b/docs/guide/src/SUMMARY.md @@ -23,6 +23,7 @@ - [Non-HTTP Event Triggers](./features/non-http-events.md) - [Multi-Tenancy](./features/multi-tenancy.md) - [Lambda Managed Instances](./features/managed-instances.md) +- [SnapStart](./features/snapstart.md) - [Graceful Shutdown](./features/graceful-shutdown.md) - [Base Path Removal](./features/base-path-removal.md) - [Authorization Header](./features/authorization-header.md) diff --git a/docs/guide/src/configuration/environment-variables.md b/docs/guide/src/configuration/environment-variables.md index 3ac78a76..38778c33 100644 --- a/docs/guide/src/configuration/environment-variables.md +++ b/docs/guide/src/configuration/environment-variables.md @@ -12,13 +12,17 @@ All configuration is done through environment variables, set either in your Dock | `AWS_LWA_READINESS_CHECK_PROTOCOL` | Readiness check protocol: `http` or `tcp` | `http` | | `AWS_LWA_READINESS_CHECK_HEALTHY_STATUS` | HTTP status codes considered healthy (e.g. `200-399` or `200,201,204,301-399`) | `100-499` | | `AWS_LWA_ASYNC_INIT` | Enable asynchronous initialization | `false` | -| `AWS_LWA_REMOVE_BASE_PATH` | Base path to remove from request path | None | +| `AWS_LWA_REMOVE_BASE_PATH` | Base path to remove from the request path. Strips **exactly one** leading occurrence and only on a path-segment boundary: with `/api`, `/api/api/order` → `/api/order` and `/apiorder` is passed through unchanged; a configured trailing slash (`/api/`) is normalized. | None | | `AWS_LWA_ENABLE_COMPRESSION` | Enable gzip/br compression (buffered mode only) | `false` | | `AWS_LWA_INVOKE_MODE` | Invoke mode: `buffered` or `response_stream` | `buffered` | | `AWS_LWA_PASS_THROUGH_PATH` | Path for non-HTTP event payloads | `/events` | | `AWS_LWA_AUTHORIZATION_SOURCE` | Header name to replace with `Authorization` | None | | `AWS_LWA_ERROR_STATUS_CODES` | HTTP status codes that cause Lambda invocation failure (e.g. `500,502-504`) | None | | `AWS_LWA_LAMBDA_RUNTIME_API_PROXY` | Proxy URL for Lambda Runtime API requests | None | +| `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` | Inner-app path the adapter POSTs to before a SnapStart snapshot | None | +| `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` | Inner-app path the adapter POSTs to after a SnapStart restore | None | +| `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` | Idle keep-alive (seconds) for the adapter's connection to your app | `4` | +| `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` | Seconds (fractional allowed, e.g. `0.5`) the adapter waits for the app to report ready (cold-start init **and** after a SnapStart restore). On expiry the adapter **fails** rather than serving: cold-start init fails (the runtime never starts) and a restore fails. Unset, `0`, or a negative value all mean **wait indefinitely** (no bound); a set-but-`<= 0` or malformed value is ignored with a `warn!`. The `async_init` path keeps its own ~9.8s bound (non-fatal) and is unaffected. | unset / `<= 0` (unbounded) | ## Deprecated Variables diff --git a/docs/guide/src/examples/overview.md b/docs/guide/src/examples/overview.md index 1cc47a1c..b050c732 100644 --- a/docs/guide/src/examples/overview.md +++ b/docs/guide/src/examples/overview.md @@ -8,6 +8,8 @@ The repository includes working examples for many popular web frameworks, packag |---------|-----------|-----------| | [FastAPI](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi) | Docker | No | | [FastAPI in Zip](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-zip) | Zip | No | +| [FastAPI SnapStart](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-snapstart) | Docker | No | +| [FastAPI SnapStart Zip](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-snapstart-zip) | Zip | No | | [FastAPI Background Tasks](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-background-tasks) | Docker | No | | [FastAPI Response Streaming](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-response-streaming) | Docker | Yes | | [FastAPI Response Streaming Zip](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-response-streaming-zip) | Zip | Yes | diff --git a/docs/guide/src/features/snapstart.md b/docs/guide/src/features/snapstart.md new file mode 100644 index 00000000..e9f66e88 --- /dev/null +++ b/docs/guide/src/features/snapstart.md @@ -0,0 +1,76 @@ +# SnapStart + +[Lambda SnapStart](https://docs.aws.amazon.com/lambda/latest/dg/snapstart.html) snapshots an initialized execution environment and restores it on later cold starts, reducing startup latency. Because the adapter runs your web application as a separate process, the application does not have direct access to the SnapStart lifecycle. The adapter bridges this gap with two optional HTTP hooks. + +## Hooks + +| Variable | When the adapter calls it | Use it to | +|----------|---------------------------|-----------| +| `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` | Before the snapshot is taken | Drain or close resources that will not survive the snapshot | +| `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` | After restore, before serving traffic | Reconnect, refresh credentials, reseed randomness, regenerate unique identifiers | + +Both hooks are opt-in and independent — each fires only when its variable is set. + +## How it works + +The adapter always registers for the SnapStart lifecycle; the Lambda runtime invokes the hooks only when your function runs under SnapStart. When it does, the adapter participates as follows: + +1. **Before checkpoint** — if `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` is set, the adapter sends an empty `POST` to that path on your application, then signals Lambda that it is ready for the snapshot. +2. **After restore** — Lambda restores the environment. The adapter first refreshes its own HTTP connection to your application (so it never reuses a connection captured in the snapshot); then, if `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` is set, sends an empty `POST` to that path; and finally re-runs the readiness check before admitting traffic. + +Each hook is an empty `POST`, and your application must respond with a `2xx` status. A non-`2xx` response, a connection failure, or taking longer than 60 seconds to respond fails the SnapStart phase — initialization for the before-checkpoint hook, restore for the after-restore hook — rather than serving traffic against an improperly prepared application. The final readiness check runs on every restore (whether or not an after-restore path is configured). By default this readiness wait is unbounded; set `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` (fractional seconds allowed, e.g. `0.5`) to bound it, in which case a restore whose application does not report ready within that timeout fails. The same variable also bounds the initial cold-start readiness check: when set and the application does not report ready within the timeout, initialization fails (the Lambda runtime never starts) rather than serving traffic against an app that never came up. + +## Why you need the hooks + +State captured in a snapshot is shared across every restored environment. Two classes of problem follow: + +- **Stale connections.** Database connections, cached DNS, and keep-alive HTTP connections captured in the snapshot are dead by the time the environment is restored. Close them in the before-checkpoint hook and re-establish them in the after-restore hook. +- **Uniqueness and entropy.** Values seeded once at initialization — random number generators, UUID seeds, security tokens — become identical across every restored environment. Reseed them in the after-restore hook. + +## Securing the hook paths + +The hook paths are control-plane operations. External requests (via API Gateway or ALB) that target a configured hook path receive `403 Forbidden` and are never forwarded to your application. The guard matches the hook route strictly: it canonicalizes both the configured path and the incoming request path (percent-decoding, collapsing `//`, `.` and `..` segments, and comparing case-insensitively) before comparing, so alternate spellings that resolve to the same route are blocked too. Choose paths your normal application traffic does not use (for example, `/snapstart/before` and `/snapstart/after`). + +Three kinds of value are rejected at startup, because the adapter cannot guard the route they name. In each case initialization fails with an error naming the offending path, rather than running with a state-mutating route left reachable: + +- **A path whose decoded form contains a percent sign** (for example `/snapstart/after%25`, which decodes to `/snapstart/after%`), or a malformed `%` escape (a trailing `%`, or `%zz`). Web frameworks disagree on how to route these — some reject them outright, others decode them leniently — so the adapter cannot guarantee it blocks every spelling that reaches the route, and a partially protected hook path is worse than an obviously invalid one. Percent-encoding that decodes to an ordinary path is fine (`/snapstart/%61fter` is accepted and guarded as `/snapstart/after`), but a plain unencoded path is clearest. +- **A path that collapses to the application root** (`/`, `//`, `/..`, `/.`, `/foo/..`, `/%2f`, …). Guarding the root would return `403` for every request to `/`, so the guard cannot cover it — and the hook would still `POST` to `/` on every lifecycle event, failing the phase on any application that does not handle `POST /`. Use a dedicated path instead. +- **A path that resolves to the same route as `AWS_LWA_PASS_THROUGH_PATH`** (default `/events`). Non-HTTP trigger events are rewritten onto the pass-through path before the guard runs, so every such event would be answered with `403` instead of reaching your application. + +Leaving a hook variable unset (or empty) is different from either: it simply means that hook does not fire. + +> **Warning:** this 403 guard exists only when the adapter is in the request path — i.e. when your app runs on Lambda behind the adapter. The hook routes are ordinary application routes that mutate state (the examples close and re-establish the connection pool), so if you run the same image or app **without** the adapter (Amazon ECS, Amazon EKS, a local Docker host), those routes are reachable and unauthenticated. In that case, do not expose them publicly, or protect them yourself. + +## Example + +```python +from fastapi import FastAPI, Response + +app = FastAPI() +pool = None # your database/connection pool + + +@app.post("/snapstart/before") +async def before_checkpoint(): + # Close resources that won't survive the snapshot. + if pool is not None: + await pool.close() + return Response(status_code=200) + + +@app.post("/snapstart/after") +async def after_restore(): + # Re-establish resources and reseed anything that must be unique. + global pool + pool = await create_pool() + return Response(status_code=200) +``` + +Configure the function with: + +``` +AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH=/snapstart/before +AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH=/snapstart/after +``` + +See the [fastapi-snapstart-zip example](https://github.com/aws/aws-lambda-web-adapter/tree/main/examples/fastapi-snapstart-zip) for a complete, deployable application. diff --git a/examples/fastapi-snapstart-zip/.gitignore b/examples/fastapi-snapstart-zip/.gitignore new file mode 100644 index 00000000..4808264d --- /dev/null +++ b/examples/fastapi-snapstart-zip/.gitignore @@ -0,0 +1,244 @@ + +# Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Build folder + +*/build/* + +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode \ No newline at end of file diff --git a/examples/fastapi-snapstart-zip/README.md b/examples/fastapi-snapstart-zip/README.md new file mode 100644 index 00000000..c7965a2e --- /dev/null +++ b/examples/fastapi-snapstart-zip/README.md @@ -0,0 +1,53 @@ +# FastAPI with Lambda SnapStart + +This example shows how to use Lambda Web Adapter's SnapStart hooks to drain and re-establish a connection pool around the snapshot/restore boundary, running a FastAPI application on the managed python runtime. + +### How does it work? + +We add the Lambda Web Adapter layer to the function and configure the wrapper script. + +1. attach Lambda Adapter layer to your function. This layer contains the Lambda Adapter binary and a wrapper script. + 1. x86_64: `arn:aws:lambda:${AWS::Region}:753240598075:layer:LambdaAdapterLayerX86:30` + 2. arm64: `arn:aws:lambda:${AWS::Region}:753240598075:layer:LambdaAdapterLayerArm64:30` +2. configure Lambda environment variable `AWS_LAMBDA_EXEC_WRAPPER` to `/opt/bootstrap`. This is a wrapper script included in the layer. +3. set function handler to a startup command: `run.sh`. The wrapper script will execute this command to boot up your application. + +To get more information of Wrapper script, please read Lambda documentation [here](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html#runtime-wrapper). + +#### SnapStart hooks + +[Lambda SnapStart](https://docs.aws.amazon.com/lambda/latest/dg/snapstart.html) initializes your function, takes a snapshot of the initialized execution environment, and restores from that snapshot to serve invocations. Some resources do not survive the snapshot (open connections) and some values must not be shared across every restored environment (unique ids, seeds). The Lambda Web Adapter bridges the SnapStart lifecycle to your inner web app through two opt-in environment variables: + +- **Before the snapshot** — `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` is set to `/snapstart/before`. The adapter sends an empty HTTP `POST` to this path. The app uses it to drain and close resources that will not survive the snapshot (in this example, it closes the connection pool). +- **After restore**, before traffic is served, the adapter performs three steps in order: + 1. It refreshes its own HTTP connection to the inner app, so it never reuses a connection captured in the snapshot (this is automatic — you do not configure or manage the adapter's client). + 2. `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` is set to `/snapstart/after`. The adapter sends an empty HTTP `POST` to this path over the refreshed connection. The app uses it to re-establish connections and regenerate per-environment unique values (in this example, it reconnects the pool and generates a fresh `connection_id`). + 3. It re-runs the readiness check against your app before admitting traffic. + +Both hook routes must return a `2xx` status code. A non-2xx response, a connection failure, or taking longer than 60 seconds to respond fails the SnapStart phase. The readiness check runs on every restore; by default it waits indefinitely for the app to recover, but you can bound it with `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` (fractional seconds allowed), in which case a restore whose app does not report ready within that timeout fails — so traffic is never served against an app that has not finished recovering. + +These hook routes are protected on the Lambda invocation path: the 403 guard lives in the adapter, which only sits in front of your app while it processes Lambda events, so external callers that reach the function (via API Gateway or ALB) and request `/snapstart/before` or `/snapstart/after` receive a `403 Forbidden`. (The guard exists only when the adapter is in the request path — if you run your app without the adapter, protect or don't expose these state-mutating routes yourself.) + +### Build and Deploy + +Run the following commands to build and deploy the application to lambda. + +```bash +sam build --use-container +sam deploy --guided +``` +When the deployment completes, take note of FastAPISnapStartApi's Value. It is the API Gateway endpoint URL. + +### Verify it works + +Open FastAPISnapStartApi's URL in a browser. The `/` response shows `connected: true` and a `connection_id`, for example: + +```json +{ + "message": "Hello from FastAPI on Lambda SnapStart", + "connected": true, + "connection_id": 482913007 +} +``` + +After a SnapStart restore, the `connection_id` is regenerated rather than shared across every restored environment, because the adapter calls the after-restore hook (`/snapstart/after`), which reconnects the pool and generates a fresh id. This is exactly the behavior you want for any per-environment value (connections, random seeds, unique identifiers) that must not be duplicated across restored snapshots. diff --git a/examples/fastapi-snapstart-zip/app/main.py b/examples/fastapi-snapstart-zip/app/main.py new file mode 100644 index 00000000..82370943 --- /dev/null +++ b/examples/fastapi-snapstart-zip/app/main.py @@ -0,0 +1,64 @@ +import random + +from fastapi import FastAPI, Response + +app = FastAPI() + + +class ConnectionPool: + """A stand-in for a real database/connection pool. + + In a real application these methods would open and close sockets to a + database. Here we just track state so the SnapStart lifecycle is observable. + """ + + def __init__(self): + self.connected = False + self.connection_id = None + + def connect(self): + # A fresh, unique id per environment — the kind of value that must be + # regenerated after a SnapStart restore so it is not shared across + # every restored environment. + self.connection_id = random.randint(1, 1_000_000_000) + self.connected = True + + def close(self): + self.connected = False + + +pool = ConnectionPool() +pool.connect() + + +@app.get("/") +async def root(): + return { + "message": "Hello from FastAPI on Lambda SnapStart", + "connected": pool.connected, + "connection_id": pool.connection_id, + } + + +@app.post("/snapstart/before") +async def before_checkpoint(): + """Called by the adapter before the snapshot is taken. + + Close resources that will not survive the snapshot. + """ + pool.close() + return Response(status_code=200) + + +@app.post("/snapstart/after") +async def after_restore(): + """Called by the adapter after the environment is restored. + + Re-establish connections and regenerate per-environment unique values. + """ + # Reseed from OS entropy first. Python's random module keeps its state in + # process memory, which is captured in the snapshot — without reseeding, + # every restored environment would draw the same "unique" value. + random.seed() + pool.connect() + return Response(status_code=200) diff --git a/examples/fastapi-snapstart-zip/app/requirements.txt b/examples/fastapi-snapstart-zip/app/requirements.txt new file mode 100644 index 00000000..e61869b3 --- /dev/null +++ b/examples/fastapi-snapstart-zip/app/requirements.txt @@ -0,0 +1,10 @@ +annotated-types==0.7.0 +anyio==4.6.2.post1 +fastapi==0.115.5 +idna==3.10 +pydantic==2.9.2 +pydantic-core==2.23.4 +sniffio==1.3.1 +starlette==0.41.2 +typing-extensions==4.12.2 +uvicorn==0.32.0 \ No newline at end of file diff --git a/examples/fastapi-snapstart-zip/app/run.sh b/examples/fastapi-snapstart-zip/app/run.sh new file mode 100755 index 00000000..a630f442 --- /dev/null +++ b/examples/fastapi-snapstart-zip/app/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +PATH=$PATH:$LAMBDA_TASK_ROOT/bin \ + PYTHONPATH=$PYTHONPATH:/opt/python:$LAMBDA_RUNTIME_DIR \ + exec python -m uvicorn --port=$PORT main:app diff --git a/examples/fastapi-snapstart-zip/events/event.json b/examples/fastapi-snapstart-zip/events/event.json new file mode 100644 index 00000000..a6197dea --- /dev/null +++ b/examples/fastapi-snapstart-zip/events/event.json @@ -0,0 +1,62 @@ +{ + "body": "{\"message\": \"hello world\"}", + "resource": "/hello", + "path": "/hello", + "httpMethod": "GET", + "isBase64Encoded": false, + "queryStringParameters": { + "foo": "bar" + }, + "pathParameters": { + "proxy": "/path/to/resource" + }, + "stageVariables": { + "baz": "qux" + }, + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, sdch", + "Accept-Language": "en-US,en;q=0.8", + "Cache-Control": "max-age=0", + "CloudFront-Forwarded-Proto": "https", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-Mobile-Viewer": "false", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Tablet-Viewer": "false", + "CloudFront-Viewer-Country": "US", + "Host": "1234567890.execute-api.us-east-1.amazonaws.com", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Custom User Agent String", + "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", + "X-Forwarded-For": "127.0.0.1, 127.0.0.2", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "requestContext": { + "accountId": "123456789012", + "resourceId": "123456", + "stage": "prod", + "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", + "requestTime": "09/Apr/2015:12:34:56 +0000", + "requestTimeEpoch": 1428582896000, + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "accessKey": null, + "sourceIp": "127.0.0.1", + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Custom User Agent String", + "user": null + }, + "path": "/prod/hello", + "resourcePath": "/hello", + "httpMethod": "POST", + "apiId": "1234567890", + "protocol": "HTTP/1.1" + } +} diff --git a/examples/fastapi-snapstart-zip/template.yaml b/examples/fastapi-snapstart-zip/template.yaml new file mode 100644 index 00000000..27d0628f --- /dev/null +++ b/examples/fastapi-snapstart-zip/template.yaml @@ -0,0 +1,44 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + FastAPI with Lambda SnapStart + +# More info about Globals: https://github.com/aws/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 10 + +Resources: + FastAPISnapStartFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: app/ + Handler: run.sh + Runtime: python3.12 + MemorySize: 256 + AutoPublishAlias: live + SnapStart: + ApplyOn: PublishedVersions + Environment: + Variables: + AWS_LAMBDA_EXEC_WRAPPER: /opt/bootstrap + PORT: 8000 + AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH: /snapstart/before + AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH: /snapstart/after + Layers: + - !Sub arn:aws:lambda:${AWS::Region}:753240598075:layer:LambdaAdapterLayerX86:30 + Events: + ApiEvent: + Type: HttpApi + + +Outputs: + FastAPISnapStartApi: + Description: "API Gateway endpoint URL for Prod stage for FastAPI SnapStart function" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.${AWS::URLSuffix}/" + FastAPISnapStartFunction: + Description: "FastAPI SnapStart Lambda Function ARN" + Value: !GetAtt FastAPISnapStartFunction.Arn + FastAPISnapStartIamRole: + Description: "Implicit IAM Role created for FastAPI SnapStart function" + Value: !GetAtt FastAPISnapStartFunctionRole.Arn diff --git a/examples/fastapi-snapstart/.gitignore b/examples/fastapi-snapstart/.gitignore new file mode 100644 index 00000000..4808264d --- /dev/null +++ b/examples/fastapi-snapstart/.gitignore @@ -0,0 +1,244 @@ + +# Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Build folder + +*/build/* + +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode \ No newline at end of file diff --git a/examples/fastapi-snapstart/README.md b/examples/fastapi-snapstart/README.md new file mode 100644 index 00000000..c51e68d0 --- /dev/null +++ b/examples/fastapi-snapstart/README.md @@ -0,0 +1,124 @@ +# FastAPI with Lambda SnapStart (container image) + +This example shows how to use the Lambda Web Adapter's SnapStart hooks to drain and +re-establish a connection pool around the snapshot/restore boundary, packaged as a +**container image** (OCI) rather than a zip. + +For the zip-packaged equivalent, see +[fastapi-snapstart-zip](../fastapi-snapstart-zip). + +## How does it work? + +The [Dockerfile](app/Dockerfile) copies the Lambda Web Adapter binary into +`/opt/extensions`: + +```dockerfile +FROM public.ecr.aws/docker/library/python:3.12-slim +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:1.1.0 /lambda-adapter /opt/extensions/lambda-adapter +ENV PORT=8000 +WORKDIR /var/task +COPY requirements.txt ./ +RUN python -m pip install -r requirements.txt +COPY *.py ./ +CMD exec uvicorn --port=$PORT main:app +``` + +The two SnapStart hook endpoints are configured as function environment variables in +[`template.yaml`](template.yaml), keeping the image itself generic: + +```yaml + Environment: + Variables: + AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH: /snapstart/before + AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH: /snapstart/after +``` + +When the function runs under SnapStart, the adapter calls your application at the +snapshot boundary: + +- **Before the snapshot** — `AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH` is set to + `/snapstart/before`. The adapter sends an empty HTTP `POST` to this path. The app uses + it to drain and close resources that will not survive the snapshot (in this example, + it closes the connection pool). +- **After restore**, before traffic is served, the adapter performs three steps in + order: + 1. It refreshes its own HTTP connection to the inner app, so it never reuses a + connection captured in the snapshot (this is automatic — you do not configure or + manage the adapter's client). + 2. `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH` is set to `/snapstart/after`. The adapter + sends an empty HTTP `POST` to this path over the refreshed connection. The app uses + it to re-establish connections and regenerate per-environment unique values (in + this example, it reconnects the pool and generates a fresh `connection_id`). + 3. It re-runs the readiness check against your app before admitting traffic. + +Both hook routes must return a `2xx` status code. A non-2xx response, a connection +failure, or taking longer than 60 seconds to respond fails the SnapStart phase. +The readiness check runs on every restore; by default it waits indefinitely for the +app to recover, but you can bound it with `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` +(fractional seconds allowed), in which case a restore whose app does not report +ready within that timeout fails — so traffic is never served against an app that has +not finished recovering. + +These hook routes are protected **on the Lambda invocation path**: the 403 guard +lives in the adapter, which only sits in front of your app when it is processing +Lambda events, so external callers that reach the function via API Gateway or ALB +and request `/snapstart/before` or `/snapstart/after` receive a `403 Forbidden`. + +> **Warning:** that guard exists only when the adapter is in the request path. +> Because the adapter is packaged inside the image, the same container also runs +> unchanged on Amazon ECS, Amazon EKS, or a local Docker host (`docker run -p +> 8000:8000` below) — but in those deployments traffic goes straight to your app +> and the guard is not present. `/snapstart/before` and `/snapstart/after` are +> state-mutating (this example closes and re-establishes the connection pool), so +> outside Lambda you must not expose them publicly, or protect them yourself. + +## Pre-requisites + +* [AWS CLI](https://aws.amazon.com/cli/) +* [SAM CLI](https://github.com/aws/aws-sam-cli) +* [Docker](https://www.docker.com/products/docker-desktop) + +## Build and Deploy + +Build the container image and deploy with SAM: + +```bash +sam build +sam deploy --guided +``` + +When the deployment completes, take note of the `FastAPISnapStartApi` output — it is +the API Gateway endpoint URL. + +## Verify it works + +Open the API URL in a browser or with `curl`: + +```bash +curl https://xxxxxxxxxx.execute-api.us-west-2.amazonaws.com/ +``` + +The response reports the connection state, for example: + +```json +{ + "message": "Hello from FastAPI on Lambda SnapStart (container image)", + "connected": true, + "connection_id": 426384719 +} +``` + +After a SnapStart restore, the `connection_id` is regenerated rather than shared across +every restored environment, because the adapter calls the after-restore hook +(`/snapstart/after`), which reconnects the pool and generates a fresh id. This is +exactly the behavior you want for any per-environment value (connections, random seeds, +unique identifiers) that must not be duplicated across restored snapshots. + +## Run the container locally + +The same image runs locally — without SnapStart, the hooks are simply never invoked: + +```bash +docker run -d -p 8000:8000 {ECR Image} +curl localhost:8000/ +``` diff --git a/examples/fastapi-snapstart/app/Dockerfile b/examples/fastapi-snapstart/app/Dockerfile new file mode 100644 index 00000000..98a4ad21 --- /dev/null +++ b/examples/fastapi-snapstart/app/Dockerfile @@ -0,0 +1,8 @@ +FROM public.ecr.aws/docker/library/python:3.12-slim +COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:1.1.0 /lambda-adapter /opt/extensions/lambda-adapter +ENV PORT=8000 +WORKDIR /var/task +COPY requirements.txt ./ +RUN python -m pip install -r requirements.txt +COPY *.py ./ +CMD exec uvicorn --port=$PORT main:app diff --git a/examples/fastapi-snapstart/app/main.py b/examples/fastapi-snapstart/app/main.py new file mode 100644 index 00000000..1476f5df --- /dev/null +++ b/examples/fastapi-snapstart/app/main.py @@ -0,0 +1,64 @@ +import random + +from fastapi import FastAPI, Response + +app = FastAPI() + + +class ConnectionPool: + """A stand-in for a real database/connection pool. + + In a real application these methods would open and close sockets to a + database. Here we just track state so the SnapStart lifecycle is observable. + """ + + def __init__(self): + self.connected = False + self.connection_id = None + + def connect(self): + # A fresh, unique id per environment — the kind of value that must be + # regenerated after a SnapStart restore so it is not shared across + # every restored environment. + self.connection_id = random.randint(1, 1_000_000_000) + self.connected = True + + def close(self): + self.connected = False + + +pool = ConnectionPool() +pool.connect() + + +@app.get("/") +async def root(): + return { + "message": "Hello from FastAPI on Lambda SnapStart (container image)", + "connected": pool.connected, + "connection_id": pool.connection_id, + } + + +@app.post("/snapstart/before") +async def before_checkpoint(): + """Called by the adapter before the snapshot is taken. + + Close resources that will not survive the snapshot. + """ + pool.close() + return Response(status_code=200) + + +@app.post("/snapstart/after") +async def after_restore(): + """Called by the adapter after the environment is restored. + + Re-establish connections and regenerate per-environment unique values. + """ + # Reseed from OS entropy first. Python's random module keeps its state in + # process memory, which is captured in the snapshot — without reseeding, + # every restored environment would draw the same "unique" value. + random.seed() + pool.connect() + return Response(status_code=200) diff --git a/examples/fastapi-snapstart/app/requirements.txt b/examples/fastapi-snapstart/app/requirements.txt new file mode 100644 index 00000000..37a49601 --- /dev/null +++ b/examples/fastapi-snapstart/app/requirements.txt @@ -0,0 +1,10 @@ +annotated-types==0.7.0 +anyio==4.6.2.post1 +fastapi==0.115.5 +idna==3.10 +pydantic==2.9.2 +pydantic-core==2.23.4 +sniffio==1.3.1 +starlette==0.41.2 +typing-extensions==4.12.2 +uvicorn==0.32.0 diff --git a/examples/fastapi-snapstart/events/event.json b/examples/fastapi-snapstart/events/event.json new file mode 100644 index 00000000..41be9620 --- /dev/null +++ b/examples/fastapi-snapstart/events/event.json @@ -0,0 +1,62 @@ +{ + "body": "{\"message\": \"hello world\"}", + "resource": "/hello", + "path": "/hello", + "httpMethod": "GET", + "isBase64Encoded": false, + "queryStringParameters": { + "foo": "bar" + }, + "pathParameters": { + "proxy": "/path/to/resource" + }, + "stageVariables": { + "baz": "qux" + }, + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, sdch", + "Accept-Language": "en-US,en;q=0.8", + "Cache-Control": "max-age=0", + "CloudFront-Forwarded-Proto": "https", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-Mobile-Viewer": "false", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Tablet-Viewer": "false", + "CloudFront-Viewer-Country": "US", + "Host": "1234567890.execute-api.us-east-1.amazonaws.com", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Custom User Agent String", + "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", + "X-Forwarded-For": "127.0.0.1, 127.0.0.2", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "requestContext": { + "accountId": "123456789012", + "resourceId": "123456", + "stage": "prod", + "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", + "requestTime": "09/Apr/2015:12:34:56 +0000", + "requestTimeEpoch": 1428582896000, + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "accessKey": null, + "sourceIp": "127.0.0.1", + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Custom User Agent String", + "user": null + }, + "path": "/prod/hello", + "resourcePath": "/hello", + "httpMethod": "POST", + "apiId": "1234567890", + "protocol": "HTTP/1.1" + } + } diff --git a/examples/fastapi-snapstart/template.yaml b/examples/fastapi-snapstart/template.yaml new file mode 100644 index 00000000..3eb6e733 --- /dev/null +++ b/examples/fastapi-snapstart/template.yaml @@ -0,0 +1,41 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + FastAPI with Lambda SnapStart (container image) + +# More info about Globals: https://github.com/aws/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 10 + +Resources: + FastAPISnapStartFunction: + Type: AWS::Serverless::Function + Properties: + PackageType: Image + MemorySize: 256 + AutoPublishAlias: live + SnapStart: + ApplyOn: PublishedVersions + Environment: + Variables: + AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH: /snapstart/before + AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH: /snapstart/after + Events: + ApiEvent: + Type: HttpApi + Metadata: + Dockerfile: Dockerfile + DockerContext: ./app + DockerTag: python3.12-v1 + +Outputs: + FastAPISnapStartApi: + Description: "API Gateway endpoint URL for Prod stage for FastAPI SnapStart function" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.${AWS::URLSuffix}/" + FastAPISnapStartFunction: + Description: "FastAPI SnapStart Lambda Function ARN" + Value: !GetAtt FastAPISnapStartFunction.Arn + FastAPISnapStartIamRole: + Description: "Implicit IAM Role created for FastAPI SnapStart function" + Value: !GetAtt FastAPISnapStartFunctionRole.Arn diff --git a/src/lib.rs b/src/lib.rs index d3eec116..fcdd7561 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ //! let mut adapter = Adapter::new(&options)?; //! //! adapter.register_default_extension(); -//! adapter.check_init_health().await; +//! adapter.check_init_health().await?; //! adapter.run().await //! }) //! } @@ -63,6 +63,7 @@ //! with `InvokeMode: RESPONSE_STREAM`. mod readiness; +mod snapstart; // Environment variable names (AWS_LWA_ prefix) const ENV_PORT: &str = "AWS_LWA_PORT"; @@ -78,7 +79,16 @@ const ENV_ENABLE_COMPRESSION: &str = "AWS_LWA_ENABLE_COMPRESSION"; const ENV_INVOKE_MODE: &str = "AWS_LWA_INVOKE_MODE"; const ENV_AUTHORIZATION_SOURCE: &str = "AWS_LWA_AUTHORIZATION_SOURCE"; const ENV_ERROR_STATUS_CODES: &str = "AWS_LWA_ERROR_STATUS_CODES"; +const ENV_SNAPSTART_BEFORE_CHECKPOINT_PATH: &str = "AWS_LWA_SNAPSTART_BEFORE_CHECKPOINT_PATH"; +const ENV_SNAPSTART_AFTER_RESTORE_PATH: &str = "AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH"; const ENV_LAMBDA_RUNTIME_API_PROXY: &str = "AWS_LWA_LAMBDA_RUNTIME_API_PROXY"; +const ENV_POOL_IDLE_TIMEOUT_SECONDS: &str = "AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS"; + +/// Default idle-connection keep-alive for the adapter's inner-app HTTP client, +/// used when [`ENV_POOL_IDLE_TIMEOUT_SECONDS`] is unset or unparseable. +const DEFAULT_POOL_IDLE_TIMEOUT_SECONDS: u64 = 4; + +const ENV_READINESS_CHECK_TIMEOUT_SECONDS: &str = "AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS"; // Deprecated environment variable names (without prefix) const ENV_PORT_DEPRECATED: &str = "PORT"; @@ -104,16 +114,21 @@ use http::{ Method, StatusCode, }; use http_body::Body as HttpBody; -use http_body_util::BodyExt; -use hyper::body::Incoming; +use http_body_util::{BodyExt, Empty}; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::client::legacy::Client; use lambda_http::request::RequestContext; pub use lambda_http::tracing; use lambda_http::Body; pub use lambda_http::Error; + +// Re-export the body types that appear in the public `Service::Response` +// (`Response>`), so downstream consumers driving the +// `Adapter` as a `tower::Service` can name that type without taking their own +// direct dependency on `bytes` / `http-body-util`. +pub use bytes::Bytes; +pub use http_body_util::combinators::BoxBody; use lambda_http::{Request, RequestExt, Response}; -use readiness::Checkpoint; use std::borrow::Cow; use std::fmt::Debug; use std::{ @@ -126,8 +141,7 @@ use std::{ }, time::Duration, }; -use tokio::{net::TcpStream, time::timeout}; -use tokio_retry::{strategy::FixedInterval, Retry}; +use tokio::time::timeout; use tower::{Service, ServiceBuilder}; use tower_http::compression::CompressionLayer; use url::Url; @@ -349,6 +363,36 @@ pub struct AdapterOptions { /// the adapter will return an error to Lambda instead of the response. /// This can be useful for triggering Lambda retry behavior. pub error_status_codes: Option>, + + /// Inner-app path POSTed before the SnapStart snapshot is taken. + /// When set, the adapter notifies the app so it can drain resources. + /// Default: `None` (phase skipped). + pub snapstart_before_checkpoint_path: Option, + + /// Inner-app path POSTed after the SnapStart restore completes. + /// When set, the adapter notifies the app so it can reconnect / reseed. + /// Default: `None` (phase skipped). + pub snapstart_after_restore_path: Option, + + /// Idle-connection keep-alive for the adapter's HTTP client to the inner app. + /// + /// Configurable via `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` (whole seconds). + /// Default: 4 seconds. + pub pool_idle_timeout: Duration, + + /// Bound on the readiness check: how long the adapter waits for the inner app + /// to report ready before giving up. Applied to both the initial (cold-start) + /// readiness check and the post-SnapStart-restore readiness check. + /// + /// Configurable via `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` (fractional + /// seconds allowed, e.g. `0.5`). + /// `None` (the default) means **unbounded**: the adapter waits indefinitely for + /// the app to become ready, preserving the historical behavior. Setting it bounds + /// both checks; on the restore check a timeout fails the restore. + /// + /// Note: the `async_init` initial-readiness path retains its own fixed ~9.8s + /// bound and is unaffected by this option. + pub readiness_check_timeout: Option, } /// Helper to get env var with deprecation warning for old name @@ -435,6 +479,14 @@ impl Default for AdapterOptions { error_status_codes: env::var(ENV_ERROR_STATUS_CODES) .ok() .map(|codes| parse_status_codes(&codes)), + snapstart_before_checkpoint_path: env::var(ENV_SNAPSTART_BEFORE_CHECKPOINT_PATH) + .ok() + .filter(|p| !p.is_empty()), + snapstart_after_restore_path: env::var(ENV_SNAPSTART_AFTER_RESTORE_PATH) + .ok() + .filter(|p| !p.is_empty()), + pool_idle_timeout: pool_idle_timeout_from_env(), + readiness_check_timeout: readiness_check_timeout_from_env(), } } } @@ -503,6 +555,236 @@ fn strip_forbidden_header_bytes(s: &str) -> Cow<'_, [u8]> { } } +/// Percent-decode `input` a single pass. Returns `None` if a `%` escape is +/// malformed (not followed by two hex digits) — the caller treats a decode +/// failure as an ambiguous input and fails closed. +fn percent_decode_once(input: &str) -> Option { + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + // Need two hex digits after '%'. + let hi = bytes.get(i + 1).copied()?; + let lo = bytes.get(i + 2).copied()?; + let h = (hi as char).to_digit(16)?; + let l = (lo as char).to_digit(16)?; + out.push((h * 16 + l) as u8); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + // The decoded bytes must remain valid UTF-8 to be a comparable path. + String::from_utf8(out).ok() +} + +/// Canonicalize a path into a list of lowercased segments for the strict, +/// fail-closed SnapStart hook guard. +/// +/// The guard must block *every* spelling that the downstream app router would +/// resolve to the configured hook route, so this over-approximates: it +/// percent-decodes (a single pass, matching the router), splits on `/`, +/// drops empty segments (collapsing `//`, leading/trailing slashes), resolves +/// `.`/`..`, and lowercases each segment. +/// +/// An encoded slash (`%2f`) is decoded to a literal `/` *before* splitting, so a +/// spelling like `/snapstart/%2fafter` collapses onto the same segment list as +/// the hook route and is caught — while a genuinely distinct route that merely +/// contains `%2f` produces a different segment list and is left alone. This keeps +/// the strictness targeted: it only bites paths that canonicalize onto the hook. +/// +/// Returns `None` only for two genuinely undecidable inputs: a malformed `%` escape, +/// or a byte sequence that is not UTF-8 once decoded. A control/null byte is *not* +/// undecidable — it is stripped before canonicalization, which *widens* the blocked +/// equivalence class (`/snapstart/af%00ter` and `/snapstart/after%0A` canonicalize +/// onto the hook route and are blocked), because a router can still resolve such a +/// path to the hook: Python's `$` matches before a trailing newline. +/// +/// [`matches_hook_path`] treats a `None` **request** path as *not the hook* and +/// passes it through (see `matches_hook_path` and +/// `test_matches_hook_path_undecidable_passes_through`); that is safe because +/// [`hook_target`] guarantees no hook route contains a literal `%`. A `None` on the +/// **configured** side is rejected outright by [`hook_target`], failing +/// initialization rather than leaving the route partially guarded. +fn canonicalize_hook_path(path: &str) -> Option> { + // Percent-decode a SINGLE pass, mirroring what the downstream app router + // does. A router decodes exactly once, so `/snapstart/%61fter` reaches the + // app as `/snapstart/after` (and must be guarded), while `/snapstart/%2561fter` + // reaches it as the literal `/snapstart/%61fter` (a different route the app + // does NOT resolve to the hook). Decoding more than once would over-decode + // relative to the router — buying no extra protection while making a validly + // single-encoded path like `/reports/100%25` (i.e. `/reports/100%`) look + // undecidable and get a false 403. A malformed escape or non-UTF-8 result + // still yields None — the only two cases that do; see the contract above. + let current = percent_decode_once(path)?; + // Control bytes are NOT "undecidable": a downstream router can still resolve a + // path that carries them (e.g. Python's `$` matches immediately before a + // trailing `\n`, so Starlette resolves `/hook\n` to the `/hook` route). Bailing + // out with `None` here would make `matches_hook_path` pass such a request + // through and leave the hook route externally reachable. So STRIP control bytes + // (including DEL) and keep canonicalizing — `/snapstart/after%0A` then + // canonicalizes to `["snapstart", "after"]` and is blocked. A malformed `%` + // escape / non-UTF-8 result is different (the `?` above already returned None + // for it) and keeps its pass-through, avoiding the `/reports/100%` false 403. + let current: String = current.chars().filter(|c| !c.is_control()).collect(); + let mut segments: Vec = Vec::new(); + for seg in current.split('/') { + // Drop matrix / path parameters (everything from the first `;` in a + // segment). Several supported frameworks strip these before routing — + // Spring MVC's UrlPathHelper defaults to removeSemicolonContent=true, and + // servlet containers strip `;jsessionid` — so `/snapstart/after;x=1` + // resolves to `/snapstart/after`. The guard must block that spelling too; + // stripping here (before empty/`.`/`..` classification) keeps the guard's + // equivalence class aligned with the app router. This runs on both sides + // via `hook_target`, so it stays symmetric. + let seg = seg.split(';').next().unwrap_or(seg); + match seg { + "" | "." => continue, // collapse empty segments and `.` + ".." => { + segments.pop(); // resolve parent segment + } + s => segments.push(s.to_ascii_lowercase()), + } + } + Some(segments) +} + +/// Treats an empty configured value as unset. +/// +/// An empty `AWS_LWA_SNAPSTART_*_PATH` means "this hook does not fire". Collapsing +/// it to `None` at the single point where [`Adapter::new`] reads it keeps the guard +/// target and the hook's POST target from disagreeing — see +/// `test_empty_hook_path_is_normalized_on_both_sides`. +fn non_empty(value: &Option) -> Option { + value.as_deref().filter(|v| !v.is_empty()).map(str::to_string) +} + +/// Precomputes the guard target for a configured hook path. +/// +/// Runs the operator-configured path through the SAME `Url::set_path` +/// transformation that `SnapStartHooks::post_hook` uses to reach the app +/// (`domain.set_path(configured)`), so the guard protects exactly the route the +/// app actually serves — not the raw env-var string. Returns: +/// +/// * `Ok(None)` — no hook configured (unset, or set to the empty string). +/// * `Ok(Some(segments))` — the normal case: the post-`set_path` route +/// canonicalized (percent-decode, collapse `//`/`.`/`..`, case-fold). +/// * `Err(_)` — the route cannot be guarded exactly. Three cases, all always a +/// misconfiguration of a control-plane path, and all rejected rather than +/// downgraded to a weaker guard or to no guard at all (the app still *serves* +/// such a route, so anything less leaves a state-mutating route reachable): +/// 1. It could not be canonicalized at all (a malformed `%` escape, or non-UTF-8 +/// after decoding). +/// 2. Its canonical form contains a literal `%`. This is what makes +/// [`matches_hook_path`]'s pass-through of an undecidable *request* path safe +/// on every framework, without modelling per-framework decoding: an +/// undecidable request path is either rejected by the router outright (Node +/// throws `URIError`, so Express answers 400; Go and Spring likewise 400) or +/// decoded leniently into a path containing a literal `%` or U+FFFD (Python's +/// `unquote`) — and neither can equal a `%`-free hook route. +/// 3. It collapses to the app root (`/`, `//`, `/..`, `/.`, `/foo/..`, `/%2f`, …). +/// Guarding the root would 403 all normal traffic, so the guard cannot cover +/// it — and `SnapStartHooks::post_hook` would still POST to `/` on every +/// lifecycle event, which is a 405 on an app that does not handle `POST /` +/// and fails the phase. A hook path must be one "your normal application +/// traffic does not use", which the root never is. +/// +/// `Adapter::new` propagates the error, failing initialization with an actionable +/// message rather than starting up with the hook route reachable or with a hook +/// that fails every restore. +fn hook_target(domain: &Url, configured: &Option) -> Result>, Error> { + let Some(configured) = configured.as_deref() else { + return Ok(None); + }; + if configured.is_empty() { + return Ok(None); + } + // Normalize the configured path exactly as post_hook will send it, so the two + // sides of the guard comparison cannot diverge by construction. + let mut u = domain.clone(); + u.set_path(configured); + let outbound = u.path().to_string(); + match canonicalize_hook_path(&outbound) { + // A configured path that canonicalizes to the root (e.g. "/", "//", "/..", + // "/.", "/foo/..", "/%2f") cannot be guarded: matching it would 403 every + // request to `/`. Reject it rather than silently disabling the guard, because + // `after_restore` POSTs the RAW configured path regardless of the guard + // target, so "no hook" here still leaves the hook firing at `/`. This check + // must live AFTER canonicalization: a raw pre-check on the configured string + // misses the spellings that only collapse to root once `..`/`.`/encoded-slash + // resolve. + Some(segments) if segments.is_empty() => Err(Error::from(format!( + "SnapStart hook path {configured:?} collapses to the application root \ + (normalized to {outbound:?}). It cannot be guarded — matching it would return 403 \ + for every request to `/` — and the hook would still POST to `/` on every SnapStart \ + lifecycle event, failing the phase on any app that does not handle `POST /`. Choose \ + a dedicated path your normal traffic does not use, such as `/snapstart/after`." + ))), + // A literal `%` anywhere in the canonical route breaks the invariant that + // lets the request side pass undecidable paths through (see case 2 above). + Some(segments) if segments.iter().any(|s| s.contains('%')) => Err(Error::from(format!( + "SnapStart hook path {configured:?} resolves to a route containing a literal `%` \ + ({outbound:?} decodes to /{}). The 403 guard cannot cover every spelling a web \ + framework resolves onto such a route, so it is rejected rather than left partially \ + protected. Choose a hook path without a percent sign.", + segments.join("/") + ))), + Some(segments) => Ok(Some(segments)), + None => Err(Error::from(format!( + "SnapStart hook path {configured:?} is not canonicalizable after URL normalization \ + (normalized to {outbound:?}): it contains a malformed % escape or a byte sequence \ + that is not UTF-8 once decoded. The 403 guard cannot cover the encoded spellings \ + of such a route, so it is rejected rather than left partially protected. Choose a \ + hook path without a percent sign." + ))), + } +} + +/// True if the outbound request path resolves to the precomputed hook route. +/// +/// Both sides derive from `Url::set_path`: `want` is computed by [`hook_target`] +/// from `domain.set_path(configured)`, and `outbound_request_path` is the request's +/// `app_url.path()` (also post-`set_path`; see `fetch_response`). Because the two +/// sides share the identical normalization, a configured value that `set_path` +/// rewrites (e.g. `/snapstart\after` → `/snapstart/after`) is guarded on its +/// rewritten form, closing the divergence where the app served a route the guard +/// did not protect. +/// +/// The request path is canonicalized and compared as segment lists. An undecidable +/// request path (a malformed escape, or non-UTF-8 once decoded) passes through. +/// That is safe — not merely a heuristic — because [`hook_target`] guarantees +/// `want` contains no literal `%`: a router either rejects an undecidable path +/// outright (400) or decodes it leniently to something containing a literal `%` or +/// U+FFFD, and neither can equal a `%`-free route. Passing through is what keeps a +/// request like `/reports/100%` from taking a false 403 under an unrelated hook. +/// +/// Single-target convenience form, used by the tests; production goes through +/// [`matches_any_hook_path`] so the request path is canonicalized only once. +#[cfg(test)] +fn matches_hook_path(want: &Option>, outbound_request_path: &str) -> bool { + matches_any_hook_path(&[want], outbound_request_path) +} + +/// [`matches_hook_path`] against several targets, canonicalizing the request path +/// **once**. +/// +/// This runs on every invocation, and both examples plus the guide configure both +/// hooks — so calling the single-target form twice would repeat the percent-decode, +/// control-byte filter, split and per-segment `to_ascii_lowercase` (and their +/// allocations) for an identical result. Costs nothing when no hook is configured: +/// the all-`None` check short-circuits before canonicalizing. +fn matches_any_hook_path(wants: &[&Option>], outbound_request_path: &str) -> bool { + if wants.iter().all(|w| w.is_none()) { + return false; + } + let Some(got) = canonicalize_hook_path(outbound_request_path) else { + return false; // undecidable: not the hook, passes through + }; + wants.iter().any(|w| w.as_ref().is_some_and(|want| want == &got)) +} + /// The Lambda Web Adapter. /// /// This is the main struct that handles forwarding Lambda events to your web application. @@ -530,13 +812,14 @@ fn strip_forbidden_header_bytes(s: &str) -> Cow<'_, [u8]> { /// let mut adapter = Adapter::new(&options)?; /// /// adapter.register_default_extension(); -/// adapter.check_init_health().await; +/// adapter.check_init_health().await?; /// adapter.run().await /// # } /// ``` #[derive(Clone)] pub struct Adapter { client: Arc>, + restored_client: Arc>>>, healthcheck_url: Url, healthcheck_protocol: Protocol, healthcheck_healthy_status: Vec, @@ -549,18 +832,236 @@ pub struct Adapter { invoke_mode: LambdaInvokeMode, authorization_source: Option, error_status_codes: Option>, + snapstart_before_checkpoint_path: Option, + snapstart_after_restore_path: Option, + /// Precomputed guard target for the before-checkpoint hook path, derived from + /// `domain.set_path(configured)` so it matches the route the app actually + /// serves (see [`hook_target`]). + hook_target_before_checkpoint: Option>, + /// Precomputed guard target for the after-restore hook path (see [`hook_target`]). + hook_target_after_restore: Option>, + pool_idle_timeout: Duration, + readiness_check_timeout: Option, +} + +/// Builds the hyper client used to talk to the inner web application. +/// +/// Shared by [`Adapter::new`] and the SnapStart after-restore hook so the +/// post-restore client is built identically to the original. `idle_timeout` is the +/// idle-connection keep-alive, resolved from [`AdapterOptions::pool_idle_timeout`] +/// (env `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS`, default 4 seconds). +/// +/// When `pooling` is [`Pooling::Disabled`] the client sets +/// `pool_max_idle_per_host(0)`, which turns hyper's pool off outright +/// (`Config::is_enabled()` is `max_idle_per_host > 0`): a finished connection is +/// dropped rather than parked, so reuse is impossible *by construction*. +/// +/// That distinction is load-bearing, and a zero `idle_timeout` is NOT a substitute. +/// With the pool enabled, reuse is decided at checkout by +/// `now.saturating_duration_since(idle_at) > timeout`; that saturates to `ZERO` when +/// the recorded instant is ahead of `now`, and `ZERO > ZERO` is false, so the entry +/// is treated as fresh and handed out. A monotonic clock that has not advanced +/// across a restore is exactly the condition hyper#3810 / rust-lang/rust#79462 +/// describe — so an expiry-based scheme would depend on the very clock the +/// workaround exists to distrust. +/// +/// This function reads no environment: the caller decides, so the post-restore +/// rebuild cannot silently inherit the pre-snapshot restriction. +fn build_client(idle_timeout: Duration, pooling: Pooling) -> Client { + let mut builder = Client::builder(hyper_util::rt::TokioExecutor::new()); + builder.pool_idle_timeout(idle_timeout); + if pooling == Pooling::Disabled { + builder.pool_max_idle_per_host(0); + } + builder.build(HttpConnector::new()) +} + +/// Builds the client used to talk to the Lambda Runtime API (RAPID) for extension +/// registration. +/// +/// Idle pooling is disabled. Under SnapStart, a connection parked here is captured in +/// the snapshot and dead after restore — the same hazard `lambda_runtime` handles by +/// calling `reset_pool()` on its own RAPID client in the restore lifecycle. Nothing +/// resets or re-establishes this one, and [`Adapter::register_default_extension`] +/// terminates the process with `exit(1)` if its request fails, so handing out a dead +/// connection would kill a restored environment before it serves anything. +/// +/// Pooling costs nothing to give up here: this client issues exactly two requests — +/// `register`, then the long poll for the first extension event — and the long poll's +/// own in-flight connection is unaffected by the idle-pool setting. +fn runtime_api_client() -> Client { + let mut builder = Client::builder(hyper_util::rt::TokioExecutor::new()); + builder.pool_max_idle_per_host(0); + builder.build(HttpConnector::new()) +} + +/// Whether a client may keep idle connections alive for reuse. See [`build_client`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Pooling { + /// Normal keep-alive, bounded by the configured idle timeout. + Enabled, + /// No connection is retained at all — required before a SnapStart snapshot. + Disabled, +} + +/// Connection-pool policy for the client [`Adapter::new`] builds — the one used +/// before a SnapStart snapshot is taken. +/// +/// Disabled under SnapStart, so no connection can be captured in the snapshot and +/// handed out — dead — after a restore (hyper#3810, rust-lang/rust#79462). +/// +/// Why the pool must be *off* rather than expiry-bounded, measured on a deployed +/// SnapStart container function: `CLOCK_MONOTONIC` does not advance across the +/// snapshot gap. One restore showed the monotonic clock moving **0.54s** while wall +/// time moved **161s**. hyper decides reuse with +/// `now.saturating_duration_since(idle_at) > idle_timeout`, so a connection pooled +/// before the snapshot reads as half a second idle after restore no matter how long +/// the snapshot actually sat — fresh under any sane timeout, and dead. No idle +/// timeout, including `Duration::ZERO`, can fix that (`ZERO > ZERO` is false). +/// +/// `run()` additionally rebuilds a fresh client in the after-restore hook, but +/// keeping this one safe by construction also protects a consumer driving the +/// `Service` impl directly, who never triggers that hook — and that consumer has no +/// other protection, so it must not depend on the clock. +/// +/// The cost is that a pre-snapshot readiness poll reconnects on every 10ms attempt +/// (measured: 27 connections per 300ms of polling, versus 1 with keep-alive). That is +/// confined to init, which under SnapStart runs once per published version rather +/// than per restore. +/// +/// The configured idle timeout is NOT lost: it is kept on +/// [`Adapter::pool_idle_timeout`] and applied to the after-restore rebuild, whose +/// pool starts empty and therefore cannot hold a snapshotted connection. That is what +/// makes `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` effective for the invocations that +/// actually serve traffic, instead of a no-op for the life of the environment. +/// +/// The two sites deliberately disagree — pooling off here, on in `after_restore` — +/// and that is sound because the clock anomaly is confined to the snapshot boundary. +/// Measured after restore on the same deployment, `CLOCK_MONOTONIC` tracks wall time +/// exactly (+6.079s / +6.059s monotonic against +6.1s / +6.0s wall), and requests +/// separated by idle gaps longer than the configured 4s keep-alive all succeeded. A +/// client built after restore holds only post-restore entries, so its expiry +/// accounting is reliable; this one may hold pre-boundary entries, so its is not. +fn base_client_pooling() -> Pooling { + if env::var("AWS_LAMBDA_INITIALIZATION_TYPE").as_deref() == Ok("snap-start") { + Pooling::Disabled + } else { + Pooling::Enabled + } +} + +/// Reads a `Duration` in seconds from environment variable `name`, falling back to +/// `default_secs` when the var is unset or unusable. Surrounding whitespace is +/// tolerated. +/// +/// Accepts fractional seconds (`0.5`), matching +/// [`readiness_check_timeout_from_env`] — the two knobs are siblings and taking +/// different numeric formats would be a trap. A value that is set but unusable (a +/// stray unit suffix like `30s`, non-numeric, negative, NaN, infinity, or an +/// overflowing magnitude) emits a `warn!` before falling back, for the same reason +/// its sibling does: a set value silently becoming the default is the opposite of +/// the operator's intent, and silence makes the misconfiguration invisible. +fn duration_secs_from_env(name: &str, default_secs: u64) -> Duration { + let default = Duration::from_secs(default_secs); + let Ok(raw) = env::var(name) else { + return default; // unset: silent, this is the normal case + }; + let trimmed = raw.trim(); + match trimmed.parse::().map(Duration::try_from_secs_f64) { + Ok(Ok(d)) => d, + // Negative, NaN, infinite, or beyond Duration's range. + Ok(Err(_)) | Err(_) => { + tracing::warn!( + variable = %name, + value = %trimmed, + default = ?default, + "environment variable is set but is not a usable number of seconds \ + (e.g. use `4` or `0.5`, not `4s`); falling back to the default" + ); + default + } + } +} + +/// Reads the inner-app connection pool idle timeout from +/// [`ENV_POOL_IDLE_TIMEOUT_SECONDS`] (`AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS`). +/// Falls back to [`DEFAULT_POOL_IDLE_TIMEOUT_SECONDS`] when unset or unparseable. +fn pool_idle_timeout_from_env() -> Duration { + duration_secs_from_env(ENV_POOL_IDLE_TIMEOUT_SECONDS, DEFAULT_POOL_IDLE_TIMEOUT_SECONDS) +} + +/// Reads the readiness-check timeout from +/// [`ENV_READINESS_CHECK_TIMEOUT_SECONDS`] (`AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS`). +/// Accepts fractional seconds (e.g. `0.5`). Returns `None` — an unbounded readiness +/// wait (historical behavior) — in these cases: +/// * **unset**: silent (the default). +/// * **`<= 0` (including `0`)**: a zero/negative value reads as "fail fast" but a +/// zero timeout would expire instantly and fail every check, so it is treated as +/// "no bound" — and this emits a `warn!` so the non-obvious mapping is visible. +/// * **set but unusable** (a stray suffix like `10s`, non-numeric, NaN, infinity, +/// or an overflowing magnitude): emits a `warn!` before falling back, because +/// silently ignoring a misconfigured bound would let a sync-init cold start hang +/// to the Lambda function timeout with no diagnostic. +fn readiness_check_timeout_from_env() -> Option { + // Unset -> silent unbounded (historical default). + let raw = env::var(ENV_READINESS_CHECK_TIMEOUT_SECONDS).ok()?; + let trimmed = raw.trim(); + + // Parse as fractional seconds; a value that does not parse as a finite, + // representable, positive Duration is REJECTED and falls back to unbounded. + // In every set-but-rejected case we WARN, because a set value silently + // becoming "wait forever" is the opposite of the operator's intent and would + // let a sync-init cold start hang to the Lambda function timeout with no + // signal. This includes `<= 0` (including `0`): a zero/negative bound reads + // naturally as "fail fast / don't wait", but a zero timeout would expire + // instantly and fail every check, so it is treated as "no bound" — and the + // warning makes that non-obvious mapping visible. + match trimmed.parse::() { + Ok(secs) if secs <= 0.0 => { + tracing::warn!( + value = %trimmed, + "AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS is <= 0; a zero/negative bound is \ + treated as no timeout (waiting for readiness indefinitely), not fail-fast" + ); + None + } + Ok(secs) => match Duration::try_from_secs_f64(secs) { + Ok(d) if !d.is_zero() => Some(d), + // secs > 0 but not representable as a Duration (NaN is caught by the + // <= 0.0 arm not matching; this covers infinity / overflow). + _ => { + tracing::warn!( + value = %trimmed, + "AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS is set but out of range; \ + ignoring it and waiting for readiness without a timeout" + ); + None + } + }, + Err(_) => { + tracing::warn!( + value = %trimmed, + "AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS is set but not a number of seconds \ + (e.g. use `10` or `0.5`, not `10s`); ignoring it and waiting for readiness \ + without a timeout" + ); + None + } + } } impl Adapter { /// Creates a new HTTP Adapter instance. /// /// This function initializes a new HTTP client configured to communicate with - /// your web application. When Lambda SnapStart is detected - /// (`AWS_LAMBDA_INITIALIZATION_TYPE=snap-start`), connection pooling is - /// disabled to prevent stale connections after restore, where - /// `CLOCK_MONOTONIC` inconsistencies can cause hyper's pool to reuse dead - /// connections. Otherwise, a 4-second idle timeout is used for connection - /// pooling. + /// your web application. The idle-connection keep-alive comes from + /// [`AdapterOptions::pool_idle_timeout`] (`AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS`, + /// default 4 seconds). Under SnapStart + /// (`AWS_LAMBDA_INITIALIZATION_TYPE=snap-start`) this client is built with idle + /// keep-alive disabled instead, so no connection can be captured in the snapshot + /// and handed out dead after a restore; the configured value is retained and + /// applied to the client rebuilt in the after-restore hook, which is what serves + /// invocations. See the private `base_client_pooling` for the rationale. /// /// # Arguments /// @@ -575,6 +1076,9 @@ impl Adapter { /// Returns an error if: /// - The configured host, port, or readiness check path contain invalid URL characters /// - TCP protocol is configured but the URL is missing host or port + /// - A SnapStart hook path cannot be guarded: it is not canonicalizable, its + /// decoded form contains a literal `%`, it collapses to the application root, + /// or it resolves to the same route as `AWS_LWA_PASS_THROUGH_PATH` /// /// # Examples /// @@ -585,19 +1089,7 @@ impl Adapter { /// let adapter = Adapter::new(&options).expect("Failed to create adapter"); /// ``` pub fn new(options: &AdapterOptions) -> Result, Error> { - let mut builder = Client::builder(hyper_util::rt::TokioExecutor::new()); - - // When running under SnapStart, CLOCK_MONOTONIC can be inconsistent after - // restore, causing hyper's pool to reuse dead connections (hyper#3810, - // rust-lang/rust#79462). Disable pooling in that case. For localhost - // communication the overhead of new TCP connections is negligible. - if env::var("AWS_LAMBDA_INITIALIZATION_TYPE").as_deref() == Ok("snap-start") { - builder.pool_max_idle_per_host(0); - } else { - builder.pool_idle_timeout(Duration::from_secs(4)); - } - - let client = builder.build(HttpConnector::new()); + let client = build_client(options.pool_idle_timeout, base_client_pooling()); let schema = "http"; @@ -622,6 +1114,56 @@ impl Adapter { )) })?; + // Normalize an empty hook path to "unset" BEFORE anything reads it, so the + // guard target and the path the hook POSTs to cannot disagree. `hook_target` + // treats `Some("")` as "no hook", but `SnapStartHooks` would still take its + // `if let Some(path)` branch and POST to `Url::set_path("")` — which is `/`, + // the unguarded application root. Env-derived options already drop empties; + // this covers an `AdapterOptions` built directly. + let snapstart_before_checkpoint_path = non_empty(&options.snapstart_before_checkpoint_path); + let snapstart_after_restore_path = non_empty(&options.snapstart_after_restore_path); + + // Precompute the SnapStart hook guard targets while `domain` is still in + // scope, so the guard compares against the route the app actually serves + // (`domain.set_path(configured)`) rather than the raw configured string. + // A hook path the guard cannot cover fails initialization here rather than + // starting up with a state-mutating route left externally reachable. + let hook_target_before_checkpoint = hook_target(&domain, &snapstart_before_checkpoint_path)?; + let hook_target_after_restore = hook_target(&domain, &snapstart_after_restore_path)?; + + // A hook path that resolves to the same route as the pass-through path is + // unguardable in a different way: `fetch_response` rewrites the path to + // `pass_through_path` for a PassThrough POST *before* the guard runs, so every + // non-HTTP trigger event would canonicalize onto the hook route and get a 403 + // instead of reaching the app. Fail here rather than silently swallowing that + // whole class of events with only a per-invocation warning. + // + // Only relevant when a hook exists. And note the `.unwrap_or(None)`: + // `pass_through_path` is unrelated configuration read straight from the + // environment, so a value `hook_target` would reject (root-collapsing, + // `%`-bearing, non-canonicalizable) must NOT fail initialization here — it is + // also incapable of colliding, since hook targets are canonicalizable and + // non-empty by construction and so nothing rewritten onto such a path can + // canonicalize onto one. + if hook_target_before_checkpoint.is_some() || hook_target_after_restore.is_some() { + let pass_through_target = hook_target(&domain, &Some(options.pass_through_path.clone())).unwrap_or(None); + for (configured, target) in [ + (&snapstart_before_checkpoint_path, &hook_target_before_checkpoint), + (&snapstart_after_restore_path, &hook_target_after_restore), + ] { + if target.is_some() && *target == pass_through_target { + return Err(Error::from(format!( + "SnapStart hook path {:?} resolves to the same route as the pass-through path \ + {:?} (AWS_LWA_PASS_THROUGH_PATH). Non-HTTP trigger events are rewritten onto \ + that path before the hook guard runs, so every such event would be rejected \ + with 403 instead of reaching your application. Choose a different hook path.", + configured.as_deref().unwrap_or_default(), + options.pass_through_path + ))); + } + } + } + // Validate TCP protocol requirements if options.readiness_check_protocol == Protocol::Tcp { if healthcheck_url.host().is_none() { @@ -641,6 +1183,7 @@ impl Adapter { Ok(Adapter { client: Arc::new(client), + restored_client: Arc::new(OnceLock::new()), healthcheck_url, healthcheck_protocol: options.readiness_check_protocol, healthcheck_healthy_status: options.readiness_check_healthy_status.clone(), @@ -653,8 +1196,20 @@ impl Adapter { invoke_mode: options.invoke_mode, authorization_source: options.authorization_source.clone(), error_status_codes: options.error_status_codes.clone(), + snapstart_before_checkpoint_path, + snapstart_after_restore_path, + hook_target_before_checkpoint, + hook_target_after_restore, + pool_idle_timeout: options.pool_idle_timeout, + readiness_check_timeout: options.readiness_check_timeout, }) } + + /// Returns the active inner-app HTTP client: the restored client if a + /// SnapStart restore has published one, otherwise the base client. + fn client(&self) -> &Arc> { + self.restored_client.get().unwrap_or(&self.client) + } } impl Adapter { @@ -693,7 +1248,7 @@ impl Adapter { Some(captured) => captured.clone().unwrap_or_else(|| "127.0.0.1:9001".to_string()), None => env::var(ENV_LAMBDA_RUNTIME_API).unwrap_or_else(|_| "127.0.0.1:9001".to_string()), }; - let client = Client::builder(hyper_util::rt::TokioExecutor::new()).build(HttpConnector::new()); + let client = runtime_api_client(); let register_req = hyper::Request::builder() .method(Method::POST) @@ -741,7 +1296,17 @@ impl Adapter { /// - Allow the application to continue booting in the background /// /// The first request will re-check readiness if the application wasn't ready - /// during initialization. + /// during initialization. The async path always returns `Ok`. + /// + /// # Synchronous Initialization + /// + /// Without `async_init`, this waits for the app to report ready before the + /// Lambda runtime starts serving. If `readiness_check_timeout` + /// (`AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS`) is set and the app does not + /// become ready within it, this returns `Err`: init fails and no traffic is + /// served against an app that never came up. When the timeout is unset the + /// wait is unbounded and this returns `Ok` once the check completes + /// (historical behavior). /// /// # Examples /// @@ -751,23 +1316,47 @@ impl Adapter { /// # async fn example() -> Result<(), lambda_web_adapter::Error> { /// let options = AdapterOptions::default(); /// let mut adapter = Adapter::new(&options)?; - /// adapter.check_init_health().await; + /// adapter.check_init_health().await?; /// # Ok(()) /// # } /// ``` - pub async fn check_init_health(&mut self) { + pub async fn check_init_health(&mut self) -> Result<(), Error> { let ready_at_init = if self.async_init { + // async_init keeps its own fixed bound, independent of + // readiness_check_timeout (see AdapterOptions::readiness_check_timeout). + // A timeout here is non-fatal: the app keeps booting and the first + // request re-checks readiness. + // `is_ok()` means the wait COMPLETED within the bound; the readiness wait + // itself never reports "not ready" (it retries until it is). timeout(Duration::from_secs_f32(9.8), self.check_readiness()) .await - .unwrap_or_default() + .is_ok() + } else if let Some(t) = self.readiness_check_timeout { + // Bound the sync-init readiness wait when configured. On expiry, refuse + // to serve: fail init rather than admit traffic to an app that never + // reported ready — this is the point of configuring the bound. + match timeout(t, self.check_readiness()).await { + Ok(()) => true, + Err(_) => { + return Err(Error::from(format!( + "web application did not become ready within {t:?} \ + (AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS); failing initialization" + ))); + } + } } else { - self.check_readiness().await + // Unset: unbounded wait (historical behavior). It only returns once the + // app is ready, so reaching this point means ready. + self.check_readiness().await; + true }; self.ready_at_init.store(ready_at_init, Ordering::SeqCst); + Ok(()) } - /// Performs a single readiness check against the configured endpoint. - async fn check_readiness(&self) -> bool { + /// Waits for the app to report ready against the configured endpoint. Returns + /// only once it does; callers impose any bound with an external timeout. + async fn check_readiness(&self) { let url = self.healthcheck_url.clone(); let protocol = self.healthcheck_protocol; self.is_web_ready(&url, &protocol).await @@ -776,61 +1365,21 @@ impl Adapter { /// Waits for the web application to become ready, with retries. /// /// Uses a fixed 10ms interval between retry attempts and logs progress - /// at increasing intervals (100ms, 500ms, 1s, 2s, 5s, 10s). - async fn is_web_ready(&self, url: &Url, protocol: &Protocol) -> bool { - let mut checkpoint = Checkpoint::new(); - Retry::spawn(FixedInterval::from_millis(10), || { - if checkpoint.lapsed() { - tracing::info!(url = %url.to_string(), "app is not ready after {}ms", checkpoint.next_ms()); - checkpoint.increment(); - } - self.check_web_readiness(url, protocol) - }) - .await - .is_ok() + /// at increasing intervals (100ms, 500ms, 1s, 2s, 5s, 10s). Returns only once + /// the app is ready — see [`readiness::wait_until_ready`]. + async fn is_web_ready(&self, url: &Url, protocol: &Protocol) { + readiness::wait_until_ready(self.client(), url, *protocol, &self.healthcheck_healthy_status).await; } /// Performs a single readiness check using the configured protocol. /// /// For HTTP: Makes a GET request and checks if the status code is in the healthy range. /// For TCP: Attempts to establish a TCP connection. + /// + /// Used by tests; `Adapter`'s own readiness path goes through [`is_web_ready`](Self::is_web_ready). + #[cfg(test)] async fn check_web_readiness(&self, url: &Url, protocol: &Protocol) -> Result<(), i8> { - match protocol { - Protocol::Http => { - // url is already validated in Adapter::new(), this conversion should always succeed - // If it fails, it indicates a programming error, not a runtime condition - let uri: http::Uri = url - .as_str() - .parse() - .expect("BUG: healthcheck_url should be valid - validated in Adapter::new()"); - - match self.client.get(uri).await { - Ok(response) if self.healthcheck_healthy_status.contains(&response.status().as_u16()) => { - tracing::debug!("app is ready"); - Ok(()) - } - _ => { - tracing::trace!("app is not ready"); - Err(-1) - } - } - } - Protocol::Tcp => { - // url is already validated in Adapter::new(), host and port should exist - // If they don't, it indicates a programming error, not a runtime condition - let host = url - .host_str() - .expect("BUG: healthcheck_url should have host - validated in Adapter::new()"); - let port = url - .port() - .expect("BUG: healthcheck_url should have port - validated in Adapter::new()"); - - match TcpStream::connect(format!("{}:{}", host, port)).await { - Ok(_) => Ok(()), - Err(_) => Err(-1), - } - } - } + readiness::check_web_readiness(self.client(), url, *protocol, &self.healthcheck_healthy_status).await } /// Starts the adapter and begins processing Lambda events. @@ -860,16 +1409,62 @@ impl Adapter { /// # } /// ``` pub async fn run(self) -> Result<(), Error> { + let hooks = Arc::new(snapstart::SnapStartHooks::new( + self.restored_client.clone(), + self.client.clone(), + self.domain.clone(), + self.snapstart_before_checkpoint_path.clone(), + self.snapstart_after_restore_path.clone(), + self.healthcheck_url.clone(), + self.healthcheck_protocol, + self.healthcheck_healthy_status.clone(), + self.pool_idle_timeout, + self.readiness_check_timeout, + )); match (self.compression, self.invoke_mode) { (true, LambdaInvokeMode::Buffered) => { let svc = ServiceBuilder::new().layer(CompressionLayer::new()).service(self); - lambda_http::run_concurrent(svc).await + Self::register_and_run(lambda_http::runtime_concurrent(svc), hooks).await + } + (_, LambdaInvokeMode::Buffered) => { + Self::register_and_run(lambda_http::runtime_concurrent(self), hooks).await + } + (_, LambdaInvokeMode::ResponseStream) => { + Self::register_and_run(lambda_http::streaming_runtime_concurrent(self), hooks).await } - (_, LambdaInvokeMode::Buffered) => lambda_http::run_concurrent(self).await, - (_, LambdaInvokeMode::ResponseStream) => lambda_http::run_with_streaming_response_concurrent(self).await, } } + /// Registers the SnapStart hooks on `runtime` and starts the concurrent event loop. + /// + /// Each `run()` arm builds a different runtime type (buffered vs. streaming), + /// so the shared "register, then run" tail lives here as a generic helper. + /// + /// Applies [`TracingLayer`](lambda_http::lambda_runtime::layers::TracingLayer) + /// before registering the SnapStart hooks. The free + /// `lambda_runtime::run_concurrent` helper adds this layer internally, but this + /// crate builds the runtime via `lambda_http::runtime_concurrent` (to attach + /// `register_snapstart_resource`), which does not — so without it every + /// per-invocation adapter log line (including the SnapStart hook-path 403 warn) + /// would lose its `requestId` / `xrayTraceId` span fields. + async fn register_and_run( + runtime: lambda_http::lambda_runtime::Runtime, + hooks: Arc, + ) -> Result<(), Error> + where + S: lambda_http::Service + + Clone + + Send + + 'static, + S::Future: Send, + { + runtime + .layer(lambda_http::lambda_runtime::layers::TracingLayer::new()) + .register_snapstart_resource(hooks) + .run_concurrent() + .await + } + /// Applies runtime API proxy configuration from environment variables. /// /// If `AWS_LWA_LAMBDA_RUNTIME_API_PROXY` is set, this method overwrites @@ -930,7 +1525,7 @@ impl Adapter { /// 4. Strips the base path if configured /// 5. Forwards the request to the web application /// 6. Returns the response (or error if status code is in error_status_codes) - async fn fetch_response(&self, event: Request) -> Result, Error> { + async fn fetch_response(&self, event: Request) -> Result>, Error> { if self.async_init && !self.ready_at_init.load(Ordering::SeqCst) { self.is_web_ready(&self.healthcheck_url, &self.healthcheck_protocol) .await; @@ -944,18 +1539,63 @@ impl Adapter { let (parts, body) = event.into_parts(); // strip away Base Path if environment variable REMOVE_BASE_PATH is set. + // Strip exactly ONE leading occurrence, and only on a path-segment boundary, + // so `/api/api/order` -> `/api/order` (not `/order`) and `/apiorder` is left + // untouched (a partial-segment prefix must not be stripped). A configured + // trailing slash is normalized away first, so `/api/` behaves like `/api` + // (otherwise `/api/order` would fail the segment-boundary check and pass + // through unstripped, a regression for trailing-slash base paths). if let Some(base_path) = self.base_path.as_deref() { - let stripped = path.trim_start_matches(base_path); - if stripped.len() != path.len() { - tracing::debug!(base_path = %base_path, original = %path, stripped = %stripped, "stripped base path"); + let base_path = base_path.strip_suffix('/').unwrap_or(base_path); + if let Some(rest) = path.strip_prefix(base_path) { + if rest.is_empty() || rest.starts_with('/') { + let stripped = if rest.is_empty() { "/" } else { rest }; + tracing::debug!(base_path = %base_path, original = %path, stripped = %stripped, "stripped base path"); + path = stripped; + } } - path = stripped; } if matches!(request_context, RequestContext::PassThrough) && parts.method == Method::POST { path = self.pass_through_path.as_str(); } + // Block external traffic to the SnapStart hook paths. These routes are + // control-plane operations driven only by the adapter's own hook calls + // (which target `domain` directly and never reach this function). + // + // Build the outbound app URL FIRST, then run the guard against the exact + // path that will be sent (`app_url.path()`). `Url::set_path` applies the + // WHATWG normalization the request actually carries — e.g. for the `http` + // scheme it rewrites `\` to `/` and resolves `.`/`..` — so guarding on the + // raw event path could diverge from what the app receives (a `\` spelling + // would sail past a raw-path guard yet reach the hook route). Guarding on + // `app_url.path()` makes the guard structurally incapable of that + // divergence; `matches_hook_path` still layers percent-decode / case-fold / + // empty-segment collapse on top, for the spellings the app router (not + // `Url`) resolves. + let mut app_url = self.domain.clone(); + app_url.set_path(path); + + // The match is strict: it canonicalizes the outbound path (percent-decode, + // strip control bytes, collapse `//`/`.`/`..`, case-fold) so that every + // spelling the downstream app router would resolve to the hook route is + // blocked — not just the exact configured string. Only a path that stays + // undecidable (a malformed `%` escape, or non-UTF-8 once decoded) is treated + // as NOT the hook and passed through rather than 403'd; that cannot reach a + // hook route, because `hook_target` rejects any route containing a literal + // `%`. See `matches_hook_path`. + let outbound_path = app_url.path(); + if matches_any_hook_path( + &[&self.hook_target_before_checkpoint, &self.hook_target_after_restore], + outbound_path, + ) { + tracing::warn!(path = %outbound_path, "rejecting external request to SnapStart hook path"); + return Ok(Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Empty::::new().map_err(Error::from).boxed())?); + } + let mut req_headers = parts.headers; // include request context in http header "x-amzn-request-context" @@ -988,8 +1628,7 @@ impl Adapter { } } - let mut app_url = self.domain.clone(); - app_url.set_path(path); + // `app_url` was built (path set + hook guard) before the header work above. app_url.set_query(parts.uri.query().filter(|q| !q.is_empty())); tracing::debug!(app_url = %app_url, req_headers = ?req_headers, "sending request to app server"); @@ -1009,7 +1648,7 @@ impl Adapter { }; let request = builder.body(Body::Binary(body_bytes))?; - let mut app_response = self.client.request(request).await?; + let mut app_response = self.client().request(request).await?; // Check if status code should trigger an error if let Some(error_codes) = &self.error_status_codes { @@ -1036,7 +1675,9 @@ impl Adapter { tracing::debug!(status = %app_response.status(), body_size = ?app_response.body().size_hint().lower(), app_headers = ?app_response.headers().clone(), "responding to lambda event"); - Ok(app_response) + // Box the body into a uniform type so synthetic responses (e.g. the 403 + // hook-path guard) can share the return type with proxied responses. + Ok(app_response.map(|body| body.map_err(Error::from).boxed())) } } @@ -1045,7 +1686,7 @@ impl Adapter { /// This allows the adapter to be used directly with the Lambda runtime, /// which expects a `Service` that can handle Lambda events. impl Service for Adapter { - type Response = Response; + type Response = Response>; type Error = Error; type Future = Pin> + Send>>; @@ -1078,6 +1719,331 @@ mod tests { assert_eq!(parse_status_codes(""), Vec::::new()); } + // Both cases live in one test because they mutate the same process-global env + // vars; splitting them lets Rust's parallel test runner interleave the + // set/remove calls and clobber each other's state. + #[test] + fn test_snapstart_paths() { + // Default case: unset env vars -> both None. + std::env::remove_var(ENV_SNAPSTART_BEFORE_CHECKPOINT_PATH); + std::env::remove_var(ENV_SNAPSTART_AFTER_RESTORE_PATH); + let options = AdapterOptions::default(); + assert_eq!(options.snapstart_before_checkpoint_path, None); + assert_eq!(options.snapstart_after_restore_path, None); + + // Set case: env vars present -> parsed into Some(..). + std::env::set_var(ENV_SNAPSTART_BEFORE_CHECKPOINT_PATH, "/snapstart/before"); + std::env::set_var(ENV_SNAPSTART_AFTER_RESTORE_PATH, "/snapstart/after"); + let options = AdapterOptions::default(); + assert_eq!( + options.snapstart_before_checkpoint_path.as_deref(), + Some("/snapstart/before") + ); + assert_eq!( + options.snapstart_after_restore_path.as_deref(), + Some("/snapstart/after") + ); + + std::env::remove_var(ENV_SNAPSTART_BEFORE_CHECKPOINT_PATH); + std::env::remove_var(ENV_SNAPSTART_AFTER_RESTORE_PATH); + } + + // All cases share one test because they mutate the same process-global env + // var; separate tests would let the parallel runner clobber each other. + #[test] + fn test_pool_idle_timeout() { + // Unset -> default 4s. + std::env::remove_var(ENV_POOL_IDLE_TIMEOUT_SECONDS); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_secs(4)); + assert_eq!(AdapterOptions::default().pool_idle_timeout, Duration::from_secs(4)); + + // Explicit value -> parsed, and surfaced on AdapterOptions. + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, "30"); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_secs(30)); + assert_eq!(AdapterOptions::default().pool_idle_timeout, Duration::from_secs(30)); + + // Zero is honored (disables idle keep-alive by timeout). + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, "0"); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_secs(0)); + + // Surrounding whitespace tolerated. + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, " 15 "); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_secs(15)); + + // Fractional seconds are accepted, matching the sibling + // AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS. Previously `0.5` failed to parse + // as u64 and silently became the 4s default. + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, "0.5"); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_millis(500)); + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, "4.5"); + assert_eq!(pool_idle_timeout_from_env(), Duration::from_millis(4500)); + + // Genuinely unusable values still fall back to the default (and now warn). + for bad in ["not-a-number", "30s", "-1", "NaN", "inf", "1e400"] { + std::env::set_var(ENV_POOL_IDLE_TIMEOUT_SECONDS, bad); + assert_eq!( + pool_idle_timeout_from_env(), + Duration::from_secs(4), + "{bad:?} must fall back to the default" + ); + } + + std::env::remove_var(ENV_POOL_IDLE_TIMEOUT_SECONDS); + } + + /// Serves `n` sequential requests over `client` and returns how many TCP + /// connections the server had to accept. One connection means the idle pool + /// was reused (keep-alive honored); `n` means every request reconnected. + async fn connections_used(client: &Client, n: usize) -> usize { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let accepted = Arc::new(AtomicUsize::new(0)); + let counter = accepted.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + counter.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let svc = hyper::service::service_fn(|_req: hyper::Request| async { + Ok::<_, std::convert::Infallible>(hyper::Response::new(String::from("ok"))) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(hyper_util::rt::TokioIo::new(stream), svc) + .await; + }); + } + }); + + for _ in 0..n { + let req = hyper::Request::builder() + .uri(format!("http://{addr}/")) + .body(Body::Empty) + .unwrap(); + let resp = client.request(req).await.unwrap(); + // Drain the body so the connection is eligible to return to the pool. + let _ = resp.into_body().collect().await.unwrap(); + tokio::time::sleep(Duration::from_millis(40)).await; + } + accepted.load(Ordering::SeqCst) + } + + /// `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` must actually take effect on the client + /// that serves invocations after a SnapStart restore. + /// + /// Regression for the bot finding: `build_client` used to apply + /// `pool_max_idle_per_host(0)` whenever `AWS_LAMBDA_INITIALIZATION_TYPE=snap-start`, + /// and that variable stays set for the whole lifetime of a restored environment. + /// Both call sites went through it, so the post-restore client — the one + /// `Adapter::client()` returns for every invocation after a restore — never + /// retained a connection, making the configured timeout a no-op on exactly the + /// functions this feature targets and reconnecting on every single invocation + /// (each one a fresh file descriptor, which Lambda limits). + /// + /// The snapshot hazard only applies to the client built BEFORE the snapshot; a + /// client built inside `after_restore` starts with an empty pool and cannot hold + /// a snapshotted connection, so it is safe for it to pool normally. + #[tokio::test] + async fn test_pool_idle_timeout_applies_under_snapstart() { + std::env::set_var("AWS_LAMBDA_INITIALIZATION_TYPE", "snap-start"); + // This is the call `SnapStartHooks::after_restore` makes. + let restored = build_client(Duration::from_secs(4), Pooling::Enabled); + let used = connections_used(&restored, 3).await; + std::env::remove_var("AWS_LAMBDA_INITIALIZATION_TYPE"); + assert_eq!( + used, 1, + "the post-restore client must honor the configured idle keep-alive and reuse \ + its connection, but it opened {used} connections for 3 requests" + ); + } + + /// The Lambda Runtime API client must not retain an idle connection either. + /// + /// `register_extension_internal` built a default-pooled client. Under SnapStart any + /// connection it parks is captured in the snapshot and dead after restore — the + /// same hazard `lambda_runtime`'s own restore path handles by calling + /// `reset_pool()` on its RAPID client. Nothing re-establishes or resets this one, + /// and its failure path is `std::process::exit(1)`, so a reused dead connection + /// would terminate the restored environment. It also has nothing to gain from + /// pooling: it makes exactly two requests, `register` and then the long poll for + /// the first extension event. + #[tokio::test] + async fn test_runtime_api_client_does_not_retain_connections() { + let retained = connection_retained_after_request(&runtime_api_client()).await; + assert!( + !retained, + "the Runtime API client must drop its connection rather than park a socket \ + that a snapshot would capture" + ); + } + + /// The pre-snapshot client must still never retain an idle connection, so nothing + /// dead can be captured in the snapshot and handed out after a restore + /// (hyper#3810). This also covers a consumer driving the `Service` impl directly, + /// who never triggers the after-restore rebuild. + /// + /// This counts reuse; `test_pre_snapshot_client_pool_is_disabled_not_merely_expiring` + /// pins the stronger, clock-independent property that the pool is off entirely. + #[tokio::test] + async fn test_adapter_new_client_never_pools_under_snapstart() { + std::env::set_var("AWS_LAMBDA_INITIALIZATION_TYPE", "snap-start"); + let options = AdapterOptions { + pool_idle_timeout: Duration::from_secs(4), + ..Default::default() + }; + let adapter = Adapter::new(&options).unwrap(); + let used = connections_used(&adapter.client, 3).await; + std::env::remove_var("AWS_LAMBDA_INITIALIZATION_TYPE"); + assert_eq!( + used, 3, + "the pre-snapshot client must not retain an idle connection, but it reused one \ + ({used} connections for 3 requests)" + ); + // The configured value is still carried through for the post-restore rebuild. + assert_eq!(adapter.pool_idle_timeout, Duration::from_secs(4)); + } + + /// Issues one request over `client`, then reports whether the connection is still + /// open afterwards (i.e. parked in hyper's idle pool). + /// + /// This distinguishes a *disabled* pool from a pool whose entries merely expire: + /// with `pool_max_idle_per_host(0)` the connection is dropped as soon as the + /// response completes, so the server side finishes; with + /// `pool_idle_timeout(Duration::ZERO)` the socket stays parked and is only + /// evicted at the next checkout. It observes the connection's lifetime rather + /// than elapsed time, so unlike a reuse count it cannot be satisfied by a clock + /// that happens to have advanced. + async fn connection_retained_after_request(client: &Client) -> bool { + use std::sync::atomic::{AtomicBool, Ordering}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let closed = Arc::new(AtomicBool::new(false)); + let flag = closed.clone(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let svc = hyper::service::service_fn(|_req: hyper::Request| async { + Ok::<_, std::convert::Infallible>(hyper::Response::new(String::from("ok"))) + }); + // Returns once the peer closes the connection. + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(hyper_util::rt::TokioIo::new(stream), svc) + .await; + flag.store(true, Ordering::SeqCst); + }); + + let req = hyper::Request::builder() + .uri(format!("http://{addr}/")) + .body(Body::Empty) + .unwrap(); + let resp = client.request(req).await.unwrap(); + let _ = resp.into_body().collect().await.unwrap(); + // Give the client a moment to either drop or park the connection. + tokio::time::sleep(Duration::from_millis(100)).await; + !closed.load(Ordering::SeqCst) + } + + /// The pre-snapshot client must make reuse impossible *by construction*, not by + /// relying on the monotonic clock. + /// + /// Regression for the bot `[BUG]` finding: `pool_idle_timeout(Duration::ZERO)` + /// leaves hyper's pool enabled and parks the connection, deciding reuse at + /// checkout via `now.saturating_duration_since(idle_at) > timeout`. That + /// saturates to `ZERO` when the recorded instant is ahead of `now`, and + /// `ZERO > ZERO` is false — so the entry counts as fresh and is handed out. A + /// monotonic clock that did not advance across a restore is exactly the + /// condition hyper#3810 / rust-lang/rust#79462 describe, and exactly what the + /// original `pool_max_idle_per_host(0)` was written to distrust. Under `run()` + /// it is masked by the after-restore rebuild, but the direct-`Service` consumer + /// this restriction exists for is the one path where it can fail. + /// + /// `pool_max_idle_per_host(0)` disables the pool, so no clock is consulted. + #[tokio::test] + async fn test_pre_snapshot_client_pool_is_disabled_not_merely_expiring() { + std::env::set_var("AWS_LAMBDA_INITIALIZATION_TYPE", "snap-start"); + let options = AdapterOptions { + pool_idle_timeout: Duration::from_secs(4), + ..Default::default() + }; + let adapter = Adapter::new(&options).unwrap(); + let retained = connection_retained_after_request(&adapter.client).await; + std::env::remove_var("AWS_LAMBDA_INITIALIZATION_TYPE"); + assert!( + !retained, + "the pre-snapshot client must DROP its connection, not park it in the idle \ + pool where a non-advancing clock could see it as fresh and reuse it" + ); + } + + /// The post-restore client, by contrast, must keep pooling: its pool starts empty + /// so it cannot hold a snapshotted connection, and retaining one is the whole + /// point of `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS`. + #[tokio::test] + async fn test_post_restore_client_retains_its_connection() { + std::env::set_var("AWS_LAMBDA_INITIALIZATION_TYPE", "snap-start"); + // The call `SnapStartHooks::after_restore` makes. + let restored = build_client(Duration::from_secs(4), Pooling::Enabled); + let retained = connection_retained_after_request(&restored).await; + std::env::remove_var("AWS_LAMBDA_INITIALIZATION_TYPE"); + assert!( + retained, + "the post-restore client must keep its connection alive for reuse" + ); + } + + /// Without SnapStart nothing changes: keep-alive to the inner app is honored, so + /// the safety mechanism above must not cost every other deployment its pooling. + #[tokio::test] + async fn test_adapter_new_client_pools_without_snapstart() { + std::env::remove_var("AWS_LAMBDA_INITIALIZATION_TYPE"); + let options = AdapterOptions { + pool_idle_timeout: Duration::from_secs(4), + ..Default::default() + }; + let adapter = Adapter::new(&options).unwrap(); + let used = connections_used(&adapter.client, 3).await; + assert_eq!(used, 1, "keep-alive must be honored without SnapStart, got {used}"); + } + + // All cases share one test because they mutate the same process-global env + // var; separate tests would let the parallel runner clobber each other. + #[test] + fn test_readiness_check_timeout() { + // Unset -> None (unbounded), surfaced on AdapterOptions. + std::env::remove_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS); + assert_eq!(readiness_check_timeout_from_env(), None); + assert_eq!(AdapterOptions::default().readiness_check_timeout, None); + + // Explicit value -> Some(secs), parsed and surfaced. + std::env::set_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS, "45"); + assert_eq!(readiness_check_timeout_from_env(), Some(Duration::from_secs(45))); + assert_eq!( + AdapterOptions::default().readiness_check_timeout, + Some(Duration::from_secs(45)) + ); + + // Fractional seconds are accepted. + std::env::set_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS, "0.5"); + assert_eq!(readiness_check_timeout_from_env(), Some(Duration::from_millis(500))); + + // Surrounding whitespace tolerated. + std::env::set_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS, " 20 "); + assert_eq!(readiness_check_timeout_from_env(), Some(Duration::from_secs(20))); + + // Unparseable / non-finite / negative -> None (unbounded), never a panic. + for bad in ["nope", "-1", "NaN", "inf", "0", "0.0", "1e300", "99999999999999999999"] { + std::env::set_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS, bad); + assert_eq!( + readiness_check_timeout_from_env(), + None, + "value {bad:?} should be rejected" + ); + } + + std::env::remove_var(ENV_READINESS_CHECK_TIMEOUT_SECONDS); + } + #[tokio::test] async fn test_status_200_is_ok() { // Start app server @@ -1142,6 +2108,36 @@ mod tests { healthcheck.assert(); } + #[tokio::test] + async fn test_check_init_health_fails_when_sync_init_readiness_timeout_expires() { + // App server that never reports ready (always 500) so the readiness + // retry loop runs until the configured timeout fires. + let app_server = MockServer::start(); + app_server.mock(|when, then| { + when.method(GET).path("/healthcheck"); + then.status(500).body("nope"); + }); + + // Sync init (async_init defaults to false) with a short configured bound. + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/healthcheck".to_string(), + readiness_check_timeout: Some(Duration::from_millis(100)), + ..Default::default() + }; + + let mut adapter = Adapter::new(&options).expect("Failed to create adapter"); + + // Refuse to serve: a configured timeout that expires fails initialization. + let result = adapter.check_init_health().await; + assert!( + result.is_err(), + "sync-init readiness timeout should fail init, got {result:?}" + ); + } + #[tokio::test] async fn test_status_403_is_bad_when_configured() { // Start app server @@ -1402,6 +2398,77 @@ mod tests { assert_eq!(200, response.status().as_u16()); } + #[tokio::test] + async fn test_external_request_to_hook_path_is_forbidden() { + // App server should NOT be called for a guarded path. + let app_server = MockServer::start(); + let guarded = app_server.mock(|when, then| { + when.path("/snapstart/after"); + then.status(200).body("should not be called"); + }); + + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/".to_string(), + snapstart_after_restore_path: Some("/snapstart/after".to_string()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + + // External request (ALB) targeting the guarded hook path. + let alb_req = lambda_http::request::LambdaRequest::Alb({ + let mut req = lambda_http::aws_lambda_events::alb::AlbTargetGroupRequest::default(); + req.http_method = Method::POST; + req.path = Some("/snapstart/after".into()); + req + }); + let mut request = Request::from(alb_req); + request.extensions_mut().insert(make_lambda_context(None)); + + let response = adapter + .fetch_response(request) + .await + .expect("guard returns Ok response"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + + // The inner app must not have been contacted. + guarded.assert_calls(0); + } + + #[tokio::test] + async fn test_non_hook_path_is_proxied_normally() { + let app_server = MockServer::start(); + let hello = app_server.mock(|when, then| { + when.path("/hello"); + then.status(200).body("OK"); + }); + + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/".to_string(), + snapstart_after_restore_path: Some("/snapstart/after".to_string()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + + let alb_req = lambda_http::request::LambdaRequest::Alb({ + let mut req = lambda_http::aws_lambda_events::alb::AlbTargetGroupRequest::default(); + req.http_method = Method::GET; + req.path = Some("/hello".into()); + req + }); + let mut request = Request::from(alb_req); + request.extensions_mut().insert(make_lambda_context(None)); + + let response = adapter.fetch_response(request).await.expect("Request failed"); + assert_eq!(response.status(), StatusCode::OK); + hello.assert(); + } + #[tokio::test] async fn test_tenant_id_header_absent_when_no_tenant() { let app_server = MockServer::start(); @@ -1557,4 +2624,589 @@ mod tests { .expect("Request failed despite control bytes in request context path"); assert_eq!(200, response.status().as_u16()); } + + #[tokio::test] + async fn test_client_helper_returns_restored_when_set() { + let options = AdapterOptions { + host: "127.0.0.1".to_string(), + port: "8080".to_string(), + readiness_check_port: "8080".to_string(), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + + // Before restore: client() returns the base client. + let base_ptr = Arc::as_ptr(adapter.client()) as *const (); + + // Publish a fresh client. + let fresh = Arc::new(build_client(Duration::from_secs(4), Pooling::Enabled)); + let fresh_ptr = Arc::as_ptr(&fresh) as *const (); + assert!(adapter.restored_client.set(fresh).is_ok(), "set should succeed once"); + + // After restore: client() returns the restored client (different pointer). + let now_ptr = Arc::as_ptr(adapter.client()) as *const (); + assert_ne!(now_ptr, base_ptr); + assert_eq!(now_ptr, fresh_ptr); + } + + // --------------------------------------------------------------------- + // Strict fail-closed SnapStart hook-path guard + // --------------------------------------------------------------------- + + /// Build an ALB request for an arbitrary method + raw path. + fn alb_request(method: Method, raw_path: &str) -> Request { + let alb_req = lambda_http::request::LambdaRequest::Alb({ + let mut req = lambda_http::aws_lambda_events::alb::AlbTargetGroupRequest::default(); + req.http_method = method; + req.path = Some(raw_path.into()); + req + }); + let mut request = Request::from(alb_req); + request.extensions_mut().insert(make_lambda_context(None)); + request + } + + /// The guard must block the ENTIRE equivalence class of spellings that the + /// downstream app router would resolve to the configured hook route — not just + /// the exact configured string. Each of these must yield 403 and never reach + /// the app. + #[tokio::test] + async fn test_hook_guard_blocks_equivalence_class() { + let blocked = [ + "/snapstart/after", // canonical + "snapstart/after", // missing leading slash (set_path still routes it) + "/snapstart/after/", // trailing slash + "//snapstart//after", // duplicate empty segments + "/snapstart/./after", // dot segment + "/foo/../snapstart/after", // parent segment resolves onto the hook + "/snapstart/%61fter", // percent-encoded 'a' + "/SnapStart/After", // case variance + "/snapstart/%2fafter", // encoded slash decodes to '/' -> matches hook route + "/snapstart\\after", // backslash: Url::set_path normalizes it to '/' + "/snapstart/after;x=1", // matrix param: stripped by Spring MVC / servlet routing + "/snapstart/after;jsessionid=abc", // servlet session param variant + "/snapstart;a=b/after", // matrix param on a non-terminal segment + "/snapstart/after%0A", // trailing LF: Starlette `$` matches before `\n` + "/snapstart/after%0a", // lowercase-encoded LF + "/snapstart/after%0d%0a", // trailing CRLF + "/snapstart/after%00", // trailing NUL + ]; + + for raw in blocked { + let app_server = MockServer::start(); + // Match ANY path; if the guard fails open, this proves the app was hit. + let hook = app_server.mock(|when, then| { + when.any_request(); + then.status(200).body("should not be reached for guarded paths"); + }); + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/".to_string(), + snapstart_after_restore_path: Some("/snapstart/after".to_string()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + let response = adapter + .fetch_response(alb_request(Method::POST, raw)) + .await + .expect("guard returns Ok response"); + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "path {raw:?} must be blocked (403) by the strict guard" + ); + hook.assert_calls(0); + } + } + + /// The guard must NOT collapse into "block everything": a genuinely distinct + /// route that merely shares a prefix with the hook path is proxied normally. + #[tokio::test] + async fn test_hook_guard_allows_distinct_route() { + let allowed = ["/snapstart/after-report", "/snapstart", "/snapstart/afterx", "/hello"]; + for raw in allowed { + let app_server = MockServer::start(); + let route = app_server.mock(|when, then| { + when.path(raw); + then.status(200).body("OK"); + }); + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/".to_string(), + snapstart_after_restore_path: Some("/snapstart/after".to_string()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + let response = adapter + .fetch_response(alb_request(Method::GET, raw)) + .await + .expect("request failed"); + assert_eq!( + response.status(), + StatusCode::OK, + "distinct route {raw:?} must NOT be blocked" + ); + route.assert(); + } + } + + // --------------------------------------------------------------------- + // Base-path strip: single occurrence, segment-aware + // --------------------------------------------------------------------- + + /// `/api/api/order` with base_path `/api` must strip exactly ONE occurrence + /// (-> `/api/order`), and a partial-segment prefix like `/apiorder` must not be + /// stripped at all. + #[tokio::test] + async fn test_base_path_strip_single_and_segment_aware() { + // (base_path, request_path, path the app should receive) + let cases = [ + ("/api", "/api/api/order", "/api/order"), // strip once, not repeatedly + ("/api", "/apiorder", "/apiorder"), // partial segment: not stripped + ("/api", "/api/order", "/order"), // normal single strip + ("/api", "/api", "/"), // exact base path -> root + ("/api", "/other", "/other"), // no prefix: untouched + ("/api/", "/api/order", "/order"), // trailing slash: normalized, still strips + ("/api/", "/api/api/order", "/api/order"), // trailing slash + repeated segment + ("/api/", "/api", "/"), // trailing slash, exact -> root + ]; + + for (base, req_path, expected) in cases { + let app_server = MockServer::start(); + let route = app_server.mock(|when, then| { + when.path(expected); + then.status(200).body("OK"); + }); + let options = AdapterOptions { + host: app_server.host(), + port: app_server.port().to_string(), + readiness_check_port: app_server.port().to_string(), + readiness_check_path: "/".to_string(), + base_path: Some(base.to_string()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("Failed to create adapter"); + let response = adapter + .fetch_response(alb_request(Method::GET, req_path)) + .await + .unwrap_or_else(|e| panic!("request for base={base} path={req_path} failed: {e}")); + assert_eq!( + response.status(), + StatusCode::OK, + "base={base} req={req_path} should proxy to {expected}" + ); + route.assert(); + } + } + + // --------------------------------------------------------------------- + // Canonicalization helper unit tests (pin the contract directly) + // --------------------------------------------------------------------- + + #[test] + fn test_percent_decode_once() { + assert_eq!( + percent_decode_once("/snapstart/%61fter").as_deref(), + Some("/snapstart/after") + ); + assert_eq!(percent_decode_once("/a%2fb").as_deref(), Some("/a/b")); // %2f -> '/' + assert_eq!(percent_decode_once("plain").as_deref(), Some("plain")); + // Malformed escapes -> None. + assert_eq!(percent_decode_once("/a%"), None); + assert_eq!(percent_decode_once("/a%2"), None); + assert_eq!(percent_decode_once("/a%zz"), None); + } + + #[test] + fn test_canonicalize_hook_path_equivalence() { + let want = canonicalize_hook_path("/snapstart/after").unwrap(); + for spelling in [ + "snapstart/after", + "/snapstart/after/", + "//snapstart//after", + "/snapstart/./after", + "/foo/../snapstart/after", + "/snapstart/%61fter", + "/SnapStart/After", + "/snapstart/%2fafter", // encoded slash decodes to a real slash + ] { + assert_eq!( + canonicalize_hook_path(spelling).as_ref(), + Some(&want), + "{spelling:?} should canonicalize onto the hook route" + ); + } + } + + #[test] + fn test_canonicalize_hook_path_distinct_and_ambiguous() { + let hook = canonicalize_hook_path("/snapstart/after").unwrap(); + // Distinct routes must NOT canonicalize onto the hook. + for distinct in [ + "/snapstart/after-report", + "/snapstart", + "/snapstart/afterx", + "/hello", + "/foo/%2fbar", + // Single-pass decode: `%2561` decodes ONCE to the literal `%61fter`, + // which the app router does NOT resolve to `/snapstart/after`. + "/snapstart/%2561fter", + ] { + assert_ne!(canonicalize_hook_path(distinct).as_ref(), Some(&hook), "{distinct:?}"); + } + // A validly single-encoded literal percent (`%25` -> `%`) is DECIDABLE and + // must not fall into the fail-closed branch (regression: decode-until-stable + // used to 403 `/reports/100%25`). It canonicalizes to a concrete route. + assert_eq!( + canonicalize_hook_path("/reports/100%25"), + Some(vec!["reports".to_string(), "100%".to_string()]), + ); + // A malformed `%` escape is undecidable -> None (request side passes through). + assert_eq!(canonicalize_hook_path("/snapstart/%2"), None); + // A control byte is NOT undecidable: it is stripped and canonicalization + // continues, so `/snapstart/af\u{0}ter` collapses onto the hook route and + // will be blocked (a router like Starlette resolves it to `/snapstart/after`). + assert_eq!( + canonicalize_hook_path("/snapstart/af\u{0}ter"), + Some(vec!["snapstart".to_string(), "after".to_string()]), + ); + } + + /// Test helper mirroring the real guard: the configured path is canonicalized + /// by [`hook_target`] (through `set_path`), and the request path is passed + /// through the same `set_path` normalization the request side uses in + /// `fetch_response` (`app_url.path()`). Both sides therefore share the identical + /// transformation, exactly as in production. + /// + /// Panics on a configured path `hook_target` rejects; that path never reaches + /// the guard in production either, because `Adapter::new` fails first (see + /// `test_non_canonicalizable_configured_hook_path_is_rejected`). + fn guard_blocks(configured: &str, request: &str) -> bool { + let domain: Url = "http://127.0.0.1:8080".parse().unwrap(); + let want = + hook_target(&domain, &Some(configured.to_string())).expect("configured hook path must be canonicalizable"); + let mut u = domain.clone(); + u.set_path(request); + matches_hook_path(&want, u.path()) + } + + /// Regression for the single-pass fix: with a hook that shares a first + /// segment, a validly single-encoded request under that segment must NOT be + /// blocked (bot finding: `/reports/100%25` vs hook `/reports/snapshot`). + #[test] + fn test_matches_hook_path_valid_encoded_percent_not_blocked() { + assert!( + !guard_blocks("/reports/snapshot", "/reports/100%25"), + "/reports/100%25 must not 403" + ); + // The genuine single-encoded hook spelling is still blocked. + assert!(guard_blocks("/reports/snapshot", "/reports/%73napshot")); + } + + /// The configured side must be normalized through `set_path` too, so a + /// configured value that `set_path` rewrites still guards the route the app + /// actually serves (bot SECURITY finding: `/snapstart\after`). + #[test] + fn test_matches_hook_path_configured_side_normalized_through_set_path() { + // Configured with a backslash: set_path rewrites it to `/snapstart/after`, + // which is the route the app serves — so requests to it must be blocked. + assert!(guard_blocks("/snapstart\\after", "/snapstart/after")); + assert!(guard_blocks("/snapstart\\after", "/snapstart\\after")); + assert!(guard_blocks("/snapstart\\after", "/SnapStart/After/")); + // A genuinely different route is still allowed. + assert!(!guard_blocks("/snapstart\\after", "/snapstart/other")); + + // Dot segments / duplicate slashes in the configured value are resolved by + // set_path + canonicalize, so the effective route is still guarded. + assert!(guard_blocks("/a/../snapstart//after", "/snapstart/after")); + } + + #[test] + fn test_matches_hook_path_undecidable_passes_through() { + assert!(guard_blocks("/snapstart/after", "/SnapStart/After/")); + assert!(!guard_blocks("/snapstart/after", "/snapstart/after-report")); + assert!(!matches_hook_path(&None, "/snapstart/after")); // no hook configured + assert!(!matches_hook_path(&None, "/snapstart/%2")); // ...and none => never match + + // A malformed `%` escape is undecidable — the app router cannot decode it + // to the hook route either — so it is NOT the hook and passes through, even + // when it shares the hook's leading segment. + assert!(!guard_blocks("/snapstart/after", "/snapstart/%2")); + assert!(!guard_blocks("/snapstart/after", "/snapstart/%")); // trailing bare % + + // A control byte is DIFFERENT: a router can still resolve the surrounding + // path to the hook (Python's `$` matches before a trailing `\n`), so the + // guard strips control bytes and BLOCKS the request rather than passing it + // through. `af\u{0}ter` canonicalizes to the `after` segment. + assert!(guard_blocks("/snapstart/after", "/snapstart/af\u{0}ter")); + + // Unrelated undecidable routes are likewise not blocked (bot findings: + // /reports/100% under a shared-prefix hook, /100%, /other/%2). + assert!(!guard_blocks("/snapstart/after", "/reports/100%")); + assert!(!guard_blocks("/snapstart/after", "/other/%2")); + assert!(!guard_blocks("/snapstart/after", "/100%")); + assert!(!guard_blocks("/snapstart/after", "/%2")); + // The exact bot-reported case: hook shares the first segment. + assert!(!guard_blocks("/reports/snapshot", "/reports/100%")); + } + + /// An empty configured hook path must mean "no hook" on BOTH sides: the guard + /// target and the path the hook actually POSTs to. + /// + /// Regression: `hook_target` short-circuits on `configured.is_empty()` and + /// returns `Ok(None)` ("no hook"), but `Adapter::new` used to store the raw + /// `Some("")`, which `run()` hands to `SnapStartHooks`. `before_snapshot` / + /// `after_restore` then take their `if let Some(path)` branch and call + /// `post_hook(.., "")` — and `Url::set_path("")` yields `/`, so the adapter + /// POSTed to the unguarded application root on every lifecycle event (a 405 on + /// both FastAPI examples, which `post_hook` treats as fatal). That is the same + /// guard-versus-POST divergence the root-collapse rejection closed; `""` slipped + /// past it by returning before canonicalization. Normalizing to `None` here + /// keeps the documented "empty means unset" semantics while making the two + /// sides agree by construction. + #[test] + fn test_empty_hook_path_is_normalized_on_both_sides() { + let options = AdapterOptions { + snapstart_before_checkpoint_path: Some(String::new()), + snapstart_after_restore_path: Some(String::new()), + ..Default::default() + }; + let adapter = Adapter::new(&options).expect("empty hook paths mean 'no hook', not an error"); + assert_eq!( + adapter.snapstart_before_checkpoint_path, None, + "an empty before-checkpoint path must not leave a hook that POSTs to /" + ); + assert_eq!( + adapter.snapstart_after_restore_path, None, + "an empty after-restore path must not leave a hook that POSTs to /" + ); + // The guard side already agreed; assert both halves together so they cannot + // drift apart again. + assert_eq!(adapter.hook_target_before_checkpoint, None); + assert_eq!(adapter.hook_target_after_restore, None); + } + + /// A configured hook path that cannot be canonicalized must be REJECTED, not + /// degraded to a raw string compare. + /// + /// Regression for the bot SECURITY finding: the old `HookTarget::Raw` fallback + /// compared raw strings on both sides, so a configured `/snapstart/after%` + /// (bare `%`, undecidable) left every encoded spelling of that same route + /// unguarded — verified against uvicorn/Starlette, which serves the route as + /// `/snapstart/after%` and resolves a request for `/snapstart/after%25` onto it. + /// A non-canonicalizable hook path is always a misconfiguration, so fail init + /// rather than ship a guard that reads as protective but is not. + #[test] + fn test_non_canonicalizable_configured_hook_path_is_rejected() { + let domain: Url = "http://127.0.0.1:8080".parse().unwrap(); + for cfg in ["/snapstart/after%", "/snapstart/%2", "/snapstart/%zz"] { + assert!( + hook_target(&domain, &Some(cfg.to_string())).is_err(), + "configured hook path {cfg:?} must be rejected, not silently degraded" + ); + } + // A canonicalizable path is still accepted. + assert_eq!( + hook_target(&domain, &Some("/snapstart/after".to_string())).unwrap(), + Some(vec!["snapstart".to_string(), "after".to_string()]) + ); + } + + /// A configured hook path whose canonical form contains a literal `%` must also + /// be rejected — this is what makes the request-side pass-through provably safe. + /// + /// Rejecting only *non-canonicalizable* configs is not enough: configuring the + /// same route the "correct" way (`/snapstart/after%25`, canonical `after%`) left + /// the bare-`%` spelling reachable, because an undecidable request path passes + /// through the guard while uvicorn/Starlette still resolves it onto the route + /// (verified end-to-end: `POST /snapstart/after%` -> 200, handler ran). + /// + /// With no `%` in any hook route, the pass-through cannot be exploited on ANY + /// framework, without the adapter modelling per-framework decoding: an + /// undecidable request path either is rejected by the router outright (Node + /// throws `URIError` -> Express 400; Go and Spring likewise 400), or is decoded + /// leniently into a path containing a literal `%` or U+FFFD (Python's + /// `unquote`) — and neither can equal a `%`-free hook route. + #[test] + fn test_configured_hook_path_with_literal_percent_is_rejected() { + let domain: Url = "http://127.0.0.1:8080".parse().unwrap(); + for cfg in ["/snapstart/after%25", "/reports/100%25", "/%25/after"] { + assert!( + hook_target(&domain, &Some(cfg.to_string())).is_err(), + "configured hook path {cfg:?} canonicalizes to a route containing `%` \ + and must be rejected" + ); + } + // Sanity: a `%`-free route is unaffected, and a request carrying a literal + // percent under an unrelated route still must NOT be blocked. + assert!(hook_target(&domain, &Some("/snapstart/after".to_string())).is_ok()); + assert!(!guard_blocks("/reports/snapshot", "/reports/100%25")); + assert!(!guard_blocks("/reports/snapshot", "/reports/100%")); + } + + /// `Adapter::new` must surface that rejection, so a misconfigured function + /// fails initialization with a clear error instead of starting with a + /// weakened guard on a state-mutating route. + #[test] + fn test_adapter_new_fails_on_non_canonicalizable_hook_path() { + let options = AdapterOptions { + snapstart_after_restore_path: Some("/snapstart/after%".to_string()), + ..Default::default() + }; + let err = Adapter::new(&options) + .err() + .expect("Adapter::new must reject a non-canonicalizable hook path"); + let msg = err.to_string(); + assert!( + msg.contains("/snapstart/after%"), + "error must name the offending path, got: {msg}" + ); + } + + #[test] + fn test_matches_hook_path_empty_config_never_matches() { + // An unset or empty configured hook path means "no hook": it must never 403 + // the app root (bot finding: AWS_LWA_..._PATH="" blocks "/"). + assert!(!guard_blocks("", "/"), "empty config must not block /"); + assert!(!guard_blocks("", "/anything"), "empty config must not block /anything"); + assert!(!matches_hook_path(&None, "/")); + } + + /// A configured hook path that collapses to the app root must be REJECTED. + /// + /// Regression for the bot `[BUG]` finding: `hook_target` returned `Ok(None)` + /// for these, silently disabling the guard, while `SnapStartHooks::after_restore` + /// still POSTs to the raw configured path (it reads `after_restore_path`, not the + /// guard target). The two therefore diverged with no diagnostic: with + /// `AWS_LWA_SNAPSTART_AFTER_RESTORE_PATH=/..` the adapter POSTs to `/` on every + /// restore, which is a 405 on both FastAPI examples (they declare only + /// `@app.get("/")`), and `post_hook` treats any non-2xx as fatal — so every + /// restore failed with nothing explaining why. + /// + /// Rejecting is the consistent resolution: the guard cannot protect the app root + /// without 403-ing all normal traffic, and the docs require a hook path "your + /// normal application traffic does not use" — which the root never is. Same rule + /// as the `%` cases: if the adapter cannot guard it, it refuses to run with it. + #[test] + fn test_root_collapsing_configured_hook_path_is_rejected() { + let domain: Url = "http://127.0.0.1:8080".parse().unwrap(); + for cfg in ["/", "//", "///", "/..", "/.", "/foo/..", "/%2f", "/a/../..", "/./"] { + assert!( + hook_target(&domain, &Some(cfg.to_string())).is_err(), + "configured hook path {cfg:?} collapses to the app root and must be rejected" + ); + } + // Unset and empty remain "no hook", not an error. + assert!(hook_target(&domain, &None).unwrap().is_none()); + assert!(hook_target(&domain, &Some(String::new())).unwrap().is_none()); + // A real route is still accepted. + assert!(hook_target(&domain, &Some("/snapstart/after".to_string())).is_ok()); + } + + /// `Adapter::new` must surface the root-collapse rejection, so the operator gets + /// one clear error at init instead of a 405-driven restore failure every restore. + #[test] + fn test_adapter_new_fails_on_root_collapsing_hook_path() { + let options = AdapterOptions { + snapstart_after_restore_path: Some("/..".to_string()), + ..Default::default() + }; + let err = Adapter::new(&options) + .err() + .expect("Adapter::new must reject a hook path that collapses to the root"); + let msg = err.to_string(); + assert!(msg.contains("/.."), "error must name the offending path, got: {msg}"); + } + + /// A hook path that collides with `AWS_LWA_PASS_THROUGH_PATH` must be rejected at + /// init, because the pass-through rewrite happens BEFORE the guard. + /// + /// Regression for the bot finding: `fetch_response` rewrites `path` to + /// `pass_through_path` for a `RequestContext::PassThrough` POST, and only then + /// runs the guard on the rewritten path. So configuring the hook at `/events` + /// (the default pass-through path) makes EVERY non-HTTP trigger event canonicalize + /// onto the guarded route and get a 403 instead of reaching the app — silently, + /// with only a per-invocation `warn!`. Init-time validation already exists for the + /// other unguardable hook paths, so this belongs there too. + #[test] + fn test_adapter_new_fails_when_hook_path_collides_with_pass_through_path() { + // The default pass-through path is `/events`. + for hook in ["/events", "/Events", "/events/", "/./events", "/%65vents"] { + let options = AdapterOptions { + snapstart_after_restore_path: Some(hook.to_string()), + ..Default::default() + }; + let err = Adapter::new(&options).err().unwrap_or_else(|| { + panic!("hook path {hook:?} collides with the pass-through path and must be rejected") + }); + let msg = err.to_string(); + assert!( + msg.contains("pass-through") || msg.contains("AWS_LWA_PASS_THROUGH_PATH"), + "error should explain the pass-through collision, got: {msg}" + ); + } + + // A custom pass-through path moves the collision with it. + let options = AdapterOptions { + pass_through_path: "/queue".to_string(), + snapstart_after_restore_path: Some("/events".to_string()), + ..Default::default() + }; + assert!( + Adapter::new(&options).is_ok(), + "/events must be fine once the pass-through path is elsewhere" + ); + let options = AdapterOptions { + pass_through_path: "/queue".to_string(), + snapstart_after_restore_path: Some("/queue".to_string()), + ..Default::default() + }; + assert!( + Adapter::new(&options).is_err(), + "the collision follows the configured value" + ); + } + + /// The pass-through collision check must not fail init on an unguardable + /// `AWS_LWA_PASS_THROUGH_PATH`, which is unrelated configuration. + /// + /// Regression for the bot `[BUG]` finding on de0ea31: the check ran + /// `hook_target(&domain, &Some(pass_through_path))?`, so a pass-through path that + /// `hook_target` rejects — `/` collapses to the root, and `AWS_LWA_PASS_THROUGH_PATH` + /// is read straight from the environment with no prior validation — aborted + /// `Adapter::new` with a SnapStart-flavored error, even with no hook configured and + /// therefore no guard and nothing to collide with. + /// + /// Such a path cannot collide: hook targets are canonicalizable and non-empty by + /// construction, so a request rewritten onto a root-collapsing, `%`-bearing, or + /// non-canonicalizable pass-through path can never canonicalize onto one. + #[test] + fn test_unguardable_pass_through_path_does_not_fail_init() { + for pass_through in ["/", "//", "/..", "/reports/100%25", "/bad/%2"] { + // No hook configured: nothing to validate against at all. + let options = AdapterOptions { + pass_through_path: pass_through.to_string(), + ..Default::default() + }; + assert!( + Adapter::new(&options).is_ok(), + "pass-through path {pass_through:?} must not fail init when no hook is configured" + ); + + // Hook configured: still no collision possible with such a path. + let options = AdapterOptions { + pass_through_path: pass_through.to_string(), + snapstart_after_restore_path: Some("/snapstart/after".to_string()), + ..Default::default() + }; + assert!( + Adapter::new(&options).is_ok(), + "pass-through path {pass_through:?} cannot collide with a canonical hook route" + ); + } + } } diff --git a/src/main.rs b/src/main.rs index f7959738..1a36c4c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,7 +30,7 @@ async fn async_main() -> Result<(), Error> { // register the adapter as an extension adapter.register_default_extension(); // check if the web application is ready - adapter.check_init_health().await; + adapter.check_init_health().await?; // start lambda runtime after the web application is ready adapter.run().await?; diff --git a/src/readiness.rs b/src/readiness.rs index a48e5096..065dda07 100644 --- a/src/readiness.rs +++ b/src/readiness.rs @@ -1,5 +1,87 @@ use std::time::Instant; +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::client::legacy::Client; +use lambda_http::Body; +use tokio::net::TcpStream; +use tokio_retry::{strategy::FixedInterval, Retry}; +use url::Url; + +use crate::Protocol; + +/// Performs a single readiness check against `url` using `protocol`. +/// +/// For HTTP: issues a GET via `client` and checks the status is in `healthy_status`. +/// For TCP: attempts to establish a TCP connection. Returns `Ok(())` when ready. +pub(crate) async fn check_web_readiness( + client: &Client, + url: &Url, + protocol: Protocol, + healthy_status: &[u16], +) -> Result<(), i8> { + match protocol { + Protocol::Http => { + // url is validated in Adapter::new(); this conversion should always succeed. + let uri: http::Uri = url + .as_str() + .parse() + .expect("BUG: healthcheck_url should be valid - validated in Adapter::new()"); + + match client.get(uri).await { + Ok(response) if healthy_status.contains(&response.status().as_u16()) => { + tracing::debug!("app is ready"); + Ok(()) + } + _ => { + tracing::trace!("app is not ready"); + Err(-1) + } + } + } + Protocol::Tcp => { + // url is validated in Adapter::new(); host and port should exist. + let host = url + .host_str() + .expect("BUG: healthcheck_url should have host - validated in Adapter::new()"); + let port = url + .port() + .expect("BUG: healthcheck_url should have port - validated in Adapter::new()"); + + match TcpStream::connect(format!("{}:{}", host, port)).await { + Ok(_) => Ok(()), + Err(_) => Err(-1), + } + } + } +} + +/// Waits for the web application to become ready, retrying on a fixed 10ms +/// interval and logging progress at increasing checkpoints. +/// +/// Returns only once the app is ready: [`FixedInterval`] is an unbounded iterator, +/// so the retry never exhausts and there is no "gave up" outcome to report. Callers +/// that need a bound must impose it externally with a timeout — an unbounded caller +/// waits indefinitely, and the only signal in that case is the escalating +/// `app is not ready after {}ms` log this emits at each checkpoint. +pub(crate) async fn wait_until_ready( + client: &Client, + url: &Url, + protocol: Protocol, + healthy_status: &[u16], +) { + let mut checkpoint = Checkpoint::new(); + // The `Err` variant is unreachable (see above), so the result is discarded + // rather than turned into a `bool` that callers would branch on pointlessly. + let _ = Retry::spawn(FixedInterval::from_millis(10), || { + if checkpoint.lapsed() { + tracing::info!(url = %url.to_string(), "app is not ready after {}ms", checkpoint.next_ms()); + checkpoint.increment(); + } + check_web_readiness(client, url, protocol, healthy_status) + }) + .await; +} + pub(crate) struct Checkpoint { start: Instant, interval_ms: u128, diff --git a/src/snapstart.rs b/src/snapstart.rs new file mode 100644 index 00000000..770fd239 --- /dev/null +++ b/src/snapstart.rs @@ -0,0 +1,422 @@ +//! SnapStart bridge: notifies the inner web application over HTTP at the +//! snapshot boundary and refreshes the adapter's HTTP client after restore. + +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::client::legacy::Client; +use lambda_http::{Body, BoxFuture, Error, SnapStartResource}; +use tokio::time::timeout; +use url::Url; + +use crate::{build_client, readiness, Pooling, Protocol}; + +/// Maximum time the adapter waits for an inner-app hook to respond before +/// failing the SnapStart phase. Bounds a hung or unresponsive hook so the +/// snapshot/restore lifecycle cannot stall indefinitely. +const HOOK_TIMEOUT: Duration = Duration::from_secs(60); + +/// A [`SnapStartResource`] that bridges the Lambda SnapStart lifecycle to the +/// inner web application running behind the adapter. +pub(crate) struct SnapStartHooks { + /// Shared with the [`Adapter`](crate::Adapter); `after_restore` publishes the + /// fresh client here so invocations stop using pre-snapshot connections. + restored_client: Arc>>>, + /// The adapter's base (pre-snapshot) client, used for the BEFORE-CHECKPOINT hook + /// only. `after_restore` deliberately uses the freshly built client instead, so + /// its hook POST never travels over a connection captured in the snapshot. + client: Arc>, + /// `http://host:port` of the inner application. + domain: Url, + before_checkpoint_path: Option, + after_restore_path: Option, + /// Readiness-check endpoint, protocol, and healthy statuses — shared with the + /// adapter so the post-restore readiness check (step 3) matches init behavior. + healthcheck_url: Url, + healthcheck_protocol: Protocol, + healthcheck_healthy_status: Vec, + /// Idle keep-alive used to rebuild the client after restore, so the + /// post-restore client honors the same `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS`. + pool_idle_timeout: Duration, + /// Bound on the post-restore readiness check (step 3), from + /// `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS`. `None` = unbounded (wait forever + /// for the app to become ready), preserving historical behavior. + readiness_timeout: Option, +} + +impl SnapStartHooks { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + restored_client: Arc>>>, + client: Arc>, + domain: Url, + before_checkpoint_path: Option, + after_restore_path: Option, + healthcheck_url: Url, + healthcheck_protocol: Protocol, + healthcheck_healthy_status: Vec, + pool_idle_timeout: Duration, + readiness_timeout: Option, + ) -> Self { + Self { + restored_client, + client, + domain, + before_checkpoint_path, + after_restore_path, + healthcheck_url, + healthcheck_protocol, + healthcheck_healthy_status, + pool_idle_timeout, + readiness_timeout, + } + } + + /// Publishes `fresh` as the post-restore client, or adopts the one already + /// published, returning whichever client invocations will actually use. + /// + /// `OnceLock::set` fails if the cell is already populated. Discarding that failure + /// and carrying on with `fresh` would leave `after_restore` validating a client no + /// request can reach: the hook POST and readiness check would report the restore + /// healthy while every invocation kept using the earlier client. Returning the + /// published client instead removes that divergence rather than reporting it, and + /// the `warn!` records the unexpected second lifecycle run. + fn publish_or_adopt( + cell: &OnceLock>>, + fresh: Arc>, + ) -> Arc> { + match cell.set(fresh.clone()) { + Ok(()) => fresh, + Err(_) => { + tracing::warn!( + "post-restore client was already published; adopting it so the hook call and \ + readiness check validate the client invocations actually use" + ); + // `set` only fails when the cell is populated, so this cannot be None. + cell.get().unwrap_or(&fresh).clone() + } + } + } + + /// POSTs an empty body to `domain + path` using `client`. A non-2xx + /// response, a transport error, or exceeding [`HOOK_TIMEOUT`] is an error. + async fn post_hook(client: &Client, domain: &Url, path: &str) -> Result<(), Error> { + Self::post_hook_with_timeout(client, domain, path, HOOK_TIMEOUT).await + } + + /// Implementation of [`post_hook`](Self::post_hook) with an explicit timeout, + /// so tests can exercise the timeout path without waiting [`HOOK_TIMEOUT`]. + async fn post_hook_with_timeout( + client: &Client, + domain: &Url, + path: &str, + hook_timeout: Duration, + ) -> Result<(), Error> { + let mut url = domain.clone(); + url.set_path(path); + let req = hyper::Request::builder() + .method(hyper::Method::POST) + .uri(url.to_string()) + .body(Body::Empty)?; + let resp = timeout(hook_timeout, client.request(req)) + .await + .map_err(|_| Error::from(format!("SnapStart hook POST {path} timed out after {hook_timeout:?}")))??; + if !resp.status().is_success() { + return Err(Error::from(format!( + "SnapStart hook POST {path} returned non-success status: {}", + resp.status() + ))); + } + Ok(()) + } +} + +impl SnapStartResource for SnapStartHooks { + fn before_snapshot(&self) -> BoxFuture<'_, Result<(), Error>> { + Box::pin(async move { + if let Some(path) = self.before_checkpoint_path.as_deref() { + Self::post_hook(&self.client, &self.domain, path).await?; + } + Ok(()) + }) + } + + fn after_restore(&self) -> BoxFuture<'_, Result<(), Error>> { + Box::pin(async move { + // 1. Publish a fresh client FIRST so the hook POST below (and all + // subsequent invocations) use post-restore connections rather than stale + // pre-snapshot ones. If one is somehow already published, adopt it, so + // steps 2 and 3 always validate the client invocations will use. + // + // Pooling is ENABLED here even though `Adapter::new` disables it under + // SnapStart (see `base_client_pooling`), and the disagreement is + // deliberate. That restriction exists because `CLOCK_MONOTONIC` does not + // advance across the snapshot gap — measured on a deployed SnapStart + // container function, 0.54s of monotonic time for 161s of wall time — so + // hyper's `elapsed > idle_timeout` test cannot be trusted for an entry + // pooled before the boundary. This client is built AFTER the restore, so + // every entry it holds is post-boundary, and monotonic time tracks wall + // time normally from here on (measured: +6.079s/+6.059s monotonic + // against +6.1s/+6.0s wall, with idle gaps beyond the keep-alive + // expiring cleanly). Keeping the pool on is therefore both safe and the + // only way `AWS_LWA_POOL_IDLE_TIMEOUT_SECONDS` has any effect on the + // invocations that actually serve traffic. + let fresh = Arc::new(build_client(self.pool_idle_timeout, Pooling::Enabled)); + let fresh = Self::publish_or_adopt(&self.restored_client, fresh); + + // 2. Notify the app over the fresh client. Failure fails the restore; + // the fresh client stays published regardless. + if let Some(path) = self.after_restore_path.as_deref() { + Self::post_hook(&fresh, &self.domain, path).await?; + } + + // 3. Confirm the app is serving again before traffic is admitted. + // A configured timeout bounds the wait and fails the restore on + // expiry; when unset the wait is unbounded (historical behavior). + match self.readiness_timeout { + Some(t) => self.check_readiness_with_timeout(&fresh, t).await?, + None => self.check_readiness_unbounded(&fresh).await, + } + + Ok(()) + }) + } +} + +impl SnapStartHooks { + /// Step 3 of [`after_restore`](SnapStartResource::after_restore): retry-until-ready + /// over `client`, bounded by `readiness_timeout`. A timeout or an unready app is an + /// error, which fails the restore (reported to `/restore/error`). Split out with an + /// explicit timeout so tests can exercise the failure path without waiting a long + /// configured timeout (`AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS`). + async fn check_readiness_with_timeout( + &self, + client: &Client, + readiness_timeout: Duration, + ) -> Result<(), Error> { + timeout(readiness_timeout, self.wait_ready(client)).await.map_err(|_| { + Error::from(format!( + "SnapStart after-restore readiness check timed out after {readiness_timeout:?}" + )) + }) + } + + /// Unbounded variant of [`check_readiness_with_timeout`](Self::check_readiness_with_timeout): + /// waits indefinitely for the app to become ready. Used when + /// `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` is unset (historical behavior). + /// + /// This cannot fail, only block: [`readiness::wait_until_ready`] retries forever. + /// An app that never recovers therefore holds the restore open until Lambda's own + /// restore timeout fires, with the escalating `app is not ready after {}ms` log as + /// the only adapter-side signal. Set `AWS_LWA_READINESS_CHECK_TIMEOUT_SECONDS` to + /// convert that into a reported `/restore/error` instead. + async fn check_readiness_unbounded(&self, client: &Client) { + self.wait_ready(client).await; + } + + /// Shared readiness wait against the configured healthcheck endpoint. + async fn wait_ready(&self, client: &Client) { + readiness::wait_until_ready( + client, + &self.healthcheck_url, + self.healthcheck_protocol, + &self.healthcheck_healthy_status, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use httpmock::MockServer; + + /// Builds hooks pointed at `server`, with the readiness check targeting + /// `health_path` on the same server. + fn hooks_with_health( + server: &MockServer, + before: Option<&str>, + after: Option<&str>, + health_path: &str, + ) -> SnapStartHooks { + let domain: Url = format!("http://{}:{}", server.host(), server.port()).parse().unwrap(); + let healthcheck_url: Url = format!("http://{}:{}{}", server.host(), server.port(), health_path) + .parse() + .unwrap(); + SnapStartHooks::new( + Arc::new(OnceLock::new()), + Arc::new(build_client(Duration::from_secs(4), Pooling::Enabled)), + domain, + before.map(str::to_string), + after.map(str::to_string), + healthcheck_url, + Protocol::Http, + (100..500).collect(), + Duration::from_secs(4), + Some(Duration::from_secs(10)), + ) + } + + /// Builds hooks with a readiness check that always passes (a mocked `/health` + /// returning 200), for tests focused on the before/after hook behavior. + fn hooks(server: &MockServer, before: Option<&str>, after: Option<&str>) -> SnapStartHooks { + server.mock(|when, then| { + when.path("/health"); + then.status(200); + }); + hooks_with_health(server, before, after, "/health") + } + + #[tokio::test] + async fn before_snapshot_posts_when_set() { + let server = MockServer::start(); + let m = server.mock(|when, then| { + when.method(httpmock::Method::POST).path("/before"); + then.status(200); + }); + let h = hooks(&server, Some("/before"), None); + assert!(h.before_snapshot().await.is_ok()); + m.assert(); + } + + #[tokio::test] + async fn before_snapshot_noop_when_unset() { + let server = MockServer::start(); + let h = hooks(&server, None, None); + assert!(h.before_snapshot().await.is_ok()); + } + + #[tokio::test] + async fn before_snapshot_non_2xx_is_error() { + let server = MockServer::start(); + server.mock(|when, then| { + when.method(httpmock::Method::POST).path("/before"); + then.status(500); + }); + let h = hooks(&server, Some("/before"), None); + assert!(h.before_snapshot().await.is_err()); + } + + #[tokio::test] + async fn after_restore_publishes_client_then_posts() { + let server = MockServer::start(); + let m = server.mock(|when, then| { + when.method(httpmock::Method::POST).path("/after"); + then.status(200); + }); + let h = hooks(&server, None, Some("/after")); + assert!(h.restored_client.get().is_none()); + assert!(h.after_restore().await.is_ok()); + assert!(h.restored_client.get().is_some(), "fresh client published"); + m.assert(); + } + + /// When the post-restore client is already published, `after_restore` must run its + /// hook POST and readiness check over THAT client — the one invocations use — not + /// over a freshly built one nobody can see. + /// + /// Regression for the bot `[ERROR_HANDLING]` finding: `let _ = ...set(fresh)` + /// discarded the "already set" case and then used `fresh` for steps 2 and 3, so a + /// second `after_restore` would report the restore healthy on the basis of a client + /// the request path never touches, with no signal anywhere. Latent today (the + /// runtime drives the lifecycle once) — this pins it so it cannot become real. + #[test] + fn publish_or_adopt_keeps_the_client_invocations_use() { + let cell: OnceLock>> = OnceLock::new(); + let first = Arc::new(build_client(Duration::from_secs(4), Pooling::Enabled)); + let adopted = SnapStartHooks::publish_or_adopt(&cell, first.clone()); + assert!( + Arc::ptr_eq(&adopted, &first), + "first call publishes and returns its own client" + ); + + // A second call must adopt the published client and discard its own. + let second = Arc::new(build_client(Duration::from_secs(4), Pooling::Enabled)); + let adopted = SnapStartHooks::publish_or_adopt(&cell, second.clone()); + assert!( + Arc::ptr_eq(&adopted, &first), + "second call must return the ALREADY-PUBLISHED client, not its own" + ); + assert!(!Arc::ptr_eq(&adopted, &second)); + assert!( + Arc::ptr_eq(cell.get().unwrap(), &first), + "published client is unchanged" + ); + } + + #[tokio::test] + async fn after_restore_publishes_client_even_when_hook_fails() { + let server = MockServer::start(); + server.mock(|when, then| { + when.method(httpmock::Method::POST).path("/after"); + then.status(503); + }); + let h = hooks(&server, None, Some("/after")); + let result = h.after_restore().await; + assert!(result.is_err(), "hook failure returns Err"); + assert!( + h.restored_client.get().is_some(), + "client published despite hook failure" + ); + } + + #[tokio::test] + async fn post_hook_times_out_when_app_is_slow() { + let server = MockServer::start(); + // The app takes far longer to respond than the timeout we pass below. + server.mock(|when, then| { + when.method(httpmock::Method::POST).path("/slow"); + then.status(200).delay(Duration::from_secs(2)); + }); + let domain: Url = format!("http://{}:{}", server.host(), server.port()).parse().unwrap(); + let client = build_client(Duration::from_secs(4), Pooling::Enabled); + + let result = + SnapStartHooks::post_hook_with_timeout(&client, &domain, "/slow", Duration::from_millis(100)).await; + + let err = result.expect_err("slow hook should time out"); + assert!(err.to_string().contains("timed out"), "unexpected error: {err}"); + } + + #[tokio::test] + async fn after_restore_publishes_client_when_path_unset() { + let server = MockServer::start(); + let h = hooks(&server, None, None); + assert!(h.after_restore().await.is_ok()); + assert!(h.restored_client.get().is_some()); + } + + #[tokio::test] + async fn after_restore_readiness_check_runs_over_fresh_client() { + // No after-restore POST configured: step 3 must still run and pass. + let server = MockServer::start(); + let health = server.mock(|when, then| { + when.path("/ready"); + then.status(200); + }); + let h = hooks_with_health(&server, None, None, "/ready"); + assert!(h.after_restore().await.is_ok()); + health.assert(); + } + + #[tokio::test] + async fn check_readiness_times_out_when_app_never_ready() { + // Health endpoint always reports unhealthy; the bounded readiness check + // should give up and fail rather than retry forever. + let server = MockServer::start(); + server.mock(|when, then| { + when.path("/never"); + then.status(503); + }); + let h = hooks_with_health(&server, None, None, "/never"); + let client = build_client(Duration::from_secs(4), Pooling::Enabled); + + let result = h + .check_readiness_with_timeout(&client, Duration::from_millis(100)) + .await; + + let err = result.expect_err("unready app should fail the readiness check"); + assert!(err.to_string().contains("timed out"), "unexpected error: {err}"); + } +} diff --git a/tests/integ_tests/main.rs b/tests/integ_tests/main.rs index 0238618e..ba0f1c5d 100644 --- a/tests/integ_tests/main.rs +++ b/tests/integ_tests/main.rs @@ -13,7 +13,6 @@ use httpmock::{ Method::{DELETE, GET, POST, PUT}, MockServer, }; -use hyper::body::Incoming; use lambda_http::Body; use lambda_http::Context; use lambda_web_adapter::{Adapter, AdapterOptions, LambdaInvokeMode, Protocol}; @@ -25,7 +24,7 @@ use flate2::Compression; use http_body_util::BodyExt; use lambda_http::lambda_runtime::Config; use serde_json::json; -use tower_http::compression::{CompressionBody, CompressionLayer}; +use tower_http::compression::CompressionLayer; #[test] fn test_adapter_options_from_env() { @@ -136,7 +135,10 @@ async fn test_http_readiness_check() { // Initialize adapter and do readiness check let mut adapter = Adapter::new(&options).expect("Failed to create adapter"); - adapter.check_init_health().await; + adapter + .check_init_health() + .await + .expect("init health check should succeed"); // Assert app server's healthcheck endpoint got called healthcheck.assert(); @@ -1110,7 +1112,10 @@ async fn test_http_async_init_ready_at_init() { .expect("Failed to create adapter"); // Perform init health check — app is already running so it should succeed - adapter.check_init_health().await; + adapter + .check_init_health() + .await + .expect("init health check should succeed"); healthcheck.assert(); @@ -1143,7 +1148,10 @@ async fn test_http_tcp_readiness_check() { .expect("Failed to create adapter"); // TCP readiness check should succeed since MockServer is listening - adapter.check_init_health().await; + adapter + .check_init_health() + .await + .expect("init health check should succeed"); // Now verify the adapter can still forward requests let hello = app_server.mock(|when, then| { @@ -1162,12 +1170,20 @@ async fn test_http_tcp_readiness_check() { assert_eq!("TCP Ready", body_to_string(response).await); } -async fn body_to_string(res: Response) -> String { +async fn body_to_string(res: Response) -> String +where + B: http_body::Body, + B::Error: std::fmt::Debug, +{ let body_bytes = res.collect().await.unwrap().to_bytes(); String::from_utf8_lossy(&body_bytes).to_string() } -async fn compressed_body_to_string(res: Response>) -> String { +async fn compressed_body_to_string(res: Response) -> String +where + B: http_body::Body, + B::Error: std::fmt::Debug, +{ let body_bytes = res.collect().await.unwrap().to_bytes(); decode_reader(&body_bytes).unwrap() }