diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 33829fc8..25392339 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,6 +28,7 @@ on: - nextjs - node - node-slim + - nonroot - onboarder-nextjs - playwright-chromium - playwright-firefox diff --git a/docker-compose.yaml b/docker-compose.yaml index dae5d519..6b66a85d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,6 +34,17 @@ services: - "8080:8080" - "3010:3010" + nonroot: + platform: linux/amd64 + build: + context: . + dockerfile: hub/nonroot/Dockerfile + env_file: + - .env + ports: + - "8080:8080" + - "3010:3010" + app-runner: platform: linux/amd64 build: diff --git a/hub/nonroot/Dockerfile b/hub/nonroot/Dockerfile new file mode 100644 index 00000000..e2fa3839 --- /dev/null +++ b/hub/nonroot/Dockerfile @@ -0,0 +1,39 @@ +ARG SANDBOX_VERSION=latest +FROM ghcr.io/blaxel-ai/sandbox:${SANDBOX_VERSION} AS sandbox-api + +FROM node:24-alpine3.21 + +RUN apk update && apk add --no-cache \ + bash \ + git \ + python3 \ + py3-pip \ + && rm -rf /var/cache/apk/* + +# The unprivileged identity every user-launched process runs as. There is +# deliberately no USER directive: the runtime would then setuid PID 1, which is +# sandbox-api itself, and it would lose the privileges it needs to mount agent +# drives (FUSE), bring up the egress tunnel and merge the CA bundle. +RUN adduser -D -u 10001 -h /blaxel app + +WORKDIR /blaxel + +COPY --from=sandbox-api /sandbox-api /usr/local/bin/sandbox-api +COPY hub/nonroot/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +EXPOSE 8080 + +ENV HOME=/blaxel + +# Consumed by the entrypoint, which passes it to sandbox-api as --user. Setting +# the variable alone is enough: sandbox-api reads it directly when no flag is +# given. The entrypoint exists so the image can also do root-only preparation +# (chown, mkdir, mounts) before the workload identity takes effect. +ENV BL_SANDBOX_USER=app + +# Processes, terminals, codegen and the startup command run as that user, and +# the filesystem API enforces its permissions. Calling the API from inside a +# sandbox process therefore grants nothing extra: it hands back the same +# identity. +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/hub/nonroot/entrypoint.sh b/hub/nonroot/entrypoint.sh new file mode 100644 index 00000000..40d256da --- /dev/null +++ b/hub/nonroot/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Entrypoint for a sandbox whose workload runs unprivileged. +# +# It runs as root (PID 1), does the root-only preparation an image needs, and +# then hands the workload identity to sandbox-api, which keeps its own +# privileges (drive mounts, WireGuard, CA bundle, keep-alive) but runs every +# process, terminal and filesystem operation as that user. +# +# The image must NOT have a USER directive: that would de-privilege PID 1 — +# sandbox-api itself — and break drive mounting. +set -eu + +# Docker USER syntax: "app", "10001", "app:app", "10001:10001". +SANDBOX_USER="${BL_SANDBOX_USER:-app}" + +# Root-only preparation goes here, before privileges are handed over. +# Anything the workload has to write to must belong to it. +chown -R "$SANDBOX_USER" "${HOME:-/blaxel}" 2>/dev/null || true + +exec /usr/local/bin/sandbox-api --user "$SANDBOX_USER" "$@" diff --git a/hub/nonroot/template.json b/hub/nonroot/template.json new file mode 100644 index 00000000..d91a26fd --- /dev/null +++ b/hub/nonroot/template.json @@ -0,0 +1,20 @@ +{ + "name": "nonroot", + "displayName": "Non-root", + "categories": ["backend"], + "description": "Sandbox API running privileged, user processes running unprivileged.", + "longDescription": "Reference image for the unprivileged execution model: the sandbox API keeps the privileges it needs for infrastructure work (agent drive FUSE mounts, egress tunnel, CA bundle, keep-alive), while every process, terminal and filesystem operation the user triggers runs as the unprivileged 'app' user configured through BL_SANDBOX_USER. Processes cannot regain privileges by calling the sandbox API back, because the API applies the same identity to anything it starts on their behalf.", + "url": "https://github.com/blaxel-ai/sandbox", + "icon": "https://blaxel.ai/logo.png", + "memory": 2048, + "ports": [ + { + "name": "sandbox-api", + "target": 8080, + "protocol": "HTTP" + } + ], + "enterprise": false, + "coming_soon": false, + "hidden": true +} diff --git a/sandbox-api/docs/UNPRIVILEGED_EXECUTION.md b/sandbox-api/docs/UNPRIVILEGED_EXECUTION.md new file mode 100644 index 00000000..dc9f3e9f --- /dev/null +++ b/sandbox-api/docs/UNPRIVILEGED_EXECUTION.md @@ -0,0 +1,81 @@ +# Unprivileged execution (`BL_SANDBOX_USER`) + +The sandbox API is the image entrypoint, and it needs privileges: agent drive +mounts are FUSE mounts (`CAP_SYS_ADMIN`), the egress tunnel is WireGuard +(`CAP_NET_ADMIN`), the MITM CA is merged into the system trust store, keep-alive +toggles scale-to-zero, and port probing inspects other processes. + +A Docker `USER` directive cannot express that split: the runtime applies it to +PID 1, which de-privileges the API itself — that is why images built with +`bl deploy --experimental` lose drive mounting. + +`BL_SANDBOX_USER` expresses it instead. The API stays privileged; everything it +does *on behalf of the user* is dropped to an unprivileged identity. + +## Enabling it + +```dockerfile +RUN adduser -D -u 10001 -h /blaxel app + +ENV BL_SANDBOX_USER=app # also accepts "10001", "app:app", "10001:10001" +``` + +Equivalently, `sandbox-api --user app` — the flag wins over the environment. +Use an entrypoint when the image also needs root-only preparation before the +workload identity applies: + +```sh +#!/bin/sh +set -eu +SANDBOX_USER="${BL_SANDBOX_USER:-app}" +chown -R "$SANDBOX_USER" "${HOME:-/blaxel}" +exec /usr/local/bin/sandbox-api --user "$SANDBOX_USER" "$@" +``` + +Do **not** add a `USER` directive: that is the mechanism this replaces. +`hub/nonroot/` is a working example of both halves. + +If the value cannot be resolved, or resolves to uid 0, the API refuses to start. +Failing open would hand every workload the privileges the feature exists to +remove. + +## What runs as the workload user + +| Surface | Mechanism | +|---|---| +| `POST /process`, `/process/{id}/exec`, restarts, MCP `processExecute`, codegen | `SysProcAttr.Credential` (uid, gid, supplementary groups) | +| `/ws/terminal` | same credential, plus the PTY slave is chowned to the user before the shell starts | +| `-c/--command` startup command | same credential | +| every `/filesystem` operation, including multipart completion | `setfsuid(2)`/`setfsgid(2)` around the operation | + +`HOME`, `USER` and `LOGNAME` are rewritten to match the identity in all spawned +environments. + +## What stays privileged + +Drive mounts, WireGuard, CA bundle, keep-alive, port/network inspection, +process supervision and log files. So that a mounted drive is still usable: + +- `-map.uid` / `-map.gid` default to the workload uid/gid (drive content is + owned by filer uid 0), overridable per request or with + `BLFS_UID_MAP`/`BLFS_GID_MAP`; +- the mount point is chowned to the workload user when it is created. + +## No escalation by calling the API back + +A process inside the sandbox can reach the API, and it may hold a valid token. +That grants it nothing extra: every execution surface applies the same +identity — there is no "run as root" parameter — and the filesystem endpoints +are checked by the kernel against the workload user, so the classic escalation +(overwrite a root-owned binary such as `blfs` or `sandbox-api`, wait for a +privileged component to run it) fails with `EACCES`. + +Two things are worth stating plainly: + +- **The microVM remains the security boundary.** This model contains what a + compromised workload process can do inside the VM; it is not a substitute for + the VM isolation, and API authentication still gates everything. +- **Supplementary groups are not applied to filesystem operations.** + `setfsgid(2)` covers the primary group only, so access granted exclusively + through a secondary group is denied inside `/filesystem` while it is allowed + for spawned processes (which do get the full group list). diff --git a/sandbox-api/main.go b/sandbox-api/main.go index b6fae154..ac97a478 100644 --- a/sandbox-api/main.go +++ b/sandbox-api/main.go @@ -20,6 +20,7 @@ import ( "github.com/blaxel-ai/sandbox-api/src/handler" "github.com/blaxel-ai/sandbox-api/src/handler/process" "github.com/blaxel-ai/sandbox-api/src/lib/blaxel" + "github.com/blaxel-ai/sandbox-api/src/lib/identity" "github.com/blaxel-ai/sandbox-api/src/lib/networking" "github.com/blaxel-ai/sandbox-api/src/lib/proxy" "github.com/blaxel-ai/sandbox-api/src/lib/sentrylib" @@ -51,8 +52,14 @@ func main() { command := flag.String("command", "", "Command to execute") shortCommand := flag.String("c", "", "Command to execute (shorthand)") disableTelemetry := flag.Bool("disable-telemetry", false, "Disable anonymous error reporting") + workloadUser := flag.String("user", "", "Run processes, terminals and filesystem operations as this user, in Docker USER syntax (also settable with "+identity.EnvUser+")") flag.Parse() + // Resolve the workload identity before anything can spawn a process, so a + // misconfigured user fails at boot instead of at first exec. + identity.SetSpec(*workloadUser) + identity.Get() + sentrylib.Version = handler.Version sentryFlush := sentrylib.Init(*disableTelemetry) defer sentryFlush() @@ -239,6 +246,8 @@ func startBackgroundCommand(ctx context.Context, command string) { cmd.Stdout = logrus.StandardLogger().Out cmd.Stderr = logrus.StandardLogger().Out cmd.Dir = "/" + cmd.Env = identity.Get().DecorateEnv(os.Environ()) + cmd.SysProcAttr = &syscall.SysProcAttr{Credential: identity.Get().Credential()} // Start the command in a goroutine so it doesn't block the server go func() { diff --git a/sandbox-api/src/handler/drive/mount.go b/sandbox-api/src/handler/drive/mount.go index 4e18faf3..7d2e9513 100644 --- a/sandbox-api/src/handler/drive/mount.go +++ b/sandbox-api/src/handler/drive/mount.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "github.com/blaxel-ai/sandbox-api/src/lib/identity" "github.com/sirupsen/logrus" ) @@ -72,13 +73,20 @@ func validateLocalID(value, name string) error { // resolveMapping returns the effective local UID/GID value. // Priority: request parameter > environment variable > empty (no mapping). -func resolveMapping(reqValue, envKey, name string) (string, error) { +func resolveMapping(reqValue, envKey, name string, workloadID int) (string, error) { value := reqValue source := "request" if value == "" { value = os.Getenv(envKey) source = "env" } + // Drive content belongs to filer uid/gid 0. Mapping it onto the workload + // identity by default is what makes the mount writable by processes, which + // no longer run as root. + if value == "" && workloadID > 0 { + value = strconv.Itoa(workloadID) + source = "workload identity" + } if value == "" { return "", nil } @@ -123,12 +131,16 @@ func MountDrive(driveName, mountPath, drivePath string, readOnly bool, uidMap, g lock.Lock() defer lock.Unlock() - // Resolve UID/GID mappings (request param > env var > none). - effectiveUidMap, err := resolveMapping(uidMap, "BLFS_UID_MAP", "uidMap") + // Resolve UID/GID mappings (request param > env var > workload identity > none). + workloadUid, workloadGid := -1, -1 + if id := identity.Get(); id != nil { + workloadUid, workloadGid = id.Uid, id.Gid + } + effectiveUidMap, err := resolveMapping(uidMap, "BLFS_UID_MAP", "uidMap", workloadUid) if err != nil { return "", "", fmt.Errorf("invalid uidMap: %w", err) } - effectiveGidMap, err := resolveMapping(gidMap, "BLFS_GID_MAP", "gidMap") + effectiveGidMap, err := resolveMapping(gidMap, "BLFS_GID_MAP", "gidMap", workloadGid) if err != nil { return "", "", fmt.Errorf("invalid gidMap: %w", err) } @@ -169,10 +181,25 @@ func MountDrive(driveName, mountPath, drivePath string, readOnly bool, uidMap, g return "", "", fmt.Errorf("failed to get filer address: %w", err) } - // Create mount directory if it doesn't exist + // Create mount directory if it doesn't exist. It is created by the API + // (root) but handed to the workload user so it is usable while the drive + // is not yet mounted over it. Only chown directories we actually create: + // re-owning a pre-existing system directory (e.g. /usr/local/bin) would be + // a privilege-escalation vector if the subsequent mount fails. + created := false + if _, err := os.Stat(mountPath); os.IsNotExist(err) { + created = true + } if err := os.MkdirAll(mountPath, 0755); err != nil { return "", "", fmt.Errorf("failed to create mount directory: %w", err) } + if created { + if id := identity.Get(); id != nil { + if err := os.Chown(mountPath, id.Uid, id.Gid); err != nil { + logrus.WithError(err).WithField("mount_path", mountPath).Warn("Failed to hand mount point to the workload user") + } + } + } // Build the filer path: /buckets/{infrastructureId}{drivePath} filerPath := fmt.Sprintf("/buckets/%s%s", infrastructureId, drivePath) diff --git a/sandbox-api/src/handler/filesystem/filesystem.go b/sandbox-api/src/handler/filesystem/filesystem.go index 90b1ba96..73b79028 100644 --- a/sandbox-api/src/handler/filesystem/filesystem.go +++ b/sandbox-api/src/handler/filesystem/filesystem.go @@ -223,7 +223,7 @@ func (fs *Filesystem) GetAbsolutePath(path string) (string, error) { } // FileExists checks if a file exists at the given path -func (fs *Filesystem) FileExists(path string) (bool, error) { +func (fs *Filesystem) fileExists(path string) (bool, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return false, err @@ -241,7 +241,7 @@ func (fs *Filesystem) FileExists(path string) (bool, error) { } // DirectoryExists checks if a directory exists at the given path -func (fs *Filesystem) DirectoryExists(path string) (bool, error) { +func (fs *Filesystem) directoryExists(path string) (bool, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return false, err @@ -260,7 +260,7 @@ func (fs *Filesystem) DirectoryExists(path string) (bool, error) { // Infos returns the information about a file or directory // Uses os.Stat to follow symlinks, so symlinks to directories are treated as directories -func (fs *Filesystem) Infos(path string) (os.FileInfo, error) { +func (fs *Filesystem) infos(path string) (os.FileInfo, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return nil, err @@ -270,7 +270,7 @@ func (fs *Filesystem) Infos(path string) (os.FileInfo, error) { } // ReadFile reads a file and returns its contents -func (fs *Filesystem) ReadFile(path string) (*FileWithContentByte, error) { +func (fs *Filesystem) readFile(path string) (*FileWithContentByte, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return nil, err @@ -314,7 +314,7 @@ func (fs *Filesystem) ReadFile(path string) (*FileWithContentByte, error) { } // WriteFile writes content to a file -func (fs *Filesystem) WriteFile(path string, content []byte, perm os.FileMode) error { +func (fs *Filesystem) writeFile(path string, content []byte, perm os.FileMode) error { absPath, err := fs.GetAbsolutePath(path) if err != nil { return err @@ -330,7 +330,7 @@ func (fs *Filesystem) WriteFile(path string, content []byte, perm os.FileMode) e } // WriteFileFromReader streams content from a reader to a file on disk -func (fs *Filesystem) WriteFileFromReader(path string, r io.Reader, perm os.FileMode) error { +func (fs *Filesystem) writeFileFromReader(path string, r io.Reader, perm os.FileMode) error { absPath, err := fs.GetAbsolutePath(path) if err != nil { return err @@ -358,7 +358,7 @@ func (fs *Filesystem) WriteFileFromReader(path string, r io.Reader, perm os.File } // CreateDirectory creates a directory at the given path -func (fs *Filesystem) CreateDirectory(path string, perm os.FileMode) error { +func (fs *Filesystem) createDirectory(path string, perm os.FileMode) error { absPath, err := fs.GetAbsolutePath(path) if err != nil { return err @@ -368,7 +368,7 @@ func (fs *Filesystem) CreateDirectory(path string, perm os.FileMode) error { } // ListDirectory lists files and directories in the given path -func (fs *Filesystem) ListDirectory(path string) (*Directory, error) { +func (fs *Filesystem) listDirectory(path string) (*Directory, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return nil, err @@ -413,7 +413,7 @@ func (fs *Filesystem) ListDirectory(path string) (*Directory, error) { } // DeleteFile deletes a file at the given path -func (fs *Filesystem) DeleteFile(path string) error { +func (fs *Filesystem) deleteFile(path string) error { absPath, err := fs.GetAbsolutePath(path) if err != nil { return err @@ -432,7 +432,7 @@ func (fs *Filesystem) DeleteFile(path string) error { } // DeleteDirectory deletes a directory at the given path -func (fs *Filesystem) DeleteDirectory(path string, recursive bool) error { +func (fs *Filesystem) deleteDirectory(path string, recursive bool) error { absPath, err := fs.GetAbsolutePath(path) if err != nil { return err @@ -454,7 +454,7 @@ func (fs *Filesystem) DeleteDirectory(path string, recursive bool) error { } // CopyFile copies a file from src to dst -func (fs *Filesystem) CopyFile(src, dst string) error { +func (fs *Filesystem) copyFile(src, dst string) error { srcAbs, err := fs.GetAbsolutePath(src) if err != nil { return err @@ -488,7 +488,7 @@ func (fs *Filesystem) CopyFile(src, dst string) error { } // MoveFile moves a file from src to dst -func (fs *Filesystem) MoveFile(src, dst string) error { +func (fs *Filesystem) moveFile(src, dst string) error { srcAbs, err := fs.GetAbsolutePath(src) if err != nil { return err @@ -542,7 +542,7 @@ func (fs *Filesystem) getFileOwnerAndGroup(path string) (string, string, error) } // GetFileInfo returns file information without reading its content -func (fs *Filesystem) GetFileInfo(path string) (*FileByte, error) { +func (fs *Filesystem) getFileInfo(path string) (*FileByte, error) { absPath, err := fs.GetAbsolutePath(path) if err != nil { return nil, err @@ -566,7 +566,7 @@ func (fs *Filesystem) GetFileInfo(path string) (*FileByte, error) { } // Walk walks the file tree rooted at root, calling fn for each file or directory -func (fs *Filesystem) Walk(root string, fn filepath.WalkFunc) error { +func (fs *Filesystem) walk(root string, fn filepath.WalkFunc) error { absRoot, err := fs.GetAbsolutePath(root) if err != nil { return err @@ -589,7 +589,7 @@ func (fs *Filesystem) Walk(root string, fn filepath.WalkFunc) error { } // CreateOrUpdateFile creates or updates a file -func (fs *Filesystem) CreateOrUpdateFile(path string, content string, isDirectory bool, permissions string) error { +func (fs *Filesystem) createOrUpdateFile(path string, content string, isDirectory bool, permissions string) error { // Parse permissions or use appropriate defaults var perm os.FileMode if permissions != "" { @@ -608,7 +608,7 @@ func (fs *Filesystem) CreateOrUpdateFile(path string, content string, isDirector } if isDirectory { - return fs.CreateDirectory(path, perm) + return fs.createDirectory(path, perm) } // Get absolute path for directory creation @@ -623,5 +623,5 @@ func (fs *Filesystem) CreateOrUpdateFile(path string, content string, isDirector return err } - return fs.WriteFile(path, []byte(content), perm) + return fs.writeFile(path, []byte(content), perm) } diff --git a/sandbox-api/src/handler/filesystem/multipart.go b/sandbox-api/src/handler/filesystem/multipart.go index 4fc72316..9210afb3 100644 --- a/sandbox-api/src/handler/filesystem/multipart.go +++ b/sandbox-api/src/handler/filesystem/multipart.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/blaxel-ai/sandbox-api/src/lib/identity" "github.com/google/uuid" ) @@ -169,16 +170,23 @@ func (m *MultipartManager) CompleteUpload(uploadID string, parts []UploadedPart) return parts[i].PartNumber < parts[j].PartNumber }) - // Create parent directories if they don't exist - dir := filepath.Dir(upload.Path) - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create parent directory: %w", err) - } - - // Create final file - finalFile, err := os.OpenFile(upload.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, upload.Permissions) - if err != nil { - return fmt.Errorf("failed to create final file: %w", err) + // The destination is caller-controlled, so it is created with the workload + // identity. The part files live in the API's own uploads directory and stay + // privileged. + var finalFile *os.File + if err := identity.Do(func() error { + dir := filepath.Dir(upload.Path) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + f, err := os.OpenFile(upload.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, upload.Permissions) + if err != nil { + return fmt.Errorf("failed to create final file: %w", err) + } + finalFile = f + return nil + }); err != nil { + return err } defer finalFile.Close() diff --git a/sandbox-api/src/handler/filesystem/unprivileged.go b/sandbox-api/src/handler/filesystem/unprivileged.go new file mode 100644 index 00000000..3fb5ddb3 --- /dev/null +++ b/sandbox-api/src/handler/filesystem/unprivileged.go @@ -0,0 +1,135 @@ +package filesystem + +import ( + "io" + "os" + "path/filepath" + + "github.com/blaxel-ai/sandbox-api/src/lib/identity" +) + +// Every exported filesystem operation runs through identity.Do, so the kernel +// checks each path against the workload user instead of the API's root +// identity. Without it the filesystem endpoints would be an escalation path out +// of the unprivileged execution model: a process could ask the API to overwrite +// a root-owned binary (blfs, the sandbox-api itself, a login shell) and get its +// code run as root by the next privileged operation. +// +// The lowercase methods hold the actual implementation and call each other, so +// no operation nests the identity switch more than once. + +func (fs *Filesystem) FileExists(path string) (bool, error) { + var exists bool + err := identity.Do(func() error { + var err error + exists, err = fs.fileExists(path) + return err + }) + return exists, err +} + +func (fs *Filesystem) DirectoryExists(path string) (bool, error) { + var exists bool + err := identity.Do(func() error { + var err error + exists, err = fs.directoryExists(path) + return err + }) + return exists, err +} + +func (fs *Filesystem) Infos(path string) (os.FileInfo, error) { + var info os.FileInfo + err := identity.Do(func() error { + var err error + info, err = fs.infos(path) + return err + }) + return info, err +} + +func (fs *Filesystem) ReadFile(path string) (*FileWithContentByte, error) { + var file *FileWithContentByte + err := identity.Do(func() error { + var err error + file, err = fs.readFile(path) + return err + }) + return file, err +} + +func (fs *Filesystem) WriteFile(path string, content []byte, perm os.FileMode) error { + return identity.Do(func() error { + return fs.writeFile(path, content, perm) + }) +} + +func (fs *Filesystem) WriteFileFromReader(path string, r io.Reader, perm os.FileMode) error { + return identity.Do(func() error { + return fs.writeFileFromReader(path, r, perm) + }) +} + +func (fs *Filesystem) CreateDirectory(path string, perm os.FileMode) error { + return identity.Do(func() error { + return fs.createDirectory(path, perm) + }) +} + +func (fs *Filesystem) ListDirectory(path string) (*Directory, error) { + var dir *Directory + err := identity.Do(func() error { + var err error + dir, err = fs.listDirectory(path) + return err + }) + return dir, err +} + +func (fs *Filesystem) DeleteFile(path string) error { + return identity.Do(func() error { + return fs.deleteFile(path) + }) +} + +func (fs *Filesystem) DeleteDirectory(path string, recursive bool) error { + return identity.Do(func() error { + return fs.deleteDirectory(path, recursive) + }) +} + +func (fs *Filesystem) CopyFile(src, dst string) error { + return identity.Do(func() error { + return fs.copyFile(src, dst) + }) +} + +func (fs *Filesystem) MoveFile(src, dst string) error { + return identity.Do(func() error { + return fs.moveFile(src, dst) + }) +} + +func (fs *Filesystem) GetFileInfo(path string) (*FileByte, error) { + var file *FileByte + err := identity.Do(func() error { + var err error + file, err = fs.getFileInfo(path) + return err + }) + return file, err +} + +// Walk keeps the identity applied for the whole traversal, including the calls +// to fn, so a callback reading the files it is given is checked the same way. +func (fs *Filesystem) Walk(root string, fn filepath.WalkFunc) error { + return identity.Do(func() error { + return fs.walk(root, fn) + }) +} + +func (fs *Filesystem) CreateOrUpdateFile(path string, content string, isDirectory bool, permissions string) error { + return identity.Do(func() error { + return fs.createOrUpdateFile(path, content, isDirectory, permissions) + }) +} diff --git a/sandbox-api/src/handler/process/process.go b/sandbox-api/src/handler/process/process.go index a884cbe6..eadcae81 100644 --- a/sandbox-api/src/handler/process/process.go +++ b/sandbox-api/src/handler/process/process.go @@ -14,6 +14,7 @@ import ( "github.com/blaxel-ai/sandbox-api/src/handler/constants" "github.com/blaxel-ai/sandbox-api/src/lib/blaxel" + "github.com/blaxel-ai/sandbox-api/src/lib/identity" "github.com/sirupsen/logrus" ) @@ -240,10 +241,11 @@ func (pm *ProcessManager) StartProcessWithName(command string, workingDir string // Set up process group to ensure all child processes can be killed together cmd.SysProcAttr = &syscall.SysProcAttr{ - Setpgid: true, + Setpgid: true, + Credential: identity.Get().Credential(), } - cmd.Env = buildProcessEnv(env) + cmd.Env = identity.Get().DecorateEnv(buildProcessEnv(env)) // Ensure log directory exists if err := ensureLogDir(); err != nil { @@ -661,13 +663,14 @@ func (pm *ProcessManager) restartProcess(oldProcess *ProcessInfo, callback func( // Set up process group to ensure all child processes can be killed together cmd.SysProcAttr = &syscall.SysProcAttr{ - Setpgid: true, + Setpgid: true, + Credential: identity.Get().Credential(), } // Re-merge the custom env vars provided at the original start with the // current system environment. Using os.Environ() alone here would drop any // custom env vars the caller passed when first starting the process. - cmd.Env = buildProcessEnv(oldProcess.Env) + cmd.Env = identity.Get().DecorateEnv(buildProcessEnv(oldProcess.Env)) // Open log files for appending - child writes directly to files stdoutFile, err := os.OpenFile(oldProcess.StdoutFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) diff --git a/sandbox-api/src/handler/terminal/terminal.go b/sandbox-api/src/handler/terminal/terminal.go index 875e845a..75d92c1f 100644 --- a/sandbox-api/src/handler/terminal/terminal.go +++ b/sandbox-api/src/handler/terminal/terminal.go @@ -9,6 +9,7 @@ import ( "sync" "syscall" + "github.com/blaxel-ai/sandbox-api/src/lib/identity" "github.com/creack/pty" ) @@ -95,20 +96,43 @@ func NewTerminalSession(shell string, workingDir string, env map[string]string, if !hasTerm { finalEnv = append(finalEnv, "TERM=xterm-256color") } - cmd.Env = finalEnv + cmd.Env = identity.Get().DecorateEnv(finalEnv) - // NOTE: Do NOT set SysProcAttr here! - // The pty.Start() function internally sets Setsid: true to create a new session, - // which is required for proper PTY operation. Setting Setpgid would conflict with Setsid. - - // Start command with PTY - ptmx, err := pty.StartWithSize(cmd, &pty.Winsize{ - Cols: cols, - Rows: rows, - }) + // The PTY pair is opened here rather than through pty.StartWithSize so the + // slave can be handed over to the workload user (as login(1) does) before + // the shell starts. Setsid/Setctty are required for PTY operation and + // conflict with Setpgid, so the credential travels in the same attributes. + ptmx, tty, err := pty.Open() if err != nil { return nil, err } + defer tty.Close() + + if err := pty.Setsize(ptmx, &pty.Winsize{Cols: cols, Rows: rows}); err != nil { + ptmx.Close() + return nil, err + } + + if id := identity.Get(); id != nil { + if err := tty.Chown(id.Uid, id.Gid); err != nil { + ptmx.Close() + return nil, err + } + } + + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + Credential: identity.Get().Credential(), + } + + if err := cmd.Start(); err != nil { + ptmx.Close() + return nil, err + } // Store only the PID to avoid FD leak (workspace rule: never store *exec.Cmd in struct) pid := cmd.Process.Pid diff --git a/sandbox-api/src/lib/identity/do_linux.go b/sandbox-api/src/lib/identity/do_linux.go new file mode 100644 index 00000000..49bf64e1 --- /dev/null +++ b/sandbox-api/src/lib/identity/do_linux.go @@ -0,0 +1,76 @@ +//go:build linux + +package identity + +import ( + "fmt" + "runtime" + "syscall" + + "github.com/sirupsen/logrus" +) + +// Do runs fn with the calling thread's filesystem identity set to the workload +// user, so every path the kernel resolves inside fn is checked against that +// user's permissions instead of root's. +// +// setfsuid(2)/setfsgid(2) are per-thread, hence the locked OS thread; they also +// drop the thread's CAP_DAC_OVERRIDE / CAP_DAC_READ_SEARCH for the duration, +// which is precisely the point: it stops the filesystem API from being used to +// overwrite root-owned files (for example the blfs binary the drive mount runs +// as root) and thereby escalate out of the unprivileged identity. +// +// Limitation: the fsgid change covers the primary group only. Access granted +// exclusively through a supplementary group of the workload user is not +// honoured inside Do. +func (id *Identity) Do(fn func() error) error { + if id == nil { + return fn() + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + previousGid := setfsgid(id.Gid) + previousUid := setfsuid(id.Uid) + + // setfsuid(2)/setfsgid(2) have no error return: they report the previous + // value whether or not they changed anything. Reading the value back is the + // only way to know the thread really lost its root filesystem privileges. + if current := setfsuid(id.Uid); current != id.Uid { + setfsuid(previousUid) + setfsgid(previousGid) + return fmt.Errorf("failed to drop filesystem uid to %d", id.Uid) + } + if current := setfsgid(id.Gid); current != id.Gid { + setfsuid(previousUid) + setfsgid(previousGid) + return fmt.Errorf("failed to drop filesystem gid to %d", id.Gid) + } + + defer func() { + // Restoring must succeed: a thread left with a non-root fsuid would + // silently break the privileged work (mounts, tunnel) that runs on it + // later. Restoring the previous value rather than 0 keeps nested calls + // correct. + setfsuid(previousUid) + setfsgid(previousGid) + if current := setfsuid(previousUid); current != previousUid { + logrus.Fatalf("Failed to restore filesystem uid to %d", previousUid) + } + }() + + return fn() +} + +// setfsuid and setfsgid wrap the raw syscalls because the syscall package +// discards their return value, which is the previous uid/gid. +func setfsuid(uid int) int { + previous, _, _ := syscall.Syscall(syscall.SYS_SETFSUID, uintptr(uid), 0, 0) + return int(previous) +} + +func setfsgid(gid int) int { + previous, _, _ := syscall.Syscall(syscall.SYS_SETFSGID, uintptr(gid), 0, 0) + return int(previous) +} diff --git a/sandbox-api/src/lib/identity/do_other.go b/sandbox-api/src/lib/identity/do_other.go new file mode 100644 index 00000000..b90b85f2 --- /dev/null +++ b/sandbox-api/src/lib/identity/do_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package identity + +// Do is a no-op on non-Linux platforms: setfsuid/setfsgid are Linux-specific +// syscalls. Without them, the sandbox cannot enforce per-thread filesystem +// identity, so fn runs under the process's existing credentials. +func (id *Identity) Do(fn func() error) error { + return fn() +} diff --git a/sandbox-api/src/lib/identity/identity.go b/sandbox-api/src/lib/identity/identity.go new file mode 100644 index 00000000..c11b03bd --- /dev/null +++ b/sandbox-api/src/lib/identity/identity.go @@ -0,0 +1,203 @@ +// Package identity resolves the unprivileged identity that user workloads run +// as, while the sandbox API itself keeps the privileges it needs (FUSE drive +// mounts, WireGuard, CA bundle, keep-alive, port probing). +// +// The identity is configured with BL_SANDBOX_USER, using Docker's USER syntax: +// "app", "10001", "app:app" or "10001:10001". When it is unset the whole +// mechanism is disabled and everything keeps running as the API user (root), +// which is the historical behaviour. +package identity + +import ( + "fmt" + "os" + "os/user" + "strconv" + "strings" + "sync" + "syscall" + + "github.com/sirupsen/logrus" +) + +// EnvUser is the environment variable holding the workload identity. +const EnvUser = "BL_SANDBOX_USER" + +// Identity is a resolved, unprivileged POSIX identity. +type Identity struct { + Uid int + Gid int + Groups []uint32 + Name string + Home string +} + +var ( + once sync.Once + resolved *Identity + spec string + source = EnvUser +) + +// SetSpec sets the workload identity from the command line, taking precedence +// over the environment. It must be called before the first Get. +func SetSpec(value string) { + if value = strings.TrimSpace(value); value != "" { + spec = value + source = "--user" + } +} + +// Get returns the workload identity, or nil when none is configured. The +// resolution happens once: the value cannot change during the lifetime of the +// process, so no request can influence which user its work runs as. +func Get() *Identity { + once.Do(func() { + if spec == "" { + spec = strings.TrimSpace(os.Getenv(EnvUser)) + } + if spec == "" { + return + } + id, err := resolve(spec) + if err != nil { + // Failing open (running as root) would silently give every + // workload the privileges this feature exists to remove. + logrus.WithError(err).Fatalf("Invalid %s=%q", source, spec) + } + if id.Uid == 0 { + logrus.Fatalf("%s=%q resolves to uid 0; the workload identity must be unprivileged", source, spec) + } + logrus.WithFields(logrus.Fields{ + "user": id.Name, + "uid": id.Uid, + "gid": id.Gid, + "home": id.Home, + }).Info("Workload identity enabled: processes, terminals and filesystem operations run unprivileged") + resolved = id + }) + return resolved +} + +// resolve parses a Docker USER string and looks the parts up in the passwd and +// group databases. +func resolve(spec string) (*Identity, error) { + userPart, groupPart, hasGroup := strings.Cut(spec, ":") + if userPart == "" { + return nil, fmt.Errorf("empty user part") + } + + id := &Identity{Gid: -1} + + if uid, err := strconv.Atoi(userPart); err == nil { + id.Uid = uid + id.Name = userPart + if u, err := user.LookupId(userPart); err == nil { + id.Name = u.Username + id.Home = u.HomeDir + id.Gid, _ = strconv.Atoi(u.Gid) + } + } else { + u, err := user.Lookup(userPart) + if err != nil { + return nil, fmt.Errorf("user %q not found: %w", userPart, err) + } + id.Uid, _ = strconv.Atoi(u.Uid) + id.Gid, _ = strconv.Atoi(u.Gid) + id.Name = u.Username + id.Home = u.HomeDir + } + + if hasGroup && groupPart != "" { + gid, err := strconv.Atoi(groupPart) + if err != nil { + g, err := user.LookupGroup(groupPart) + if err != nil { + return nil, fmt.Errorf("group %q not found: %w", groupPart, err) + } + gid, _ = strconv.Atoi(g.Gid) + } + id.Gid = gid + } + if id.Gid < 0 { + id.Gid = id.Uid + } + if id.Home == "" { + id.Home = "/" + } + + id.Groups = supplementaryGroups(id.Name, id.Gid) + return id, nil +} + +// supplementaryGroups mirrors initgroups(3): every group the user is a member +// of, plus its primary group. +func supplementaryGroups(name string, gid int) []uint32 { + groups := []uint32{uint32(gid)} + if name == "" { + return groups + } + u, err := user.Lookup(name) + if err != nil { + return groups + } + ids, err := u.GroupIds() + if err != nil { + return groups + } + for _, raw := range ids { + g, err := strconv.Atoi(raw) + if err != nil || g == gid { + continue + } + groups = append(groups, uint32(g)) + } + return groups +} + +// Credential returns the syscall credential to attach to a spawned process, or +// nil when no identity is configured. +func (id *Identity) Credential() *syscall.Credential { + if id == nil { + return nil + } + return &syscall.Credential{ + Uid: uint32(id.Uid), + Gid: uint32(id.Gid), + Groups: id.Groups, + } +} + +// DecorateEnv overrides the identity-related variables of an environment slice +// so a spawned process sees a coherent HOME/USER/LOGNAME. +func (id *Identity) DecorateEnv(env []string) []string { + if id == nil { + return env + } + overrides := map[string]string{ + "HOME": id.Home, + "USER": id.Name, + "LOGNAME": id.Name, + } + out := make([]string, 0, len(env)+len(overrides)) + for _, kv := range env { + key, _, ok := strings.Cut(kv, "=") + if ok { + if _, replaced := overrides[key]; replaced { + continue + } + } + out = append(out, kv) + } + for key, value := range overrides { + if value != "" { + out = append(out, key+"="+value) + } + } + return out +} + +// Do runs fn under the process-wide workload identity when one is configured. +func Do(fn func() error) error { + return Get().Do(fn) +} diff --git a/sandbox-api/src/lib/identity/identity_test.go b/sandbox-api/src/lib/identity/identity_test.go new file mode 100644 index 00000000..22288a1c --- /dev/null +++ b/sandbox-api/src/lib/identity/identity_test.go @@ -0,0 +1,114 @@ +package identity + +import ( + "os" + "os/user" + "path/filepath" + "slices" + "strconv" + "testing" +) + +func currentUser(t *testing.T) *user.User { + t.Helper() + u, err := user.Current() + if err != nil { + t.Fatalf("user.Current: %v", err) + } + return u +} + +func TestResolve(t *testing.T) { + u := currentUser(t) + gid, _ := strconv.Atoi(u.Gid) + uid, _ := strconv.Atoi(u.Uid) + + for _, spec := range []string{u.Username, u.Uid, u.Username + ":" + u.Gid, u.Uid + ":" + u.Gid} { + id, err := resolve(spec) + if err != nil { + t.Fatalf("resolve(%q): %v", spec, err) + } + if id.Uid != uid || id.Gid != gid { + t.Fatalf("resolve(%q) = uid %d gid %d, want uid %d gid %d", spec, id.Uid, id.Gid, uid, gid) + } + if !slices.Contains(id.Groups, uint32(gid)) { + t.Fatalf("resolve(%q) groups %v missing primary gid %d", spec, id.Groups, gid) + } + } +} + +func TestResolveRejectsUnknown(t *testing.T) { + for _, spec := range []string{"", "no-such-user-9d1f", currentUser(t).Username + ":no-such-group-9d1f"} { + if _, err := resolve(spec); err == nil { + t.Fatalf("resolve(%q) succeeded, want error", spec) + } + } +} + +func TestDecorateEnv(t *testing.T) { + id := &Identity{Uid: 10001, Gid: 10001, Name: "app", Home: "/home/app"} + env := id.DecorateEnv([]string{"HOME=/root", "USER=root", "LOGNAME=root", "PATH=/usr/bin"}) + + want := map[string]bool{"HOME=/home/app": false, "USER=app": false, "LOGNAME=app": false, "PATH=/usr/bin": false} + for _, kv := range env { + if _, ok := want[kv]; !ok { + t.Fatalf("unexpected entry %q in %v", kv, env) + } + want[kv] = true + } + for kv, seen := range want { + if !seen { + t.Fatalf("missing entry %q in %v", kv, env) + } + } +} + +func TestDecorateEnvDisabled(t *testing.T) { + var id *Identity + env := []string{"HOME=/root"} + if got := id.DecorateEnv(env); len(got) != 1 || got[0] != "HOME=/root" { + t.Fatalf("DecorateEnv on nil identity = %v, want unchanged", got) + } +} + +func TestCredentialDisabled(t *testing.T) { + var id *Identity + if got := id.Credential(); got != nil { + t.Fatalf("Credential on nil identity = %v, want nil", got) + } +} + +// TestDoDropsRootAccess is the actual guarantee: inside Do, a root-owned 0600 +// file must be unreadable, and outside it must be readable again. +func TestDoDropsRootAccess(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root") + } + + secret := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(secret, []byte("root only"), 0600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + id := &Identity{Uid: 65534, Gid: 65534, Name: "nobody", Home: "/"} + + err := id.Do(func() error { + if _, err := os.ReadFile(secret); err == nil { + t.Error("root-owned 0600 file was readable under the workload identity") + } + // Nested calls must not restore root access early. + return id.Do(func() error { + if _, err := os.ReadFile(secret); err == nil { + t.Error("root-owned 0600 file was readable inside a nested Do") + } + return nil + }) + }) + if err != nil { + t.Fatalf("Do: %v", err) + } + + if _, err := os.ReadFile(secret); err != nil { + t.Fatalf("root access was not restored after Do: %v", err) + } +}