Speed up deploys with cached builds and worker-first rollout

Dockerfile

@@ -12,32 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# --- Build stage -------------------------------------------------------------
-# The full python:3.13 image carries the compilers/headers that a dependency
-# without a prebuilt cp313 wheel would need. Install everything into a
-# relocatable prefix (/install) that the slim runtime can drop in as-is, so the
-# build can never fail for lack of a compiler on the slim base.
-FROM python:3.13@sha256:e72bfff2ccf413e3c329074d643fac616d7e1dfe85ac57e527f1d13cd8e0ee6c AS builder
-
-ENV PYTHONUNBUFFERED=1
-
-COPY requirements.txt .
-RUN pip install --no-cache-dir --require-hashes --prefix=/install -r requirements.txt
-
-# --- Runtime stage -----------------------------------------------------------
-# python:3.13-slim is ~850 MB smaller than the full image: faster to push to the
-# registry and faster to cold-start. It carries only ffmpeg, the dependencies
-# built above, and the app — no compilers or build cruft.
+# --- Single-stage slim image -------------------------------------------------
+# python:3.13-slim is ~850 MB smaller than the full python:3.13 image: faster to
+# pull, build, push to Artifact Registry, and cold-start on Cloud Run. Every
+# package in requirements.txt publishes a prebuilt cp313 manylinux wheel
+# (--only-binary=:all:), so no C compiler or multi-stage /install copy is needed.
 FROM python:3.13-slim@sha256:c33f0bc4364a6881bed1ec0cc2665e6c53c87a43e774aaeab88e6f17af105e4f
 
 ENV PYTHONUNBUFFERED=1
 
-# ffmpeg is required by the worker's video actions (combine/convert). One layer,
-# no recommended extras, apt lists dropped to keep the image small.
+# ffmpeg + ffprobe are required by the worker's video actions (combine/convert).
+# Installed from official Debian packages with --no-install-recommends and apt
+# lists dropped in the same layer.
 RUN apt-get update \
   && apt-get install -y --no-install-recommends ffmpeg \
   && rm -rf /var/lib/apt/lists/*
 
+# Install locked Python dependencies from prebuilt binary wheels directly into
+# /usr/local so the entire dependency layer is cached in the final pushed image.
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir --only-binary=:all: --require-hashes -r /tmp/requirements.txt \
+  && rm -f /tmp/requirements.txt
+
 # Run as a non-root user with a real home, and give it a writable app dir it
 # owns. The worker's video actions write temp files using bare relative names
 # into the process CWD (== WORKDIR), so WORKDIR must be owned by this user.
@@ -47,10 +43,6 @@ RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \
 
 WORKDIR /app
 
-# Drop in the dependencies built in the full image (same python 3.13, so the
-# installed packages and gunicorn entry point land on /usr/local and PATH).
-# Left root-owned and world-readable — import/exec only need read access.
-COPY --from=builder /install /usr/local
 
 # Runtime files only (not the whole repo): explicit copies keep docs, examples,
 # tests, deploy scripts, and .git out of the image. Root-owned but world-readable

deploy.sh

@@ -407,6 +407,34 @@ echo "════════════════════════
 # human think-time at the prompt doesn't pollute the timing deliverable.
 SCRIPT_START=$(date +%s)
 
+# Render UI env + config and kick off the local Angular UI build in the
+# background right away so local CPU work (npm ci / ng build) overlaps with
+# cloud API, service-account, IAM, Cloud Tasks, bucket, and Firestore setup.
+export UI_CONTROL_PLANE_MODE="iap"
+envsubst < ./ui/src/env.template.txt > ./ui/src/env.ts
+generate_config
+if grep -q "controlPlaneMode: 'none'" ./ui/src/env.ts; then
+  echo "ERROR: ui/src/env.ts rendered with controlPlaneMode 'none' (sign-in disabled)." >&2
+  exit 1
+fi
+UI_BUILD_PID=""
+UI_BUILD_LOG=""
+if [ "$SKIP_UI_BUILD" != "1" ]; then
+  UI_BUILD_LOG=$(mktemp)
+  (
+    export NG_CLI_ANALYTICS=ci
+    if [ -f ui/node_modules/.package-lock.stamp ] \
+        && cmp -s ui/package-lock.json ui/node_modules/.package-lock.stamp; then
+      echo "  ✓ ui/package-lock.json unchanged — skipping npm ci."
+    else
+      ( cd ui && npm ci )
+      cp ui/package-lock.json ui/node_modules/.package-lock.stamp
+    fi
+    ( cd ui && npx ng build --configuration production )
+  ) >"$UI_BUILD_LOG" 2>&1 &
+  UI_BUILD_PID=$!
+fi
+
 # --- Enable services ---------------------------------------------------------
 # Note: compute.googleapis.com is enabled here so the default Compute Engine
 # service account (used for role bindings below) is guaranteed to exist.
@@ -650,6 +678,9 @@ echo "Granting the runtime SA self-impersonation (signBlob + Cloud Tasks OIDC)..
 add_sa_iam_binding "${RUNTIME_SA}" "serviceAccount:${RUNTIME_SA}" "roles/iam.serviceAccountTokenCreator" "$PROJECT"
 add_sa_iam_binding "${RUNTIME_SA}" "serviceAccount:${RUNTIME_SA}" "roles/iam.serviceAccountUser" "$PROJECT"
 
+# --- Non-IAM runtime infrastructure + Firestore seed (overlapped with Cloud Build) ---
+INFRA_SETUP_LOG=$(mktemp)
+(
 # --- Cloud Tasks queues -------------------------------------------------------
 phase "Setting up Cloud Tasks queues..."
 QUEUES=("Other" "Gemini" "Veo")
@@ -764,6 +795,74 @@ else
     gcloud firestore databases describe --database="$FIRESTORE_DB_UI" --project=$PROJECT --format="value(locationId)"
 fi
 
+# --- SceneMachineUser custom role -----------------------------------------------
+phase "Ensuring SceneMachineUser custom role matches user-role.yaml..."
+if ! gcloud iam roles describe SceneMachineUser --project=$PROJECT &> /dev/null; then
+  echo "SceneMachineUser role doesn't exist. Creating it..."
+  gcloud iam roles create SceneMachineUser --project=$PROJECT --file=./user-role.yaml
+else
+  echo "SceneMachineUser role exists. Syncing it to user-role.yaml..."
+  gcloud iam roles update SceneMachineUser --project=$PROJECT --file=./user-role.yaml --quiet || true
+fi
+
+# --- Seed Firestore config (front-door topology) --------------------------------
+phase "Adding default Scene Machine configurations to Firestore..."
+ADC_TOKEN=$(gcloud auth application-default print-access-token)
+CONFIG_SEED_STATUS=$(curl -s -X PATCH \
+"https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config/global" \
+  -H "Authorization: Bearer ${ADC_TOKEN}" \
+  -H "x-goog-user-project: ${PROJECT}" \
+  -H "Content-Type: application/json" \
+  -o /dev/null -w '%{http_code}' \
+  -d @<(envsubst < ./firestore_config_frontdoor.template.json))
+if [ "$CONFIG_SEED_STATUS" != "200" ]; then
+  echo "ERROR: seeding the UI config (config/global) failed (HTTP ${CONFIG_SEED_STATUS:-no response})." >&2
+  echo "       The app's backend wiring was not written; aborting." >&2
+  exit 1
+fi
+
+MODELS_SEED_STATUS=$(python3 scripts/seed_config_models.py convert < ui/definitions/models.json | curl -s -X PATCH \
+"https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config/models" \
+  -H "Authorization: Bearer ${ADC_TOKEN}" \
+  -H "x-goog-user-project: ${PROJECT}" \
+  -H "Content-Type: application/json" \
+  -o /dev/null -w '%{http_code}' \
+  -d @-)
+if [ "$MODELS_SEED_STATUS" != "200" ]; then
+  echo "ERROR: seeding the model catalog (config/models) failed (HTTP ${MODELS_SEED_STATUS:-no response})." >&2
+  echo "       The runtime model catalog was not written; aborting." >&2
+  exit 1
+fi
+
+if ! ANNOUNCEMENT_SEED_STATUS=$(GOOGLE_CLOUD_PROJECT="$PROJECT" \
+  GOOGLE_OAUTH_ACCESS_TOKEN="${ADC_TOKEN}" \
+  python3 scripts/seed_announcement.py seed \
+  "https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config?documentId=announcement" \
+  "$ANNOUNCEMENT_MARKDOWN_FILE" "$ANNOUNCEMENT_ENABLED"); then
+  echo "ERROR: seeding homepage announcement failed." >&2
+  echo "       Existing operator content was not overwritten; aborting." >&2
+  exit 1
+fi
+
+for template in creative_templates/*.json; do
+  [ -e "$template" ] || continue
+  template_name=$(basename "$template" .json)
+
+  TEMPLATE_SEED_STATUS=$(curl -s -X PATCH \
+  "https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/creativeTemplates/${template_name}" \
+    -H "Authorization: Bearer ${ADC_TOKEN}" \
+    -H "x-goog-user-project: ${PROJECT}" \
+    -H "Content-Type: application/json" \
+    -o /dev/null -w '%{http_code}' \
+    -d @"$template")
+  if [ "$TEMPLATE_SEED_STATUS" != "200" ]; then
+    echo "ERROR: seeding creative template '${template_name}' failed (HTTP ${TEMPLATE_SEED_STATUS:-no response})." >&2
+    exit 1
+  fi
+done
+) >"$INFRA_SETUP_LOG" 2>&1 &
+INFRA_SETUP_PID=$!
+
 # --- Render UI env + config (must precede the single image build) -------------
 # Order matters: these artifacts are baked into the image (Dockerfile
 # `COPY . .`), so they must exist before `gcloud builds submit`.
@@ -817,12 +916,13 @@ if [ "$SKIP_UI_BUILD" = "1" ]; then
   echo "[skip] Building the Angular UI — skipped (--skip-ui-build); reusing ui/dist."
 else
   phase "Building the Angular UI (npm ci + ng build)..."
-  export NG_CLI_ANALYTICS=ci
-  (
-    cd ui \
-      && npm ci \
-      && npx ng build --configuration production
-  )
+  if ! wait "$UI_BUILD_PID"; then
+    cat "$UI_BUILD_LOG" >&2
+    rm -f "$UI_BUILD_LOG"
+    exit 1
+  fi
+  cat "$UI_BUILD_LOG"
+  rm -f "$UI_BUILD_LOG"
 fi
 
 # --- Version stamp + Artifact Registry + ONE image build -----------------------
@@ -833,9 +933,11 @@ COMMIT_DATE=$(git log -1 --format=%cI 2>/dev/null || echo "unknown")
 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
 echo "${GIT_BRANCH}/${COMMIT_DATE}" > deployed_version.txt
 sync
+BUILD_MACHINE_ARGS=()
 if ! gcloud artifacts repositories describe "${ARTIFACT_REPO}" --project=$PROJECT --location="$REGION" &> /dev/null; then
   echo "Creating artifact repository: $ARTIFACT_REPO"
   gcloud artifacts repositories create "${ARTIFACT_REPO}" --repository-format=docker --project=$PROJECT --location="$REGION"
+  BUILD_MACHINE_ARGS=(--machine-type=e2-highcpu-8)
 fi
 # NOTE: the repo's .gcloudignore excludes ui/* but re-includes ui/dist/ and
 # ui/remix-engine-status-viewer/ — both are LOAD-BEARING for this build: the
@@ -858,15 +960,27 @@ echo "  (this step is quiet — the build runs remotely; a heartbeat prints belo
 BUILD_SUBS="_IMAGE=${IMAGE}"
 if [ "$NO_BUILD_CACHE" = "1" ]; then
   BUILD_SUBS="${BUILD_SUBS},_USE_CACHE=0"
+  BUILD_MACHINE_ARGS=(--machine-type=e2-highcpu-8)
   echo "  Docker layer cache: OFF (--no-build-cache; forcing a cold rebuild)."
 else
   echo "  Docker layer cache: ON (reuses unchanged layers from the previous image)."
 fi
 run_with_heartbeat "Cloud Build" \
   gcloud builds submit . --config=cloudbuild.yaml --substitutions="$BUILD_SUBS" \
-    --project=$PROJECT --region=$REGION
+    "${BUILD_MACHINE_ARGS[@]}" --project=$PROJECT --region=$REGION
+
+phase "Completing overlapped infrastructure & Firestore setup..."
+if ! wait "$INFRA_SETUP_PID"; then
+  cat "$INFRA_SETUP_LOG" >&2
+  rm -f "$INFRA_SETUP_LOG"
+  exit 1
+fi
+cat "$INFRA_SETUP_LOG"
+rm -f "$INFRA_SETUP_LOG"
 
 # --- Cloud Run: worker (private, Cloud-Tasks-invoked) --------------------------
+WORKER_DEPLOY_PID=""
+WORKER_ERR_FILE=""
 if [ "$APP_ONLY" = "1" ]; then
   phase "Reusing existing 'worker' Cloud Run service (--app-only)..."
   echo "[skip] Deploying 'worker' — skipped (--app-only); reusing the live service."
@@ -879,27 +993,23 @@ if [ "$APP_ONLY" = "1" ]; then
   fi
   echo "  Reusing worker: ${WORKER_URL}"
 else
-  phase "Deploying 'worker' Cloud Run service (private)..."
-  # GUNICORN_TIMEOUT just above the worker's 1800s Cloud Run request timeout so
-  # gunicorn reaps a thread only AFTER Cloud Run has already returned, never
-  # killing a legitimate long render mid-flight. (D7)
-  gcloud run deploy worker --image "$IMAGE" --region $REGION --project $PROJECT \
-    --cpu=8 --memory=16G --timeout=1800 --no-allow-unauthenticated \
-    --service-account="$RUNTIME_SA" \
-    --set-env-vars=ROLE=worker,GUNICORN_TIMEOUT=1830
-  WORKER_URL=$(gcloud run services describe worker --region=$REGION --project=$PROJECT --format='value(status.url)')
-  echo "✓ Worker deployed: ${WORKER_URL}"
-
-  # The only run.invoker grant the runtime SA gets: service-scoped to the
-  # worker, exactly what the Cloud-Tasks-minted OIDC tokens need to invoke it.
-  # (There is no project-wide run.invoker, so the app cannot invoke other
-  # Cloud Run services.)
-  echo "Granting service-scoped run.invoker on 'worker' to ${RUNTIME_SA}..."
-  add_run_invoker_binding worker "$REGION" "$PROJECT" "serviceAccount:${RUNTIME_SA}"
+  EXISTING_WORKER_URL=$(gcloud run services describe worker --region=$REGION --project=$PROJECT --format='value(status.url)' 2>/dev/null || true)
+  WORKER_URL="${EXISTING_WORKER_URL:-https://worker-${PROJECT_NUMBER}.${REGION}.run.app}"
+  WORKER_ERR_FILE=$(mktemp)
+  (
+    if ! gcloud run deploy worker --image "$IMAGE" --region "$REGION" --project "$PROJECT" \
+      --cpu=8 --memory=16G --timeout=1800 --no-allow-unauthenticated \
+      --service-account="$RUNTIME_SA" \
+      --set-env-vars=ROLE=worker,GUNICORN_TIMEOUT=1830 >/dev/null 2>"$WORKER_ERR_FILE"; then
+      exit 1
+    fi
+    add_run_invoker_binding worker "$REGION" "$PROJECT" "serviceAccount:${RUNTIME_SA}" >/dev/null 2>>"$WORKER_ERR_FILE"
+  ) &
+  WORKER_DEPLOY_PID=$!
 fi
 
 # --- Cloud Run: app (UI + same-origin /api control plane) -----------------------
-phase "Deploying 'app' Cloud Run service (AUTH_MODE=${AUTH_MODE})..."
+phase "Deploying 'worker' and 'app' Cloud Run services in parallel (AUTH_MODE=${AUTH_MODE})..."
 IAP_FLAG_AVAILABLE=true
 # IAP front door. The --iap flag (built-in IAP for Cloud Run, GA March 2026) may
 # not exist on older gcloud installs — gate it behind a CLI capability check and
@@ -939,6 +1049,16 @@ fi
 IAP_SA="service-${PROJECT_NUMBER}@gcp-sa-iap.iam.gserviceaccount.com"
 echo "Granting service-scoped run.invoker on 'app' to the IAP service agent..."
 add_run_invoker_binding app "$REGION" "$PROJECT" "serviceAccount:${IAP_SA}"
+if [ -n "$WORKER_DEPLOY_PID" ]; then
+  if ! wait "$WORKER_DEPLOY_PID"; then
+    cat "$WORKER_ERR_FILE" >&2
+    rm -f "$WORKER_ERR_FILE"
+    exit 1
+  fi
+  rm -f "$WORKER_ERR_FILE"
+  WORKER_URL=$(gcloud run services describe worker --region=$REGION --project=$PROJECT --format='value(status.url)')
+  echo "✓ Worker deployed: ${WORKER_URL}"
+fi
 APP_URL=$(gcloud run services describe app --region=$REGION --project=$PROJECT --format='value(status.url)')
 echo "✓ App deployed: ${APP_URL}"
 # Nothing was predicted: the image carries only same-origin URLs, so the app and
@@ -957,92 +1077,6 @@ export UI_CORS_ORIGINS
 envsubst < ./gcs-cors-config.template.json > ./gcs-cors-config.json
 gcloud storage buckets update gs://$GCS_BUCKET --cors-file=./gcs-cors-config.json --project=$PROJECT
 
-# --- SceneMachineUser custom role -----------------------------------------------
-phase "Ensuring SceneMachineUser custom role matches user-role.yaml..."
-if ! gcloud iam roles describe SceneMachineUser --project=$PROJECT &> /dev/null; then
-  echo "SceneMachineUser role doesn't exist. Creating it..."
-  gcloud iam roles create SceneMachineUser --project=$PROJECT --file=./user-role.yaml
-else
-  # Update (not skip) so an edited user-role.yaml — e.g. the slimmed
-  # IAP-access-only permission set — actually takes effect on a project where the
-  # role already exists, instead of being silently ignored. '|| true' tolerates
-  # the benign "no changes to apply" case on a re-deploy; the role keeps its
-  # IAP-access permission regardless, so user admission is never at risk here.
-  echo "SceneMachineUser role exists. Syncing it to user-role.yaml..."
-  gcloud iam roles update SceneMachineUser --project=$PROJECT --file=./user-role.yaml --quiet || true
-fi
-
-# --- Seed Firestore config (front-door topology) --------------------------------
-# Uses firestore_config_frontdoor.template.json: the UI Firestore config doc with
-# the backend base fixed to the same-origin '/api' and no API key. Owner-
-# credential REST writes bypass the (deliberately read-only) config rules — by
-# design.
-phase "Adding default Scene Machine configurations to Firestore..."
-# Capture the HTTP status (as the other REST calls in this script do): a non-200
-# here means the UI's same-origin '/api' wiring was NOT written, so fail loudly
-# instead of reporting a successful deploy with a broken app config.
-CONFIG_SEED_STATUS=$(curl -s -X PATCH \
-"https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config/global" \
-  -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
-  -H "x-goog-user-project: ${PROJECT}" \
-  -H "Content-Type: application/json" \
-  -o /dev/null -w '%{http_code}' \
-  -d @<(envsubst < ./firestore_config_frontdoor.template.json))
-if [ "$CONFIG_SEED_STATUS" != "200" ]; then
-  echo "ERROR: seeding the UI config (config/global) failed (HTTP ${CONFIG_SEED_STATUS:-no response})." >&2
-  echo "       The app's backend wiring was not written; aborting." >&2
-  exit 1
-fi
-
-# The model catalog: config/models is overwritten from the repo file on every
-# deploy. Operators may edit the live doc between deploys; the pre-flight
-# preview above showed what this write replaces. Same fail-loudly contract as
-# the config/global seed.
-MODELS_SEED_STATUS=$(python3 scripts/seed_config_models.py convert < ui/definitions/models.json | curl -s -X PATCH \
-"https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config/models" \
-  -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
-  -H "x-goog-user-project: ${PROJECT}" \
-  -H "Content-Type: application/json" \
-  -o /dev/null -w '%{http_code}' \
-  -d @-)
-if [ "$MODELS_SEED_STATUS" != "200" ]; then
-  echo "ERROR: seeding the model catalog (config/models) failed (HTTP ${MODELS_SEED_STATUS:-no response})." >&2
-  echo "       The runtime model catalog was not written; aborting." >&2
-  exit 1
-fi
-
-# The announcement is operator-authored after the first deploy. Firestore's
-# create operation makes the initial seed race-safe and returns 409 when an
-# operator document already exists; either result is a successful deploy.
-if ! ANNOUNCEMENT_SEED_STATUS=$(GOOGLE_CLOUD_PROJECT="$PROJECT" \
-  GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth application-default print-access-token)" \
-  python3 scripts/seed_announcement.py seed \
-  "https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/config?documentId=announcement" \
-  "$ANNOUNCEMENT_MARKDOWN_FILE" "$ANNOUNCEMENT_ENABLED"); then
-  echo "ERROR: seeding homepage announcement failed." >&2
-  echo "       Existing operator content was not overwritten; aborting." >&2
-  exit 1
-fi
-
-for template in creative_templates/*.json; do
-  # Skip cleanly if the directory is empty/absent: without 'nullglob' the glob
-  # would otherwise stay literal and run the body once on a non-existent file.
-  [ -e "$template" ] || continue
-  template_name=$(basename "$template" .json)
-
-  TEMPLATE_SEED_STATUS=$(curl -s -X PATCH \
-  "https://firestore.googleapis.com/v1/projects/${PROJECT}/databases/${FIRESTORE_DB_UI}/documents/creativeTemplates/${template_name}" \
-    -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
-    -H "x-goog-user-project: ${PROJECT}" \
-    -H "Content-Type: application/json" \
-    -o /dev/null -w '%{http_code}' \
-    -d @"$template")
-  if [ "$TEMPLATE_SEED_STATUS" != "200" ]; then
-    echo "ERROR: seeding creative template '${template_name}' failed (HTTP ${TEMPLATE_SEED_STATUS:-no response})." >&2
-    exit 1
-  fi
-done
-
 # --- Automated provisioning complete: timing checkpoint ----------------------------
 close_phase
 PROVISIONING_END=$(date +%s)