ci(deploy): deploy ingest to render with 1Password-sourced secrets

.github/deploy.env.tpl

@@ -0,0 +1,63 @@
+
+GQL_PORT=op://webops-prod/kong/GQL_PORT
+GQL_ENABLE_CACHE=op://webops-prod/kong/GQL_ENABLE_CACHE
+MQ_CONCURRENCY_MAX_PER_PROCESSOR=op://webops-prod/kong/MQ_CONCURRENCY_MAX_PER_PROCESSOR
+MQ_CONCURRENCY_THRESHOLD=op://webops-prod/kong/MQ_CONCURRENCY_THRESHOLD
+MQ_CONCURRENCY_MAX_PER_PROCESSOR_100=op://webops-prod/kong/MQ_CONCURRENCY_MAX_PER_PROCESSOR_100
+MQ_CONCURRENCY_THRESHOLD_100=op://webops-prod/kong/MQ_CONCURRENCY_THRESHOLD_100
+MQ_CONCURRENCY_MAX_PER_PROCESSOR_137=op://webops-prod/kong/MQ_CONCURRENCY_MAX_PER_PROCESSOR_137
+MQ_CONCURRENCY_THRESHOLD_137=op://webops-prod/kong/MQ_CONCURRENCY_THRESHOLD_137
+
+DEFAULT_START_DAYS_AGO=op://webops-prod/kong/DEFAULT_START_DAYS_AGO
+
+POSTGRES_HOST=op://webops-prod/kong/POSTGRES_HOST
+POSTGRES_DATABASE=op://webops-prod/kong/POSTGRES_DATABASE
+POSTGRES_USER=op://webops-prod/kong/POSTGRES_USER
+POSTGRES_PASSWORD=op://webops-prod/kong/POSTGRES_PASSWORD
+POSTGRES_SSL=op://webops-prod/kong/POSTGRES_SSL
+POSTGRES_SSL_REJECT_UNAUTHORIZED=op://webops-prod/kong/POSTGRES_SSL_REJECT_UNAUTHORIZED
+POSTGRES_PORT=op://webops-prod/kong/POSTGRES_PORT
+
+HTTP_ARCHIVE_1=op://webops-prod/kong/HTTP_ARCHIVE_1
+HTTP_ARCHIVE_10=op://webops-prod/kong/HTTP_ARCHIVE_10
+HTTP_ARCHIVE_100=op://webops-prod/kong/HTTP_ARCHIVE_100
+HTTP_ARCHIVE_137=op://webops-prod/kong/HTTP_ARCHIVE_137
+HTTP_ARCHIVE_250=op://webops-prod/kong/HTTP_ARCHIVE_250
+HTTP_ARCHIVE_8453=op://webops-prod/kong/HTTP_ARCHIVE_8453
+HTTP_ARCHIVE_34443=op://webops-prod/kong/HTTP_ARCHIVE_34443
+HTTP_ARCHIVE_42161=op://webops-prod/kong/HTTP_ARCHIVE_42161
+HTTP_ARCHIVE_80094=op://webops-prod/kong/HTTP_ARCHIVE_80094
+HTTP_ARCHIVE_747474=op://webops-prod/kong/HTTP_ARCHIVE_747474
+
+HTTP_FULLNODE_1=op://webops-prod/kong/HTTP_FULLNODE_1
+HTTP_FULLNODE_10=op://webops-prod/kong/HTTP_FULLNODE_10
+HTTP_FULLNODE_100=op://webops-prod/kong/HTTP_FULLNODE_100
+HTTP_FULLNODE_137=op://webops-prod/kong/HTTP_FULLNODE_137
+HTTP_FULLNODE_250=op://webops-prod/kong/HTTP_FULLNODE_250
+HTTP_FULLNODE_8453=op://webops-prod/kong/HTTP_FULLNODE_8453
+HTTP_FULLNODE_34443=op://webops-prod/kong/HTTP_FULLNODE_34443
+HTTP_FULLNODE_42161=op://webops-prod/kong/HTTP_FULLNODE_42161
+HTTP_FULLNODE_80094=op://webops-prod/kong/HTTP_FULLNODE_80094
+HTTP_FULLNODE_747474=op://webops-prod/kong/HTTP_FULLNODE_747474
+
+YDAEMON_API=op://webops-prod/kong/YDAEMON_API
+
+YPRICE_ENABLED=op://webops-prod/kong/YPRICE_ENABLED
+YPRICE_API=op://webops-prod/kong/YPRICE_API
+YPRICE_API_X_SIGNER=op://webops-prod/kong/YPRICE_API_X_SIGNER
+YPRICE_API_X_SIGNATURE=op://webops-prod/kong/YPRICE_API_X_SIGNATURE
+
+GITHUB_PERSONAL_ACCESS_TOKEN=op://webops-prod/kong/GITHUB_PERSONAL_ACCESS_TOKEN
+APE_TAX_VAULTS=op://webops-prod/kong/APE_TAX_VAULTS
+WAVEYDB_HOST=op://webops-prod/kong/WAVEYDB_HOST
+WAVEYDB_NAME=op://webops-prod/kong/WAVEYDB_NAME
+WAVEYDB_USER=op://webops-prod/kong/WAVEYDB_USER
+WAVEYDB_PASSWORD=op://webops-prod/kong/WAVEYDB_PASSWORD
+
+WEBHOOK_SECRET_S_0E70A395=op://webops-prod/kong/WEBHOOK_SECRET_S_0E70A395
+WEBHOOK_SECRET_S_C2795BC0=op://webops-prod/kong/WEBHOOK_SECRET_S_C2795BC0
+WEBHOOK_SECRET_S_YVUSD_APR=op://webops-prod/kong/WEBHOOK_SECRET_S_YVUSD_APR
+WEBHOOK_SECRET_S_KATANA_APR=op://webops-prod/kong/WEBHOOK_SECRET_S_KATANA_APR
+SENTRY_DSN=op://webops-prod/kong/SENTRY_DSN
+MQ_INVENTORY=op://webops-prod/kong/MQ_INVENTORY
+PRICE_SERVICE_API_KEY=op://webops-prod/kong/PRICE_SERVICE_API_KEY

.github/workflows/deploy.yml

@@ -0,0 +1,103 @@
+name: deploy
+
+# Deploy the ingest worker to Render with 1Password as the source of truth for
+# secrets. Secrets live in the 1Password `kong` item (vault webops-prod) and are
+# declared in .github/deploy.env.tpl. This workflow resolves them with
+# 1Password/load-secrets-action, writes them as the `.env` secret file on the
+# Render env group `kong` (fromGroup: kong in render.yaml; the worker reads it via
+# dotenv), then triggers a deploy via the Render API.
+#
+# IMPORTANT: turn OFF Render's own Auto-Deploy for the ingest-v-2 service so this
+# workflow is the only thing that deploys (otherwise the git push and this API
+# call both fire a deploy).
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-render
+  cancel-in-progress: false
+
+env:
+  ENV_TEMPLATE: .github/deploy.env.tpl
+  RENDER_ENV_GROUP_ID: evg-d3dkc7jipnbc73ce23lg
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Load secrets from 1Password
+        uses: 1password/load-secrets-action@v4
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          OP_ENV_FILE: ${{ env.ENV_TEMPLATE }}
+
+      - name: Sync secrets to Render env group `.env` secret file
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_ENV_GROUP_ID: ${{ env.RENDER_ENV_GROUP_ID }}
+        run: |
+          set -euo pipefail
+
+          # Build the .env body from the same template (so the two can never drift),
+          # pulling each resolved value from the env load-secrets-action exported.
+          # One PUT replaces the whole secret file -> keys removed from the template
+          # are pruned, keeping the file a true mirror of this template.
+          # ponytail: KEY=value assumes single-line values (RPC URLs, creds, tokens
+          # qualify); a value with a literal newline would corrupt the file.
+          content="$(grep -oE '^[A-Z][A-Z0-9_]*' "$ENV_TEMPLATE" | while read -r key; do
+            value="$(printenv "$key" || true)"
+            [ -n "$value" ] && printf '%s=%s\n' "$key" "$value"
+          done)"
+
+          curl -fsS -X PUT \
+            "https://api.render.com/v1/env-groups/${RENDER_ENV_GROUP_ID}/secret-files/.env" \
+            -H "Authorization: Bearer ${RENDER_API_KEY}" \
+            -H "Content-Type: application/json" \
+            -d "$(jq -n --arg c "$content" '{content: $c}')" \
+            -o /dev/null
+          echo "synced .env secret file ($(printf '%s\n' "$content" | grep -cE '^[A-Z][A-Z0-9_]*=') vars)"
+
+      - name: Trigger Render deploy
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_SERVICE_ID: ${{ secrets.RENDER_SERVICE_ID }}
+        run: |
+          set -euo pipefail
+
+          deploy_id="$(curl -fsS -X POST \
+            "https://api.render.com/v1/services/${RENDER_SERVICE_ID}/deploys" \
+            -H "Authorization: Bearer ${RENDER_API_KEY}" \
+            -H "Content-Type: application/json" \
+            -d '{"clearCache":"do_not_clear"}' | jq -r '.id')"
+
+          if [ -z "$deploy_id" ] || [ "$deploy_id" = "null" ]; then
+            echo "failed to create deploy" >&2
+            exit 1
+          fi
+          echo "deploy $deploy_id created; waiting for it to go live"
+
+          for _ in $(seq 1 80); do
+            status="$(curl -fsS \
+              "https://api.render.com/v1/services/${RENDER_SERVICE_ID}/deploys/${deploy_id}" \
+              -H "Authorization: Bearer ${RENDER_API_KEY}" | jq -r '.status')"
+            echo "status=$status"
+            case "$status" in
+              live) echo "deploy succeeded"; exit 0 ;;
+              build_failed|update_failed|pre_deploy_failed|canceled|deactivated)
+                echo "deploy failed: $status" >&2; exit 1 ;;
+            esac
+            sleep 15
+          done
+
+          echo "timed out waiting for deploy $deploy_id" >&2
+          exit 1

.gitignore

@@ -9,3 +9,4 @@ gql-cache*
 __pycache__
 output/
 .mcp.json
+.secrets