feat: seed the http database on up

cmd/build_db/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"github.com/thd-spatial-ai/ignis/internal/config"
 	importer "github.com/thd-spatial-ai/ignis/internal/db"
@@ -14,6 +15,9 @@ import (
 )
 
 func main() {
+	ifEmpty := flag.Bool("if-empty", false, "skip the rebuild when the tabula schema already holds tables")
+	flag.Parse()
+
 	startTime := time.Now()
 	fmt.Println("============================================================")
 	fmt.Println("=== ignis Database Rebuild Tool ===")
@@ -48,6 +52,17 @@ func main() {
 	fmt.Println("Database connection successful")
 	fmt.Println("")
 
+	if *ifEmpty {
+		seeded, err := importer.IsSeeded(context.Background(), pool, cfg.DB.Schemas.Tabula)
+		if err != nil {
+			log.Fatalf("Checking whether the database is seeded: %v", err)
+		}
+		if seeded {
+			fmt.Printf("Schema %s already holds tables, leaving it unchanged (-if-empty)\n", cfg.DB.Schemas.Tabula)
+			return
+		}
+	}
+
 	// Run table constructor
 	fmt.Println("Starting table construction...")
 	fmt.Println("WARNING: This will DROP and recreate all country tables!")

docs/getting-started.md

@@ -105,9 +105,9 @@ To build from this checkout instead, so local code changes are picked up, drop t
 | `HOST_PORT` | Host port the app is published on | `8088` |
 | `APP_PORT` | The app's internal listen port | `8080` |
 
-### 2. Seed and verify
+### 2. Verify
 
-Follow [Seeding the database](#seeding-the-database), then [Verifying](#verifying). The base URL is `http://localhost:8088`.
+`up` seeds an empty database before it starts `ignis`, so there is no separate seed step (see [Seeding the database](#seeding-the-database)). Follow [Verifying](#verifying). The base URL is `http://localhost:8088`.
 
 ---
 
@@ -176,16 +176,30 @@ Then [Seeding the database](#seeding-the-database) and [Verifying](#verifying).
 
 ## Seeding the database
 
+The TABULA workbook is baked into the `build-db` image, so there is nothing to download. A seed runs in one transaction: if it fails, nothing is committed and any previous tables stay.
+
+`<compose prefix>` below is `docker compose` for `docker-compose.yml`, or `docker compose -f <file>` for the others.
+
+### environment/http
+
+`up` runs `build-db -if-empty` and starts `ignis` only once it exits successfully. An empty database is seeded, a populated one is left unchanged, and a failed seed fails `up` with `service "build-db" didn't complete successfully`.
+
+To rebuild a populated database from the workbook, which drops and recreates all country tables:
+
+```bash
+<compose prefix> run --rm build-db -if-empty=false
+```
+
+### environment/https
+
 !!! warning "Required before first use, and destructive"
     A fresh `db` volume is empty. Seeding drops and recreates all country tables, so it is gated behind the `seed` profile and never runs automatically. Until it has run once, every endpoint that reads the schema will fail.
 
 ```bash
 <compose prefix> --profile seed run --rm build-db
 ```
 
-`<compose prefix>` is `docker compose` for `docker-compose.yml`, or `docker compose -f <file>` for the others. On a path that pulls images, `--profile seed` is also needed on the `pull`, since profile-gated services are otherwise skipped.
-
-The TABULA workbook is baked into the `build-db` image, so there is nothing to download.
+On a path that pulls images, `--profile seed` is also needed on the `pull`, since profile-gated services are otherwise skipped.
 
 ## Verifying
 
@@ -208,7 +222,7 @@ Both return `200`. Add `-k` on the https quickstart path, where the certificate
 <compose prefix> down -v
 ```
 
-The `-v` removes the database volume, so the next start needs seeding again. Omit it to keep the seeded data.
+The `-v` removes the database volume, so the next start seeds again (http) or needs seeding again (https). Omit it to keep the seeded data.
 
 ## Pinning a version
 
@@ -299,7 +313,7 @@ Copy across the whole directory: the compose file, `.env`, the `env/` directory,
     | Compose service | Published image | Role |
     |---|---|---|
     | `ignis` | `ghcr.io/thd-spatial-ai/ignis` | HTTP API server |
-    | `build-db` | `ghcr.io/thd-spatial-ai/ignis-build-db` | one-off TABULA seeder, `seed` profile only |
+    | `build-db` | `ghcr.io/thd-spatial-ai/ignis-build-db` | TABULA seeder: on `up` in `environment/http`, `seed` profile only in `environment/https` |
     | `db` | `postgres:17-alpine` (not built here) | PostgreSQL database |
 
     Service names and image names are independent: the seeder's image keeps the `ignis-` prefix it publishes under. `IGNIS_IMAGE_TAG` pins both `ghcr.io` images to one release. The service is named `ignis` because a service name is registered as a DNS alias on every network it joins, `tentacron-net` included, so it has to be unique across the workspace; `db` and `build-db` never join it and so stay short.
@@ -328,16 +342,25 @@ Set `IGNIS_IMAGE_TAG` to pin a release. This is strongly advised for any deploym
 
 ### 4. Pull, start, seed
 
+For `environment/http`, `up` seeds an empty database itself:
+
+```bash
+docker compose -f docker-compose.prod.yml pull
+docker compose -f docker-compose.prod.yml up -d
+```
+
+For `environment/https`:
+
 ```bash
 docker compose -f docker-compose.prod.yml --profile seed pull
 docker compose -f docker-compose.prod.yml up -d
 docker compose -f docker-compose.prod.yml --profile seed run --rm build-db
 ```
 
-!!! warning "Pull first, and include the seed profile"
-    With `IGNIS_IMAGE_TAG` unset, a host that already has `latest` cached keeps running the old build after a release, with nothing on that host revealing it. The `--profile seed` on the pull is what fetches `build-db`, since profile-gated services are otherwise skipped.
+!!! warning "Pull first"
+    With `IGNIS_IMAGE_TAG` unset, a host that already has `latest` cached keeps running the old build after a release, with nothing on that host revealing it. On `environment/https`, the `--profile seed` on the pull is what fetches `build-db`, since profile-gated services are otherwise skipped.
 
-Seed only on first deployment. Running it against a populated database drops every country table.
+On `environment/https`, seed only on first deployment. Running it against a populated database drops every country table.
 
 ### 5. Verify
 

environment/http/docker-compose.prod.yml

@@ -41,12 +41,11 @@
 # by IGNIS_IMAGE_TAG together, since the app and the seed job must come from
 # the same release.
 #
-# Seeding is a one-off admin operation, gated behind the `seed` profile so it
-# never runs on `up`. It DROPS and recreates all country tables from the
-# TABULA workbook baked into the ignis-build-db image, so it needs no source
-# checkout, but run it deliberately and never against a populated database you
-# care about:
-#   docker compose -f docker-compose.prod.yml --profile seed run --rm build-db
+# `up` seeds an empty database from the TABULA workbook baked into the
+# ignis-build-db image and leaves a populated one alone. ignis starts only
+# after build-db exits successfully, so a failed seed fails `up`. Rebuilding a
+# populated database DROPS and recreates all country tables:
+#   docker compose -f docker-compose.prod.yml run --rm build-db -if-empty=false
 name: ignis-http
 
 services:
@@ -75,6 +74,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      build-db:
+        condition: service_completed_successfully
     # See the header: loopback by default, HOST_BIND=0.0.0.0 to publish wider.
     # 8088 rather than 8080, which the EnerPlanET platform's Keycloak holds on
     # every interface wherever that stack is also running.
@@ -89,7 +90,7 @@ services:
 
   build-db:
     image: ghcr.io/thd-spatial-ai/ignis-build-db:${IGNIS_IMAGE_TAG:-latest}
-    profiles: [seed]
+    command: ["-if-empty"]   # needs an image from v0.7.0 or later
     env_file: [env/app.env]
     depends_on:
       db:

environment/http/docker-compose.yml

@@ -4,7 +4,6 @@
 #
 # ---
 # Usage:  docker compose up -d
-#         docker compose --profile seed run --rm build-db
 # ---
 #
 # No reverse proxy, no TLS, no API key, and nothing to install on the host but
@@ -68,6 +67,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      build-db:
+        condition: service_completed_successfully
     # Bound to the loopback interface by default: nothing outside this host can
     # connect. Set HOST_BIND=0.0.0.0 in .env to publish on every interface, and
     # only where something else (a firewall, an upstream proxy) controls who
@@ -82,14 +83,12 @@ services:
       retries: 3
       start_period: 10s
 
-  # One-off DB seed/rebuild job, NOT part of the normal `up` path (see
-  # `profiles` below). build_db DROPS and recreates all country tables from
-  # the Excel workbook — invoke deliberately, never automatically on startup:
-  #   docker compose --profile seed run --rm build-db
-  # No `container_name`: this is only ever started via `run`, which assigns
-  # its own name, and a fixed one would collide rather than help.
+  # Seeds an empty database on `up` and exits; a populated one is left alone
+  # (-if-empty) and ignis waits for it to succeed. Rebuilding a populated
+  # database DROPS and recreates all country tables:
+  #   docker compose run --rm build-db -if-empty=false
   build-db:
-    profiles: ["seed"]
+    command: ["-if-empty"]
     build:
       context: ./../..
       dockerfile: environment/ignis-db.dockerfile

environment/ignis-db.dockerfile

@@ -1,10 +1,9 @@
 # syntax=docker/dockerfile:1
 
-# One-off DB rebuild job — NOT a long-running service. build_db DROPS and
-# recreates all country tables from the Excel workbook (see cmd/build_db's
-# own startup banner). Never wire this into a container's default startup;
-# it must only run when explicitly invoked (see the compose files'
-# `ignis-build-db` service, gated behind the `seed` profile).
+# One-off DB rebuild job, not a long-running service. With no arguments
+# build_db DROPS and recreates all country tables from the Excel workbook.
+# environment/http runs it on `up` with -if-empty, which leaves a populated
+# database alone; the other environments gate it behind the `seed` profile.
 #
 # Despite the filename, this builds the `ignis-build-db` seeder image, not
 # the database. The database is the `ignis-db` service, plain postgres with

internal/db/table_constructor.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/xuri/excelize/v2"
 )
@@ -54,6 +55,7 @@ type HeaderInfo struct {
 
 type TableConstructor struct {
 	conn          *pgxpool.Pool
+	tx            pgx.Tx // open only for the duration of Run
 	cfg           *config.Config
 	xlsxFile      *excelize.File
 	headers       map[string]*HeaderInfo
@@ -70,6 +72,22 @@ func NewTableConstructor(conn *pgxpool.Pool, cfg *config.Config) *TableConstruct
 	}
 }
 
+// IsSeeded reports whether schema holds any table. Run writes in a single
+// transaction, so a schema with tables is one a Run completed.
+func IsSeeded(ctx context.Context, pool *pgxpool.Pool, schema string) (bool, error) {
+	var seeded bool
+	err := pool.QueryRow(ctx,
+		`SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = $1)`,
+		schema,
+	).Scan(&seeded)
+	if err != nil {
+		return false, fmt.Errorf("checking for tables in schema %s: %w", schema, err)
+	}
+	return seeded, nil
+}
+
+// Run drops and recreates every country table from the workbook in one
+// transaction: on any error nothing is committed and the previous tables stay.
 func (tc *TableConstructor) Run() error {
 	defer tc.close()
 
@@ -84,9 +102,26 @@ func (tc *TableConstructor) Run() error {
 
 	tc.extractHeaders(rows)
 	tc.extractCountryCodes(rows)
-	tc.createTables()
-	tc.insertData(rows)
-	tc.updateDropdownColumns()
+
+	ctx := context.Background()
+	tc.tx, err = tc.conn.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("starting seed transaction: %w", err)
+	}
+	defer tc.tx.Rollback(ctx) // no-op once committed
+
+	if err := tc.createTables(); err != nil {
+		return err
+	}
+	if err := tc.insertData(rows); err != nil {
+		return err
+	}
+	if err := tc.updateDropdownColumns(); err != nil {
+		return err
+	}
+	if err := tc.tx.Commit(ctx); err != nil {
+		return fmt.Errorf("committing seed transaction: %w", err)
+	}
 
 	utils.Info.Println("Table construction completed successfully")
 	return nil
@@ -205,8 +240,11 @@ func (tc *TableConstructor) extractCountryCodes(rows [][]string) {
 	sort.Strings(tc.countryCodes)
 }
 
-func (tc *TableConstructor) createTables() {
-	tc.conn.Exec(context.Background(), fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", tc.cfg.DB.Schemas.Tabula))
+func (tc *TableConstructor) createTables() error {
+	schema := tc.cfg.DB.Schemas.Tabula
+	if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", schema)); err != nil {
+		return fmt.Errorf("creating schema %s: %w", schema, err)
+	}
 
 	var cols []string
 	for header, info := range tc.headers {
@@ -219,12 +257,17 @@ func (tc *TableConstructor) createTables() {
 
 	for _, code := range tc.countryCodes {
 		table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(code))
-		tc.conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table))
-		tc.conn.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, %s)", table, strings.Join(cols, ", ")))
+		if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table)); err != nil {
+			return fmt.Errorf("dropping table %s: %w", table, err)
+		}
+		if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, %s)", table, strings.Join(cols, ", "))); err != nil {
+			return fmt.Errorf("creating table %s: %w", table, err)
+		}
 	}
+	return nil
 }
 
-func (tc *TableConstructor) insertData(rows [][]string) {
+func (tc *TableConstructor) insertData(rows [][]string) error {
 	headerMap := make(map[string]int)
 	for i, header := range rows[0] {
 		headerMap[header] = i
@@ -243,11 +286,14 @@ func (tc *TableConstructor) insertData(rows [][]string) {
 			continue
 		}
 
-		tc.insertRow(code, headerMap, rows[i])
+		if err := tc.insertRow(code, headerMap, rows[i]); err != nil {
+			return fmt.Errorf("workbook row %d: %w", i+1, err)
+		}
 	}
+	return nil
 }
 
-func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]int, dataRow []string) {
+func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]int, dataRow []string) error {
 	table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(countryCode))
 
 	var cols, placeholders []string
@@ -290,13 +336,13 @@ func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]i
 	}
 
 	query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(cols, ", "), strings.Join(placeholders, ", "))
-	_, err := tc.conn.Exec(context.Background(), query, vals...)
-	if err != nil {
-		utils.Error.Printf("Failed to insert row into %s: %v\n", table, err)
+	if _, err := tc.tx.Exec(context.Background(), query, vals...); err != nil {
+		return fmt.Errorf("inserting into %s: %w", table, err)
 	}
+	return nil
 }
 
-func (tc *TableConstructor) updateDropdownColumns() {
+func (tc *TableConstructor) updateDropdownColumns() error {
 	for _, code := range tc.countryCodes {
 		table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(code))
 
@@ -312,9 +358,12 @@ func (tc *TableConstructor) updateDropdownColumns() {
 		}
 
 		if len(updates) > 0 {
-			tc.conn.Exec(context.Background(), fmt.Sprintf("UPDATE %s SET %s", table, strings.Join(updates, ", ")))
+			if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("UPDATE %s SET %s", table, strings.Join(updates, ", "))); err != nil {
+				return fmt.Errorf("setting dropdown columns on %s: %w", table, err)
+			}
 		}
 	}
+	return nil
 }
 
 func (tc *TableConstructor) close() error {