diff --git a/cmd/build_db/main.go b/cmd/build_db/main.go index 3909a90..192d7ed 100644 --- a/cmd/build_db/main.go +++ b/cmd/build_db/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "flag" "fmt" "github.com/thd-spatial-ai/ignis/internal/config" importer "github.com/thd-spatial-ai/ignis/internal/db" @@ -14,6 +15,9 @@ import ( ) func main() { + ifEmpty := flag.Bool("if-empty", false, "skip the rebuild when the tabula schema already holds tables") + flag.Parse() + startTime := time.Now() fmt.Println("============================================================") fmt.Println("=== ignis Database Rebuild Tool ===") @@ -48,6 +52,17 @@ func main() { fmt.Println("Database connection successful") fmt.Println("") + if *ifEmpty { + seeded, err := importer.IsSeeded(context.Background(), pool, cfg.DB.Schemas.Tabula) + if err != nil { + log.Fatalf("Checking whether the database is seeded: %v", err) + } + if seeded { + fmt.Printf("Schema %s already holds tables, leaving it unchanged (-if-empty)\n", cfg.DB.Schemas.Tabula) + return + } + } + // Run table constructor fmt.Println("Starting table construction...") fmt.Println("WARNING: This will DROP and recreate all country tables!") diff --git a/docs/getting-started.md b/docs/getting-started.md index a66beaf..4664c03 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -105,9 +105,9 @@ To build from this checkout instead, so local code changes are picked up, drop t | `HOST_PORT` | Host port the app is published on | `8088` | | `APP_PORT` | The app's internal listen port | `8080` | -### 2. Seed and verify +### 2. Verify -Follow [Seeding the database](#seeding-the-database), then [Verifying](#verifying). The base URL is `http://localhost:8088`. +`up` seeds an empty database before it starts `ignis`, so there is no separate seed step (see [Seeding the database](#seeding-the-database)). Follow [Verifying](#verifying). The base URL is `http://localhost:8088`. --- @@ -176,6 +176,22 @@ Then [Seeding the database](#seeding-the-database) and [Verifying](#verifying). ## Seeding the database +The TABULA workbook is baked into the `build-db` image, so there is nothing to download. A seed runs in one transaction: if it fails, nothing is committed and any previous tables stay. + +`` below is `docker compose` for `docker-compose.yml`, or `docker compose -f ` for the others. + +### environment/http + +`up` runs `build-db -if-empty` and starts `ignis` only once it exits successfully. An empty database is seeded, a populated one is left unchanged, and a failed seed fails `up` with `service "build-db" didn't complete successfully`. + +To rebuild a populated database from the workbook, which drops and recreates all country tables: + +```bash + run --rm build-db -if-empty=false +``` + +### environment/https + !!! warning "Required before first use, and destructive" A fresh `db` volume is empty. Seeding drops and recreates all country tables, so it is gated behind the `seed` profile and never runs automatically. Until it has run once, every endpoint that reads the schema will fail. @@ -183,9 +199,7 @@ Then [Seeding the database](#seeding-the-database) and [Verifying](#verifying). --profile seed run --rm build-db ``` -`` is `docker compose` for `docker-compose.yml`, or `docker compose -f ` for the others. On a path that pulls images, `--profile seed` is also needed on the `pull`, since profile-gated services are otherwise skipped. - -The TABULA workbook is baked into the `build-db` image, so there is nothing to download. +On a path that pulls images, `--profile seed` is also needed on the `pull`, since profile-gated services are otherwise skipped. ## Verifying @@ -208,7 +222,7 @@ Both return `200`. Add `-k` on the https quickstart path, where the certificate down -v ``` -The `-v` removes the database volume, so the next start needs seeding again. Omit it to keep the seeded data. +The `-v` removes the database volume, so the next start seeds again (http) or needs seeding again (https). Omit it to keep the seeded data. ## Pinning a version @@ -299,7 +313,7 @@ Copy across the whole directory: the compose file, `.env`, the `env/` directory, | Compose service | Published image | Role | |---|---|---| | `ignis` | `ghcr.io/thd-spatial-ai/ignis` | HTTP API server | - | `build-db` | `ghcr.io/thd-spatial-ai/ignis-build-db` | one-off TABULA seeder, `seed` profile only | + | `build-db` | `ghcr.io/thd-spatial-ai/ignis-build-db` | TABULA seeder: on `up` in `environment/http`, `seed` profile only in `environment/https` | | `db` | `postgres:17-alpine` (not built here) | PostgreSQL database | Service names and image names are independent: the seeder's image keeps the `ignis-` prefix it publishes under. `IGNIS_IMAGE_TAG` pins both `ghcr.io` images to one release. The service is named `ignis` because a service name is registered as a DNS alias on every network it joins, `tentacron-net` included, so it has to be unique across the workspace; `db` and `build-db` never join it and so stay short. @@ -328,16 +342,25 @@ Set `IGNIS_IMAGE_TAG` to pin a release. This is strongly advised for any deploym ### 4. Pull, start, seed +For `environment/http`, `up` seeds an empty database itself: + +```bash +docker compose -f docker-compose.prod.yml pull +docker compose -f docker-compose.prod.yml up -d +``` + +For `environment/https`: + ```bash docker compose -f docker-compose.prod.yml --profile seed pull docker compose -f docker-compose.prod.yml up -d docker compose -f docker-compose.prod.yml --profile seed run --rm build-db ``` -!!! warning "Pull first, and include the seed profile" - With `IGNIS_IMAGE_TAG` unset, a host that already has `latest` cached keeps running the old build after a release, with nothing on that host revealing it. The `--profile seed` on the pull is what fetches `build-db`, since profile-gated services are otherwise skipped. +!!! warning "Pull first" + With `IGNIS_IMAGE_TAG` unset, a host that already has `latest` cached keeps running the old build after a release, with nothing on that host revealing it. On `environment/https`, the `--profile seed` on the pull is what fetches `build-db`, since profile-gated services are otherwise skipped. -Seed only on first deployment. Running it against a populated database drops every country table. +On `environment/https`, seed only on first deployment. Running it against a populated database drops every country table. ### 5. Verify diff --git a/environment/http/docker-compose.prod.yml b/environment/http/docker-compose.prod.yml index 8847f42..1702cc7 100644 --- a/environment/http/docker-compose.prod.yml +++ b/environment/http/docker-compose.prod.yml @@ -41,12 +41,11 @@ # by IGNIS_IMAGE_TAG together, since the app and the seed job must come from # the same release. # -# Seeding is a one-off admin operation, gated behind the `seed` profile so it -# never runs on `up`. It DROPS and recreates all country tables from the -# TABULA workbook baked into the ignis-build-db image, so it needs no source -# checkout, but run it deliberately and never against a populated database you -# care about: -# docker compose -f docker-compose.prod.yml --profile seed run --rm build-db +# `up` seeds an empty database from the TABULA workbook baked into the +# ignis-build-db image and leaves a populated one alone. ignis starts only +# after build-db exits successfully, so a failed seed fails `up`. Rebuilding a +# populated database DROPS and recreates all country tables: +# docker compose -f docker-compose.prod.yml run --rm build-db -if-empty=false name: ignis-http services: @@ -75,6 +74,8 @@ services: depends_on: db: condition: service_healthy + build-db: + condition: service_completed_successfully # See the header: loopback by default, HOST_BIND=0.0.0.0 to publish wider. # 8088 rather than 8080, which the EnerPlanET platform's Keycloak holds on # every interface wherever that stack is also running. @@ -89,7 +90,7 @@ services: build-db: image: ghcr.io/thd-spatial-ai/ignis-build-db:${IGNIS_IMAGE_TAG:-latest} - profiles: [seed] + command: ["-if-empty"] # needs an image from v0.7.0 or later env_file: [env/app.env] depends_on: db: diff --git a/environment/http/docker-compose.yml b/environment/http/docker-compose.yml index 796e372..5ffa135 100644 --- a/environment/http/docker-compose.yml +++ b/environment/http/docker-compose.yml @@ -4,7 +4,6 @@ # # --- # Usage: docker compose up -d -# docker compose --profile seed run --rm build-db # --- # # No reverse proxy, no TLS, no API key, and nothing to install on the host but @@ -68,6 +67,8 @@ services: depends_on: db: condition: service_healthy + build-db: + condition: service_completed_successfully # Bound to the loopback interface by default: nothing outside this host can # connect. Set HOST_BIND=0.0.0.0 in .env to publish on every interface, and # only where something else (a firewall, an upstream proxy) controls who @@ -82,14 +83,12 @@ services: retries: 3 start_period: 10s - # One-off DB seed/rebuild job, NOT part of the normal `up` path (see - # `profiles` below). build_db DROPS and recreates all country tables from - # the Excel workbook — invoke deliberately, never automatically on startup: - # docker compose --profile seed run --rm build-db - # No `container_name`: this is only ever started via `run`, which assigns - # its own name, and a fixed one would collide rather than help. + # Seeds an empty database on `up` and exits; a populated one is left alone + # (-if-empty) and ignis waits for it to succeed. Rebuilding a populated + # database DROPS and recreates all country tables: + # docker compose run --rm build-db -if-empty=false build-db: - profiles: ["seed"] + command: ["-if-empty"] build: context: ./../.. dockerfile: environment/ignis-db.dockerfile diff --git a/environment/ignis-db.dockerfile b/environment/ignis-db.dockerfile index 3be73dd..a5350c2 100644 --- a/environment/ignis-db.dockerfile +++ b/environment/ignis-db.dockerfile @@ -1,10 +1,9 @@ # syntax=docker/dockerfile:1 -# One-off DB rebuild job — NOT a long-running service. build_db DROPS and -# recreates all country tables from the Excel workbook (see cmd/build_db's -# own startup banner). Never wire this into a container's default startup; -# it must only run when explicitly invoked (see the compose files' -# `ignis-build-db` service, gated behind the `seed` profile). +# One-off DB rebuild job, not a long-running service. With no arguments +# build_db DROPS and recreates all country tables from the Excel workbook. +# environment/http runs it on `up` with -if-empty, which leaves a populated +# database alone; the other environments gate it behind the `seed` profile. # # Despite the filename, this builds the `ignis-build-db` seeder image, not # the database. The database is the `ignis-db` service, plain postgres with diff --git a/internal/db/table_constructor.go b/internal/db/table_constructor.go index 56789f9..b961cb6 100644 --- a/internal/db/table_constructor.go +++ b/internal/db/table_constructor.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/xuri/excelize/v2" ) @@ -54,6 +55,7 @@ type HeaderInfo struct { type TableConstructor struct { conn *pgxpool.Pool + tx pgx.Tx // open only for the duration of Run cfg *config.Config xlsxFile *excelize.File headers map[string]*HeaderInfo @@ -70,6 +72,22 @@ func NewTableConstructor(conn *pgxpool.Pool, cfg *config.Config) *TableConstruct } } +// IsSeeded reports whether schema holds any table. Run writes in a single +// transaction, so a schema with tables is one a Run completed. +func IsSeeded(ctx context.Context, pool *pgxpool.Pool, schema string) (bool, error) { + var seeded bool + err := pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = $1)`, + schema, + ).Scan(&seeded) + if err != nil { + return false, fmt.Errorf("checking for tables in schema %s: %w", schema, err) + } + return seeded, nil +} + +// Run drops and recreates every country table from the workbook in one +// transaction: on any error nothing is committed and the previous tables stay. func (tc *TableConstructor) Run() error { defer tc.close() @@ -84,9 +102,26 @@ func (tc *TableConstructor) Run() error { tc.extractHeaders(rows) tc.extractCountryCodes(rows) - tc.createTables() - tc.insertData(rows) - tc.updateDropdownColumns() + + ctx := context.Background() + tc.tx, err = tc.conn.Begin(ctx) + if err != nil { + return fmt.Errorf("starting seed transaction: %w", err) + } + defer tc.tx.Rollback(ctx) // no-op once committed + + if err := tc.createTables(); err != nil { + return err + } + if err := tc.insertData(rows); err != nil { + return err + } + if err := tc.updateDropdownColumns(); err != nil { + return err + } + if err := tc.tx.Commit(ctx); err != nil { + return fmt.Errorf("committing seed transaction: %w", err) + } utils.Info.Println("Table construction completed successfully") return nil @@ -205,8 +240,11 @@ func (tc *TableConstructor) extractCountryCodes(rows [][]string) { sort.Strings(tc.countryCodes) } -func (tc *TableConstructor) createTables() { - tc.conn.Exec(context.Background(), fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", tc.cfg.DB.Schemas.Tabula)) +func (tc *TableConstructor) createTables() error { + schema := tc.cfg.DB.Schemas.Tabula + if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", schema)); err != nil { + return fmt.Errorf("creating schema %s: %w", schema, err) + } var cols []string for header, info := range tc.headers { @@ -219,12 +257,17 @@ func (tc *TableConstructor) createTables() { for _, code := range tc.countryCodes { table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(code)) - tc.conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table)) - tc.conn.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, %s)", table, strings.Join(cols, ", "))) + if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table)); err != nil { + return fmt.Errorf("dropping table %s: %w", table, err) + } + if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id SERIAL PRIMARY KEY, %s)", table, strings.Join(cols, ", "))); err != nil { + return fmt.Errorf("creating table %s: %w", table, err) + } } + return nil } -func (tc *TableConstructor) insertData(rows [][]string) { +func (tc *TableConstructor) insertData(rows [][]string) error { headerMap := make(map[string]int) for i, header := range rows[0] { headerMap[header] = i @@ -243,11 +286,14 @@ func (tc *TableConstructor) insertData(rows [][]string) { continue } - tc.insertRow(code, headerMap, rows[i]) + if err := tc.insertRow(code, headerMap, rows[i]); err != nil { + return fmt.Errorf("workbook row %d: %w", i+1, err) + } } + return nil } -func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]int, dataRow []string) { +func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]int, dataRow []string) error { table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(countryCode)) var cols, placeholders []string @@ -290,13 +336,13 @@ func (tc *TableConstructor) insertRow(countryCode string, headerMap map[string]i } query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(cols, ", "), strings.Join(placeholders, ", ")) - _, err := tc.conn.Exec(context.Background(), query, vals...) - if err != nil { - utils.Error.Printf("Failed to insert row into %s: %v\n", table, err) + if _, err := tc.tx.Exec(context.Background(), query, vals...); err != nil { + return fmt.Errorf("inserting into %s: %w", table, err) } + return nil } -func (tc *TableConstructor) updateDropdownColumns() { +func (tc *TableConstructor) updateDropdownColumns() error { for _, code := range tc.countryCodes { table := fmt.Sprintf("%s.%s", tc.cfg.DB.Schemas.Tabula, tc.countryHelper.CodeToCountry(code)) @@ -312,9 +358,12 @@ func (tc *TableConstructor) updateDropdownColumns() { } if len(updates) > 0 { - tc.conn.Exec(context.Background(), fmt.Sprintf("UPDATE %s SET %s", table, strings.Join(updates, ", "))) + if _, err := tc.tx.Exec(context.Background(), fmt.Sprintf("UPDATE %s SET %s", table, strings.Join(updates, ", "))); err != nil { + return fmt.Errorf("setting dropdown columns on %s: %w", table, err) + } } } + return nil } func (tc *TableConstructor) close() error { diff --git a/internal/db/table_constructor_integration_test.go b/internal/db/table_constructor_integration_test.go index 6e48c59..6f34573 100644 --- a/internal/db/table_constructor_integration_test.go +++ b/internal/db/table_constructor_integration_test.go @@ -181,3 +181,72 @@ func TestTableConstructor_Run_missingWorkbook_returnsError(t *testing.T) { t.Fatal("expected error for missing workbook") } } + +// withUninsertableRow adds a Date column to the fixture at path whose first +// DE row holds a value Postgres rejects, so the insert for that row fails. +func withUninsertableRow(t *testing.T, path string) { + t.Helper() + f, err := excelize.OpenFile(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + sheet := "Calc.Set.Building" + for cell, v := range map[string]string{"F1": "Date_Test", "F6": "Date", "F13": "not-a-date"} { + if err := f.SetCellValue(sheet, cell, v); err != nil { + t.Fatal(err) + } + } + if err := f.Save(); err != nil { + t.Fatal(err) + } +} + +func TestTableConstructor_Run_failedInsert_returnsErrorAndLeavesNoTables(t *testing.T) { + xlsxPath := writeFixtureWorkbook(t) + withUninsertableRow(t, xlsxPath) + cfg := &config.Config{ + Data: &config.DataPaths{ExcelFile: xlsxPath}, + DB: &config.DBConfig{Schemas: &config.Schemas{Tabula: "tc_failed"}}, + } + + if err := importer.NewTableConstructor(testPool, cfg).Run(); err == nil { + t.Fatal("Run() returned nil for a row Postgres rejects") + } + + seeded, err := importer.IsSeeded(context.Background(), testPool, "tc_failed") + if err != nil { + t.Fatalf("IsSeeded: %v", err) + } + if seeded { + t.Error("a failed Run left tables behind in tc_failed") + } +} + +func TestIsSeeded_reportsWhetherSchemaHasTables(t *testing.T) { + ctx := context.Background() + cfg := &config.Config{ + Data: &config.DataPaths{ExcelFile: writeFixtureWorkbook(t)}, + DB: &config.DBConfig{Schemas: &config.Schemas{Tabula: "tc_seeded"}}, + } + + seeded, err := importer.IsSeeded(ctx, testPool, "tc_seeded") + if err != nil { + t.Fatalf("IsSeeded before Run: %v", err) + } + if seeded { + t.Fatal("IsSeeded = true before any Run") + } + + if err := importer.NewTableConstructor(testPool, cfg).Run(); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + seeded, err = importer.IsSeeded(ctx, testPool, "tc_seeded") + if err != nil { + t.Fatalf("IsSeeded after Run: %v", err) + } + if !seeded { + t.Error("IsSeeded = false after a successful Run") + } +}