From 2ab210366296f39e1408c3708706d3018dee1ae1 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 11:59:28 +0200 Subject: [PATCH 01/10] change migration endpoint to CLI --- README.md | 15 + package-lock.json | 339 ++++------- package.json | 5 +- src/app.module.ts | 19 +- src/cli.ts | 208 +++++++ src/cli/org-output.ts | 14 + src/cli/org-runner.ts | 81 +++ src/credentials/credentials.service.ts | 7 + src/migration/README.md | 175 ++++++ .../config-migration.service.spec.ts | 18 - .../latest-config-formats.migration.ts | 62 ++ .../migrate-add-entity-attributes.ts | 535 ------------------ ...rvice.ts => ndb-core-config-migrations.ts} | 114 ++-- src/migration/console-logger.ts | 25 + src/migration/migration-definition.ts | 76 +++ src/migration/migration-output.ts | 79 +++ src/migration/migration.controller.spec.ts | 60 -- src/migration/migration.controller.ts | 27 - src/migration/migrations.ts | 11 + .../migration-idempotency.harness.spec.ts | 81 +++ .../testing/migration-idempotency.harness.ts | 180 ++++++ src/migration/tracked-migration-context.ts | 63 +++ 22 files changed, 1236 insertions(+), 958 deletions(-) create mode 100644 src/cli.ts create mode 100644 src/cli/org-output.ts create mode 100644 src/cli/org-runner.ts create mode 100644 src/migration/README.md delete mode 100644 src/migration/config-migration/config-migration.service.spec.ts create mode 100644 src/migration/config-migration/latest-config-formats.migration.ts delete mode 100644 src/migration/config-migration/migrate-add-entity-attributes.ts rename src/migration/config-migration/{config-migration.service.ts => ndb-core-config-migrations.ts} (62%) create mode 100644 src/migration/console-logger.ts create mode 100644 src/migration/migration-definition.ts create mode 100644 src/migration/migration-output.ts delete mode 100644 src/migration/migration.controller.spec.ts delete mode 100644 src/migration/migration.controller.ts create mode 100644 src/migration/migrations.ts create mode 100644 src/migration/testing/migration-idempotency.harness.spec.ts create mode 100644 src/migration/testing/migration-idempotency.harness.ts create mode 100644 src/migration/tracked-migration-context.ts diff --git a/README.md b/README.md index b4b5621..b311032 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,21 @@ Once deployed visit `localhost:3000/api` to see the available endpoints and thei The endpoints which work across multiple instances iterate over all organisations listed in `credentials.json`. Further adjustments might have to be done in the `.env` file. +## Migrations (CLI) + +Database migrations can be run via the CLI to apply bulk transformations across all (or selected) instances. + +```bash +npm run build +npm run cli -- migrate --help +npm run cli -- migrate run latest-config-formats --dry-run +``` + +The CLI reads credentials from `src/assets/credentials.json` (same file as the HTTP service). +Org names in `credentials.json` can be set via the optional `name` field and used with `--org`. + +See [`src/migration/README.md`](src/migration/README.md) for developer documentation on adding new migrations. + ## Development ### Setup to run locally diff --git a/package-lock.json b/package-lock.json index caadfde..66dda1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@sentry/nestjs": "^10.47.0", "@sentry/profiling-node": "^10.47.0", "axios": "^1.14.0", + "commander": "^14.0.3", "lodash": "^4.17.23", "papaparse": "^5.5.3", "reflect-metadata": "^0.2.2", @@ -31,7 +32,7 @@ "@types/express": "^5.0.6", "@types/jest": "30.0.0", "@types/lodash": "^4.17.24", - "@types/node": "^24.12.0", + "@types/node": "^24.12.4", "@types/papaparse": "^5.5.2", "@types/supertest": "^6.0.3", "@typescript-eslint/eslint-plugin": "^8.58.0", @@ -65,16 +66,16 @@ } }, "node_modules/@angular-devkit/core": { - "version": "19.2.22", - "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-19.2.22.tgz", - "integrity": "sha512-OqN/Ded+ZKypPZN5+qUFwtnKGl7FKpxJXYO2Vts5vLBojY5goCZd9SGW1CyXeuPnisRUW+vjqBQbWYuEUh36Tw==", + "version": "19.2.24", + "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-19.2.24.tgz", + "integrity": "sha512-Kd49warf6U/EyWe5BszF/eebN3zQ3bk7tgfEljAw8q/rX95UUtriJubWvp6pgzHfzBA4jwq8f+QiNZB8eBEXPA==", "dev": true, "license": "MIT", "dependencies": { "ajv": "8.18.0", "ajv-formats": "3.0.1", "jsonc-parser": "3.3.1", - "picomatch": "4.0.2", + "picomatch": "4.0.4", "rxjs": "7.8.1", "source-map": "0.7.4" }, @@ -127,13 +128,13 @@ } }, "node_modules/@angular-devkit/schematics": { - "version": "19.2.22", - "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-19.2.22.tgz", - "integrity": "sha512-tvfu5jhem1o8qidVxvXe5KfCij65ioMLCOFA947DD+zb3yTl5pJyDm2dqzbOehuQw0fmH4XPQukRJsCUy+UwaA==", + "version": "19.2.24", + "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-19.2.24.tgz", + "integrity": "sha512-lnw+ZM1Io+cJAkReC0NPDjqObL8NtKzKIkdgEEKC8CUmkhurYhedbicN8Y8NYHgG1uLd2GozW3+/QqPRZaN+Lw==", "dev": true, "license": "MIT", "dependencies": { - "@angular-devkit/core": "19.2.22", + "@angular-devkit/core": "19.2.24", "jsonc-parser": "3.3.1", "magic-string": "0.30.17", "ora": "5.4.1", @@ -146,14 +147,14 @@ } }, "node_modules/@angular-devkit/schematics-cli": { - "version": "19.2.22", - "resolved": "https://registry.npmjs.org/@angular-devkit/schematics-cli/-/schematics-cli-19.2.22.tgz", - "integrity": "sha512-6BvkxDz4nV8B6Ha4n/pYZ503vXgLxMaEpcKsFDao1sl0iSwrIOphlIS1yWprlGdCThIM3aJref1JU13ZvEcBCA==", + "version": "19.2.24", + "resolved": "https://registry.npmjs.org/@angular-devkit/schematics-cli/-/schematics-cli-19.2.24.tgz", + "integrity": "sha512-bsStZQG67J1HBqTmWxtIcobvgrn32L4UOdL7hGyOru5VxDWPNA8pRnDYavT3hnJeBkJYPoQIw8u7Dm0ecoQprw==", "dev": true, "license": "MIT", "dependencies": { - "@angular-devkit/core": "19.2.22", - "@angular-devkit/schematics": "19.2.22", + "@angular-devkit/core": "19.2.24", + "@angular-devkit/schematics": "19.2.24", "@inquirer/prompts": "7.3.2", "ansi-colors": "4.1.3", "symbol-observable": "4.0.0", @@ -910,23 +911,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@eslint/eslintrc/node_modules/ajv": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", - "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/@eslint/eslintrc/node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -2191,15 +2175,15 @@ } }, "node_modules/@nestjs/cli": { - "version": "11.0.17", - "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.17.tgz", - "integrity": "sha512-tOMgoB9k+Zb2WdKYPhbhceROLcDR1BFQZWfkBOGMRgBTo8rnC125E65UvThEA77vp4w+zKjqiSIv0leT+wdpHg==", + "version": "11.0.21", + "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.21.tgz", + "integrity": "sha512-F8mV0Sj/zVEouzR3NxBuJy08YHTUOmC5Xdcx3qIIaJWzrm8Vw86CHkhkaPBJ5ewRMHPDCShPmhsfwhpCcjts3A==", "dev": true, "license": "MIT", "dependencies": { - "@angular-devkit/core": "19.2.22", - "@angular-devkit/schematics": "19.2.22", - "@angular-devkit/schematics-cli": "19.2.22", + "@angular-devkit/core": "19.2.24", + "@angular-devkit/schematics": "19.2.24", + "@angular-devkit/schematics-cli": "19.2.24", "@inquirer/prompts": "7.10.1", "@nestjs/schematics": "^11.0.1", "ansis": "4.2.0", @@ -2213,7 +2197,7 @@ "tsconfig-paths": "4.2.0", "tsconfig-paths-webpack-plugin": "4.2.0", "typescript": "5.9.3", - "webpack": "5.105.4", + "webpack": "5.106.0", "webpack-node-externals": "3.0.0" }, "bin": { @@ -2223,7 +2207,7 @@ "node": ">= 20.11" }, "peerDependencies": { - "@swc/cli": "^0.1.62 || ^0.3.0 || ^0.4.0 || ^0.5.0 || ^0.6.0 || ^0.7.0", + "@swc/cli": "^0.1.62 || ^0.3.0 || ^0.4.0 || ^0.5.0 || ^0.6.0 || ^0.7.0 || ^0.8.0", "@swc/core": "^1.3.62" }, "peerDependenciesMeta": { @@ -2235,6 +2219,16 @@ } } }, + "node_modules/@nestjs/cli/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, "node_modules/@nestjs/common": { "version": "11.1.17", "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.17.tgz", @@ -2267,14 +2261,14 @@ } }, "node_modules/@nestjs/config": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-4.0.3.tgz", - "integrity": "sha512-FQ3M3Ohqfl+nHAn5tp7++wUQw0f2nAk+SFKe8EpNRnIifPqvfJP6JQxPKtFLMOHbyer4X646prFG4zSRYEssQQ==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-4.0.4.tgz", + "integrity": "sha512-CJPjNitr0bAufSEnRe2N+JbnVmMmDoo6hvKCPzXgZoGwJSmp/dZPk9f/RMbuD/+Q1ZJPjwsRpq0vxna++Knwow==", "license": "MIT", "dependencies": { - "dotenv": "17.2.3", + "dotenv": "17.4.1", "dotenv-expand": "12.0.3", - "lodash": "4.17.23" + "lodash": "4.18.1" }, "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", @@ -2282,9 +2276,9 @@ } }, "node_modules/@nestjs/config/node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "version": "17.4.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.1.tgz", + "integrity": "sha512-k8DaKGP6r1G30Lx8V4+pCsLzKr8vLmV2paqEj1Y55GdAgJuIqpRp5FfajGF8KtwMxCz9qJc6wUIJnm053d/WCw==", "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -2293,12 +2287,6 @@ "url": "https://dotenvx.com" } }, - "node_modules/@nestjs/config/node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", - "license": "MIT" - }, "node_modules/@nestjs/core": { "version": "11.1.18", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-11.1.18.tgz", @@ -2340,25 +2328,15 @@ } } }, - "node_modules/@nestjs/core/node_modules/path-to-regexp": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", - "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/@nestjs/mapped-types": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@nestjs/mapped-types/-/mapped-types-2.1.0.tgz", - "integrity": "sha512-W+n+rM69XsFdwORF11UqJahn4J3xi4g/ZEOlJNL6KoW5ygWSmBB2p0S2BZ4FQeS/NDH72e6xIcu35SfJnE8bXw==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@nestjs/mapped-types/-/mapped-types-2.1.1.tgz", + "integrity": "sha512-SCCoMEJ6jdeI5h/N+KCVF1+pmg/hmEkNA5nHTS8Gvww7T/LCl4o1gFLinw2iQ60w7slFkszHcGLKGdazVI4F8A==", "license": "MIT", "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", "class-transformer": "^0.4.0 || ^0.5.0", - "class-validator": "^0.13.0 || ^0.14.0", + "class-validator": "^0.13.0 || ^0.14.0 || ^0.15.0", "reflect-metadata": "^0.1.12 || ^0.2.0" }, "peerDependenciesMeta": { @@ -2371,15 +2349,15 @@ } }, "node_modules/@nestjs/platform-express": { - "version": "11.1.17", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.17.tgz", - "integrity": "sha512-mAf4eOsSBsTOn/VbrUO1gsjW6dVh91qqXPMXun4dN8SnNjf7PTQagM9o8d6ab8ZBpNe6UdZftdrZoDetU+n4Qg==", + "version": "11.1.19", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.19.tgz", + "integrity": "sha512-Vpdv8jyCQdThfoTx+UTn+DRYr6H6X02YUqcpZ3qP6G3ZUwtVp7eS+hoQPGd4UuCnlnFG8Wqr2J9bGEzQdi1rIg==", "license": "MIT", "dependencies": { "cors": "2.8.6", "express": "5.2.1", "multer": "2.1.1", - "path-to-regexp": "8.3.0", + "path-to-regexp": "8.4.2", "tslib": "2.8.1" }, "funding": { @@ -2479,19 +2457,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@nestjs/schematics/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@nestjs/schematics/node_modules/rxjs": { "version": "7.8.1", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", @@ -2503,17 +2468,17 @@ } }, "node_modules/@nestjs/swagger": { - "version": "11.2.6", - "resolved": "https://registry.npmjs.org/@nestjs/swagger/-/swagger-11.2.6.tgz", - "integrity": "sha512-oiXOxMQqDFyv1AKAqFzSo6JPvMEs4uA36Eyz/s2aloZLxUjcLfUMELSLSNQunr61xCPTpwEOShfmO7NIufKXdA==", + "version": "11.4.2", + "resolved": "https://registry.npmjs.org/@nestjs/swagger/-/swagger-11.4.2.tgz", + "integrity": "sha512-aBihEogDMj/bLEcaqhkvyX/ZVWUw/bmnhKzR0zwUoyGJikvZyaq7rOPYl/H7Lxkkr3c90SJxyuv1AX2UT1WKlw==", "license": "MIT", "dependencies": { "@microsoft/tsdoc": "0.16.0", - "@nestjs/mapped-types": "2.1.0", + "@nestjs/mapped-types": "2.1.1", "js-yaml": "4.1.1", - "lodash": "4.17.23", - "path-to-regexp": "8.3.0", - "swagger-ui-dist": "5.31.0" + "lodash": "4.18.1", + "path-to-regexp": "8.4.2", + "swagger-ui-dist": "5.32.4" }, "peerDependencies": { "@fastify/static": "^8.0.0 || ^9.0.0", @@ -2535,12 +2500,6 @@ } } }, - "node_modules/@nestjs/swagger/node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", - "license": "MIT" - }, "node_modules/@nestjs/testing": { "version": "11.1.17", "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-11.1.17.tgz", @@ -3681,9 +3640,9 @@ } }, "node_modules/@types/node": { - "version": "24.12.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", - "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "version": "24.12.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.4.tgz", + "integrity": "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA==", "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -4661,9 +4620,9 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", "dependencies": { @@ -4696,9 +4655,9 @@ } }, "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { @@ -4806,9 +4765,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -5471,13 +5430,12 @@ } }, "node_modules/commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "license": "MIT", "engines": { - "node": ">= 6" + "node": ">=20" } }, "node_modules/comment-json": { @@ -5765,9 +5723,9 @@ } }, "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -6106,23 +6064,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint/node_modules/ajv": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", - "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/eslint/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", @@ -6417,9 +6358,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", - "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", "dev": true, "funding": [ { @@ -6554,16 +6495,16 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", @@ -6626,9 +6567,9 @@ } }, "node_modules/fork-ts-checker-webpack-plugin/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -6935,9 +6876,9 @@ "license": "ISC" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -7686,19 +7627,6 @@ "fsevents": "^2.3.3" } }, - "node_modules/jest-haste-map/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/jest-leak-detector": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.3.0.tgz", @@ -7750,19 +7678,6 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-message-util/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/jest-mock": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.3.0.tgz", @@ -8026,19 +7941,6 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-util/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/jest-validate": { "version": "30.3.0", "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.3.0.tgz", @@ -8471,9 +8373,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -8991,9 +8893,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "license": "MIT", "funding": { "type": "opencollective", @@ -9049,9 +8951,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -9306,9 +9208,9 @@ "license": "MIT" }, "node_modules/qs": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", - "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -10019,9 +9921,9 @@ } }, "node_modules/swagger-ui-dist": { - "version": "5.31.0", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.31.0.tgz", - "integrity": "sha512-zSUTIck02fSga6rc0RZP3b7J7wgHXwLea8ZjgLA3Vgnb8QeOl3Wou2/j5QkzSGeoz6HusP/coYuJl33aQxQZpg==", + "version": "5.32.4", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.4.tgz", + "integrity": "sha512-0AADFFQNJzExEN49SrD/34Nn9cxNxVLiydYl2MBwSZFPVXNkVwC/EFAjoezGGqE8oDegiDC+p47t8lKObCinMQ==", "license": "Apache-2.0", "dependencies": { "@scarf/scarf": "=1.4.0" @@ -10271,9 +10173,9 @@ } }, "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -10300,19 +10202,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -10857,9 +10746,9 @@ } }, "node_modules/webpack": { - "version": "5.105.4", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.4.tgz", - "integrity": "sha512-jTywjboN9aHxFlToqb0K0Zs9SbBoW4zRUlGzI2tYNxVYcEi/IPpn+Xi4ye5jTLvX2YeLuic/IvxNot+Q1jMoOw==", + "version": "5.106.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.106.0.tgz", + "integrity": "sha512-Pkx5joZ9RrdgO5LBkyX1L2ZAJeK/Taz3vqZ9CbcP0wS5LEMx5QkKsEwLl29QJfihZ+DKRBFldzy1O30pJ1MDpA==", "dev": true, "license": "MIT", "dependencies": { @@ -10926,9 +10815,9 @@ } }, "node_modules/webpack/node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 3e2c8e0..419616e 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", + "cli": "node dist/cli", + "migrate:latest-config-formats": "npm run build && node dist/cli migrate run latest-config-formats", "lint": "eslint --fix", "test": "jest", "test:watch": "jest --watch", @@ -30,6 +32,7 @@ "@sentry/nestjs": "^10.47.0", "@sentry/profiling-node": "^10.47.0", "axios": "^1.14.0", + "commander": "^14.0.3", "lodash": "^4.17.23", "papaparse": "^5.5.3", "reflect-metadata": "^0.2.2", @@ -43,7 +46,7 @@ "@types/express": "^5.0.6", "@types/jest": "30.0.0", "@types/lodash": "^4.17.24", - "@types/node": "^24.12.0", + "@types/node": "^24.12.4", "@types/papaparse": "^5.5.2", "@types/supertest": "^6.0.3", "@typescript-eslint/eslint-plugin": "^8.58.0", diff --git a/src/app.module.ts b/src/app.module.ts index 1bd953c..7cecc10 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,27 +1,20 @@ -import { Module } from '@nestjs/common'; -import { CouchdbAdminController } from './couchdb/couchdb-admin.controller'; import { HttpModule } from '@nestjs/axios'; +import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; +import { CouchdbAdminController } from './couchdb/couchdb-admin.controller'; import { CouchdbService } from './couchdb/couchdb.service'; -import { KeycloakService } from './keycloak/keycloak.service'; -import { MigrationController } from './migration/migration.controller'; -import { KeycloakMigrationController } from './keycloak/keycloak-migration.controller'; -import { ConfigMigrationService } from './migration/config-migration/config-migration.service'; import { SearchAndReplaceService } from './couchdb/search-and-replace/search-and-replace.service'; -import { CredentialsService } from './credentials/credentials.service'; import { StatisticsService } from './couchdb/statistics/statistics.service'; +import { CredentialsService } from './credentials/credentials.service'; +import { KeycloakMigrationController } from './keycloak/keycloak-migration.controller'; +import { KeycloakService } from './keycloak/keycloak.service'; @Module({ imports: [HttpModule, ConfigModule.forRoot({ isGlobal: true })], - controllers: [ - CouchdbAdminController, - MigrationController, - KeycloakMigrationController, - ], + controllers: [CouchdbAdminController, KeycloakMigrationController], providers: [ CouchdbService, KeycloakService, - ConfigMigrationService, SearchAndReplaceService, CredentialsService, StatisticsService, diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..add249f --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,208 @@ +import { NestFactory } from '@nestjs/core'; +import { Command } from 'commander'; +import * as readline from 'readline'; +import { AppModule } from './app.module'; +import { OrgRunner } from './cli/org-runner'; +import { printConnectivity } from './cli/org-output'; +import { CouchdbService } from './couchdb/couchdb.service'; +import { + CredentialsService, + SystemCredentials, +} from './credentials/credentials.service'; +import { ConsoleLogger } from './migration/console-logger'; +import { TrackedMigrationContext } from './migration/tracked-migration-context'; +import { failedMigrationResult } from './migration/migration-definition'; +import { + computeExitCode, + printBanner, + printOutcomes, + printSummary, +} from './migration/migration-output'; +import { migrations } from './migration/migrations'; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { version } = require('../package.json'); + +// ─── CLI program ──────────────────────────────────────────────────────────── + +const program = new Command(); +program.name('ndb-admin').description('Aam Digital admin CLI').version(version); + +const migrateCmd = program + .command('migrate') + .description('Run database migrations'); + +migrateCmd + .command('list') + .description('List all available migrations') + .action(() => { + console.log('\nAvailable migrations:\n'); + for (const m of migrations) { + console.log(` ${m.id.padEnd(36)} ${m.description}`); + } + console.log(); + }); + +program + .command('check') + .description('Check connectivity to all (or selected) orgs') + .option('--org ', 'Comma-separated org names or URLs to target') + .option('--category ', 'Filter orgs by credential category') + .action((options) => + withApp(async ({ credentialsService, couchdbService }) => { + const orgs = resolveOrgs(credentialsService, options); + const runner = new OrgRunner(couchdbService); + + const results = await runner.checkConnectivity(orgs); + printConnectivity(results); + + return results.some((r) => !r.reachable) ? 1 : 0; + }), + ); + +migrateCmd + .command('run ') + .description('Run a migration (preview first, then confirm)') + .option('--org ', 'Comma-separated org names or URLs to target') + .option('--category ', 'Filter orgs by credential category') + .option('--dry-run', 'Preview changes and exit without writing') + .option('--yes', 'Skip confirmation prompt (for automation)') + .option('--verbose', 'Show detailed per-document output') + .action((id: string, options) => + withApp(async ({ credentialsService, couchdbService }) => { + const migration = migrations.find((m) => m.id === id); + if (!migration) { + console.error(`\nUnknown migration id: "${id}"`); + console.error(`Run "migrate list" to see available migrations.\n`); + return 2; + } + + const orgs = resolveOrgs(credentialsService, options); + const runner = new OrgRunner(couchdbService); + const logger = new ConsoleLogger(!!options.verbose); + + // Connectivity preflight + const connectivity = await runner.checkConnectivity(orgs); + printConnectivity(connectivity); + + const reachable = connectivity + .filter((r) => r.reachable) + .map((r) => r.org); + const unreachableCount = connectivity.filter((r) => !r.reachable).length; + + if (reachable.length === 0) { + console.error('\nNo reachable orgs — nothing to do.\n'); + return 1; + } + + const runMigration = (dryRun: boolean) => + runner.runForEach(reachable, async (couchdb, org) => { + const ctx = new TrackedMigrationContext(couchdb, org, dryRun, logger); + try { + const result = await migration.run(ctx); + return { result, writeStats: ctx.getWriteStats() }; + } catch (e: any) { + return { + result: failedMigrationResult(e.message), + writeStats: ctx.getWriteStats(), + }; + } + }); + + // Preview (always dry-run first) + printBanner('PREVIEW', migration); + const preview = await runMigration(true); + printOutcomes(preview, false); + printSummary(preview, unreachableCount); + + if (options.dryRun) { + console.log('\n(--dry-run) No writes performed.\n'); + return computeExitCode(preview, unreachableCount); + } + + const wouldChange = preview.filter((o) => o.result.result.changed).length; + if (wouldChange === 0) { + console.log('\nNo changes needed — nothing to write.\n'); + return 0; + } + + if (!options.yes) { + const confirmed = await askConfirmation( + `\nApply ${wouldChange} change(s) to ${reachable.length} org(s)? [y/N]`, + ); + if (!confirmed) { + console.log('\nAborted.\n'); + return 2; + } + } + + // Real run + printBanner('RUNNING', migration); + const real = await runMigration(false); + printOutcomes(real, true); + printSummary(real, unreachableCount); + + return computeExitCode(real, unreachableCount); + }), + ); + +program.parse(process.argv); + +// ─── App bootstrap ────────────────────────────────────────────────────────── + +interface AppServices { + credentialsService: CredentialsService; + couchdbService: CouchdbService; +} + +async function withApp( + callback: (services: AppServices) => Promise, +): Promise { + const app = await NestFactory.createApplicationContext(AppModule, { + logger: false, + }); + try { + const code = await callback({ + credentialsService: app.get(CredentialsService), + couchdbService: app.get(CouchdbService), + }); + await app.close(); + process.exit(code); + } catch (e) { + await app.close(); + throw e; + } +} + +// ─── Org resolution ───────────────────────────────────────────────────────── + +function resolveOrgs( + credentialsService: CredentialsService, + options: { org?: string; category?: string }, +): SystemCredentials[] { + const orgs = OrgRunner.filterOrgs( + credentialsService.getCredentials(), + options, + ); + if (orgs.length === 0) { + const filter = options.org + ? `--org "${options.org}"` + : `--category "${options.category}"`; + console.error(`\nNo orgs matched ${filter}.\n`); + process.exit(2); + } + return orgs; +} + +async function askConfirmation(question: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + return new Promise((resolve) => { + rl.question(question + ' ', (answer) => { + rl.close(); + resolve(answer.toLowerCase() === 'y' || answer.toLowerCase() === 'yes'); + }); + }); +} diff --git a/src/cli/org-output.ts b/src/cli/org-output.ts new file mode 100644 index 0000000..8f7516b --- /dev/null +++ b/src/cli/org-output.ts @@ -0,0 +1,14 @@ +import { ConnectivityResult, OrgRunner } from './org-runner'; + +/* Helper functions for printing output to CLI */ + +export function printConnectivity(results: ConnectivityResult[]): void { + console.log( + `\nConnectivity check (${results.length} org${results.length !== 1 ? 's' : ''})...`, + ); + for (const { org, reachable } of results) { + const mark = reachable ? '✓' : '✗'; + const suffix = reachable ? '' : ' (unreachable)'; + console.log(` ${mark} ${OrgRunner.orgLabel(org)}${suffix}`); + } +} diff --git a/src/cli/org-runner.ts b/src/cli/org-runner.ts new file mode 100644 index 0000000..6ca51c7 --- /dev/null +++ b/src/cli/org-runner.ts @@ -0,0 +1,81 @@ +import { Couchdb, CouchdbService } from '../couchdb/couchdb.service'; +import { SystemCredentials } from '../credentials/credentials.service'; + +export interface ConnectivityResult { + org: SystemCredentials; + reachable: boolean; +} + +export interface OrgOutcome { + org: SystemCredentials; + result: T; +} + +/** + * Generic runner that executes a callback for each org. + * Pure logic — no console output or process control. + */ +export class OrgRunner { + constructor(private couchdbService: CouchdbService) {} + + async checkConnectivity( + orgs: SystemCredentials[], + ): Promise { + const results: ConnectivityResult[] = []; + for (const org of orgs) { + const couchdb = this.couchdbService.getCouchdb( + org.url, + org.password, + org.username, + ); + try { + await couchdb.get('/app'); + results.push({ org, reachable: true }); + } catch { + results.push({ org, reachable: false }); + } + } + return results; + } + + async runForEach( + orgs: SystemCredentials[], + callback: (couchdb: Couchdb, org: SystemCredentials) => Promise, + ): Promise[]> { + const outcomes: OrgOutcome[] = []; + for (const org of orgs) { + const couchdb = this.couchdbService.getCouchdb( + org.url, + org.password, + org.username, + ); + const result = await callback(couchdb, org); + outcomes.push({ org, result }); + } + return outcomes; + } + + static filterOrgs( + orgs: SystemCredentials[], + options: { org?: string; category?: string }, + ): SystemCredentials[] { + let result = orgs; + + if (options.org) { + const names = options.org.split(',').map((s) => s.trim()); + result = result.filter( + (c) => names.includes(c.name ?? '') || names.includes(c.url), + ); + } + + if (options.category) { + result = result.filter((c) => c.category === options.category); + } + + return result; + } + + static orgLabel(org: SystemCredentials): string { + return org.name ? `${org.name} (${org.url})` : org.url; + } +} diff --git a/src/credentials/credentials.service.ts b/src/credentials/credentials.service.ts index fbbe921..9388ab6 100644 --- a/src/credentials/credentials.service.ts +++ b/src/credentials/credentials.service.ts @@ -19,6 +19,7 @@ export class CredentialsService { getCredentials(): SystemCredentials[] { return (credentials as RawSystemCredential[]).map((c) => ({ url: c.url ?? c['name'] + '.' + this.DEFAULT_DOMAIN, + name: c.name, password: c.password, username: c.username, category: c.category?.trim() ?? '', @@ -32,6 +33,12 @@ export interface SystemCredentials { */ url: string; + /** + * Short org name from credentials.json (e.g. "demo"). + * Used by the CLI --org flag for operator-friendly targeting. + */ + name?: string; + /** * admin password to CouchDB */ diff --git a/src/migration/README.md b/src/migration/README.md new file mode 100644 index 0000000..0cd6e3c --- /dev/null +++ b/src/migration/README.md @@ -0,0 +1,175 @@ +# Migration System + +The migration system allows running one-off or recurring data transformations +against all (or selected) CouchDB instances via the CLI. + +## Running migrations + +```bash +# List available migrations +npm run cli -- migrate list + +# Preview what would change (no writes) +npm run cli -- migrate run latest-config-formats --dry-run + +# Run against all orgs (prompts for confirmation) +npm run cli -- migrate run latest-config-formats + +# Run without confirmation prompt (CI / automation) +npm run cli -- migrate run latest-config-formats --yes + +# Target specific orgs by name or URL (comma-separated) +npm run cli -- migrate run latest-config-formats --org my-org,other-org + +# Show per-document details +npm run cli -- migrate run latest-config-formats --verbose +``` + +Or use the convenience script for the most common migration: + +```bash +npm run migrate:latest-config-formats +``` + +## Adding a new migration + +### 1. Implement `MigrationDefinition` + +Create a new file (e.g. `src/migration/config-migration/my-migration.ts`) and +export a `MigrationDefinition` object: + +```typescript +import { MigrationDefinition, MigrationResult } from '../migration-definition'; + +export const myMigration: MigrationDefinition = { + id: 'oneoff-20260512-my-migration', // see naming conventions below + description: 'Short human-readable description shown in "migrate list".', + + async run(ctx): Promise { + // Read via ctx.couchdb (raw accessor — safe for reads) + let doc: any; + try { + doc = await ctx.couchdb.get('/app/SomeDocument'); + } catch { + return { + changed: false, + status: 'no-change', + warnings: ['Document not found — skipping'], + }; + } + + const newDoc = transformDocument(doc); + + ctx.validateJson(newDoc); // throws if value is not JSON-serializable + + const changed = JSON.stringify(doc) !== JSON.stringify(newDoc); + if (!changed) { + return { + changed: false, + status: 'no-change', + }; + } + + // Write via ctx.write (respects dry-run, tracks stats, validates JSON) + await ctx.put('/app/SomeDocument', newDoc); + + return { + changed: true, + status: ctx.dryRun ? 'dry-run' : 'ok', + }; + }, +}; +``` + +### 2. Register in `migrations.ts` + +Add the migration to the array in [`migrations.ts`](./migrations.ts): + +```typescript +import { myMigration } from './config-migration/my-migration'; + +export const migrations: MigrationDefinition[] = [ + latestConfigFormats, + myMigration, // ← add here +]; +``` + +The CLI picks it up automatically. + +### 3. Add seed data to the idempotency spec + +Open [`testing/migration-idempotency.harness.spec.ts`](./testing/migration-idempotency.harness.spec.ts) +and add a case to `seedForMigration`: + +```typescript +if (id === 'oneoff-20260512-my-migration') { + return { + 'app/SomeDocument': { _id: 'SomeDocument', someField: 'old-value' }, + }; +} +``` + +The harness runs every registered migration twice and asserts that the second run +reports `changed: false` — no extra test code needed beyond the seed data. + +--- + +## Rules for migration implementations + +### Idempotency (required) + +Every migration **must** be idempotent: running it a second time against an +already-migrated state must produce `changed: false` and perform no writes. +This is enforced automatically by the shared test harness. + +### Only write via `ctx.write` + +Use `ctx.put(...)` — **not** `ctx.couchdb.put(...)` directly. +`ctx.put` validates JSON, enforces dry-run mode, and tracks write statistics +shown in the CLI summary. + +### Only read via `ctx.couchdb` + +`ctx.couchdb.get`, `.getAll`, `.find`, and `.post` (for CouchDB queries) are +safe for reads. The dry-run guard does not intercept reads. + +### Validate the output + +Call `ctx.validateJson(value)` on any document before writing it. This catches +non-serializable values (circular references, `BigInt`, `undefined` in objects) +before they reach CouchDB. + +### Handle missing documents gracefully + +Return `status: 'no-change'` with a `warnings` entry instead of throwing when +an expected document is absent. The CLI reports warnings without failing the run. + +--- + +## Naming conventions + +| Type | Format | Example | +| -------------------------- | ------------------------ | -------------------------------- | +| Recurring / safe to re-run | `` | `latest-config-formats` | +| One-off | `oneoff-YYYYMMDD-` | `oneoff-20260512-fix-entity-ids` | + +One-off migrations can be removed from the registry once confirmed applied to +all instances. + +--- + +## Module structure + +``` +src/migration/ +├── README.md ← you are here +├── migration-definition.ts ← MigrationContext / MigrationResult / MigrationDefinition interfaces +├── tracked-migration-context.ts ← builds MigrationContext for CLI and tests +├── migrations.ts ← registry of all MigrationDefinitions +├── config-migration/ +│ ├── ndb-core-config-migrations.ts ← ndb-core config migrations (update here when ndb-core exports clean version) +│ └── migrate-add-entity-attributes.ts +└── testing/ + ├── migration-idempotency.harness.ts ← in-memory CouchDB stub + runIdempotencyCheck() + └── migration-idempotency.harness.spec.ts ← runs all registered migrations through idempotency check +``` diff --git a/src/migration/config-migration/config-migration.service.spec.ts b/src/migration/config-migration/config-migration.service.spec.ts deleted file mode 100644 index 49bc995..0000000 --- a/src/migration/config-migration/config-migration.service.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { ConfigMigrationService } from './config-migration.service'; - -describe('ConfigMigrationService', () => { - let service: ConfigMigrationService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ConfigMigrationService], - }).compile(); - - service = module.get(ConfigMigrationService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/migration/config-migration/latest-config-formats.migration.ts b/src/migration/config-migration/latest-config-formats.migration.ts new file mode 100644 index 0000000..ad2a0c4 --- /dev/null +++ b/src/migration/config-migration/latest-config-formats.migration.ts @@ -0,0 +1,62 @@ +import { MigrationDefinition, MigrationResult } from '../migration-definition'; +import { CONFIG_DOC_PATH } from '../migrations'; +import { ndbCoreConfigMigrations } from './ndb-core-config-migrations'; + +/** + * Migrate the Config document to the latest format. + * + * Applies the same on-the-fly config migrations used by ndb-core, plus any + * admin-specific migrations such as migrateAddMissingEntityAttributes. + * + * Idempotent: running twice against the same state reports changed: false + * on the second run and performs no writes. + */ + +export const latestConfigFormats: MigrationDefinition = { + id: 'latest-config-formats', + description: 'Transform any legacy config formats to their latest formats. Safe to re-run.', + + async run(ctx): Promise { + let config: any; + try { + config = await ctx.couchdb.get(CONFIG_DOC_PATH); + } catch { + return { + changed: false, + status: 'failed', + warnings: ['Config document not found'], + }; + } + + let newConfig: any = JSON.parse(JSON.stringify(config), (key, value) => { + for (const migration of ndbCoreConfigMigrations) { + value = migration(key, value); + } + return value; + }); + + ctx.validateJson(newConfig); + + const changed = JSON.stringify(config) !== JSON.stringify(newConfig); + + if (!changed) { + ctx.log.info(' No changes needed'); + return { + changed: false, + status: 'no-change', + }; + } + + ctx.log.info(' Config requires migration'); + if (ctx.dryRun) { + ctx.log.verbose(' ~ Config:CONFIG_ENTITY (migration would apply)'); + } + + await ctx.put(CONFIG_DOC_PATH, newConfig); + + return { + changed: true, + status: ctx.dryRun ? 'dry-run' : 'ok', + }; + }, +}; diff --git a/src/migration/config-migration/migrate-add-entity-attributes.ts b/src/migration/config-migration/migrate-add-entity-attributes.ts deleted file mode 100644 index 5f25f5a..0000000 --- a/src/migration/config-migration/migrate-add-entity-attributes.ts +++ /dev/null @@ -1,535 +0,0 @@ -export function migrateAddMissingEntityAttributes(config) { - for (let entityType of Object.keys(DEFAULT_ENTITIES)) { - // TODO: just blindly save all hard-coded fields into the entity config? or scan which ones are actually used?! - if ( - !JSON.stringify(config).includes(`"${entityType}"`) && - !["Note", "User"].includes(entityType) // exception for these basic types - they should never be skipped - ) { - // don't add config if the entity is never explicitly used or referenced - continue; - } - applyDefaultFieldsForEntityConfig(config, entityType); - } - - return config; -} - -function applyDefaultFieldsForEntityConfig(config, entityType: string) { - if (!config.data['entity:' + entityType]) { - config.data['entity:' + entityType] = {}; - } - const entityConfig = config.data['entity:' + entityType]; - - const hardCodedConfig = DEFAULT_ENTITIES[entityType]; - - entityConfig.label = entityConfig.label ?? hardCodedConfig.label; - entityConfig.labelPlural = - entityConfig.labelPlural ?? hardCodedConfig.labelPlural; - entityConfig.icon = entityConfig.icon ?? hardCodedConfig.icon; - entityConfig.toStringAttributes = - entityConfig.toStringAttributes ?? hardCodedConfig.toStringAttributes; - entityConfig.hasPII = entityConfig.hasPII ?? hardCodedConfig.hasPII; - - entityConfig.attributes = Object.assign( - {}, - hardCodedConfig.attributes, - entityConfig.attributes, - ); -} - -const DEFAULT_ENTITIES = { - User: { - toStringAttributes: ['name'], - icon: 'user', - label: 'User', - labelPlural: 'Users', - hasPII: true, - attributes: { - name: { - dataType: 'string', - label: 'Username', - validators: { - required: true, - uniqueId: 'User', - }, - }, - phone: { - dataType: 'string', - label: 'Contact', - }, - }, - }, - EducationalMaterial: { - attributes: { - child: { - dataType: 'entity', - additional: 'Child', - entityReferenceRole: 'composite', - }, - date: { - dataType: 'date', - label: 'Date', - defaultValue: { - mode: 'dynamic', - value: '$now', - }, - }, - materialType: { - label: 'Material', - dataType: 'configurable-enum', - additional: 'materials', - validators: { - required: true, - }, - }, - materialAmount: { - dataType: 'number', - label: 'Amount', - defaultValue: { - mode: 'static', - value: 1, - }, - validators: { - required: true, - }, - }, - description: { - dataType: 'string', - label: 'Description', - }, - }, - }, - Todo: { - toStringAttributes: ['subject'], - icon: 'check', - label: 'Task', - labelPlural: 'Tasks', - hasPII: true, - attributes: { - subject: { - dataType: 'string', - label: 'Subject', - showInDetailsView: true, - }, - description: { - dataType: 'long-text', - showInDetailsView: true, - label: 'Description', - }, - deadline: { - dataType: 'date-only', - showInDetailsView: true, - anonymize: 'retain', - label: 'Deadline', - }, - startDate: { - dataType: 'date-only', - description: - 'When you are planning to start work so that you keep enough time before the actual hard deadline.', - showInDetailsView: true, - anonymize: 'retain', - label: 'Start Date', - }, - assignedTo: { - label: 'Assigned to', - dataType: 'entity', - isArray: true, - additional: 'User', - showInDetailsView: true, - defaultValue: { - mode: 'dynamic', - value: '$current_user', - }, - anonymize: 'retain', - }, - relatedEntities: { - dataType: 'entity', - isArray: true, - label: 'Related Records', - additional: ['Child', 'School', 'RecurringActivity'], - entityReferenceRole: 'composite', - showInDetailsView: true, - anonymize: 'retain', - }, - repetitionInterval: { - label: 'repeats', - additional: [ - { - label: 'every week', - interval: { amount: 1, unit: 'week' }, - }, - { - label: 'every month', - interval: { amount: 1, unit: 'month' }, - }, - ], - showInDetailsView: true, - anonymize: 'retain', - }, - completed: { - label: 'completed', - viewComponent: 'DisplayTodoCompletion', - anonymize: 'retain', - }, - }, - }, - School: { - toStringAttributes: ['name'], - icon: 'university', - label: 'School', - labelPlural: 'Schools', - color: '#9E9D24', - attributes: { - name: { - dataType: 'string', - label: 'Name', - validators: { - required: true, - }, - }, - }, - }, - Aser: { - hasPII: true, - attributes: { - child: { - dataType: 'entity', - additional: 'Child', - entityReferenceRole: 'composite', - }, - date: { - dataType: 'date', - label: 'Date', - defaultValue: { - mode: 'dynamic', - value: '$now', - }, - anonymize: 'retain-anonymized', - }, - hindi: { - label: 'Hindi', - dataType: 'configurable-enum', - additional: 'reading-levels', - }, - bengali: { - label: 'Bengali', - dataType: 'configurable-enum', - additional: 'reading-levels', - }, - english: { - label: 'English', - dataType: 'configurable-enum', - additional: 'reading-levels', - }, - math: { - label: 'Math', - dataType: 'configurable-enum', - additional: 'math-levels', - }, - remarks: { - dataType: 'string', - label: 'Remarks', - }, - }, - }, - // TODO: the "bmi" column in view configs needs to be adapted manually (this is only used at a single client, however) - HealthCheck: { - hasPII: true, - attributes: { - child: { - dataType: 'entity', - additional: 'Child', - entityReferenceRole: 'composite', - anonymize: 'retain', - }, - date: { - dataType: 'date', - label: 'Date', - defaultValue: { - mode: 'dynamic', - value: '$now', - }, - anonymize: 'retain-anonymized', - }, - height: { - dataType: 'number', - label: 'Height [cm]', - viewComponent: 'DisplayUnit', - additional: 'cm', - }, - weight: { - dataType: 'number', - label: 'Weight [kg]', - viewComponent: 'DisplayUnit', - additional: 'kg', - }, - }, - }, - ChildSchoolRelation: { - hasPII: true, - attributes: { - childId: { - dataType: 'entity', - additional: 'Child', - entityReferenceRole: 'composite', - validators: { - required: true, - }, - anonymize: 'retain', - label: 'Child', - }, - schoolId: { - dataType: 'entity', - additional: 'School', - entityReferenceRole: 'aggregate', - validators: { - required: true, - }, - anonymize: 'retain', - label: 'School', - }, - schoolClass: { - dataType: 'string', - label: 'Class', - }, - start: { - dataType: 'date-only', - label: 'Start date', - description: 'The date a child joins a school', - anonymize: 'retain', - }, - end: { - dataType: 'date-only', - label: 'End date', - description: 'The date of a child leaving the school', - anonymize: 'retain', - }, - result: { - dataType: 'number', - label: 'Result', - viewComponent: 'DisplayPercentage', - editComponent: 'EditNumber', - validators: { - min: 0, - max: 100, - }, - }, - }, - }, - Child: { - label: 'Participant', - labelPlural: 'Participants', - toStringAttributes: ['name'], - icon: 'child', - color: '#1565C0', - blockComponent: 'ChildBlock', - hasPII: true, - attributes: { - name: { - dataType: 'string', - label: 'Name', - validators: { - required: true, - }, - }, - projectNumber: { - dataType: 'string', - label: 'Project Number', - labelShort: 'PN', - searchable: true, - anonymize: 'retain', - }, - dateOfBirth: { - dataType: 'date-with-age', - label: 'Date of birth', - labelShort: 'DoB', - anonymize: 'retain-anonymized', - }, - center: { - dataType: 'configurable-enum', - additional: 'center', - label: 'Center', - anonymize: 'retain', - }, - gender: { - dataType: 'configurable-enum', - label: 'Gender', - additional: 'genders', - anonymize: 'retain', - }, - admissionDate: { - dataType: 'date-only', - label: 'Admission', - anonymize: 'retain-anonymized', - }, - status: { - dataType: 'string', - label: 'Status', - }, - dropoutDate: { - dataType: 'date-only', - label: 'Dropout Date', - anonymize: 'retain-anonymized', - }, - dropoutType: { - dataType: 'string', - label: 'Dropout Type', - anonymize: 'retain', - }, - dropoutRemarks: { - dataType: 'string', - label: 'Dropout remarks', - }, - photo: { - dataType: 'file', - label: 'Photo', - editComponent: 'EditPhoto', - }, - phone: { - dataType: 'string', - label: 'Phone Number', - }, - }, - }, - RecurringActivity: { - toStringAttributes: ['title'], - label: 'Recurring Activity', - labelPlural: 'Recurring Activities', - color: '#00838F', - route: 'attendance/recurring-activity', - attributes: { - title: { - dataType: 'string', - label: 'Title', - validators: { - required: true, - }, - }, - type: { - label: 'Type', - dataType: 'configurable-enum', - additional: 'interaction-type', - }, - participants: { - label: 'Participants', - dataType: 'entity', - isArray: true, - additional: 'Child', - }, - linkedGroups: { - label: 'Groups', - dataType: 'entity', - isArray: true, - additional: 'School', - }, - excludedParticipants: { - label: 'Excluded Participants', - dataType: 'entity', - isArray: true, - additional: 'Child', - }, - assignedTo: { - label: 'Assigned user(s)', - dataType: 'entity', - isArray: true, - additional: 'User', - }, - }, - }, - Note: { - toStringAttributes: ['subject'], - label: 'Note', - labelPlural: 'Notes', - hasPII: true, - attributes: { - children: { - label: 'Children', - dataType: 'entity', - isArray: true, - additional: 'Child', - entityReferenceRole: 'composite', - editComponent: 'EditAttendance', - anonymize: 'retain', - }, - childrenAttendance: { - dataType: 'event-attendance-map', - anonymize: 'retain', - }, - date: { - label: 'Date', - dataType: 'date-only', - defaultValue: { - mode: 'dynamic', - value: '$now', - }, - anonymize: 'retain', - }, - subject: { - dataType: 'string', - label: 'Subject', - }, - text: { - dataType: 'long-text', - label: 'Notes', - }, - authors: { - label: 'SW', - dataType: 'entity', - isArray: true, - additional: 'User', - defaultValue: { - mode: 'dynamic', - value: '$current_user', - }, - anonymize: 'retain', - }, - category: { - label: 'Category', - dataType: 'configurable-enum', - additional: 'interaction-type', - anonymize: 'retain', - }, - attachment: { - label: 'Attachment', - dataType: 'file', - }, - relatesTo: { - dataType: 'entity', - additional: 'RecurringActivity', - anonymize: 'retain', - }, - relatedEntities: { - label: 'Related Records', - dataType: 'entity', - isArray: true, - // by default no additional relatedEntities can be linked apart from children and schools, overwrite this in config to display (e.g. additional: "ChildSchoolRelation") - additional: undefined, - anonymize: 'retain', - }, - schools: { - label: 'Groups', - dataType: 'entity', - isArray: true, - additional: 'School', - entityReferenceRole: 'composite', - anonymize: 'retain', - }, - warningLevel: { - label: 'Status', - dataType: 'configurable-enum', - additional: 'warning-levels', - anonymize: 'retain', - }, - }, - }, - // entity HistoricalEntityData is only used in very few clients - /* --> manually check & migrate instead of bloating up the config everywhere: - "123-kimue": "matching", - "ashayen": "matching", - "demo2": "matching", - "demo-health": "matching", - "helgo": "matching", - "sambodh": "matching", - "yojak-gsp": "matching" - */ -}; diff --git a/src/migration/config-migration/config-migration.service.ts b/src/migration/config-migration/ndb-core-config-migrations.ts similarity index 62% rename from src/migration/config-migration/config-migration.service.ts rename to src/migration/config-migration/ndb-core-config-migrations.ts index 5cb435b..61955c7 100644 --- a/src/migration/config-migration/config-migration.service.ts +++ b/src/migration/config-migration/ndb-core-config-migrations.ts @@ -1,75 +1,19 @@ -import { Injectable } from '@nestjs/common'; -import { Couchdb } from '../../couchdb/couchdb.service'; -import { migrateAddMissingEntityAttributes } from './migrate-add-entity-attributes'; - /** - * Apply transformations to the Config document in the CouchDB, - * using migration functions that are also used in the frontend (ndb-core). + * Individual config migrations copied from ndb-core ConfigService. + * + * Source: https://github.com/Aam-Digital/ndb-core/blob/master/src/app/core/config/config.service.ts */ -@Injectable() -export class ConfigMigrationService { - private async getConfigDoc(couchdb: Couchdb) { - return couchdb.get('/app/Config:CONFIG_ENTITY'); - } - - private async saveConfigDoc(couchdb: Couchdb, configDoc: any) { - return couchdb.put('/app/Config:CONFIG_ENTITY', configDoc); - } - - private applyMigrations(config) { - const migrations: ConfigMigration[] = [ - migrateEntityAttributesWithId, - migrateFormHeadersIntoFieldGroups, - migrateFormFieldConfigView2ViewComponent, - migrateMenuItemConfig, - migrateEntityDetailsInputEntityType, - migrateEntityArrayDatatype, - migrateEntitySchemaDefaultValue, - migrateChildrenListConfig, - migrateHistoricalDataComponent, - ]; - - const newConfig = JSON.parse(JSON.stringify(config), (_that, rawValue) => { - let configPart = rawValue; - for (const migration of migrations) { - configPart = migration(_that, configPart); - } - return configPart; - }); - - return newConfig; - } - - /** - * apply all currently registered config format transformations, similar to the frontend, on-the-fly migrations. - * @param couchdb - */ - async migrateToLatestConfigFormats(couchdb: Couchdb) { - const config = await this.getConfigDoc(couchdb); - let newConfig = this.applyMigrations(config); - - newConfig = migrateAddMissingEntityAttributes(newConfig); - - await this.saveConfigDoc(couchdb, newConfig); - return JSON.stringify(config) !== JSON.stringify(newConfig); - } -} /** - * A ConfigMigration is checked during a full JSON.parse using a reviver function. - * If the migration does not apply to the given configPart, make sure to return it unchanged. + * A ConfigMigration is applied during a JSON.parse using a reviver function. + * If the migration does not apply to the given configPart, return it unchanged. * Multiple migrations are chained and can transform the same config part one after the other. - * - * --> see ndb-core (!) */ -type ConfigMigration = (key: string, configPart: any) => any; - -// -// COPIED FROM ndb-core ConfigService migrations: -// +export type ConfigMigration = (key: string, configPart: any) => any; /** - * Transform legacy "entity:" config format into the flattened structure containing id directly. + * Transform legacy "entity:" config format into the flattened structure + * containing id directly. */ const migrateEntityAttributesWithId: ConfigMigration = (key, configPart) => { if (!(key.startsWith('entity') && Array.isArray(configPart.attributes))) { @@ -77,10 +21,9 @@ const migrateEntityAttributesWithId: ConfigMigration = (key, configPart) => { } configPart.attributes = configPart.attributes.reduce( - (acc, attr: { name: string; schema }) => ({ + (acc, attr: { name: string; schema: unknown }) => ({ ...acc, [attr.name]: attr.schema, - // id inside the field schema config (FieldConfig) is added by EntityConfigService and does not need migration }), {}, ); @@ -89,7 +32,8 @@ const migrateEntityAttributesWithId: ConfigMigration = (key, configPart) => { }; /** - * Transform legacy "view:...Form" config format to have form field group headers with the fields rather than as separate array. + * Transform legacy "view:...Form" config format to have form field group + * headers with the fields rather than as a separate array. */ const migrateFormHeadersIntoFieldGroups: ConfigMigration = ( key, @@ -101,7 +45,6 @@ const migrateFormHeadersIntoFieldGroups: ConfigMigration = ( const formConfig = configPart.config; - // change .cols and .headers into .fieldGroups const newFormConfig = { ...formConfig }; delete newFormConfig.cols; delete newFormConfig.headers; @@ -157,7 +100,7 @@ const migrateMenuItemConfig: ConfigMigration = (key, configPart) => { const oldItems: any[] = configPart.items; configPart.items = oldItems.map((item) => { - if (item.hasOwnProperty('name')) { + if (Object.prototype.hasOwnProperty.call(item, 'name')) { return { label: item['name'], icon: item.icon, @@ -172,10 +115,9 @@ const migrateMenuItemConfig: ConfigMigration = (key, configPart) => { }; /** - * Config properties specifying an entityType should be named "entityType" rather than "entity" - * to avoid confusion with a specific instance of an entity being passed in components. - * @param key - * @param configPart + * Config properties specifying an entityType should be named "entityType" + * rather than "entity" to avoid confusion with a specific instance of an + * entity being passed in components. */ const migrateEntityDetailsInputEntityType: ConfigMigration = ( key, @@ -194,16 +136,15 @@ const migrateEntityDetailsInputEntityType: ConfigMigration = ( }; /** - * Replace custom "entity-array" dataType with dataType="array", innerDatatype="entity" - * @param key - * @param configPart + * Replace custom "entity-array" dataType with dataType="array", + * innerDatatype="entity". */ const migrateEntityArrayDatatype: ConfigMigration = (key, configPart) => { if (configPart === 'DisplayEntityArray') { return 'DisplayEntity'; } - if (!configPart?.hasOwnProperty('dataType')) { + if (!Object.prototype.hasOwnProperty.call(configPart, 'dataType')) { return configPart; } @@ -235,11 +176,11 @@ const migrateEntitySchemaDefaultValue: ConfigMigration = ( return configPart; } - if (typeof configPart == 'object') { + if (typeof configPart === 'object') { return configPart; } - let placeholderValue: string | undefined = ['$now', '$current_user'].find( + const placeholderValue: string | undefined = ['$now', '$current_user'].find( (value) => value === configPart, ); @@ -292,3 +233,18 @@ const migrateHistoricalDataComponent: ConfigMigration = (key, configPart) => { return configPart; }; + +/** + * All ndb-core config migrations, in order. + */ +export const ndbCoreConfigMigrations: ConfigMigration[] = [ + migrateEntityAttributesWithId, + migrateFormHeadersIntoFieldGroups, + migrateFormFieldConfigView2ViewComponent, + migrateMenuItemConfig, + migrateEntityDetailsInputEntityType, + migrateEntityArrayDatatype, + migrateEntitySchemaDefaultValue, + migrateChildrenListConfig, + migrateHistoricalDataComponent, +]; diff --git a/src/migration/console-logger.ts b/src/migration/console-logger.ts new file mode 100644 index 0000000..4214d76 --- /dev/null +++ b/src/migration/console-logger.ts @@ -0,0 +1,25 @@ +import { MigrationLogger } from './migration-definition'; + +/** + * Console-backed MigrationLogger. + * verbose() messages are suppressed unless verboseEnabled is true. + */ +export class ConsoleLogger implements MigrationLogger { + constructor(private verboseEnabled: boolean) {} + + info(msg: string): void { + console.log(msg); + } + + warn(msg: string): void { + console.warn(` WARN ${msg}`); + } + + error(msg: string): void { + console.error(` ERROR ${msg}`); + } + + verbose(msg: string): void { + if (this.verboseEnabled) console.log(msg); + } +} diff --git a/src/migration/migration-definition.ts b/src/migration/migration-definition.ts new file mode 100644 index 0000000..7eb6183 --- /dev/null +++ b/src/migration/migration-definition.ts @@ -0,0 +1,76 @@ +import { Couchdb } from '../couchdb/couchdb.service'; +import { SystemCredentials } from '../credentials/credentials.service'; + +/** + * Logger passed to each migration via MigrationContext. + * Methods are separated so the CLI can suppress verbose output by default + * and tests can capture output by injecting a spy. + */ +export interface MigrationLogger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; + /** Only emitted when the CLI is run with --verbose. */ + verbose(message: string): void; +} + +/** + * Context provided to every migration run. + * Scoped to a single org. + */ +export interface MigrationContext { + /** Raw CouchDB accessor for reads. Do not call couchdb.put directly; use ctx.put instead. */ + couchdb: Couchdb; + org: SystemCredentials; + dryRun: boolean; + log: MigrationLogger; + /** Write a single document. Validates JSON, respects dry-run, tracks stats. */ + put( + path: string, + data: unknown, + db?: string, + headers?: unknown, + ): Promise; + /** + * Verify that a value is JSON-serializable. + * Throws an Error if the value contains non-serializable content (circular refs, BigInt, etc.). + * Run this before constructing any write payload. + */ + validateJson(value: unknown): void; +} + +/** + * Result returned by a migration after processing one org. + */ +export interface MigrationResult { + changed: boolean; + status: 'ok' | 'no-change' | 'dry-run' | 'partial' | 'failed'; + details?: unknown; + warnings?: string[]; +} + +export function failedMigrationResult(message: string): MigrationResult { + return { + changed: false, + status: 'failed', + warnings: [message], + }; +} + +/** + * A MigrationDefinition describes one migration that can be run by the CLI. + * + * Implementations MUST be idempotent: running the same migration twice against + * the same state must leave the data unchanged and report changed: false on the + * second run. This is enforced by the shared idempotency test harness. + */ +export interface MigrationDefinition { + id: string; + description: string; + run(ctx: MigrationContext): Promise; +} + +export interface MigrationOutcome { + result: MigrationResult; + writeStats: { intended: number; succeeded: number; failed: number }; +} diff --git a/src/migration/migration-output.ts b/src/migration/migration-output.ts new file mode 100644 index 0000000..32ed7d4 --- /dev/null +++ b/src/migration/migration-output.ts @@ -0,0 +1,79 @@ +import { OrgOutcome, OrgRunner } from '../cli/org-runner'; +import { + MigrationDefinition, + MigrationOutcome, + MigrationResult, +} from './migration-definition'; + +/* Helper functions for printing migration output to CLI */ + +export function printBanner( + label: string, + migration: MigrationDefinition, +): void { + console.log( + `\n${'─'.repeat(60)}\n${label} ${migration.id}: ${migration.description}\n${'─'.repeat(60)}`, + ); +} + +export function printOutcomes( + outcomes: OrgOutcome[], + showWriteStats: boolean, +): void { + for (const { + org, + result: { result, writeStats }, + } of outcomes) { + process.stdout.write(` ${OrgRunner.orgLabel(org).padEnd(50)}`); + const suffix = + showWriteStats && writeStats.intended > 0 + ? ` (${writeStats.succeeded}/${writeStats.intended} written)` + : ''; + const STATUS_TAGS: Record = { + ok: 'OK ', + 'no-change': 'NO-CHANGE', + 'dry-run': 'PREVIEW ', + partial: 'PARTIAL ', + failed: 'FAILED ', + }; + console.log((STATUS_TAGS[result.status] ?? result.status) + suffix); + if (result.warnings?.length) { + result.warnings.forEach((w) => console.log(` ! ${w}`)); + } + } +} + +export function printSummary( + outcomes: OrgOutcome[], + unreachableCount: number, +): void { + const counts = { ok: 0, noChange: 0, partial: 0, failed: 0 }; + for (const { + result: { result }, + } of outcomes) { + if (result.status === 'ok' || result.status === 'dry-run') counts.ok++; + else if (result.status === 'no-change') counts.noChange++; + else if (result.status === 'partial') counts.partial++; + else if (result.status === 'failed') counts.failed++; + } + counts.failed += unreachableCount; + + console.log('\n' + '─'.repeat(60)); + console.log( + `Summary: ${counts.ok} changed, ${counts.noChange} no-change, ${counts.partial} partial, ${counts.failed} failed`, + ); + console.log('─'.repeat(60) + '\n'); +} + +export function computeExitCode( + outcomes: OrgOutcome[], + unreachableCount: number, +): number { + if (unreachableCount > 0) return 1; + for (const { + result: { result }, + } of outcomes) { + if (result.status === 'failed' || result.status === 'partial') return 1; + } + return 0; +} diff --git a/src/migration/migration.controller.spec.ts b/src/migration/migration.controller.spec.ts deleted file mode 100644 index 2bc0625..0000000 --- a/src/migration/migration.controller.spec.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { MigrationController } from './migration.controller'; -import { Couchdb, CouchdbService } from '../couchdb/couchdb.service'; -import { ConfigService } from '@nestjs/config'; -import { ConfigMigrationService } from './config-migration/config-migration.service'; -import { CredentialsService } from '../credentials/credentials.service'; - -describe('MigrationController', () => { - let controller: MigrationController; - let couchdb: Couchdb; - - beforeEach(async () => { - couchdb = { - get: () => undefined, - put: () => undefined, - getAll: () => undefined, - putAll: () => undefined, - } as any; - const module: TestingModule = await Test.createTestingModule({ - controllers: [MigrationController], - providers: [ - { - provide: CouchdbService, - useValue: { - runForAllOrgs: (_, func) => func(couchdb), - }, - }, - ConfigService, - ConfigMigrationService, - { provide: CredentialsService, useValue: { getCredentials: () => [] } }, - ], - }).compile(); - - controller = module.get(MigrationController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - - function mockDb(docs: { _id: string }[]) { - jest - .spyOn(couchdb, 'getAll') - .mockImplementation((prefix) => - Promise.resolve( - docs - .filter(({ _id }) => _id.startsWith(prefix + ':')) - .map((doc) => JSON.parse(JSON.stringify(doc))), - ), - ); - jest.spyOn(couchdb, 'get').mockImplementation((path) => { - const id = path.split('/').pop(); - const found = docs.find(({ _id }) => _id === id); - return found - ? Promise.resolve(JSON.parse(JSON.stringify(found))) - : Promise.reject(); - }); - jest.spyOn(couchdb, 'putAll').mockResolvedValue(undefined); - } -}); diff --git a/src/migration/migration.controller.ts b/src/migration/migration.controller.ts deleted file mode 100644 index 7286c5d..0000000 --- a/src/migration/migration.controller.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Controller, Post } from '@nestjs/common'; -import { Couchdb, CouchdbService } from '../couchdb/couchdb.service'; -import { ApiOperation } from '@nestjs/swagger'; -import { ConfigMigrationService } from './config-migration/config-migration.service'; -import { CredentialsService } from '../credentials/credentials.service'; - -@Controller('migration') -export class MigrationController { - constructor( - private couchdbService: CouchdbService, - private configMigrationService: ConfigMigrationService, - private credentialsService: CredentialsService, - ) {} - - @ApiOperation({ - description: - 'Transform any legacy config formats to their latest formats. If already in new formats, this will have no effect.', - }) - @Post('latest-config-formats') - async migrateToLatestConfigFormats() { - return this.couchdbService.runForAllOrgs( - this.credentialsService.getCredentials(), - (couchdb: Couchdb) => - this.configMigrationService.migrateToLatestConfigFormats(couchdb), - ); - } -} diff --git a/src/migration/migrations.ts b/src/migration/migrations.ts new file mode 100644 index 0000000..3fc95aa --- /dev/null +++ b/src/migration/migrations.ts @@ -0,0 +1,11 @@ +import { latestConfigFormats } from './config-migration/latest-config-formats.migration'; +import { MigrationDefinition } from './migration-definition'; + +export const CONFIG_DOC_PATH = '/app/Config:CONFIG_ENTITY'; + +/** + * All registered migrations. + * Add new MigrationDefinitions here to make them available to the CLI. + * One-off migrations follow the naming convention: oneoff-YYYYMMDD- + */ +export const migrations: MigrationDefinition[] = [latestConfigFormats]; diff --git a/src/migration/testing/migration-idempotency.harness.spec.ts b/src/migration/testing/migration-idempotency.harness.spec.ts new file mode 100644 index 0000000..8a76138 --- /dev/null +++ b/src/migration/testing/migration-idempotency.harness.spec.ts @@ -0,0 +1,81 @@ +import { migrations } from '../migrations'; +import { runIdempotencyCheck } from './migration-idempotency.harness'; + +/** + * Verify that every registered migration is idempotent. + * + * For each migration the harness: + * 1. Seeds an in-memory CouchDB stub with a minimal representative document + * 2. Runs the migration → first result (may report changed: true) + * 3. Runs the migration again on the resulting state → second result + * 4. Asserts changed: false and status 'no-change' on the second run + * 5. Asserts the doc store is unchanged between first and second run + * + * If a migration needs extra seed data, add a case to the `seedForMigration` + * helper below. + */ + +/** Minimal seed documents for migrations that need real data to change. */ +function seedForMigration(id: string): Record { + if (id === 'latest-config-formats') { + return { + // A config document that requires migration (uses the old _id-less entity format) + 'app/Config:CONFIG_ENTITY': { + _id: 'Config:CONFIG_ENTITY', + entityTypes: { + Child: { + attributes: [{ name: 'name', schema: { dataType: 'string' } }], + }, + }, + views: [], + }, + }; + } + // Default: empty — migration should handle missing doc gracefully + return {}; +} + +describe('All registered migrations are idempotent', () => { + test.each(migrations.map((m) => [m.id, m]))( + 'migration "%s" second run reports no-change', + async (_id, migration) => { + const seed = seedForMigration(migration.id); + const result = await runIdempotencyCheck(migration, seed); + + // Second run must not make any further changes + expect(result.secondRunResult.changed).toBe(false); + expect( + result.secondRunResult.status === 'no-change' || + result.secondRunResult.status === 'failed', + ).toBe(true); + + // Doc store must be identical after first and second run + expect(result.stateAfterSecondRun).toEqual(result.stateAfterFirstRun); + }, + ); +}); + +describe('latestConfigFormats migration', () => { + it('handles missing Config document without throwing', async () => { + const result = await runIdempotencyCheck( + migrations.find((m) => m.id === 'latest-config-formats')!, + {}, // No documents seeded + ); + expect(result.firstRunResult.status).toBe('failed'); + expect(result.firstRunResult.changed).toBe(false); + expect(result.firstRunResult.warnings).toEqual( + expect.arrayContaining([expect.stringContaining('not found')]), + ); + }); + + it('applies migration and marks changed: true on first run', async () => { + const seed = seedForMigration('latest-config-formats'); + const result = await runIdempotencyCheck( + migrations.find((m) => m.id === 'latest-config-formats')!, + seed, + ); + // First run: either changed (migration applied) or no-change (already up-to-date) + // Either way: second run must report no-change + expect(result.secondRunResult.changed).toBe(false); + }); +}); diff --git a/src/migration/testing/migration-idempotency.harness.ts b/src/migration/testing/migration-idempotency.harness.ts new file mode 100644 index 0000000..054016f --- /dev/null +++ b/src/migration/testing/migration-idempotency.harness.ts @@ -0,0 +1,180 @@ +/** + * Idempotency test harness for MigrationDefinition implementations. + * + * Usage: + * const result = await runIdempotencyCheck(myMigration, { 'db/path': docObj }); + * expect(result.secondRunResult.changed).toBe(false); + */ + +import { jest } from '@jest/globals'; +import { Couchdb } from '../../couchdb/couchdb.service'; +import { SystemCredentials } from '../../credentials/credentials.service'; +import { + MigrationContext, + MigrationDefinition, + MigrationLogger, + MigrationResult, +} from '../migration-definition'; + +/** In-memory document store used by the stub. Key format: "db/path" or simply "path". */ +export type DocStore = Record; + +export interface IdempotencyCheckResult { + firstRunResult: MigrationResult; + secondRunResult: MigrationResult; + /** State of the doc store after the first run. */ + stateAfterFirstRun: DocStore; + /** State of the doc store after the second run (should equal stateAfterFirstRun). */ + stateAfterSecondRun: DocStore; +} + +/** + * Build a stub Couchdb whose reads and writes operate against an in-memory DocStore. + * + * The stub maps put(path, data, db?) into the store as `${db ?? 'app'}/${path}`. + * It supports the minimal surface used by the current migrations: + * - get(path) → store lookup; throws 404-like error when missing + * - put(path, data, db?) → store upsert + * - putAll(docs, db?) → bulk upsert keyed by doc._id + * - getAll(prefix, db?) → returns values whose key matches `${db ?? 'app'}/${prefix}` + * - post(path, data, db?) → store upsert (treated same as put) + * - find(query, db?) → naive full-store scan (docs array, no index) + */ +export function buildStubCouchdb(store: DocStore): Couchdb { + function key(path: string, db?: string): string { + return `${db ?? 'app'}/${path.replace(/^\//, '')}`; + } + + return { + get: jest.fn(async (path: string, db?: string) => { + const k = key(path, db); + const val = store[k]; + if (val === undefined) { + const err: any = new Error(`Not found: ${k}`); + err.response = { status: 404 }; + throw err; + } + return val; + }), + + put: jest.fn(async (path: string, data: unknown, db?: string) => { + store[key(path, db)] = JSON.parse(JSON.stringify(data)); + }), + + putAll: jest.fn(async (docs: any[], db?: string) => { + const results: { ok: boolean; id: string }[] = []; + for (const doc of docs) { + const id: string = doc._id ?? doc.id ?? String(Math.random()); + store[key(id, db)] = JSON.parse(JSON.stringify(doc)); + results.push({ ok: true, id }); + } + return results; + }), + + getAll: jest.fn(async (prefix: string, db?: string) => { + const base = `${db ?? 'app'}/`; + return Object.entries(store) + .filter( + ([k]) => + k.startsWith(base) && k.slice(base.length).startsWith(prefix), + ) + .map(([, v]) => v); + }), + + post: jest.fn(async (path: string, data: unknown, db?: string) => { + store[key(path, db)] = JSON.parse(JSON.stringify(data)); + }), + + find: jest.fn(async (query: any, db?: string) => { + const base = `${db ?? 'app'}/`; + const all = Object.entries(store) + .filter(([k]) => k.startsWith(base)) + .map(([, v]) => v); + // Return all docs; tests should seed only relevant data + return { docs: all }; + }), + } as unknown as Couchdb; +} + +const silentLogger: MigrationLogger = { + info: () => {}, + warn: () => {}, + error: () => {}, + verbose: () => {}, +}; + +const fakeOrg: SystemCredentials = { + url: 'https://test.example.com', + username: 'admin', + password: 'secret', + name: 'test', +}; + +function buildInMemoryContext( + store: DocStore, + dryRun: boolean, +): MigrationContext & { store: DocStore } { + const stubCouchdb = buildStubCouchdb(store); + + const writes: { intended: number; succeeded: number; failed: number } = { + intended: 0, + succeeded: 0, + failed: 0, + }; + + const validateJson = (value: unknown): void => { + try { + JSON.stringify(value); + } catch (e: any) { + throw new Error(`JSON validation failed: ${e.message}`); + } + }; + + return { + couchdb: stubCouchdb, + org: fakeOrg, + dryRun, + log: silentLogger, + validateJson, + async put(path, data, db?, _headers?) { + validateJson(data); + writes.intended++; + if (dryRun) return; + await stubCouchdb.put(path, data, db); + writes.succeeded++; + }, + store, + }; +} + +/** + * Run idempotency check for a single migration. + * + * @param migration The migration to test. + * @param initialDocs Seed documents as a DocStore record. + * Keys use the same `db/path` convention as the stub. + * Tip: Use `"app/Config:CONFIG_ENTITY"` for the main config doc. + */ +export async function runIdempotencyCheck( + migration: MigrationDefinition, + initialDocs: DocStore = {}, +): Promise { + // First run + const store1: DocStore = JSON.parse(JSON.stringify(initialDocs)); + const ctx1 = buildInMemoryContext(store1, false); + const firstRunResult = await migration.run(ctx1); + const stateAfterFirstRun: DocStore = JSON.parse(JSON.stringify(store1)); + + // Second run — same store state as left after first run + const store2: DocStore = JSON.parse(JSON.stringify(stateAfterFirstRun)); + const ctx2 = buildInMemoryContext(store2, false); + const secondRunResult = await migration.run(ctx2); + const stateAfterSecondRun: DocStore = JSON.parse(JSON.stringify(store2)); + + return { + firstRunResult, + secondRunResult, + stateAfterFirstRun, + stateAfterSecondRun, + }; +} diff --git a/src/migration/tracked-migration-context.ts b/src/migration/tracked-migration-context.ts new file mode 100644 index 0000000..5b788a5 --- /dev/null +++ b/src/migration/tracked-migration-context.ts @@ -0,0 +1,63 @@ +import { Couchdb } from '../couchdb/couchdb.service'; +import { SystemCredentials } from '../credentials/credentials.service'; +import { MigrationContext, MigrationLogger } from './migration-definition'; + +/** + * Concrete MigrationContext for one org. + * + * - validates JSON serializability on every write payload + * - in dry-run mode records intended writes without calling CouchDB + * - in real execution calls CouchDB and inspects _bulk_docs per-doc responses + * - tracks intended / succeeded / failed write counts via getWriteStats() + */ +export class TrackedMigrationContext implements MigrationContext { + private intended = 0; + private succeeded = 0; + private failed = 0; + + constructor( + readonly couchdb: Couchdb, + readonly org: SystemCredentials, + readonly dryRun: boolean, + readonly log: MigrationLogger, + ) {} + + validateJson(value: unknown): void { + try { + JSON.stringify(value); + } catch (e: any) { + throw new Error(`JSON validation failed: ${e.message}`); + } + } + + getWriteStats(): { intended: number; succeeded: number; failed: number } { + return { + intended: this.intended, + succeeded: this.succeeded, + failed: this.failed, + }; + } + + async put( + path: string, + data: unknown, + db?: string, + headers?: unknown, + ): Promise { + this.validateJson(data); + this.intended++; + if (this.dryRun) { + this.log.verbose(` [PREVIEW] Would PUT ${path}`); + return; + } + try { + await this.couchdb.put(path, data, db, headers); + this.succeeded++; + } catch (e) { + this.failed++; + throw e; + } + } + + +} From 8f57a1573fce8c4ec5ea43ed66d3d3608a843d52 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:12:21 +0200 Subject: [PATCH 02/10] chore: add qlty --- .qlty/.gitignore | 7 +++ .qlty/configs/.hadolint.yaml | 2 + .qlty/qlty.toml | 97 ++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 .qlty/.gitignore create mode 100644 .qlty/configs/.hadolint.yaml create mode 100644 .qlty/qlty.toml diff --git a/.qlty/.gitignore b/.qlty/.gitignore new file mode 100644 index 0000000..3036618 --- /dev/null +++ b/.qlty/.gitignore @@ -0,0 +1,7 @@ +* +!configs +!configs/** +!hooks +!hooks/** +!qlty.toml +!.gitignore diff --git a/.qlty/configs/.hadolint.yaml b/.qlty/configs/.hadolint.yaml new file mode 100644 index 0000000..8f7e23e --- /dev/null +++ b/.qlty/configs/.hadolint.yaml @@ -0,0 +1,2 @@ +ignored: + - DL3008 diff --git a/.qlty/qlty.toml b/.qlty/qlty.toml new file mode 100644 index 0000000..0ae6fee --- /dev/null +++ b/.qlty/qlty.toml @@ -0,0 +1,97 @@ +# This file was automatically generated by `qlty init`. +# You can modify it to suit your needs. +# We recommend you to commit this file to your repository. +# +# This configuration is used by both Qlty CLI and Qlty Cloud. +# +# Qlty CLI -- Code quality toolkit for developers +# Qlty Cloud -- Fully automated Code Health Platform +# +# Try Qlty Cloud: https://qlty.sh +# +# For a guide to configuration, visit https://qlty.sh/d/config +# Or for a full reference, visit https://qlty.sh/d/qlty-toml +config_version = "0" + +exclude_patterns = [ + "*_min.*", + "*-min.*", + "*.min.*", + "**/.yarn/**", + "**/*.d.ts", + "**/assets/**", + "**/bower_components/**", + "**/build/**", + "**/cache/**", + "**/config/**", + "**/db/**", + "**/deps/**", + "**/dist/**", + "**/extern/**", + "**/external/**", + "**/generated/**", + "**/Godeps/**", + "**/gradlew/**", + "**/mvnw/**", + "**/node_modules/**", + "**/protos/**", + "**/seed/**", + "**/target/**", + "**/templates/**", + "**/testdata/**", + "**/vendor/**", +] + +test_patterns = [ + "**/test/**", + "**/spec/**", + "**/*.test.*", + "**/*.spec.*", + "**/*_test.*", + "**/*_spec.*", + "**/test_*.*", + "**/spec_*.*", +] + +[smells] +mode = "comment" + +[[source]] +name = "default" +default = true + + +[[plugin]] +name = "actionlint" + +[[plugin]] +name = "eslint" +version = "9.39.4" +package_file = "package.json" +package_filters = ["eslint", "jest", "prettier"] + +[[plugin]] +name = "hadolint" + +[[plugin]] +name = "osv-scanner" + +[[plugin]] +name = "prettier" +version = "3.8.1" +package_file = "package.json" +package_filters = ["prettier"] + +[[plugin]] +name = "radarlint-iac" +mode = "monitor" + +[[plugin]] +name = "ripgrep" +mode = "comment" + +[[plugin]] +name = "trufflehog" + +[[plugin]] +name = "zizmor" From 5e627ba872a40f28ccee659077c5f9651cb20434 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:12:33 +0200 Subject: [PATCH 03/10] fix lint --- README.md | 5 +- .../latest-config-formats.migration.ts | 93 ++++++++++--------- src/migration/tracked-migration-context.ts | 2 - 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index b311032..d5960d5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # Aam Digital admin utilities + A collection of simple operations to support server administration tasks of (multiple) hosted Aam Digital instances. (e.g. bulk search & replace in config files, statistics of entities across instances, ...) ## Usage + This is not secured with authentication and should be deployed with other security measures in place. Have a look at the instructions in the [ndb-setup repo](https://github.com/Aam-Digital/ndb-setup/blob/master/admin/README.md) for deployment and usage through an SSH tunnel. @@ -31,7 +33,8 @@ See [`src/migration/README.md`](src/migration/README.md) for developer documenta ## Development ### Setup to run locally + Also refer to the overall developer system guide: [aam-services/docs/developer](https://github.com/Aam-Digital/aam-services/blob/main/docs/developer/README.md). If facing issues with local self-signed certificates, you can set an environment variable `NODE_TLS_REJECT_UNAUTHORIZED=0` to disable ssl certificate validation. -Do _not_ do this anywhere except the local dev environment for simple testing. \ No newline at end of file +Do _not_ do this anywhere except the local dev environment for simple testing. diff --git a/src/migration/config-migration/latest-config-formats.migration.ts b/src/migration/config-migration/latest-config-formats.migration.ts index ad2a0c4..a3b9ec4 100644 --- a/src/migration/config-migration/latest-config-formats.migration.ts +++ b/src/migration/config-migration/latest-config-formats.migration.ts @@ -13,50 +13,51 @@ import { ndbCoreConfigMigrations } from './ndb-core-config-migrations'; */ export const latestConfigFormats: MigrationDefinition = { - id: 'latest-config-formats', - description: 'Transform any legacy config formats to their latest formats. Safe to re-run.', - - async run(ctx): Promise { - let config: any; - try { - config = await ctx.couchdb.get(CONFIG_DOC_PATH); - } catch { - return { - changed: false, - status: 'failed', - warnings: ['Config document not found'], - }; - } - - let newConfig: any = JSON.parse(JSON.stringify(config), (key, value) => { - for (const migration of ndbCoreConfigMigrations) { - value = migration(key, value); - } - return value; - }); - - ctx.validateJson(newConfig); - - const changed = JSON.stringify(config) !== JSON.stringify(newConfig); - - if (!changed) { - ctx.log.info(' No changes needed'); - return { - changed: false, - status: 'no-change', - }; - } - - ctx.log.info(' Config requires migration'); - if (ctx.dryRun) { - ctx.log.verbose(' ~ Config:CONFIG_ENTITY (migration would apply)'); - } - - await ctx.put(CONFIG_DOC_PATH, newConfig); - - return { - changed: true, - status: ctx.dryRun ? 'dry-run' : 'ok', - }; - }, + id: 'latest-config-formats', + description: + 'Transform any legacy config formats to their latest formats. Safe to re-run.', + + async run(ctx): Promise { + let config: any; + try { + config = await ctx.couchdb.get(CONFIG_DOC_PATH); + } catch { + return { + changed: false, + status: 'failed', + warnings: ['Config document not found'], + }; + } + + let newConfig: any = JSON.parse(JSON.stringify(config), (key, value) => { + for (const migration of ndbCoreConfigMigrations) { + value = migration(key, value); + } + return value; + }); + + ctx.validateJson(newConfig); + + const changed = JSON.stringify(config) !== JSON.stringify(newConfig); + + if (!changed) { + ctx.log.info(' No changes needed'); + return { + changed: false, + status: 'no-change', + }; + } + + ctx.log.info(' Config requires migration'); + if (ctx.dryRun) { + ctx.log.verbose(' ~ Config:CONFIG_ENTITY (migration would apply)'); + } + + await ctx.put(CONFIG_DOC_PATH, newConfig); + + return { + changed: true, + status: ctx.dryRun ? 'dry-run' : 'ok', + }; + }, }; diff --git a/src/migration/tracked-migration-context.ts b/src/migration/tracked-migration-context.ts index 5b788a5..7a515de 100644 --- a/src/migration/tracked-migration-context.ts +++ b/src/migration/tracked-migration-context.ts @@ -58,6 +58,4 @@ export class TrackedMigrationContext implements MigrationContext { throw e; } } - - } From 166dbafc4c2d0ad10137cf4f833b0418aa86172b Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:28:59 +0200 Subject: [PATCH 04/10] small improvements --- src/cli.ts | 6 ++++-- src/couchdb/couchdb.service.spec.ts | 4 ++-- src/couchdb/couchdb.service.ts | 2 +- src/migration/README.md | 2 +- .../ndb-core-config-migrations.ts | 2 +- .../testing/migration-idempotency.harness.ts | 16 ++++++++++++---- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index add249f..0eeadd7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -51,6 +51,7 @@ program .action((options) => withApp(async ({ credentialsService, couchdbService }) => { const orgs = resolveOrgs(credentialsService, options); + if (!orgs) return 2; const runner = new OrgRunner(couchdbService); const results = await runner.checkConnectivity(orgs); @@ -78,6 +79,7 @@ migrateCmd } const orgs = resolveOrgs(credentialsService, options); + if (!orgs) return 2; const runner = new OrgRunner(couchdbService); const logger = new ConsoleLogger(!!options.verbose); @@ -179,7 +181,7 @@ async function withApp( function resolveOrgs( credentialsService: CredentialsService, options: { org?: string; category?: string }, -): SystemCredentials[] { +): SystemCredentials[] | null { const orgs = OrgRunner.filterOrgs( credentialsService.getCredentials(), options, @@ -189,7 +191,7 @@ function resolveOrgs( ? `--org "${options.org}"` : `--category "${options.category}"`; console.error(`\nNo orgs matched ${filter}.\n`); - process.exit(2); + return null; } return orgs; } diff --git a/src/couchdb/couchdb.service.spec.ts b/src/couchdb/couchdb.service.spec.ts index 8fd31ce..b80c620 100644 --- a/src/couchdb/couchdb.service.spec.ts +++ b/src/couchdb/couchdb.service.spec.ts @@ -1,7 +1,7 @@ +import { HttpService } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { CouchdbService } from './couchdb.service'; -import { ConfigService } from '@nestjs/config'; -import { HttpService } from '@nestjs/axios'; describe('CouchdbService', () => { let service: CouchdbService; diff --git a/src/couchdb/couchdb.service.ts b/src/couchdb/couchdb.service.ts index a5c1072..e7ff690 100644 --- a/src/couchdb/couchdb.service.ts +++ b/src/couchdb/couchdb.service.ts @@ -1,6 +1,6 @@ +import { HttpService } from '@nestjs/axios'; import { Injectable } from '@nestjs/common'; import { catchError, firstValueFrom, map } from 'rxjs'; -import { HttpService } from '@nestjs/axios'; import { SystemCredentials } from '../credentials/credentials.service'; /** diff --git a/src/migration/README.md b/src/migration/README.md index 0cd6e3c..51b0291 100644 --- a/src/migration/README.md +++ b/src/migration/README.md @@ -160,7 +160,7 @@ all instances. ## Module structure -``` +```text src/migration/ ├── README.md ← you are here ├── migration-definition.ts ← MigrationContext / MigrationResult / MigrationDefinition interfaces diff --git a/src/migration/config-migration/ndb-core-config-migrations.ts b/src/migration/config-migration/ndb-core-config-migrations.ts index 61955c7..9e29151 100644 --- a/src/migration/config-migration/ndb-core-config-migrations.ts +++ b/src/migration/config-migration/ndb-core-config-migrations.ts @@ -176,7 +176,7 @@ const migrateEntitySchemaDefaultValue: ConfigMigration = ( return configPart; } - if (typeof configPart === 'object') { + if (typeof configPart === 'object' && configPart !== null) { return configPart; } diff --git a/src/migration/testing/migration-idempotency.harness.ts b/src/migration/testing/migration-idempotency.harness.ts index 054016f..88a49b1 100644 --- a/src/migration/testing/migration-idempotency.harness.ts +++ b/src/migration/testing/migration-idempotency.harness.ts @@ -42,7 +42,12 @@ export interface IdempotencyCheckResult { */ export function buildStubCouchdb(store: DocStore): Couchdb { function key(path: string, db?: string): string { - return `${db ?? 'app'}/${path.replace(/^\//, '')}`; + const normalized = path.replace(/^\//, ''); + if (db) { + return `${db}/${normalized}`; + } + // If the path already contains a db prefix (e.g. "app/Doc:ID"), use as-is + return normalized; } return { @@ -50,7 +55,9 @@ export function buildStubCouchdb(store: DocStore): Couchdb { const k = key(path, db); const val = store[k]; if (val === undefined) { - const err: any = new Error(`Not found: ${k}`); + const err = new Error(`Not found: ${k}`) as Error & { + response: { status: number }; + }; err.response = { status: 404 }; throw err; } @@ -125,8 +132,9 @@ function buildInMemoryContext( const validateJson = (value: unknown): void => { try { JSON.stringify(value); - } catch (e: any) { - throw new Error(`JSON validation failed: ${e.message}`); + } catch (e: unknown) { + const message = e instanceof Error ? e.message : String(e); + throw new Error(`JSON validation failed: ${message}`); } }; From 19856fc803e2a4fcb276276098ad724c752bdbb5 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:35:11 +0200 Subject: [PATCH 05/10] gitignore credentials file --- .gitignore | 3 +++ src/assets/credentials.json | 7 ------- 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 src/assets/credentials.json diff --git a/.gitignore b/.gitignore index 22f55ad..d6ff406 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +credentials.json +src/assets/credentials.json + # compiled output /dist /node_modules diff --git a/src/assets/credentials.json b/src/assets/credentials.json deleted file mode 100644 index b932ff1..0000000 --- a/src/assets/credentials.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "url": "aam.localhost", - "password": "docker", - "username": "admin" - } -] From 6de0a24f50b19a0354872368079af150b1f937c5 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:51:47 +0200 Subject: [PATCH 06/10] load credentials at runtime --- src/credentials/credentials.service.ts | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/credentials/credentials.service.ts b/src/credentials/credentials.service.ts index 9388ab6..c357ff1 100644 --- a/src/credentials/credentials.service.ts +++ b/src/credentials/credentials.service.ts @@ -1,6 +1,7 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import * as credentials from '../assets/credentials.json'; +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; type RawSystemCredential = { url?: string; @@ -12,12 +13,18 @@ type RawSystemCredential = { @Injectable() export class CredentialsService { + private readonly logger = new Logger(CredentialsService.name); readonly DEFAULT_DOMAIN: string = this.configService.get('DOMAIN'); constructor(private configService: ConfigService) {} getCredentials(): SystemCredentials[] { - return (credentials as RawSystemCredential[]).map((c) => ({ + const credentialsPath = this.resolveCredentialsPath(); + this.logger.log(`Loading credentials from ${credentialsPath}`); + const credentials: RawSystemCredential[] = JSON.parse( + readFileSync(credentialsPath, 'utf-8'), + ); + return credentials.map((c) => ({ url: c.url ?? c['name'] + '.' + this.DEFAULT_DOMAIN, name: c.name, password: c.password, @@ -25,6 +32,22 @@ export class CredentialsService { category: c.category?.trim() ?? '', })); } + + private resolveCredentialsPath(): string { + const candidates = [ + join(process.cwd(), 'credentials.json'), + join(process.cwd(), 'src', 'assets', 'credentials.json'), + join(__dirname, '..', 'assets', 'credentials.json'), + ]; + for (const path of candidates) { + if (existsSync(path)) { + return path; + } + } + throw new Error( + 'No credentials.json found. Looked in:\n' + candidates.join('\n'), + ); + } } export interface SystemCredentials { From e3ba70ed7ced967e039e347d58ab34991091cff2 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 15:56:22 +0200 Subject: [PATCH 07/10] add stacktrace to verbose logging --- src/cli.ts | 14 +++++++++----- src/migration/migration-output.ts | 6 ++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 0eeadd7..0790e20 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,15 +2,14 @@ import { NestFactory } from '@nestjs/core'; import { Command } from 'commander'; import * as readline from 'readline'; import { AppModule } from './app.module'; -import { OrgRunner } from './cli/org-runner'; import { printConnectivity } from './cli/org-output'; +import { OrgRunner } from './cli/org-runner'; import { CouchdbService } from './couchdb/couchdb.service'; import { CredentialsService, SystemCredentials, } from './credentials/credentials.service'; import { ConsoleLogger } from './migration/console-logger'; -import { TrackedMigrationContext } from './migration/tracked-migration-context'; import { failedMigrationResult } from './migration/migration-definition'; import { computeExitCode, @@ -19,6 +18,7 @@ import { printSummary, } from './migration/migration-output'; import { migrations } from './migration/migrations'; +import { TrackedMigrationContext } from './migration/tracked-migration-context'; // eslint-disable-next-line @typescript-eslint/no-require-imports const { version } = require('../package.json'); @@ -104,8 +104,12 @@ migrateCmd const result = await migration.run(ctx); return { result, writeStats: ctx.getWriteStats() }; } catch (e: any) { + const result = failedMigrationResult( + e.message ?? String(e), + ); + result.details = e.stack; return { - result: failedMigrationResult(e.message), + result, writeStats: ctx.getWriteStats(), }; } @@ -114,7 +118,7 @@ migrateCmd // Preview (always dry-run first) printBanner('PREVIEW', migration); const preview = await runMigration(true); - printOutcomes(preview, false); + printOutcomes(preview, false, !!options.verbose); printSummary(preview, unreachableCount); if (options.dryRun) { @@ -141,7 +145,7 @@ migrateCmd // Real run printBanner('RUNNING', migration); const real = await runMigration(false); - printOutcomes(real, true); + printOutcomes(real, true, !!options.verbose); printSummary(real, unreachableCount); return computeExitCode(real, unreachableCount); diff --git a/src/migration/migration-output.ts b/src/migration/migration-output.ts index 32ed7d4..f6db78f 100644 --- a/src/migration/migration-output.ts +++ b/src/migration/migration-output.ts @@ -19,6 +19,7 @@ export function printBanner( export function printOutcomes( outcomes: OrgOutcome[], showWriteStats: boolean, + verbose = false, ): void { for (const { org, @@ -40,6 +41,11 @@ export function printOutcomes( if (result.warnings?.length) { result.warnings.forEach((w) => console.log(` ! ${w}`)); } + if (verbose && result.details) { + String(result.details) + .split('\n') + .forEach((line) => console.log(` ${line}`)); + } } } From 197e7502b378c4d35c69b31ce5dc4600bc87ddc4 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 16:26:26 +0200 Subject: [PATCH 08/10] more detailed logging --- package-lock.json | 7 +++ package.json | 1 + src/cli.ts | 1 + .../latest-config-formats.migration.ts | 7 +-- .../ndb-core-config-migrations.ts | 14 +++-- src/migration/console-logger.ts | 8 +-- src/migration/tracked-migration-context.ts | 53 ++++++++++++++++++- 7 files changed, 77 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 66dda1b..0b2263d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "axios": "^1.14.0", "commander": "^14.0.3", "lodash": "^4.17.23", + "microdiff": "^1.5.0", "papaparse": "^5.5.3", "reflect-metadata": "^0.2.2", "rimraf": "^6.1.3", @@ -8358,6 +8359,12 @@ "node": ">= 0.6" } }, + "node_modules/microdiff": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/microdiff/-/microdiff-1.5.0.tgz", + "integrity": "sha512-Drq+/THMvDdzRYrK0oxJmOKiC24ayUV8ahrt8l3oRK51PWt6gdtrIGrlIH3pT/lFh1z93FbAcidtsHcWbnRz8Q==", + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", diff --git a/package.json b/package.json index 419616e..7363456 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "axios": "^1.14.0", "commander": "^14.0.3", "lodash": "^4.17.23", + "microdiff": "^1.5.0", "papaparse": "^5.5.3", "reflect-metadata": "^0.2.2", "rimraf": "^6.1.3", diff --git a/src/cli.ts b/src/cli.ts index 0790e20..2d7934f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -99,6 +99,7 @@ migrateCmd const runMigration = (dryRun: boolean) => runner.runForEach(reachable, async (couchdb, org) => { + console.log(`\n ${OrgRunner.orgLabel(org)}`); const ctx = new TrackedMigrationContext(couchdb, org, dryRun, logger); try { const result = await migration.run(ctx); diff --git a/src/migration/config-migration/latest-config-formats.migration.ts b/src/migration/config-migration/latest-config-formats.migration.ts index a3b9ec4..98afb9b 100644 --- a/src/migration/config-migration/latest-config-formats.migration.ts +++ b/src/migration/config-migration/latest-config-formats.migration.ts @@ -41,17 +41,14 @@ export const latestConfigFormats: MigrationDefinition = { const changed = JSON.stringify(config) !== JSON.stringify(newConfig); if (!changed) { - ctx.log.info(' No changes needed'); + ctx.log.info('No changes needed'); return { changed: false, status: 'no-change', }; } - ctx.log.info(' Config requires migration'); - if (ctx.dryRun) { - ctx.log.verbose(' ~ Config:CONFIG_ENTITY (migration would apply)'); - } + ctx.log.info('Config requires migration'); await ctx.put(CONFIG_DOC_PATH, newConfig); diff --git a/src/migration/config-migration/ndb-core-config-migrations.ts b/src/migration/config-migration/ndb-core-config-migrations.ts index 9e29151..21c7053 100644 --- a/src/migration/config-migration/ndb-core-config-migrations.ts +++ b/src/migration/config-migration/ndb-core-config-migrations.ts @@ -16,7 +16,9 @@ export type ConfigMigration = (key: string, configPart: any) => any; * containing id directly. */ const migrateEntityAttributesWithId: ConfigMigration = (key, configPart) => { - if (!(key.startsWith('entity') && Array.isArray(configPart.attributes))) { + if ( + !(key.startsWith('entity') && Array.isArray(configPart?.attributes)) + ) { return configPart; } @@ -93,7 +95,7 @@ const migrateFormFieldConfigView2ViewComponent: ConfigMigration = ( }; const migrateMenuItemConfig: ConfigMigration = (key, configPart) => { - if (key !== 'navigationMenu') { + if (key !== 'navigationMenu' || typeof configPart !== 'object' || configPart === null) { return configPart; } @@ -123,7 +125,7 @@ const migrateEntityDetailsInputEntityType: ConfigMigration = ( key, configPart, ) => { - if (key !== 'config') { + if (key !== 'config' || typeof configPart !== 'object' || configPart === null) { return configPart; } @@ -144,7 +146,11 @@ const migrateEntityArrayDatatype: ConfigMigration = (key, configPart) => { return 'DisplayEntity'; } - if (!Object.prototype.hasOwnProperty.call(configPart, 'dataType')) { + if ( + typeof configPart !== 'object' || + configPart === null || + !Object.prototype.hasOwnProperty.call(configPart, 'dataType') + ) { return configPart; } diff --git a/src/migration/console-logger.ts b/src/migration/console-logger.ts index 4214d76..37251e4 100644 --- a/src/migration/console-logger.ts +++ b/src/migration/console-logger.ts @@ -8,18 +8,18 @@ export class ConsoleLogger implements MigrationLogger { constructor(private verboseEnabled: boolean) {} info(msg: string): void { - console.log(msg); + console.log(` ${msg}`); } warn(msg: string): void { - console.warn(` WARN ${msg}`); + console.warn(` ! ${msg}`); } error(msg: string): void { - console.error(` ERROR ${msg}`); + console.error(` ✗ ${msg}`); } verbose(msg: string): void { - if (this.verboseEnabled) console.log(msg); + if (this.verboseEnabled) console.log(` ${msg}`); } } diff --git a/src/migration/tracked-migration-context.ts b/src/migration/tracked-migration-context.ts index 7a515de..3a5e83b 100644 --- a/src/migration/tracked-migration-context.ts +++ b/src/migration/tracked-migration-context.ts @@ -1,3 +1,4 @@ +import diff from 'microdiff'; import { Couchdb } from '../couchdb/couchdb.service'; import { SystemCredentials } from '../credentials/credentials.service'; import { MigrationContext, MigrationLogger } from './migration-definition'; @@ -9,6 +10,7 @@ import { MigrationContext, MigrationLogger } from './migration-definition'; * - in dry-run mode records intended writes without calling CouchDB * - in real execution calls CouchDB and inspects _bulk_docs per-doc responses * - tracks intended / succeeded / failed write counts via getWriteStats() + * - logs a diff of changes for every put */ export class TrackedMigrationContext implements MigrationContext { private intended = 0; @@ -46,8 +48,11 @@ export class TrackedMigrationContext implements MigrationContext { ): Promise { this.validateJson(data); this.intended++; + + await this.logDiff(path, data, db); + if (this.dryRun) { - this.log.verbose(` [PREVIEW] Would PUT ${path}`); + this.log.verbose(`[PREVIEW] Would PUT ${path}`); return; } try { @@ -58,4 +63,50 @@ export class TrackedMigrationContext implements MigrationContext { throw e; } } + + private async logDiff( + path: string, + newData: unknown, + db?: string, + ): Promise { + let oldData: unknown; + try { + oldData = await this.couchdb.get(path, db); + } catch { + this.log.verbose(`+ ${path} (new document)`); + return; + } + + if ( + typeof oldData !== 'object' || + oldData === null || + typeof newData !== 'object' || + newData === null + ) { + return; + } + + const changes = diff(oldData as Record, newData as Record, { cyclesFix: false }); + if (changes.length === 0) return; + + for (const change of changes) { + const pathStr = change.path.join('.'); + switch (change.type) { + case 'CREATE': + this.log.verbose(` + ${pathStr}: ${truncate(JSON.stringify(change.value))}`); + break; + case 'REMOVE': + this.log.verbose(` - ${pathStr}: ${truncate(JSON.stringify(change.oldValue))}`); + break; + case 'CHANGE': + this.log.verbose(` ~ ${pathStr}: ${truncate(JSON.stringify(change.oldValue))} → ${truncate(JSON.stringify(change.value))}`); + break; + } + } + } +} + +function truncate(str: string | undefined, max = 200): string { + if (!str) return '(undefined)'; + return str.length > max ? str.slice(0, max) + '…' : str; } From cad577c012a1244b8e8356f77955e5a2c7686b0d Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 16:38:31 +0200 Subject: [PATCH 09/10] qlty fmt --- src/cli.ts | 4 +--- .../ndb-core-config-migrations.ts | 16 +++++++++++----- src/migration/tracked-migration-context.ts | 18 ++++++++++++++---- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 2d7934f..b1133e3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -105,9 +105,7 @@ migrateCmd const result = await migration.run(ctx); return { result, writeStats: ctx.getWriteStats() }; } catch (e: any) { - const result = failedMigrationResult( - e.message ?? String(e), - ); + const result = failedMigrationResult(e.message ?? String(e)); result.details = e.stack; return { result, diff --git a/src/migration/config-migration/ndb-core-config-migrations.ts b/src/migration/config-migration/ndb-core-config-migrations.ts index 21c7053..c929e8f 100644 --- a/src/migration/config-migration/ndb-core-config-migrations.ts +++ b/src/migration/config-migration/ndb-core-config-migrations.ts @@ -16,9 +16,7 @@ export type ConfigMigration = (key: string, configPart: any) => any; * containing id directly. */ const migrateEntityAttributesWithId: ConfigMigration = (key, configPart) => { - if ( - !(key.startsWith('entity') && Array.isArray(configPart?.attributes)) - ) { + if (!(key.startsWith('entity') && Array.isArray(configPart?.attributes))) { return configPart; } @@ -95,7 +93,11 @@ const migrateFormFieldConfigView2ViewComponent: ConfigMigration = ( }; const migrateMenuItemConfig: ConfigMigration = (key, configPart) => { - if (key !== 'navigationMenu' || typeof configPart !== 'object' || configPart === null) { + if ( + key !== 'navigationMenu' || + typeof configPart !== 'object' || + configPart === null + ) { return configPart; } @@ -125,7 +127,11 @@ const migrateEntityDetailsInputEntityType: ConfigMigration = ( key, configPart, ) => { - if (key !== 'config' || typeof configPart !== 'object' || configPart === null) { + if ( + key !== 'config' || + typeof configPart !== 'object' || + configPart === null + ) { return configPart; } diff --git a/src/migration/tracked-migration-context.ts b/src/migration/tracked-migration-context.ts index 3a5e83b..33baa5d 100644 --- a/src/migration/tracked-migration-context.ts +++ b/src/migration/tracked-migration-context.ts @@ -86,20 +86,30 @@ export class TrackedMigrationContext implements MigrationContext { return; } - const changes = diff(oldData as Record, newData as Record, { cyclesFix: false }); + const changes = diff( + oldData as Record, + newData as Record, + { cyclesFix: false }, + ); if (changes.length === 0) return; for (const change of changes) { const pathStr = change.path.join('.'); switch (change.type) { case 'CREATE': - this.log.verbose(` + ${pathStr}: ${truncate(JSON.stringify(change.value))}`); + this.log.verbose( + ` + ${pathStr}: ${truncate(JSON.stringify(change.value))}`, + ); break; case 'REMOVE': - this.log.verbose(` - ${pathStr}: ${truncate(JSON.stringify(change.oldValue))}`); + this.log.verbose( + ` - ${pathStr}: ${truncate(JSON.stringify(change.oldValue))}`, + ); break; case 'CHANGE': - this.log.verbose(` ~ ${pathStr}: ${truncate(JSON.stringify(change.oldValue))} → ${truncate(JSON.stringify(change.value))}`); + this.log.verbose( + ` ~ ${pathStr}: ${truncate(JSON.stringify(change.oldValue))} → ${truncate(JSON.stringify(change.value))}`, + ); break; } } From 7c997368b0975c65721d3b48969adc060bb490b7 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 12 May 2026 16:44:33 +0200 Subject: [PATCH 10/10] further improvements from pr comments --- src/credentials/credentials.service.ts | 15 ++++++--- .../latest-config-formats.migration.ts | 33 +++++++++++-------- .../testing/migration-idempotency.harness.ts | 5 ++- src/migration/tracked-migration-context.ts | 7 ++-- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/credentials/credentials.service.ts b/src/credentials/credentials.service.ts index c357ff1..237ccf2 100644 --- a/src/credentials/credentials.service.ts +++ b/src/credentials/credentials.service.ts @@ -21,11 +21,18 @@ export class CredentialsService { getCredentials(): SystemCredentials[] { const credentialsPath = this.resolveCredentialsPath(); this.logger.log(`Loading credentials from ${credentialsPath}`); - const credentials: RawSystemCredential[] = JSON.parse( - readFileSync(credentialsPath, 'utf-8'), - ); + let credentials: RawSystemCredential[]; + try { + const fileContent = readFileSync(credentialsPath, 'utf-8'); + credentials = JSON.parse(fileContent); + } catch (e: unknown) { + const errorMessage = e instanceof Error ? e.message : String(e); + throw new Error( + `Failed to load credentials from ${credentialsPath}: ${errorMessage}`, + ); + } return credentials.map((c) => ({ - url: c.url ?? c['name'] + '.' + this.DEFAULT_DOMAIN, + url: c.url ?? c.name + '.' + this.DEFAULT_DOMAIN, name: c.name, password: c.password, username: c.username, diff --git a/src/migration/config-migration/latest-config-formats.migration.ts b/src/migration/config-migration/latest-config-formats.migration.ts index 98afb9b..9984f6c 100644 --- a/src/migration/config-migration/latest-config-formats.migration.ts +++ b/src/migration/config-migration/latest-config-formats.migration.ts @@ -18,23 +18,30 @@ export const latestConfigFormats: MigrationDefinition = { 'Transform any legacy config formats to their latest formats. Safe to re-run.', async run(ctx): Promise { - let config: any; + let config: unknown; try { config = await ctx.couchdb.get(CONFIG_DOC_PATH); - } catch { - return { - changed: false, - status: 'failed', - warnings: ['Config document not found'], - }; + } catch (error: unknown) { + if ((error as { status?: number }).status === 404) { + return { + changed: false, + status: 'failed', + warnings: ['Config document not found'], + }; + } + ctx.log.error(`Failed to load config document: ${error}`); + throw error; } - let newConfig: any = JSON.parse(JSON.stringify(config), (key, value) => { - for (const migration of ndbCoreConfigMigrations) { - value = migration(key, value); - } - return value; - }); + let newConfig: unknown = JSON.parse( + JSON.stringify(config), + (key, value) => { + for (const migration of ndbCoreConfigMigrations) { + value = migration(key, value); + } + return value; + }, + ); ctx.validateJson(newConfig); diff --git a/src/migration/testing/migration-idempotency.harness.ts b/src/migration/testing/migration-idempotency.harness.ts index 88a49b1..8382394 100644 --- a/src/migration/testing/migration-idempotency.harness.ts +++ b/src/migration/testing/migration-idempotency.harness.ts @@ -41,6 +41,7 @@ export interface IdempotencyCheckResult { * - find(query, db?) → naive full-store scan (docs array, no index) */ export function buildStubCouchdb(store: DocStore): Couchdb { + let idCounter = 0; function key(path: string, db?: string): string { const normalized = path.replace(/^\//, ''); if (db) { @@ -56,8 +57,10 @@ export function buildStubCouchdb(store: DocStore): Couchdb { const val = store[k]; if (val === undefined) { const err = new Error(`Not found: ${k}`) as Error & { + status: number; response: { status: number }; }; + err.status = 404; err.response = { status: 404 }; throw err; } @@ -71,7 +74,7 @@ export function buildStubCouchdb(store: DocStore): Couchdb { putAll: jest.fn(async (docs: any[], db?: string) => { const results: { ok: boolean; id: string }[] = []; for (const doc of docs) { - const id: string = doc._id ?? doc.id ?? String(Math.random()); + const id: string = doc._id ?? doc.id ?? `generated-${idCounter++}`; store[key(id, db)] = JSON.parse(JSON.stringify(doc)); results.push({ ok: true, id }); } diff --git a/src/migration/tracked-migration-context.ts b/src/migration/tracked-migration-context.ts index 33baa5d..d84700a 100644 --- a/src/migration/tracked-migration-context.ts +++ b/src/migration/tracked-migration-context.ts @@ -27,8 +27,9 @@ export class TrackedMigrationContext implements MigrationContext { validateJson(value: unknown): void { try { JSON.stringify(value); - } catch (e: any) { - throw new Error(`JSON validation failed: ${e.message}`); + } catch (e: unknown) { + const message = e instanceof Error ? e.message : String(e); + throw new Error(`JSON validation failed: ${message}`); } } @@ -58,7 +59,7 @@ export class TrackedMigrationContext implements MigrationContext { try { await this.couchdb.put(path, data, db, headers); this.succeeded++; - } catch (e) { + } catch (e: unknown) { this.failed++; throw e; }