From a5ba8b89f45498f2c4bfe1f6b1a04700e369dec1 Mon Sep 17 00:00:00 2001 From: Jonathan Loh Date: Sat, 20 Jun 2026 22:17:25 +0100 Subject: [PATCH 1/4] Add demand allocation scraper --- MAINTENANCE.md | 2 + pnpm-lock.yaml | 71 +- scrapers/README.md | 1 + scrapers/demand-allocation-scraper/.gitignore | 5 + scrapers/demand-allocation-scraper/README.md | 95 +++ .../archive/README.md | 16 + .../oxlint.config.mjs | 34 + .../demand-allocation-scraper/package.json | 30 + .../scripts/coursereg_pdf_to_csv.py | 58 ++ .../scripts/import_courserekt_pdfs.py | 95 +++ .../demand-allocation-scraper/src/index.ts | 100 +++ .../src/scraper.test.ts | 623 ++++++++++++++++ .../demand-allocation-scraper/src/scraper.ts | 701 ++++++++++++++++++ .../demand-allocation-scraper/src/types.ts | 29 + .../demand-allocation-scraper/tsconfig.json | 16 + .../vitest.config.ts | 13 + 16 files changed, 1863 insertions(+), 26 deletions(-) create mode 100644 scrapers/demand-allocation-scraper/.gitignore create mode 100644 scrapers/demand-allocation-scraper/README.md create mode 100644 scrapers/demand-allocation-scraper/archive/README.md create mode 100644 scrapers/demand-allocation-scraper/oxlint.config.mjs create mode 100644 scrapers/demand-allocation-scraper/package.json create mode 100755 scrapers/demand-allocation-scraper/scripts/coursereg_pdf_to_csv.py create mode 100755 scrapers/demand-allocation-scraper/scripts/import_courserekt_pdfs.py create mode 100644 scrapers/demand-allocation-scraper/src/index.ts create mode 100644 scrapers/demand-allocation-scraper/src/scraper.test.ts create mode 100644 scrapers/demand-allocation-scraper/src/scraper.ts create mode 100644 scrapers/demand-allocation-scraper/src/types.ts create mode 100644 scrapers/demand-allocation-scraper/tsconfig.json create mode 100644 scrapers/demand-allocation-scraper/vitest.config.ts diff --git a/MAINTENANCE.md b/MAINTENANCE.md index f2b2259233..0609d150f4 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -39,6 +39,8 @@ Reference PRs: [PR #3286](https://github.com/nusmodifications/nusmods/pull/3286) - [ ] In `app-config.json`, add semester to `examAvailability` to indicate exam information is available for the semester - [ ] Update the ModReg schedule in `website/src/data/modreg-schedule.json`, and make sure the correct version is pointed to in `website/src/config/index.ts` - Reference PR: [PR #2764](https://github.com/nusmodifications/nusmods/pull/2764) +- [ ] After each CourseReg round is released, manually upload that round's vacancy, UG demand allocation, and GD demand allocation PDFs under `scrapers/demand-allocation-scraper/archive/pdfs//semesters//`, then run `pnpm dev --round --pdfDir archive/pdfs//semesters/` from `scrapers/demand-allocation-scraper` + - Verify that the PDFs contain released data, not "Information not available yet", and that `scrapers/nus-v2/data//semesters//courseRegHistory.json` is generated with non-empty module history for the academic-year archive ## CPEx diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61340b8568..112a153df2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -262,6 +262,39 @@ importers: specifier: 'catalog:' version: 4.1.0(@types/node@22.19.19)(jsdom@24.1.3)(vite@7.3.1(@types/node@22.19.19)(jiti@1.21.7)(sass@1.83.1)(terser@5.46.0)(yaml@2.8.2)) + scrapers/demand-allocation-scraper: + devDependencies: + '@nkzw/eslint-plugin': + specifier: 'catalog:' + version: 2.0.0(eslint@8.57.1) + '@nkzw/oxlint-config': + specifier: 'catalog:' + version: 1.0.1(@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(oxlint@1.55.0)(typescript@5.9.3) + '@types/node': + specifier: 22.19.19 + version: 22.19.19 + '@vitest/coverage-v8': + specifier: 'catalog:' + version: 4.1.0(vitest@4.1.0(@types/node@22.19.19)(jsdom@24.1.3)(vite@7.3.1(@types/node@22.19.19)(jiti@1.21.7)(sass@1.83.1)(terser@5.46.0)(yaml@2.8.2))) + eslint-plugin-no-only-tests: + specifier: 'catalog:' + version: 3.3.0 + eslint-plugin-perfectionist: + specifier: 'catalog:' + version: 5.6.0(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-unused-imports: + specifier: 'catalog:' + version: 4.4.1(@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1) + oxlint: + specifier: 'catalog:' + version: 1.55.0 + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.0(@types/node@22.19.19)(jsdom@24.1.3)(vite@7.3.1(@types/node@22.19.19)(jiti@1.21.7)(sass@1.83.1)(terser@5.46.0)(yaml@2.8.2)) + scrapers/nus-v2: dependencies: '@elastic/elasticsearch': @@ -9630,10 +9663,6 @@ packages: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} - tinyglobby@0.2.15: - resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} - engines: {node: '>=12.0.0'} - tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} @@ -9746,6 +9775,7 @@ packages: tsconfck@3.1.6: resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==} engines: {node: ^18 || >=20} + deprecated: unmaintained hasBin: true peerDependencies: typescript: ^5.0.0 @@ -10721,7 +10751,6 @@ snapshots: '@babel/parser@7.29.3': dependencies: '@babel/types': 7.29.0 - optional: true '@babel/plugin-bugfix-firefox-class-in-computed-class-key@7.28.5(@babel/core@7.26.0)': dependencies: @@ -12151,7 +12180,7 @@ snapshots: detect-libc: 2.1.2 is-glob: 4.0.3 node-addon-api: 7.1.1 - picomatch: 4.0.3 + picomatch: 4.0.4 optionalDependencies: '@parcel/watcher-android-arm64': 2.5.6 '@parcel/watcher-darwin-arm64': 2.5.6 @@ -13200,8 +13229,8 @@ snapshots: '@typescript-eslint/visitor-keys': 8.56.1 debug: 4.4.3 minimatch: 10.2.4 - semver: 7.7.4 - tinyglobby: 0.2.15 + semver: 7.8.1 + tinyglobby: 0.2.16 ts-api-utils: 2.4.0(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: @@ -13313,7 +13342,7 @@ snapshots: '@vue/compiler-core@3.5.29': dependencies: - '@babel/parser': 7.29.0 + '@babel/parser': 7.29.3 '@vue/shared': 3.5.29 entities: 7.0.1 estree-walker: 2.0.2 @@ -15105,7 +15134,7 @@ snapshots: '@one-ini/wasm': 0.1.1 commander: 10.0.1 minimatch: 9.0.9 - semver: 7.7.4 + semver: 7.8.1 ee-first@1.1.1: {} @@ -15606,10 +15635,6 @@ snapshots: dependencies: pend: 1.2.0 - fdir@6.5.0(picomatch@4.0.3): - optionalDependencies: - picomatch: 4.0.3 - fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -17193,7 +17218,7 @@ snapshots: magicast@0.5.2: dependencies: - '@babel/parser': 7.29.0 + '@babel/parser': 7.29.3 '@babel/types': 7.29.0 source-map-js: 1.2.1 @@ -17208,7 +17233,7 @@ snapshots: make-dir@4.0.0: dependencies: - semver: 7.7.4 + semver: 7.8.1 make-error@1.3.6: {} @@ -19717,7 +19742,7 @@ snapshots: rollup-plugin-visualizer@5.14.0(rollup@2.80.0): dependencies: open: 8.4.2 - picomatch: 4.0.3 + picomatch: 4.0.4 source-map: 0.7.6 yargs: 17.7.2 optionalDependencies: @@ -19920,8 +19945,7 @@ snapshots: semver@7.7.4: {} - semver@7.8.1: - optional: true + semver@7.8.1: {} send@0.19.2: dependencies: @@ -20664,11 +20688,6 @@ snapshots: tinyexec@1.0.2: {} - tinyglobby@0.2.15: - dependencies: - fdir: 6.5.0(picomatch@4.0.3) - picomatch: 4.0.3 - tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) @@ -21160,11 +21179,11 @@ snapshots: magic-string: 0.30.21 obug: 2.1.1 pathe: 2.0.3 - picomatch: 4.0.3 + picomatch: 4.0.4 std-env: 4.0.0 tinybench: 2.9.0 tinyexec: 1.0.2 - tinyglobby: 0.2.15 + tinyglobby: 0.2.16 tinyrainbow: 3.0.3 vite: 7.3.1(@types/node@22.19.19)(jiti@1.21.7)(sass@1.83.1)(terser@5.46.0)(yaml@2.8.2) why-is-node-running: 2.3.0 diff --git a/scrapers/README.md b/scrapers/README.md index 9a123ca013..7e27364d77 100644 --- a/scrapers/README.md +++ b/scrapers/README.md @@ -11,6 +11,7 @@ universities. However, we have only implemented an NUS scraper. More details on can be found in its folder. 1. [NUS Scraper](nus-v2) +2. [Demand Allocation Scraper](demand-allocation-scraper) If you are from another university and would like to implement a scraper for your uni, feel free to file an issue or just contact us! We already have some diff --git a/scrapers/demand-allocation-scraper/.gitignore b/scrapers/demand-allocation-scraper/.gitignore new file mode 100644 index 0000000000..bb5a3ba7b6 --- /dev/null +++ b/scrapers/demand-allocation-scraper/.gitignore @@ -0,0 +1,5 @@ +/archive/pdfs/ +/build/ +/coverage/ +/node_modules/ +*.log diff --git a/scrapers/demand-allocation-scraper/README.md b/scrapers/demand-allocation-scraper/README.md new file mode 100644 index 0000000000..54fcc85738 --- /dev/null +++ b/scrapers/demand-allocation-scraper/README.md @@ -0,0 +1,95 @@ +# Demand Allocation Scraper + +This scraper builds CourseReg demand allocation and vacancy history for the NUSMods API. + +The scraper is intentionally separate from `scrapers/nus-v2` because it works from CourseReg PDFs rather than the module/timetable APIs. It still writes generated API data into the NUS v2 data tree: + +```text +../nus-v2/data//semesters//courseRegHistory.json +``` + +## Setup + +Install workspace dependencies from the repository root: + +```sh +pnpm install +``` + +PDF conversion requires Python with `tabula-py` installed and Java available through `JAVA_HOME`. + +## Source PDF Archive + +NUS CourseReg PDFs are not reliably downloadable with a plain scraper request because NUS may return an HTML bot-protection page instead of PDF bytes. Treat the PDFs as manual/operator input unless NUS provides a supported download path. + +Store source PDFs under: + +```text +archive/pdfs//semesters// + vacancy/round_.pdf + ug/round_.pdf + gd/round_.pdf +``` + +For example: + +```text +archive/pdfs/2025-2026/semesters/2/vacancy/round_1.pdf +archive/pdfs/2025-2026/semesters/2/ug/round_1.pdf +archive/pdfs/2025-2026/semesters/2/gd/round_1.pdf +``` + +`archive/pdfs/` is gitignored. Generated JSON belongs under `../nus-v2/data/` and is the only CourseReg artifact intended for the public API sync. + +To import an existing CourseRekt checkout into this archive layout: + +```sh +python3 scripts/import_courserekt_pdfs.py /path/to/courserekt +``` + +## Run + +Build CourseReg history after manually staging one newly released round: + +```sh +JAVA_HOME=/path/to/java \ +pnpm dev 2 2025/2026 \ + --round 1 \ + --pdfDir archive/pdfs/2025-2026/semesters/2 +``` + +By default, output goes to: + +```text +../nus-v2/data/2025-2026/semesters/2/courseRegHistory.json +``` + +When `--round` is provided, the scraper updates that round in the existing +`courseRegHistory.json` if one exists, while preserving previously scraped rounds. +Classes that existed in earlier rounds but are missing from the newly scraped +round are recorded as `"notAvailable"` for that round. + +To rebuild all available rounds from staged PDFs, omit `--round`: + +```sh +JAVA_HOME=/path/to/java \ +pnpm dev 2 2025/2026 \ + --pdfDir archive/pdfs/2025-2026/semesters/2 +``` + +To test against a temporary output directory: + +```sh +JAVA_HOME=/path/to/java \ +pnpm dev 2 2025/2026 \ + --round 1 \ + --pdfDir archive/pdfs/2025-2026/semesters/2 \ + --inputDir /tmp/demand-allocation-csv \ + --outputDir /tmp/demand-allocation-output +``` + +## Data Semantics + +Class identity is keyed by course/module code plus class code. Titles and departments from the PDFs are not used as identifiers because they can vary or be repeated. + +Slot values in the generated JSON are non-negative integers, `"unlimited"`, or `"notAvailable"`. `0` is preserved as a real slot count; `"unlimited"` represents uncapped vacancies; `"notAvailable"` represents a class or student type that is not listed for that round/report. diff --git a/scrapers/demand-allocation-scraper/archive/README.md b/scrapers/demand-allocation-scraper/archive/README.md new file mode 100644 index 0000000000..49c1d4d7a0 --- /dev/null +++ b/scrapers/demand-allocation-scraper/archive/README.md @@ -0,0 +1,16 @@ +# Demand Allocation PDF Archive + +This directory is for operator-staged CourseReg source PDFs used by the demand +allocation scraper. + +PDFs should be stored under: + +```text +archive/pdfs//semesters// + vacancy/round_.pdf + ug/round_.pdf + gd/round_.pdf +``` + +The PDF files are ignored by git. Generated API data is written separately to +`../nus-v2/data//semesters//courseRegHistory.json`. diff --git a/scrapers/demand-allocation-scraper/oxlint.config.mjs b/scrapers/demand-allocation-scraper/oxlint.config.mjs new file mode 100644 index 0000000000..00d15883f3 --- /dev/null +++ b/scrapers/demand-allocation-scraper/oxlint.config.mjs @@ -0,0 +1,34 @@ +import nkzw from '@nkzw/oxlint-config'; +import { defineConfig } from 'oxlint'; + +const config = { ...nkzw }; +config.jsPlugins = config.jsPlugins?.filter( + (p) => + p !== '@nkzw/eslint-plugin' && + p !== 'eslint-plugin-unused-imports' && + !(typeof p === 'object' && p.name === 'react-hooks-js') && + p !== 'eslint-plugin-react-hooks', +); +config.rules = Object.fromEntries( + Object.entries(config.rules ?? {}).filter( + ([key]) => + !key.startsWith('@nkzw/') && + !key.startsWith('@typescript-eslint/') && + !key.startsWith('react-hooks-js/') && + !key.startsWith('react-hooks/') && + !key.startsWith('unused-imports/') && + key !== '@typescript-eslint/no-unused-vars', + ), +); + +export default defineConfig({ + extends: [config], + rules: { + 'import-x/no-namespace': 'off', + 'no-console': 'off', + 'perfectionist/sort-object-types': 'off', + 'perfectionist/sort-objects': 'off', + 'unicorn/prefer-string-replace-all': 'off', + 'unicorn/prefer-top-level-await': 'off', + }, +}); diff --git a/scrapers/demand-allocation-scraper/package.json b/scrapers/demand-allocation-scraper/package.json new file mode 100644 index 0000000000..0462e5ef32 --- /dev/null +++ b/scrapers/demand-allocation-scraper/package.json @@ -0,0 +1,30 @@ +{ + "name": "nusmods-demand-allocation-scraper", + "version": "1.0.0", + "description": "NUSMods scraper for CourseReg demand allocation and vacancy history", + "license": "MIT", + "repository": "https://github.com/nusmodifications/nusmods", + "main": "src/index.ts", + "scripts": { + "scrape": "node build/src/index.js", + "dev": "pnpm build && node build/src/index.js", + "build": "tsc", + "lint": "oxlint -c oxlint.config.mjs src", + "typecheck": "tsc --noEmit", + "test": "vitest run --coverage", + "test:watch": "vitest", + "check": "run-s lint typecheck test" + }, + "devDependencies": { + "@nkzw/eslint-plugin": "catalog:", + "@nkzw/oxlint-config": "catalog:", + "@types/node": "catalog:", + "@vitest/coverage-v8": "catalog:", + "eslint-plugin-no-only-tests": "catalog:", + "eslint-plugin-perfectionist": "catalog:", + "eslint-plugin-unused-imports": "catalog:", + "oxlint": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/scrapers/demand-allocation-scraper/scripts/coursereg_pdf_to_csv.py b/scrapers/demand-allocation-scraper/scripts/coursereg_pdf_to_csv.py new file mode 100755 index 0000000000..543f6b254e --- /dev/null +++ b/scrapers/demand-allocation-scraper/scripts/coursereg_pdf_to_csv.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import argparse +import os +import shutil +import tempfile +from pathlib import Path + +from tabula.io import convert_into_by_batch + + +def ensure_pdf(pdf_file: Path) -> None: + with pdf_file.open("rb") as file: + if file.read(4) != b"%PDF": + raise ValueError(f"{pdf_file} is not a PDF") + + +def convert_pdf_dir(pdf_dir: Path, csv_dir: Path) -> None: + pdf_files = sorted(pdf_dir.rglob("*.pdf")) + if not pdf_files: + raise ValueError(f"No PDF files found in {pdf_dir}") + + csv_dir.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="nusmods-demand-allocation-pdfs-") as tmp_dir_raw: + tmp_dir = Path(tmp_dir_raw) + + for pdf_file in pdf_files: + ensure_pdf(pdf_file) + relative_path = pdf_file.relative_to(pdf_dir) + encoded_name = str(relative_path).replace(os.sep, "||") + shutil.copy2(pdf_file, tmp_dir / encoded_name) + + convert_into_by_batch( + str(tmp_dir), + output_format="csv", + pages="all", + lattice=True, + silent=True, + ) + + for csv_file in sorted(tmp_dir.glob("*.csv")): + relative_csv = Path(*csv_file.name.split("||")).with_suffix(".csv") + output_file = csv_dir / relative_csv + output_file.parent.mkdir(parents=True, exist_ok=True) + shutil.move(csv_file, output_file) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Convert CourseReg PDFs to CSVs") + parser.add_argument("--pdf-dir", required=True, type=Path) + parser.add_argument("--csv-dir", required=True, type=Path) + args = parser.parse_args() + + convert_pdf_dir(args.pdf_dir, args.csv_dir) + + +if __name__ == "__main__": + main() diff --git a/scrapers/demand-allocation-scraper/scripts/import_courserekt_pdfs.py b/scrapers/demand-allocation-scraper/scripts/import_courserekt_pdfs.py new file mode 100755 index 0000000000..a80a3218a2 --- /dev/null +++ b/scrapers/demand-allocation-scraper/scripts/import_courserekt_pdfs.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +import argparse +import shutil +from pathlib import Path + + +def academic_year_path(short_year: str) -> str: + if len(short_year) != 4 or not short_year.isdigit(): + raise ValueError(f"Expected CourseRekt academic year like 2223, got {short_year}") + + start = 2000 + int(short_year[:2]) + end = 2000 + int(short_year[2:]) + return f"{start}-{end}" + + +def ensure_pdf(pdf_file: Path) -> None: + with pdf_file.open("rb") as file: + if file.read(4) != b"%PDF": + raise ValueError(f"{pdf_file} is not a PDF") + + +def copy_pdf(source_file: Path, output_file: Path) -> None: + ensure_pdf(source_file) + output_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_file, output_file) + + +def import_courserekt_pdfs(courserekt_dir: Path, output_dir: Path) -> int: + history_dir = courserekt_dir / "src" / "history" + coursereg_pdf_dir = history_dir / "coursereg_history" / "data" / "pdfs" + vacancy_pdf_dir = history_dir / "vacancy_history" / "data" / "pdfs" + + if not coursereg_pdf_dir.exists() or not vacancy_pdf_dir.exists(): + raise ValueError( + "CourseRekt PDF archive not found. Expected " + "src/history/{coursereg_history,vacancy_history}/data/pdfs under " + f"{courserekt_dir}" + ) + + copied = 0 + + for source_file in sorted(coursereg_pdf_dir.glob("*/*/*/round_*.pdf")): + short_year, semester, student_type, file_name = source_file.relative_to( + coursereg_pdf_dir + ).parts + copy_pdf( + source_file, + output_dir + / academic_year_path(short_year) + / "semesters" + / semester + / student_type + / file_name, + ) + copied += 1 + + for source_file in sorted(vacancy_pdf_dir.glob("*/*/round_*.pdf")): + short_year, semester, file_name = source_file.relative_to(vacancy_pdf_dir).parts + copy_pdf( + source_file, + output_dir + / academic_year_path(short_year) + / "semesters" + / semester + / "vacancy" + / file_name, + ) + copied += 1 + + return copied + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Import CourseRekt PDF archives into the demand allocation scraper archive" + ) + parser.add_argument( + "courserekt_dir", + type=Path, + help="path to a local CourseRekt checkout", + ) + parser.add_argument( + "--output-dir", + default=Path(__file__).resolve().parents[1] / "archive" / "pdfs", + type=Path, + help="output CourseReg PDF archive directory", + ) + args = parser.parse_args() + + copied = import_courserekt_pdfs(args.courserekt_dir, args.output_dir) + print(f"Imported {copied} PDFs into {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/scrapers/demand-allocation-scraper/src/index.ts b/scrapers/demand-allocation-scraper/src/index.ts new file mode 100644 index 0000000000..f94bf5d9c0 --- /dev/null +++ b/scrapers/demand-allocation-scraper/src/index.ts @@ -0,0 +1,100 @@ +import { parseArgs } from 'node:util'; + +import { DemandAllocationScraper, normalizeAcademicYearForPath } from './scraper'; +import type { CourseRegRound, Semester } from './types'; + +const usage = `Usage: + pnpm dev [options] + +Options: + --archiveDir Directory used to archive staged/downloaded PDFs + --download Try to download current PDFs before conversion + --inputDir Directory containing extracted CSVs + --outputDir API output academic-year directory + --pdfDir Directory containing staged PDFs + --python Python executable for PDF conversion + --round <0|1|2|3> CourseReg round to process + --help Show this help +`; + +const normalizeAcademicYear = (value: string) => { + if (value.length === 2) { + return `20${value}/20${Number(value) + 1}`; + } + if (value.length === 4) { + return `${value}/${Number(value) + 1}`; + } + return value.replace('-', '/'); +}; + +const parseSemester = (value: string): Semester => { + const semester = Number(value); + if (![1, 2, 3, 4].includes(semester)) { + throw new Error(`Invalid semester: ${value}`); + } + return semester as Semester; +}; + +const parseRound = (value: string | undefined): CourseRegRound | undefined => { + if (value === undefined) { + return undefined; + } + + const round = Number(value); + if (![0, 1, 2, 3].includes(round)) { + throw new Error(`Invalid CourseReg round: ${value}`); + } + return round as CourseRegRound; +}; + +async function main() { + const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + archiveDir: { type: 'string' }, + download: { type: 'boolean' }, + help: { short: 'h', type: 'boolean' }, + inputDir: { type: 'string' }, + outputDir: { type: 'string' }, + pdfDir: { type: 'string' }, + python: { default: 'python3', type: 'string' }, + round: { type: 'string' }, + }, + }); + + if (values.help) { + console.log(usage); + return; + } + + if (positionals.length < 2) { + throw new Error(`Semester and academic year are required.\n\n${usage}`); + } + + const semester = parseSemester(positionals[0]); + const academicYear = normalizeAcademicYear(positionals[1]); + const scraper = new DemandAllocationScraper(semester, academicYear, { + archiveDir: values.archiveDir, + download: values.download, + inputDir: values.inputDir, + outputDir: values.outputDir, + pdfDir: values.pdfDir, + python: values.python, + round: parseRound(values.round), + }); + + console.info( + JSON.stringify({ + academicYear, + academicYearPath: normalizeAcademicYearForPath(academicYear), + semester, + message: 'Running demand allocation scraper', + }), + ); + await scraper.run(); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scrapers/demand-allocation-scraper/src/scraper.test.ts b/scrapers/demand-allocation-scraper/src/scraper.test.ts new file mode 100644 index 0000000000..2cae2db18a --- /dev/null +++ b/scrapers/demand-allocation-scraper/src/scraper.test.ts @@ -0,0 +1,623 @@ +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { + DemandAllocationScraper, + getCourseRegRounds, + groupCourseRegHistory, + mergeCourseRegHistories, + mergeCourseRegRound, + parseDemandAllocationCsv, + parseVacancyCsv, +} from './scraper'; + +const makeTempDir = () => mkdtemp(path.join(os.tmpdir(), 'demand-allocation-scraper-test-')); + +const writeFixture = async (root: string, file: string, contents: string) => { + const outputPath = path.join(root, file); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, contents, { flag: 'w' }); +}; + +describe(getCourseRegRounds, () => { + test('includes Round 0 before AY24/25', () => { + expect(getCourseRegRounds('2023/2024')).toEqual([0, 1, 2, 3]); + }); + + test('omits Round 0 from AY24/25 onward', () => { + expect(getCourseRegRounds('2024/2025')).toEqual([1, 2, 3]); + expect(getCourseRegRounds('2025/2026')).toEqual([1, 2, 3]); + }); +}); + +describe(parseDemandAllocationCsv, () => { + test('removes headers, repairs split rows, and parses capacity values', () => { + const rows = + parseDemandAllocationCsv(`Course Host Faculty/School,Course Host Department,Course Code,Course Title,Course Class,Vacancy,Demand,Successful Allocations,Unsuccessful Allocations due to:,,,, +Faculty of Science,Statistics,ST2131,Probability,L1,200,180,180,0,0,0,0 +Yale-NUS College,Yale-NUS College,YSS4206C,"Topics in Psychology: The Pursuit of",E1,15,9,9,2,0,0,0,0 +,,,Happiness,,,,,,,, +FoL,FoL Dean's Office,LL4002V,Admiralty Law,E1,-,3,3,0,0,0,0,0 +Engineering,Common Engineering,EG1001,Engineering Practice,L1,,5,5,0,0,0,0,0 +Engineering,Common Engineering,EG1002,Engineering Practice II,L1,,0,0,0,0,0,0,0 +`); + + expect(rows).toEqual([ + { + allocatedSlots: 200, + classNo: 'L1', + moduleCode: 'ST2131', + registered: 180, + }, + { + allocatedSlots: 15, + classNo: 'E1', + moduleCode: 'YSS4206C', + registered: 9, + }, + { + allocatedSlots: 'unlimited', + classNo: 'E1', + moduleCode: 'LL4002V', + registered: 3, + }, + { + allocatedSlots: 'unlimited', + classNo: 'L1', + moduleCode: 'EG1001', + registered: 5, + }, + { + allocatedSlots: 'notAvailable', + classNo: 'L1', + moduleCode: 'EG1002', + registered: 0, + }, + ]); + }); +}); + +describe(parseVacancyCsv, () => { + test('deduplicates rows, trims class labels, and parses unavailable capacity', () => { + const rows = + parseVacancyCsv(`Faculty/School,Department,Course Code,Course Title,Course Class,UG,GD,DK,NG,CPE +NUS,NUS Enterprise Academy,BSN3701,Technological Innovation,Sectional Teaching - SA1 - 123,17,-1,-1,30,3 +NUS Business School,Strategy and Policy,BSN3701,Technological Innovation,Sectional Teaching - SA1 - 123,17,-1,-1,30,3 +Faculty of Science,Statistics and Data Science,ST2131,Probability,Lecture - L1,200,-,-1,-1,-1 +Faculty of Law,FoL Dean's Office,LL4002V,Admiralty Law,Seminar - E1,,3,-1,-1,-1 +`); + + expect(rows).toEqual([ + { + classNo: 'SA1', + forecastedSlots: { + GD: 'notAvailable', + UG: 17, + }, + moduleCode: 'BSN3701', + }, + { + classNo: 'L1', + forecastedSlots: { + GD: 'unlimited', + UG: 200, + }, + moduleCode: 'ST2131', + }, + { + classNo: 'E1', + forecastedSlots: { + GD: 3, + UG: 'notAvailable', + }, + moduleCode: 'LL4002V', + }, + ]); + }); +}); + +describe(mergeCourseRegRound, () => { + test('full joins demand and vacancy rows by module and class', () => { + const demandRows = + parseDemandAllocationCsv(`Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others +Computing,Computer Science,CS2030S,Programming Methodology II,L1,772,772,772,0,0,0,0,0 +Business,Accounting,ACC1701A,Accounting,SA1,40,35,35,0,0,0,0,0 +`); + const vacancyRows = parseVacancyCsv(`Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE +Computing,Computer Science,CS2030S,Programming Methodology II,L1,750,-1,-1,-1,-1 +Science,Statistics,ST2131,Probability,L1,200,-1,-1,-1,-1 +`); + + expect(mergeCourseRegRound(1, 'UG', demandRows, vacancyRows)).toEqual([ + { + classNo: 'SA1', + moduleCode: 'ACC1701A', + roundHistory: { + allocatedSlots: 40, + forecastedSlots: 'notAvailable', + registered: 35, + round: 1, + }, + studentType: 'UG', + }, + { + classNo: 'L1', + moduleCode: 'CS2030S', + roundHistory: { + allocatedSlots: 772, + forecastedSlots: 750, + registered: 772, + round: 1, + }, + studentType: 'UG', + }, + { + classNo: 'L1', + moduleCode: 'ST2131', + roundHistory: { + allocatedSlots: 'notAvailable', + forecastedSlots: 200, + registered: null, + round: 1, + }, + studentType: 'UG', + }, + ]); + }); + + test('ignores vacancy-only rows unavailable to the selected student type', () => { + const vacancyRows = parseVacancyCsv(`Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE +Computing,Computer Science,CS2030S,Programming Methodology II,L1,750,-1,-1,-1,-1 +`); + + expect(mergeCourseRegRound(1, 'GD', [], vacancyRows)).toEqual([]); + expect(mergeCourseRegRound(1, 'UG', [], vacancyRows)).toEqual([ + { + classNo: 'L1', + moduleCode: 'CS2030S', + roundHistory: { + allocatedSlots: 'notAvailable', + forecastedSlots: 750, + registered: null, + round: 1, + }, + studentType: 'UG', + }, + ]); + }); +}); + +describe(groupCourseRegHistory, () => { + test('groups round histories by module, student type, and class', () => { + const rows = [ + ...mergeCourseRegRound( + 1, + 'UG', + parseDemandAllocationCsv( + 'Faculty,Department,Code,Title,Class,Vacancy,Demand\nComputing,CS,CS2030S,Programming,L1,772,772', + ), + parseVacancyCsv( + 'Faculty,Department,Code,Title,Class,UG,GD\nComputing,CS,CS2030S,Programming,L1,750,-1', + ), + ), + ...mergeCourseRegRound( + 2, + 'UG', + parseDemandAllocationCsv( + 'Faculty,Department,Code,Title,Class,Vacancy,Demand\nComputing,CS,CS2030S,Programming,L1,772,760', + ), + parseVacancyCsv( + 'Faculty,Department,Code,Title,Class,UG,GD\nComputing,CS,CS2030S,Programming,L1,772,-1', + ), + ), + ]; + + expect(groupCourseRegHistory('2025/2026', 2, rows)).toEqual([ + { + acadYear: '2025/2026', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 772, + forecastedSlots: 750, + registered: 772, + round: 1, + }, + { + allocatedSlots: 772, + forecastedSlots: 772, + registered: 760, + round: 2, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ]); + }); + + test('matches CourseRekt known AY22/23 Sem 2 UG CS2030S history', () => { + const demandCsvs = [ + 'Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,700,693,693,0,0,0,0,0', + 'Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,31,52,31,0,21,0,0,0', + 'Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,7,18,7,0,11,0,0,0', + 'Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,2,6,0,0,0,0,0,6', + ]; + const vacancyCsvs = [ + 'Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,580,-1,-1,-1,-1', + 'Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,24,-1,-1,-1,-1', + 'Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,3,-1,-1,-1,-1', + 'Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE\nSchool of Computing,Computer Science,CS2030S,Programming Methodology II,L1,0,-1,-1,-1,-1', + ]; + const rows = demandCsvs.flatMap((demandCsv, round) => + mergeCourseRegRound( + round as 0 | 1 | 2 | 3, + 'UG', + parseDemandAllocationCsv(demandCsv), + parseVacancyCsv(vacancyCsvs[round]), + ), + ); + + expect(groupCourseRegHistory('2022/2023', 2, rows)).toEqual([ + { + acadYear: '2022/2023', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 700, + forecastedSlots: 580, + registered: 693, + round: 0, + }, + { + allocatedSlots: 31, + forecastedSlots: 24, + registered: 52, + round: 1, + }, + { + allocatedSlots: 7, + forecastedSlots: 3, + registered: 18, + round: 2, + }, + { + allocatedSlots: 2, + forecastedSlots: 0, + registered: 6, + round: 3, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ]); + }); +}); + +describe(mergeCourseRegHistories, () => { + test('replaces only the selected round and preserves existing rounds', () => { + expect( + mergeCourseRegHistories( + [ + { + acadYear: '2025/2026', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 700, + forecastedSlots: 700, + registered: 650, + round: 1, + }, + { + allocatedSlots: 50, + forecastedSlots: 50, + registered: 17, + round: 2, + }, + ], + studentType: 'UG', + }, + { + classNo: 'T1', + rounds: [ + { + allocatedSlots: 10, + forecastedSlots: 10, + registered: 8, + round: 1, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ], + [ + { + acadYear: '2025/2026', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 60, + forecastedSlots: 60, + registered: 20, + round: 2, + }, + ], + studentType: 'UG', + }, + { + classNo: 'L2', + rounds: [ + { + allocatedSlots: 25, + forecastedSlots: 25, + registered: 10, + round: 2, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ], + [2], + ), + ).toEqual([ + { + acadYear: '2025/2026', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 700, + forecastedSlots: 700, + registered: 650, + round: 1, + }, + { + allocatedSlots: 60, + forecastedSlots: 60, + registered: 20, + round: 2, + }, + ], + studentType: 'UG', + }, + { + classNo: 'L2', + rounds: [ + { + allocatedSlots: 25, + forecastedSlots: 25, + registered: 10, + round: 2, + }, + ], + studentType: 'UG', + }, + { + classNo: 'T1', + rounds: [ + { + allocatedSlots: 10, + forecastedSlots: 10, + registered: 8, + round: 1, + }, + { + allocatedSlots: 'notAvailable', + forecastedSlots: 'notAvailable', + registered: null, + round: 2, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ]); + }); +}); + +describe(DemandAllocationScraper, () => { + test('writes semester-level CourseReg history JSON from staged CSVs', async () => { + const inputDir = await makeTempDir(); + const outputDir = await makeTempDir(); + await writeFixture( + inputDir, + path.join('vacancy', 'round_1.csv'), + `Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE +Computing,Computer Science,CS2030S,Programming Methodology II,L1,750,-1,-1,-1,-1 +`, + ); + await writeFixture( + inputDir, + path.join('ug', 'round_1.csv'), + `Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others +Computing,Computer Science,CS2030S,Programming Methodology II,L1,772,772,772,0,0,0,0,0 +`, + ); + + const histories = await new DemandAllocationScraper(2, '2025/2026', { + inputDir, + outputDir, + round: 1, + }).run(); + const outputJson = JSON.parse( + await readFile(path.join(outputDir, 'semesters', '2', 'courseRegHistory.json'), 'utf8'), + ); + + expect(histories).toHaveLength(1); + expect(outputJson).toEqual(histories); + }); + + test('honors a constructor-level round option, including Round 0', async () => { + const inputDir = await makeTempDir(); + const outputDir = await makeTempDir(); + await writeFixture( + inputDir, + path.join('vacancy', 'round_0.csv'), + `Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE +Computing,Computer Science,CS2030S,Programming Methodology II,L1,580,-1,-1,-1,-1 +`, + ); + await writeFixture( + inputDir, + path.join('ug', 'round_0.csv'), + `Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others +Computing,Computer Science,CS2030S,Programming Methodology II,L1,700,693,693,0,0,0,0,0 +`, + ); + + const histories = await new DemandAllocationScraper(2, '2022/2023', { + inputDir, + outputDir, + round: 0, + }).run(); + + expect(histories[0].classes[0].rounds).toEqual([ + { + allocatedSlots: 700, + forecastedSlots: 580, + registered: 693, + round: 0, + }, + ]); + }); + + test('merges a single-round scrape into existing semester history JSON', async () => { + const inputDir = await makeTempDir(); + const outputDir = await makeTempDir(); + await writeFixture( + outputDir, + path.join('semesters', '2', 'courseRegHistory.json'), + JSON.stringify([ + { + acadYear: '2025/2026', + classes: [ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 700, + forecastedSlots: 700, + registered: 650, + round: 1, + }, + ], + studentType: 'UG', + }, + { + classNo: 'T1', + rounds: [ + { + allocatedSlots: 10, + forecastedSlots: 10, + registered: 8, + round: 1, + }, + ], + studentType: 'UG', + }, + ], + moduleCode: 'CS2030S', + semester: 2, + }, + ]), + ); + await writeFixture( + inputDir, + path.join('vacancy', 'round_2.csv'), + `Faculty,Department,Code,Title,Class,UG,GD,DK,NG,CPE +Computing,Computer Science,CS2030S,Programming Methodology II,L1,50,-1,-1,-1,-1 +`, + ); + await writeFixture( + inputDir, + path.join('ug', 'round_2.csv'), + `Faculty,Department,Code,Title,Class,Vacancy,Demand,Successful_Main,Successful_Reserve,Quota_Exceeded,Timetable_Clashes,Workload_Exceeded,Others +Computing,Computer Science,CS2030S,Programming Methodology II,L1,50,17,17,0,0,0,0,0 +`, + ); + + const histories = await new DemandAllocationScraper(2, '2025/2026', { + inputDir, + outputDir, + round: 2, + }).run(); + + expect(histories[0].classes).toEqual([ + { + classNo: 'L1', + rounds: [ + { + allocatedSlots: 700, + forecastedSlots: 700, + registered: 650, + round: 1, + }, + { + allocatedSlots: 50, + forecastedSlots: 50, + registered: 17, + round: 2, + }, + ], + studentType: 'UG', + }, + { + classNo: 'T1', + rounds: [ + { + allocatedSlots: 10, + forecastedSlots: 10, + registered: 8, + round: 1, + }, + { + allocatedSlots: 'notAvailable', + forecastedSlots: 'notAvailable', + registered: null, + round: 2, + }, + ], + studentType: 'UG', + }, + ]); + }); + + test('fails when no CourseReg histories can be extracted', async () => { + await expect( + new DemandAllocationScraper(2, '2025/2026', { + inputDir: await makeTempDir(), + outputDir: await makeTempDir(), + round: 1, + }).run(), + ).rejects.toThrow('No CourseReg history extracted'); + }); +}); diff --git a/scrapers/demand-allocation-scraper/src/scraper.ts b/scrapers/demand-allocation-scraper/src/scraper.ts new file mode 100644 index 0000000000..314d4a7305 --- /dev/null +++ b/scrapers/demand-allocation-scraper/src/scraper.ts @@ -0,0 +1,701 @@ +import { spawn } from 'node:child_process'; +import { readFile, mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import type { + CourseRegClassHistory, + CourseRegModuleSemesterHistory, + CourseRegRound, + CourseRegRoundHistory, + CourseRegSlotValue, + CourseRegStudentType, + ModuleCode, + Semester, +} from './types'; + +const COURSE_REG_ROUNDS_BEFORE_AY_2425: Array = [0, 1, 2, 3]; +const COURSE_REG_ROUNDS_AFTER_AY_2425: Array = [1, 2, 3]; +const STUDENT_TYPES: Array = ['UG', 'GD']; + +type CsvRow = Array; + +export type DemandAllocationScraperOptions = { + archiveDir?: string; + download?: boolean; + inputDir?: string; + outputDir?: string; + pdfDir?: string; + python?: string; + round?: CourseRegRound; +}; + +type DemandAllocationRow = { + allocatedSlots: CourseRegSlotValue; + classNo: string; + moduleCode: ModuleCode; + registered: number | null; +}; + +type VacancyRow = { + classNo: string; + forecastedSlots: Record; + moduleCode: ModuleCode; +}; + +type MergedClassRound = { + classNo: string; + moduleCode: ModuleCode; + roundHistory: CourseRegRoundHistory; + studentType: CourseRegStudentType; +}; + +type CourseClassKey = `${ModuleCode}:${string}`; + +const COURSE_REG_PDF_URLS = { + gd: (round: CourseRegRound) => + `https://www.nus.edu.sg/CourseReg/docs/DemandAllocationRptGD_R${round}.pdf`, + ug: (round: CourseRegRound) => + `https://www.nus.edu.sg/CourseReg/docs/DemandAllocationRptUG_R${round}.pdf`, + vacancy: (round: CourseRegRound) => + `https://www.nus.edu.sg/coursereg/docs/VacancyRpt_R${round}.pdf`, +}; + +export const normalizeAcademicYearForPath = (academicYear: string) => + academicYear.replace('/', '-'); + +export const getDefaultPdfArchiveRoot = () => path.resolve(process.cwd(), 'archive', 'pdfs'); + +export const getDefaultApiOutputRoot = (academicYear: string) => + path.resolve(process.cwd(), '..', 'nus-v2', 'data', normalizeAcademicYearForPath(academicYear)); + +export function getCourseRegRounds(academicYear: string): Array { + const yearStart = Number(academicYear.replace('/', '').replace('-', '').slice(0, 4)); + return yearStart <= 2023 ? COURSE_REG_ROUNDS_BEFORE_AY_2425 : COURSE_REG_ROUNDS_AFTER_AY_2425; +} + +export function parseCsv(text: string): Array { + const rows: Array = []; + let cell = ''; + let row: CsvRow = []; + let inQuotes = false; + + for (let i = 0; i < text.length; i += 1) { + const char = text[i]; + const nextChar = text[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + cell += '"'; + i += 1; + } else { + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + row.push(cell); + cell = ''; + } else if ((char === '\n' || char === '\r') && !inQuotes) { + if (char === '\r' && nextChar === '\n') { + i += 1; + } + row.push(cell); + rows.push(row); + cell = ''; + row = []; + } else { + cell += char; + } + } + + if (cell || row.length) { + row.push(cell); + rows.push(row); + } + + return rows; +} + +const cleanCell = (cell: string) => cell.split(/\s+/).join(' ').trim(); + +const cleanRow = (row: CsvRow) => row.map(cleanCell); + +const isBlankTailRow = (row: CsvRow, tailLength: number) => + row.length >= tailLength && row.slice(-tailLength).every((cell) => cleanCell(cell) === ''); + +const mergeOverflowRows = (rows: Array, tailLength: number): Array => { + const mergedRows: Array = []; + + for (let i = 0; i < rows.length; i += 1) { + const row = rows[i]; + const nextRow = rows[i + 1]; + + if (nextRow && isBlankTailRow(nextRow, tailLength)) { + mergedRows.push(row.map((cell, index) => cleanCell(`${cell} ${nextRow[index] ?? ''}`))); + i += 1; + } else { + mergedRows.push(row); + } + } + + return mergedRows; +}; + +const normalizedRowText = (row: CsvRow) => + row.map((cell) => cleanCell(cell).toLowerCase()).join('|'); + +const isDemandHeaderRow = (row: CsvRow) => { + const cells = row.map((cell) => cleanCell(cell).toLowerCase()); + const text = normalizedRowText(row); + return ( + ['code', 'course code', 'module code'].includes(cells[2]) || + ['class', 'course class', 'module class'].includes(cells[4]) || + text.includes('successful allocations') || + text.includes('quota exceeded') + ); +}; + +const isVacancyHeaderRow = (row: CsvRow) => { + const cells = row.map((cell) => cleanCell(cell).toLowerCase()); + const text = normalizedRowText(row); + return ( + ['code', 'course code', 'module code'].includes(cells[2]) || + ['class', 'course class', 'module class'].includes(cells[4]) || + text.includes('faculty/school') || + text === '|||||ug|gd|dk|ng|cpe' + ); +}; + +const getCourseClassKey = (moduleCode: ModuleCode, classNo: string): CourseClassKey => + `${moduleCode}:${classNo}`; + +const parseCount = (rawValue: string | undefined): number | null => { + const value = cleanCell(rawValue ?? ''); + if (!value || value === '-' || value.toLowerCase() === 'x') { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +}; + +const parseSlotValue = (rawValue: string | undefined): CourseRegSlotValue => { + const value = cleanCell(rawValue ?? '').toLowerCase(); + + if (!value || value === 'x' || value === 'na' || value === '-1') { + return 'notAvailable'; + } + if (value === '-') { + return 'unlimited'; + } + + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return 'notAvailable'; + } + if (parsed >= 2_147_483_647) { + return 'unlimited'; + } + return parsed; +}; + +const parseDemandSlotValue = ( + rawSlotValue: string | undefined, + rawDemandValue: string | undefined, +): CourseRegSlotValue => { + const slotValue = cleanCell(rawSlotValue ?? ''); + if (slotValue) { + return parseSlotValue(slotValue); + } + + const demand = parseCount(rawDemandValue); + return demand && demand > 0 ? 'unlimited' : 'notAvailable'; +}; + +const normalizeClassNo = (rawValue: string | undefined) => { + const classNo = cleanCell(rawValue ?? ''); + const parts = classNo.split(' - '); + return parts.length >= 2 ? parts[1] : classNo; +}; + +export function parseDemandAllocationCsv(text: string): Array { + const cleanedRows = mergeOverflowRows( + parseCsv(text) + .map(cleanRow) + .filter((row) => row.some(Boolean)) + .filter((row) => !isDemandHeaderRow(row)), + 8, + ); + + return cleanedRows + .filter((row) => row.length >= 7) + .map((row) => ({ + allocatedSlots: parseDemandSlotValue(row[5], row[6]), + classNo: normalizeClassNo(row[4]), + moduleCode: cleanCell(row[2]).toUpperCase(), + registered: parseCount(row[6]), + })) + .filter((row) => row.moduleCode && row.classNo); +} + +export function parseVacancyCsv(text: string): Array { + const rows = mergeOverflowRows( + parseCsv(text) + .map(cleanRow) + .filter((row) => row.some(Boolean)) + .filter((row) => !isVacancyHeaderRow(row)), + 5, + ); + const seenRows = new Set(); + const vacancyRows: Array = []; + + for (const row of rows) { + if (row.length < 7) { + continue; + } + + const moduleCode = cleanCell(row[2]).toUpperCase(); + const classNo = normalizeClassNo(row[4]); + const key = getCourseClassKey(moduleCode, classNo); + + if (!moduleCode || !classNo || seenRows.has(key)) { + continue; + } + + seenRows.add(key); + vacancyRows.push({ + classNo, + forecastedSlots: { + GD: parseSlotValue(row[6]), + UG: parseSlotValue(row[5]), + }, + moduleCode, + }); + } + + return vacancyRows; +} + +const mapByClass = (rows: Array) => + new Map(rows.map((row) => [getCourseClassKey(row.moduleCode, row.classNo), row])); + +export function mergeCourseRegRound( + round: CourseRegRound, + studentType: CourseRegStudentType, + demandRows: Array, + vacancyRows: Array, +): Array { + const demandByClass = mapByClass(demandRows); + const vacancyByClass = mapByClass( + vacancyRows.filter((row) => row.forecastedSlots[studentType] !== 'notAvailable'), + ); + const keys = Array.from(new Set([...demandByClass.keys(), ...vacancyByClass.keys()])).sort(); + + return keys.map((key) => { + const demandRow = demandByClass.get(key); + const vacancyRow = vacancyByClass.get(key); + const [moduleCode, classNo] = key.split(':'); + + return { + classNo, + moduleCode, + roundHistory: { + allocatedSlots: demandRow?.allocatedSlots ?? 'notAvailable', + forecastedSlots: vacancyRow?.forecastedSlots[studentType] ?? 'notAvailable', + registered: demandRow?.registered ?? null, + round, + }, + studentType, + }; + }); +} + +export function groupCourseRegHistory( + academicYear: string, + semester: Semester, + mergedRows: Array, +): Array { + const modules = new Map>(); + + for (const row of mergedRows) { + if (!modules.has(row.moduleCode)) { + modules.set(row.moduleCode, new Map()); + } + + const classes = modules.get(row.moduleCode)!; + const classKey = `${row.studentType}:${row.classNo}`; + if (!classes.has(classKey)) { + classes.set(classKey, { + classNo: row.classNo, + rounds: [], + studentType: row.studentType, + }); + } + + classes.get(classKey)!.rounds.push(row.roundHistory); + } + + return Array.from(modules.entries()) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([moduleCode, classes]) => ({ + acadYear: academicYear, + classes: Array.from(classes.values()) + .map((historyClass) => ({ + ...historyClass, + rounds: historyClass.rounds.sort((left, right) => left.round - right.round), + })) + .sort( + (left, right) => + left.studentType.localeCompare(right.studentType) || + left.classNo.localeCompare(right.classNo), + ), + moduleCode, + semester, + })); +} + +const sortClasses = (classes: Array) => + classes + .map((historyClass) => ({ + ...historyClass, + rounds: historyClass.rounds.sort((left, right) => left.round - right.round), + })) + .sort( + (left, right) => + left.studentType.localeCompare(right.studentType) || + left.classNo.localeCompare(right.classNo), + ); + +const getHistoryClassKey = (historyClass: CourseRegClassHistory) => + `${historyClass.studentType}:${historyClass.classNo}`; + +const notAvailableRound = (round: CourseRegRound): CourseRegRoundHistory => ({ + allocatedSlots: 'notAvailable', + forecastedSlots: 'notAvailable', + registered: null, + round, +}); + +export function mergeCourseRegHistories( + existingHistories: Array, + incomingHistories: Array, + replaceRounds: Array, +): Array { + const replaceRoundSet = new Set(replaceRounds); + const moduleMap = new Map(); + const incomingClassKeys = new Set(); + + for (const moduleHistory of existingHistories) { + moduleMap.set(moduleHistory.moduleCode, { + ...moduleHistory, + classes: moduleHistory.classes.map((historyClass) => ({ + ...historyClass, + rounds: historyClass.rounds.filter((round) => !replaceRoundSet.has(round.round)), + })), + }); + } + + for (const moduleHistory of incomingHistories) { + const existingModule = moduleMap.get(moduleHistory.moduleCode); + const classMap = new Map( + (existingModule?.classes ?? []).map((historyClass) => [ + getHistoryClassKey(historyClass), + historyClass, + ]), + ); + + for (const historyClass of moduleHistory.classes) { + const classKey = getHistoryClassKey(historyClass); + incomingClassKeys.add(`${moduleHistory.moduleCode}:${classKey}`); + classMap.set(classKey, { + ...historyClass, + rounds: [ + ...(classMap.get(classKey)?.rounds ?? []), + ...historyClass.rounds.filter((round) => replaceRoundSet.has(round.round)), + ], + }); + } + + moduleMap.set(moduleHistory.moduleCode, { + acadYear: moduleHistory.acadYear, + classes: Array.from(classMap.values()), + moduleCode: moduleHistory.moduleCode, + semester: moduleHistory.semester, + }); + } + + for (const [moduleCode, moduleHistory] of moduleMap) { + moduleHistory.classes = moduleHistory.classes.map((historyClass) => { + const classKey = getHistoryClassKey(historyClass); + if (incomingClassKeys.has(`${moduleCode}:${classKey}`)) { + return historyClass; + } + + return { + ...historyClass, + rounds: [...historyClass.rounds, ...replaceRounds.map(notAvailableRound)], + }; + }); + } + + return Array.from(moduleMap.values()) + .sort((left, right) => left.moduleCode.localeCompare(right.moduleCode)) + .map((moduleHistory) => ({ + ...moduleHistory, + classes: sortClasses(moduleHistory.classes), + })); +} + +const resolveInputFile = ( + inputDir: string, + round: CourseRegRound, + reportType: 'vacancy' | CourseRegStudentType, +) => { + if (reportType === 'vacancy') { + return path.join(inputDir, 'vacancy', `round_${round}.csv`); + } + + return path.join(inputDir, reportType.toLowerCase(), `round_${round}.csv`); +}; + +const resolvePdfFile = ( + pdfDir: string, + round: CourseRegRound, + reportType: 'vacancy' | CourseRegStudentType, +) => { + if (reportType === 'vacancy') { + return path.join(pdfDir, 'vacancy', `round_${round}.pdf`); + } + + return path.join(pdfDir, reportType.toLowerCase(), `round_${round}.pdf`); +}; + +const isPdf = (buffer: Buffer, contentType?: string | null) => + contentType?.toLowerCase().includes('application/pdf') || + buffer.subarray(0, 4).toString() === '%PDF'; + +const outputFile = async (file: string, data: Buffer | string) => { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, data); +}; + +const outputJson = async (file: string, data: unknown) => { + const spaces = process.env.NODE_ENV === 'production' ? 0 : 2; + await outputFile(file, `${JSON.stringify(data, null, spaces)}\n`); +}; + +const readJsonIfExists = async (file: string): Promise => { + try { + return JSON.parse(await readFile(file, 'utf8')) as T; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return undefined; + } + throw error; + } +}; + +const pathExists = async (file: string) => { + try { + await readFile(file); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ENOENT'; + } +}; + +const downloadPdf = async (url: string, outputPath: string) => { + const response = await fetch(url, { + signal: AbortSignal.timeout(30_000), + }); + const buffer = Buffer.from(await response.arrayBuffer()); + const contentType = response.headers.get('content-type'); + + if (response.status !== 200 || !isPdf(buffer, contentType)) { + console.warn( + JSON.stringify({ + contentType, + outputPath, + size: buffer.length, + status: response.status, + url, + warning: 'CourseReg PDF download did not return a PDF', + }), + ); + return false; + } + + await outputFile(outputPath, buffer); + return true; +}; + +const downloadCurrentPdfs = async (pdfDir: string, rounds: Array) => { + let downloaded = 0; + + for (const round of rounds) { + const downloads = await Promise.all([ + downloadPdf(COURSE_REG_PDF_URLS.vacancy(round), resolvePdfFile(pdfDir, round, 'vacancy')), + downloadPdf(COURSE_REG_PDF_URLS.ug(round), resolvePdfFile(pdfDir, round, 'UG')), + downloadPdf(COURSE_REG_PDF_URLS.gd(round), resolvePdfFile(pdfDir, round, 'GD')), + ]); + downloaded += downloads.filter(Boolean).length; + } + + if (!downloaded) { + throw new Error('No CourseReg PDFs could be downloaded'); + } + + console.info(JSON.stringify({ downloaded, pdfDir, message: 'CourseReg PDFs downloaded' })); +}; + +const getPdfToCsvScriptPath = () => + path.resolve(process.cwd(), 'scripts', 'coursereg_pdf_to_csv.py'); + +const convertPdfsToCsvs = (pdfDir: string, csvDir: string, python: string) => + new Promise((resolve, reject) => { + const child = spawn( + python, + [getPdfToCsvScriptPath(), '--pdf-dir', pdfDir, '--csv-dir', csvDir], + { + stdio: 'inherit', + }, + ); + + child.on('error', reject); + child.on('exit', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`CourseReg PDF conversion failed with exit code ${code}`)); + } + }); + }); + +export class DemandAllocationScraper { + private readonly academicYear: string; + + private readonly archiveDir: string; + + private readonly download: boolean; + + private readonly inputDir: string; + + private readonly outputDir: string; + + private readonly pdfDir?: string; + + private readonly python: string; + + private readonly round?: CourseRegRound; + + private readonly semester: Semester; + + constructor( + semester: Semester, + academicYear: string, + options: DemandAllocationScraperOptions = {}, + ) { + const academicYearPath = normalizeAcademicYearForPath(academicYear); + this.academicYear = academicYear; + this.semester = semester; + this.archiveDir = + options.archiveDir ?? + path.join(getDefaultPdfArchiveRoot(), academicYearPath, 'semesters', String(semester)); + this.download = options.download ?? false; + this.inputDir = + options.inputDir ?? + path.resolve(process.cwd(), 'data', academicYearPath, 'semesters', String(semester), 'csv'); + this.outputDir = options.outputDir ?? getDefaultApiOutputRoot(academicYear); + this.pdfDir = options.pdfDir; + this.python = options.python ?? 'python3'; + this.round = options.round; + } + + async run(options: DemandAllocationScraperOptions = {}) { + const inputDir = options.inputDir ?? this.inputDir; + const outputDir = options.outputDir ?? this.outputDir; + const pdfDir = options.pdfDir ?? this.pdfDir ?? this.archiveDir; + const python = options.python ?? this.python; + const round = options.round ?? this.round; + const rounds = round === undefined ? getCourseRegRounds(this.academicYear) : [round]; + const mergedRows: Array = []; + + if (options.download ?? this.download) { + console.info( + JSON.stringify({ pdfDir, rounds, message: 'Downloading current CourseReg PDFs' }), + ); + await downloadCurrentPdfs(pdfDir, rounds); + } + + if (options.pdfDir || options.download || this.pdfDir || this.download) { + console.info( + JSON.stringify({ inputDir, pdfDir, message: 'Converting CourseReg PDFs to CSVs' }), + ); + await convertPdfsToCsvs(pdfDir, inputDir, python); + } + + for (const round of rounds) { + const vacancyFile = resolveInputFile(inputDir, round, 'vacancy'); + if (!(await pathExists(vacancyFile))) { + console.warn( + JSON.stringify({ + round, + vacancyFile, + warning: 'CourseReg vacancy CSV not found, skipping round', + }), + ); + continue; + } + + const vacancyRows = parseVacancyCsv(await readFile(vacancyFile, 'utf8')); + + for (const studentType of STUDENT_TYPES) { + const demandFile = resolveInputFile(inputDir, round, studentType); + if (!(await pathExists(demandFile))) { + console.warn( + JSON.stringify({ + demandFile, + round, + studentType, + warning: 'CourseReg demand CSV not found, skipping student type', + }), + ); + continue; + } + + const demandRows = parseDemandAllocationCsv(await readFile(demandFile, 'utf8')); + mergedRows.push(...mergeCourseRegRound(round, studentType, demandRows, vacancyRows)); + } + } + + const extractedHistories = groupCourseRegHistory(this.academicYear, this.semester, mergedRows); + if (!extractedHistories.length) { + throw new Error( + 'No CourseReg history extracted. Check that the staged PDFs contain released CourseReg data.', + ); + } + + const outputPath = path.join( + outputDir, + 'semesters', + String(this.semester), + 'courseRegHistory.json', + ); + const existingHistories = + round === undefined + ? undefined + : await readJsonIfExists>(outputPath); + const histories = existingHistories + ? mergeCourseRegHistories(existingHistories, extractedHistories, rounds) + : extractedHistories; + + await outputJson(outputPath, histories); + + console.info( + JSON.stringify({ + modules: histories.length, + outputPath, + message: 'CourseReg history written', + }), + ); + return histories; + } +} diff --git a/scrapers/demand-allocation-scraper/src/types.ts b/scrapers/demand-allocation-scraper/src/types.ts new file mode 100644 index 0000000000..8d0ab8b0fa --- /dev/null +++ b/scrapers/demand-allocation-scraper/src/types.ts @@ -0,0 +1,29 @@ +export type CourseRegStudentType = 'UG' | 'GD'; + +export type CourseRegRound = 0 | 1 | 2 | 3; + +export type CourseRegSlotValue = number | 'notAvailable' | 'unlimited'; + +export type ModuleCode = string; + +export type Semester = 1 | 2 | 3 | 4; + +export type CourseRegRoundHistory = { + allocatedSlots: CourseRegSlotValue; + forecastedSlots: CourseRegSlotValue; + registered: number | null; + round: CourseRegRound; +}; + +export type CourseRegClassHistory = { + classNo: string; + rounds: Array; + studentType: CourseRegStudentType; +}; + +export type CourseRegModuleSemesterHistory = { + acadYear: string; + classes: Array; + moduleCode: ModuleCode; + semester: Semester; +}; diff --git a/scrapers/demand-allocation-scraper/tsconfig.json b/scrapers/demand-allocation-scraper/tsconfig.json new file mode 100644 index 0000000000..2fd82baa84 --- /dev/null +++ b/scrapers/demand-allocation-scraper/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "outDir": "build", + "target": "es2022", + "lib": ["es2022"], + "module": "commonjs", + "resolveJsonModule": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitAny": true, + "skipLibCheck": true, + "types": ["node", "vitest/globals"] + }, + "include": ["src", "vitest.config.ts"] +} diff --git a/scrapers/demand-allocation-scraper/vitest.config.ts b/scrapers/demand-allocation-scraper/vitest.config.ts new file mode 100644 index 0000000000..708bdfe185 --- /dev/null +++ b/scrapers/demand-allocation-scraper/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + coverage: { + exclude: ['build/**', 'coverage/**', 'vitest.config.ts'], + provider: 'v8', + reporter: ['text', 'html', 'clover'], + }, + exclude: ['build/**', 'coverage/**', 'node_modules/**'], + globals: true, + }, +}); From 2b0b572898645ecdd07cdb1b97e2e433277b2ba6 Mon Sep 17 00:00:00 2001 From: Jonathan Loh Date: Sat, 20 Jun 2026 22:21:09 +0100 Subject: [PATCH 2/4] Expose CourseReg history API endpoint --- scrapers/nus-v2/src/types/course-reg.ts | 27 +++++++ scrapers/nus-v2/swagger.yaml | 96 +++++++++++++++++++++++++ website/src/apis/nusmods.js | 10 +++ website/src/apis/nusmods.test.js | 9 +++ 4 files changed, 142 insertions(+) create mode 100644 scrapers/nus-v2/src/types/course-reg.ts create mode 100644 website/src/apis/nusmods.test.js diff --git a/scrapers/nus-v2/src/types/course-reg.ts b/scrapers/nus-v2/src/types/course-reg.ts new file mode 100644 index 0000000000..7a038b8fb4 --- /dev/null +++ b/scrapers/nus-v2/src/types/course-reg.ts @@ -0,0 +1,27 @@ +import type { ModuleCode, Semester } from './modules'; + +export type CourseRegStudentType = 'UG' | 'GD'; + +export type CourseRegRound = 0 | 1 | 2 | 3; + +export type CourseRegSlotValue = number | 'notAvailable' | 'unlimited'; + +export type CourseRegRoundHistory = { + allocatedSlots: CourseRegSlotValue; + forecastedSlots: CourseRegSlotValue; + registered: number | null; + round: CourseRegRound; +}; + +export type CourseRegClassHistory = { + classNo: string; + rounds: Array; + studentType: CourseRegStudentType; +}; + +export type CourseRegModuleSemesterHistory = { + acadYear: string; + classes: Array; + moduleCode: ModuleCode; + semester: Semester; +}; diff --git a/scrapers/nus-v2/swagger.yaml b/scrapers/nus-v2/swagger.yaml index e57dee2742..c8d6ee7cf1 100644 --- a/scrapers/nus-v2/swagger.yaml +++ b/scrapers/nus-v2/swagger.yaml @@ -230,6 +230,7 @@ servers: tags: - name: 'Modules' - name: 'Venues' + - name: 'CourseReg' paths: /{acadYear}/moduleList.json: @@ -388,6 +389,30 @@ paths: 404: $ref: '#/components/responses/404' + /{acadYear}/semesters/{semester}/courseRegHistory.json: + get: + summary: 'Get CourseReg vacancy and demand allocation history' + parameters: + - $ref: '#/components/parameters/acadYear' + - $ref: '#/components/parameters/semester' + description: | + Get CourseReg vacancy and demand allocation history for modules in a semester, grouped by module, student type, class number, and CourseReg round. + + This data is extracted from CourseReg vacancy and demand allocation PDFs after each round is released. Slot values preserve the difference between `0`, `unlimited`, and `notAvailable`. The `registered` value is `null` when CourseReg did not publish a registered count for that class and round. + tags: + - 'CourseReg' + responses: + 200: + description: 'Success' + content: + application/json: + schema: + type: 'array' + items: + $ref: '#/components/schemas/CourseRegModuleSemesterHistory' + 404: + $ref: '#/components/responses/404' + components: responses: 404: @@ -458,6 +483,77 @@ components: type: 'string' example: 'CS1010S' + CourseRegSlotValue: + oneOf: + - type: 'number' + example: 50 + - type: 'string' + enum: + - 'notAvailable' + - 'unlimited' + + CourseRegRoundHistory: + required: + - 'allocatedSlots' + - 'forecastedSlots' + - 'registered' + - 'round' + properties: + allocatedSlots: + $ref: '#/components/schemas/CourseRegSlotValue' + forecastedSlots: + $ref: '#/components/schemas/CourseRegSlotValue' + registered: + nullable: true + type: 'number' + example: 17 + round: + type: 'number' + enum: [0, 1, 2, 3] + example: 2 + + CourseRegClassHistory: + required: + - 'classNo' + - 'rounds' + - 'studentType' + properties: + classNo: + type: 'string' + example: '1' + rounds: + type: 'array' + items: + $ref: '#/components/schemas/CourseRegRoundHistory' + studentType: + type: 'string' + enum: + - 'UG' + - 'GD' + example: 'UG' + + CourseRegModuleSemesterHistory: + required: + - 'acadYear' + - 'classes' + - 'moduleCode' + - 'semester' + properties: + acadYear: + type: 'string' + example: '2025/2026' + classes: + type: 'array' + items: + $ref: '#/components/schemas/CourseRegClassHistory' + moduleCode: + type: 'string' + example: 'CS2030S' + semester: + type: 'number' + enum: [1, 2, 3, 4] + example: 2 + PrereqTree: oneOf: - type: 'string' diff --git a/website/src/apis/nusmods.js b/website/src/apis/nusmods.js index 624786ce65..07ad2747b1 100644 --- a/website/src/apis/nusmods.js +++ b/website/src/apis/nusmods.js @@ -68,6 +68,16 @@ class NUSModsApi { return `${NUSModsApi.baseUrl(academicYear)}/semesters/${semester}/venueInformation.json`; } + /** + * CourseReg vacancy and demand allocation history for one semester. + * @param {number} semester + * @param {string} academicYear + * @returns {string} + */ + static courseRegHistoryUrl(semester, academicYear = config.academicYear) { + return `${NUSModsApi.baseUrl(academicYear)}/semesters/${semester}/courseRegHistory.json`; + } + /** * List of departments mapped to faculties * @param {string} academicYear diff --git a/website/src/apis/nusmods.test.js b/website/src/apis/nusmods.test.js new file mode 100644 index 0000000000..e4659a5b0a --- /dev/null +++ b/website/src/apis/nusmods.test.js @@ -0,0 +1,9 @@ +import NUSModsApi from './nusmods'; + +describe(NUSModsApi.courseRegHistoryUrl, () => { + test('should return the CourseReg history API URL for a semester', () => { + expect(NUSModsApi.courseRegHistoryUrl(2, '2025/2026')).toBe( + 'https://api.nusmods.com/v2/2025-2026/semesters/2/courseRegHistory.json', + ); + }); +}); From f459ff7b5f9a378fc38cc76fc8676f5f9612af9d Mon Sep 17 00:00:00 2001 From: Jonathan Loh Date: Sat, 20 Jun 2026 22:36:07 +0100 Subject: [PATCH 3/4] Match CourseRekt vacancy-only round semantics --- scrapers/demand-allocation-scraper/src/scraper.test.ts | 8 ++++---- scrapers/demand-allocation-scraper/src/scraper.ts | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapers/demand-allocation-scraper/src/scraper.test.ts b/scrapers/demand-allocation-scraper/src/scraper.test.ts index 2cae2db18a..66fdfdeabb 100644 --- a/scrapers/demand-allocation-scraper/src/scraper.test.ts +++ b/scrapers/demand-allocation-scraper/src/scraper.test.ts @@ -156,9 +156,9 @@ Science,Statistics,ST2131,Probability,L1,200,-1,-1,-1,-1 classNo: 'L1', moduleCode: 'ST2131', roundHistory: { - allocatedSlots: 'notAvailable', + allocatedSlots: 200, forecastedSlots: 200, - registered: null, + registered: 0, round: 1, }, studentType: 'UG', @@ -177,9 +177,9 @@ Computing,Computer Science,CS2030S,Programming Methodology II,L1,750,-1,-1,-1,-1 classNo: 'L1', moduleCode: 'CS2030S', roundHistory: { - allocatedSlots: 'notAvailable', + allocatedSlots: 750, forecastedSlots: 750, - registered: null, + registered: 0, round: 1, }, studentType: 'UG', diff --git a/scrapers/demand-allocation-scraper/src/scraper.ts b/scrapers/demand-allocation-scraper/src/scraper.ts index 314d4a7305..38857fff5e 100644 --- a/scrapers/demand-allocation-scraper/src/scraper.ts +++ b/scrapers/demand-allocation-scraper/src/scraper.ts @@ -293,14 +293,15 @@ export function mergeCourseRegRound( const demandRow = demandByClass.get(key); const vacancyRow = vacancyByClass.get(key); const [moduleCode, classNo] = key.split(':'); + const forecastedSlots = vacancyRow?.forecastedSlots[studentType] ?? 'notAvailable'; return { classNo, moduleCode, roundHistory: { - allocatedSlots: demandRow?.allocatedSlots ?? 'notAvailable', - forecastedSlots: vacancyRow?.forecastedSlots[studentType] ?? 'notAvailable', - registered: demandRow?.registered ?? null, + allocatedSlots: demandRow?.allocatedSlots ?? forecastedSlots, + forecastedSlots, + registered: demandRow?.registered ?? (forecastedSlots === 'notAvailable' ? null : 0), round, }, studentType, From 31a972305f44873a11662a82c1fb95affca38731 Mon Sep 17 00:00:00 2001 From: Jonathan Loh Date: Sat, 20 Jun 2026 22:44:51 +0100 Subject: [PATCH 4/4] Pin demand allocation Python dependencies --- scrapers/demand-allocation-scraper/.gitignore | 2 ++ scrapers/demand-allocation-scraper/README.md | 19 +++++++++++++++---- .../requirements.txt | 10 ++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 scrapers/demand-allocation-scraper/requirements.txt diff --git a/scrapers/demand-allocation-scraper/.gitignore b/scrapers/demand-allocation-scraper/.gitignore index bb5a3ba7b6..18e8e253c1 100644 --- a/scrapers/demand-allocation-scraper/.gitignore +++ b/scrapers/demand-allocation-scraper/.gitignore @@ -2,4 +2,6 @@ /build/ /coverage/ /node_modules/ +/.venv/ +__pycache__/ *.log diff --git a/scrapers/demand-allocation-scraper/README.md b/scrapers/demand-allocation-scraper/README.md index 54fcc85738..8716240dae 100644 --- a/scrapers/demand-allocation-scraper/README.md +++ b/scrapers/demand-allocation-scraper/README.md @@ -16,7 +16,15 @@ Install workspace dependencies from the repository root: pnpm install ``` -PDF conversion requires Python with `tabula-py` installed and Java available through `JAVA_HOME`. +PDF conversion requires Java available through `JAVA_HOME` and a Python environment with the pinned dependencies: + +```sh +cd scrapers/demand-allocation-scraper +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +``` + +Then pass the pinned environment to scraper runs with `--python .venv/bin/python`. ## Source PDF Archive @@ -55,7 +63,8 @@ Build CourseReg history after manually staging one newly released round: JAVA_HOME=/path/to/java \ pnpm dev 2 2025/2026 \ --round 1 \ - --pdfDir archive/pdfs/2025-2026/semesters/2 + --pdfDir archive/pdfs/2025-2026/semesters/2 \ + --python .venv/bin/python ``` By default, output goes to: @@ -74,7 +83,8 @@ To rebuild all available rounds from staged PDFs, omit `--round`: ```sh JAVA_HOME=/path/to/java \ pnpm dev 2 2025/2026 \ - --pdfDir archive/pdfs/2025-2026/semesters/2 + --pdfDir archive/pdfs/2025-2026/semesters/2 \ + --python .venv/bin/python ``` To test against a temporary output directory: @@ -85,7 +95,8 @@ pnpm dev 2 2025/2026 \ --round 1 \ --pdfDir archive/pdfs/2025-2026/semesters/2 \ --inputDir /tmp/demand-allocation-csv \ - --outputDir /tmp/demand-allocation-output + --outputDir /tmp/demand-allocation-output \ + --python .venv/bin/python ``` ## Data Semantics diff --git a/scrapers/demand-allocation-scraper/requirements.txt b/scrapers/demand-allocation-scraper/requirements.txt new file mode 100644 index 0000000000..515fe6a94f --- /dev/null +++ b/scrapers/demand-allocation-scraper/requirements.txt @@ -0,0 +1,10 @@ +distro==1.8.0 +JPype1==1.5.0 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.4 +python-dateutil==2.8.2 +pytz==2023.3.post1 +six==1.16.0 +tabula-py==2.9.0 +tzdata==2023.3