From 8d2d2f10c94911c88f630fc5af2c061d2f57351d Mon Sep 17 00:00:00 2001 From: "exoplanethub-builder[bot]" Date: Sun, 30 Aug 2026 18:24:03 +0000 Subject: [PATCH 01/29] fix: delete records the NASA archive no longer lists The sync Lambda only upserted, so retracted and renamed planets accumulated in the table forever -- 13 stale records over ten months, inflating every count the UI derives from /api/planets. Adds a sweep pass that runs after a successful ingest, diffing the archive's pl_name set against a projection-only paginated scan and deleting the difference. It refuses to delete more than 5% of the table in one run, so a truncated fetch aborts loudly rather than emptying the table, and it never raises -- the upserts have already committed and must still report success. Co-Authored-By: Claude Opus 5 --- aws-backend/README.md | 8 ++ aws-backend/lambda/sync/app.py | 15 ++- aws-backend/lambda/sync/sweep.py | 54 +++++++++++ aws-backend/tests/test_sweep.py | 157 +++++++++++++++++++++++++++++++ 4 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 aws-backend/lambda/sync/sweep.py create mode 100644 aws-backend/tests/test_sweep.py diff --git a/aws-backend/README.md b/aws-backend/README.md index 25aa1b6..0a5235a 100644 --- a/aws-backend/README.md +++ b/aws-backend/README.md @@ -1,5 +1,13 @@ # AWS Backend +## Data Sync + +The sync Lambda upserts every record the NASA archive returns, then deletes stored records the +archive no longer lists, so the table cannot drift above the archive's count. The sweep refuses to +delete more than 5% of the table in one run — a truncated fetch aborts it with an error log rather +than emptying the table, and the upserts still stand. The invocation result reports `total_synced`, +`total_removed` and `sweep_aborted`; every removed `pl_name` is logged individually. + ## Local Deployment ```bash diff --git a/aws-backend/lambda/sync/app.py b/aws-backend/lambda/sync/app.py index ccb85e7..ffb686e 100644 --- a/aws-backend/lambda/sync/app.py +++ b/aws-backend/lambda/sync/app.py @@ -1,10 +1,14 @@ import json +import logging import os import urllib.request import boto3 from decimal import Decimal from datetime import datetime from esi import compute_esi +from sweep import sweep_removed + +logging.getLogger().setLevel(logging.INFO) dynamodb = boto3.resource('dynamodb') table = dynamodb.Table(os.environ['TABLE_NAME']) @@ -16,7 +20,8 @@ def lambda_handler(event, context): data = json.loads(response.read()) timestamp = datetime.utcnow().isoformat() - + archive_names = {planet['pl_name'] for planet in data} + with table.batch_writer() as batch: for planet in data: item = { @@ -48,11 +53,15 @@ def lambda_handler(event, context): item['esi'] = score batch.put_item(Item=item) - + + sweep = sweep_removed(table, archive_names) + return { 'statusCode': 200, 'body': json.dumps({ - 'total_synced': len(data) + 'total_synced': len(data), + 'total_removed': len(sweep.deleted), + 'sweep_aborted': sweep.aborted }) } diff --git a/aws-backend/lambda/sync/sweep.py b/aws-backend/lambda/sync/sweep.py new file mode 100644 index 0000000..8a95b9a --- /dev/null +++ b/aws-backend/lambda/sync/sweep.py @@ -0,0 +1,54 @@ +import logging +from dataclasses import dataclass + +logger = logging.getLogger(__name__) + +# A truncated archive fetch would otherwise sweep the table; no real run drops this much at once. +MAXIMUM_DELETION_FRACTION = 0.05 + + +@dataclass(frozen=True) +class SweepResult: + deleted: tuple[str, ...] + aborted: bool + + +# Never raises: the ingest has already committed and must still report success. +def sweep_removed(table, archive_names): + try: + return _delete_missing_from(table, archive_names) + except Exception: + logger.exception('sweep aborted: removal pass failed against %d archive records', len(archive_names)) + return SweepResult(deleted=(), aborted=True) + + +def _delete_missing_from(table, archive_names): + stored_names = _scan_names(table) + stale = sorted(stored_names - archive_names) + + if len(stale) > MAXIMUM_DELETION_FRACTION * len(stored_names): + logger.error( + 'sweep aborted: %d of %d stored records are absent from the archive, above the %.0f%% ceiling', + len(stale), + len(stored_names), + 100 * MAXIMUM_DELETION_FRACTION, + ) + return SweepResult(deleted=(), aborted=True) + + with table.batch_writer() as batch: + for name in stale: + logger.info('removing %s: no longer listed in the NASA archive', name) + batch.delete_item(Key={'pl_name': name}) + + return SweepResult(deleted=tuple(stale), aborted=False) + + +def _scan_names(table): + request = {'ProjectionExpression': 'pl_name'} + names = set() + while True: + page = table.scan(**request) + names.update(item['pl_name'] for item in page['Items']) + if not page.get('LastEvaluatedKey'): + return names + request['ExclusiveStartKey'] = page['LastEvaluatedKey'] diff --git a/aws-backend/tests/test_sweep.py b/aws-backend/tests/test_sweep.py new file mode 100644 index 0000000..38b9bbf --- /dev/null +++ b/aws-backend/tests/test_sweep.py @@ -0,0 +1,157 @@ +import logging + +import pytest + +from sweep import MAXIMUM_DELETION_FRACTION, sweep_removed + +# Wide enough that a couple of stale records stay under the deletion ceiling. +ARCHIVE = {'Kepler-22 b', 'TRAPPIST-1e', 'Proxima Cen b'} | {f'Filler {index}' for index in range(97)} + + +class FakeTable: + def __init__(self, stored_names, page_size=1000, scan_error=None, delete_error=None): + self.stored_names = list(stored_names) + self.page_size = page_size + self.scan_error = scan_error + self.delete_error = delete_error + self.deleted = [] + self.scan_requests = [] + + def scan(self, **request): + if self.scan_error: + raise self.scan_error + self.scan_requests.append(request) + start = self.stored_names.index(request['ExclusiveStartKey']['pl_name']) + 1 if 'ExclusiveStartKey' in request else 0 + page = self.stored_names[start:start + self.page_size] + response = {'Items': [{'pl_name': name} for name in page]} + if start + self.page_size < len(self.stored_names): + response['LastEvaluatedKey'] = {'pl_name': page[-1]} + return response + + def batch_writer(self): + return self + + def __enter__(self): + return self + + def __exit__(self, *exception): + return False + + def delete_item(self, Key): + if self.delete_error: + raise self.delete_error + self.deleted.append(Key['pl_name']) + + +def test_deletes_only_records_absent_from_the_archive(): + table = FakeTable(ARCHIVE | {'Retracted b', 'Renamed c'}) + + result = sweep_removed(table, ARCHIVE) + + assert set(result.deleted) == {'Retracted b', 'Renamed c'} + assert sorted(table.deleted) == ['Renamed c', 'Retracted b'] + assert result.aborted is False + + +def test_deletes_nothing_when_the_table_matches_the_archive(): + table = FakeTable(ARCHIVE) + + result = sweep_removed(table, ARCHIVE) + + assert result.deleted == () + assert table.deleted == [] + assert result.aborted is False + + +def test_ignores_archive_records_the_table_has_not_stored_yet(): + table = FakeTable({'Kepler-22 b'}) + + result = sweep_removed(table, ARCHIVE) + + assert result.deleted == () + assert result.aborted is False + + +def test_scan_pages_until_the_table_is_exhausted(): + stored = [f'Planet {index}' for index in range(250)] + table = FakeTable(stored + ['Retracted b'], page_size=100) + + result = sweep_removed(table, set(stored)) + + assert result.deleted == ('Retracted b',) + assert len(table.scan_requests) == 3 + assert 'ExclusiveStartKey' not in table.scan_requests[0] + + +# 1 of 20 stored records is exactly the 5% ceiling; 2 of 20 is over it. +@pytest.mark.parametrize( + 'stale_count, expected_aborted', + [(1, False), (2, True)], + ids=['at-the-ceiling', 'over-the-ceiling'], +) +def test_ceiling_bounds_how_much_one_sweep_may_delete(stale_count, expected_aborted): + assert MAXIMUM_DELETION_FRACTION == 0.05, 'the counts below are hand-computed against a 5% ceiling' + + survivors = {f'Planet {index}' for index in range(20 - stale_count)} + stale = {f'Retracted {index}' for index in range(stale_count)} + table = FakeTable(survivors | stale) + + result = sweep_removed(table, survivors) + + assert result.aborted is expected_aborted + assert table.deleted == ([] if expected_aborted else list(stale)) + + +def test_truncated_archive_fetch_aborts_instead_of_emptying_the_table(): + table = FakeTable(ARCHIVE) + + result = sweep_removed(table, set()) + + assert result.aborted is True + assert result.deleted == () + assert table.deleted == [] + + +def test_empty_table_is_not_a_division_by_zero(): + table = FakeTable([]) + + result = sweep_removed(table, ARCHIVE) + + assert result.deleted == () + assert result.aborted is False + + +@pytest.mark.parametrize( + 'failure', + [ + pytest.param({'scan_error': RuntimeError('throttled')}, id='scan-fails'), + pytest.param({'delete_error': RuntimeError('throttled')}, id='delete-fails'), + ], +) +def test_dynamodb_failure_aborts_the_sweep_without_raising(failure): + table = FakeTable(ARCHIVE | {'Retracted b'}, **failure) + + result = sweep_removed(table, ARCHIVE) + + assert result.aborted is True + assert result.deleted == () + + +def test_logs_every_deleted_name(caplog): + table = FakeTable(ARCHIVE | {'Retracted b', 'Renamed c'}) + + with caplog.at_level(logging.INFO, logger='sweep'): + sweep_removed(table, ARCHIVE) + + logged = '\n'.join(record.getMessage() for record in caplog.records) + assert 'Retracted b' in logged + assert 'Renamed c' in logged + + +def test_logs_an_error_when_the_ceiling_aborts_the_sweep(caplog): + table = FakeTable(ARCHIVE) + + with caplog.at_level(logging.INFO, logger='sweep'): + sweep_removed(table, set()) + + assert [record.levelno for record in caplog.records] == [logging.ERROR] From ff8bce4c37fdad6f938db853090e73f675e4e3bc Mon Sep 17 00:00:00 2001 From: aronnaxp Date: Sun, 30 Aug 2026 18:43:54 +0000 Subject: [PATCH 02/29] =?UTF-8?q?spec:=20explore=20filters=20=E2=80=94=20s?= =?UTF-8?q?earch,=20ranges,=20method,=20star=20type=20in=20the=20URL=20(#1?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .specs/issue-14-explore-filters.md | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .specs/issue-14-explore-filters.md diff --git a/.specs/issue-14-explore-filters.md b/.specs/issue-14-explore-filters.md new file mode 100644 index 0000000..71b7b00 --- /dev/null +++ b/.specs/issue-14-explore-filters.md @@ -0,0 +1,89 @@ +# Spec: Explore Filters — search, ranges, method, star type, all in the URL +Issue: #14 +Status: draft + +## Problem Statement +ROADMAP item 3 says "Improve Advanced Filters", but there is little to improve: the explore +page's only controls are a search box and a single-select discovery-method dropdown that live +as component-local state inside `PlanetTable` — they apply only to the table view (grid view is +unfiltered), vanish on reload, and can't be shared. A visitor who narrows ~6,000 planets down +to "temperate, roughly Earth-sized, around a red dwarf" has no way to do it, and no way to send +that view to anyone. This is a build, not an enhancement (#14). + +## Goals +- Filter the explore list by name/host search, radius, mass, orbital period, discovery method(s), and star type. +- Every control's state lives in the URL — shareable, bookmarkable, reload-safe. +- One module owns filter semantics; grid and table views always show the same filtered set. +- Controls are keyboard- and screen-reader-accessible, with a live results count. + +## Out of Scope +- localStorage persistence — the URL is the single share primitive (per #14 discussion). +- Server-side filtering or API query params (see Technical Approach). +- Comparison tool, saved searches, 3D visualisation — downstream, part of the #36 direction discussion. +- Fixing #6 (keyboard access to table rows/sort headers) — separate task issue; should land before this epic's UI tasks. + +## Acceptance Criteria +- [ ] Search box filters by planet name or host star (case-insensitive substring, current semantics), reflected in `?q=`, and applies to **both** grid and table views. +- [ ] Radius, mass, and orbital period each have a min/max range control on a log scale, reflected in the URL; a planet lacking that measurement is excluded **only while that filter is active**, and the UI says so (e.g. "planets without a measured mass are hidden"). +- [ ] Discovery method is a multi-select whose options are the distinct values present in the data, reflected as `?method=` (comma-separated). +- [ ] Star type filter offers O/B/A/F/G/K/M classes with plain-language labels ("M — red dwarf"), reflected as `?star=`. +- [ ] "Clear all" resets every filter and the URL; the results count ("N of 6,023 planets") updates in an `aria-live=polite` region. +- [ ] Malformed or unknown URL params are silently ignored — the page never shows an error state because of a bad query string. +- [ ] Changing any filter clamps/resets pagination; no empty page-37-of-2 states. +- [ ] `lib/planetFilters.ts` is pure and unit-tested: URL codec round-trips, null-value semantics, and teff→class banding. +- [ ] `/api/planets` projection includes `pl_orbper` and `st_teff`; `route.test.ts` updated accordingly. +- [ ] All range/select controls are native HTML inputs or have full keyboard operation with visible focus; labels associated via `