diff --git a/.github/actions/load-img/action.yml b/.github/actions/load-img/action.yml index 7a1e52d0148..c3461b96454 100644 --- a/.github/actions/load-img/action.yml +++ b/.github/actions/load-img/action.yml @@ -27,7 +27,7 @@ runs: - name: Install `@actions/artifact` shell: bash run: | - pnpm install @actions/artifact -w + pnpm install @actions/artifact@^2.3.2 -w - name: Download images uses: actions/github-script@v7 diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 58bdde527f3..1c0a701fac3 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -113,6 +113,7 @@ jobs: setup_python: "true" # Node.js is needed by lint actions. install_recipe: "node-install" + locales: "test" - name: Cache pre-commit envs uses: actions/cache@v4 diff --git a/.gitignore b/.gitignore index 6efa4f00af7..96f9779f18b 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,4 @@ dist # ov development environment v8-compile-cache* +.vale/.styles diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b23a35860d..0767adf4b9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: files: ^frontend/.*$ # Check if the i18n files have been downloaded by checking if the Arabic translation exists # Download the i18n files if they do not exist - entry: bash -c 'if [ ! -f "$(dirname "$dir")"/frontend/src/locales/ar.json ]; then just frontend/run i18n; fi' + entry: bash -c 'if [ ! -f frontend/i18n/locales/ar.json ]; then just frontend/run i18n; fi' language: system pass_filenames: false diff --git a/.vale/justfile b/.vale/justfile index e74863cd6f4..2165c699d6d 100644 --- a/.vale/justfile +++ b/.vale/justfile @@ -26,7 +26,7 @@ _files separator="\n": echo "$files" VALE_BIN_NAME := "vale_" + os() + "_" + arch() -VALE_STYLES_PATH := "/opt/.local/share/vale/styles" +VALE_STYLES_PATH := env_var_or_default("VALE_STYLES_PATH", justfile_directory() + "/.styles") @_link_openverse_styles: mkdir -p {{ VALE_STYLES_PATH }} diff --git a/api/api/examples/image_responses.py b/api/api/examples/image_responses.py index cd0fb5bca43..a5519a61e1b 100644 --- a/api/api/examples/image_responses.py +++ b/api/api/examples/image_responses.py @@ -89,6 +89,13 @@ "logo_url": None, "media_count": 2500, }, + { + "source_name": "saint_tech", + "display_name": "SAiNT (IMG Saxony-Anhalt)", + "source_url": "https://saint.tech/en", + "logo_url": None, + "media_count": 2500, + }, ] image_detail_200_example = base_image diff --git a/api/test/factory/models/media.py b/api/test/factory/models/media.py index f586e14d961..c9c4ae66a1f 100644 --- a/api/test/factory/models/media.py +++ b/api/test/factory/models/media.py @@ -53,7 +53,7 @@ class Meta: """The foreign identifier isn't necessarily a UUID but for test purposes it's fine if it looks like one""" license = Faker("random_element", elements=ALL_LICENSES) - provider = Faker("random_element", elements=("flickr", "stocksnap")) + provider = Faker("random_element", elements=("flickr", "stocksnap", "saint_tech")) foreign_landing_url = Faker("globally_unique_url") url = Faker("globally_unique_url") diff --git a/api/test/fixtures/media_type_config.py b/api/test/fixtures/media_type_config.py index 69d2b71f902..98a6cb0afd7 100644 --- a/api/test/fixtures/media_type_config.py +++ b/api/test/fixtures/media_type_config.py @@ -86,7 +86,7 @@ def indexes(self): report_factory=model_factories.ImageReportFactory, sensitive_class=SensitiveImage, deleted_class=DeletedImage, - providers=("flickr", "stocksnap"), + providers=("flickr", "stocksnap", "saint_tech"), categories=("photograph",), tags=("cat", "Cat"), q="dog", diff --git a/catalog/dags/common/loader/provider_details.py b/catalog/dags/common/loader/provider_details.py index dcd20c97362..40a87653a64 100644 --- a/catalog/dags/common/loader/provider_details.py +++ b/catalog/dags/common/loader/provider_details.py @@ -38,6 +38,7 @@ WORDPRESS_DEFAULT_PROVIDER = "wordpress" PHYLOPIC_DEFAULT_PROVIDER = "phylopic" CC_MIXTER_DEFAULT_PROVIDER = "ccmixter" +SAINT_DEFAULT_PROVIDER = "saint_tech" # Finnish parameters FINNISH_SUB_PROVIDERS = { @@ -157,6 +158,7 @@ class AudioCategory: # Default image category by source DEFAULT_IMAGE_CATEGORY = { "stocksnap": ImageCategory.PHOTOGRAPH, + "saint_tech": ImageCategory.PHOTOGRAPH, # Remains to be assigned "animaldiversity": ImageCategory.PHOTOGRAPH, "brooklynmuseum": ImageCategory.DIGITIZED_ARTWORK, diff --git a/catalog/dags/providers/provider_api_scripts/saint.py b/catalog/dags/providers/provider_api_scripts/saint.py new file mode 100644 index 00000000000..99a0ae75e9a --- /dev/null +++ b/catalog/dags/providers/provider_api_scripts/saint.py @@ -0,0 +1,98 @@ +""" +Content Provider: SAiNT (IMG Saxony-Anhalt) + +ETL Process: Use the API to identify all CC licensed media. + +Output: TSV file containing the media and the respective meta-data. + +Notes: https://saint.tech/api/docs +""" + +import logging + +from airflow.models import Variable + +from common.licenses import get_license_info +from common.loader import provider_details as prov +from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester + + +logger = logging.getLogger(__name__) + + +class SaintDataIngester(ProviderDataIngester): + providers = { + "image": prov.SAINT_DEFAULT_PROVIDER, + } + endpoint = "https://saint.tech/api/poi" + creator = "IMG Saxony-Anhalt" + creator_url = "https://saint.tech/en" + + def get_next_query_params(self, prev_query_params: dict | None) -> dict: + if not prev_query_params: + return { + "page": 1, + "pageSize": 100, + "api_key": Variable.get("API_KEY_SAINT", default_var=""), + } + else: + return { + **prev_query_params, + "page": prev_query_params["page"] + 1, + } + + def get_batch_data(self, response_json) -> list[dict] | None: + if response_json and (data := response_json.get("data")): + return data + return None + + def get_record_data(self, data: dict) -> dict | None: + # Expected fields based on typical Swagger UI schemas for POI + if not (foreign_identifier := data.get("id")): + return None + + # Look for image + if not (image := data.get("PrimaryImage")): + return None + + if not (url := image.get("url")): + return None + + # Try to find license + license_url = (image.get("license") or {}).get("url") + if not license_url: + return None + + license_info = get_license_info(license_url) + if license_info is None: + return None + + foreign_landing_url = f"https://saint.tech/poi/{foreign_identifier}" + + title = data.get("title") + + raw_record_data = { + "foreign_landing_url": foreign_landing_url, + "url": url, + "license_info": license_info, + "foreign_identifier": str(foreign_identifier), + "title": title, + "creator": self.creator, + "creator_url": self.creator_url, + } + + if width := image.get("width"): + raw_record_data["width"] = width + if height := image.get("height"): + raw_record_data["height"] = height + + return {k: v for k, v in raw_record_data.items() if v is not None} + + +def main(): + ingester = SaintDataIngester() + ingester.ingest_records() + + +if __name__ == "__main__": + main() diff --git a/catalog/dags/providers/provider_workflows.py b/catalog/dags/providers/provider_workflows.py index 0c1f54858eb..a8cfd9f02df 100644 --- a/catalog/dags/providers/provider_workflows.py +++ b/catalog/dags/providers/provider_workflows.py @@ -26,6 +26,7 @@ from providers.provider_api_scripts.phylopic import PhylopicDataIngester from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester from providers.provider_api_scripts.rawpixel import RawpixelDataIngester +from providers.provider_api_scripts.saint import SaintDataIngester from providers.provider_api_scripts.science_museum import ScienceMuseumDataIngester from providers.provider_api_scripts.smithsonian import SmithsonianDataIngester from providers.provider_api_scripts.smk import SmkDataIngester @@ -320,6 +321,11 @@ def _process_configuration_overrides(self): ingester_class=RawpixelDataIngester, pull_timeout=timedelta(hours=12), ), + ProviderWorkflow( + ingester_class=SaintDataIngester, + start_date=datetime(2024, 1, 1), + schedule_string="@monthly", + ), ProviderWorkflow( ingester_class=ScienceMuseumDataIngester, start_date=datetime(2020, 1, 1), diff --git a/catalog/tests/dags/providers/provider_api_scripts/test_saint.py b/catalog/tests/dags/providers/provider_api_scripts/test_saint.py new file mode 100644 index 00000000000..9e64708b500 --- /dev/null +++ b/catalog/tests/dags/providers/provider_api_scripts/test_saint.py @@ -0,0 +1,126 @@ +from unittest.mock import patch + +import pytest + +from common.licenses import LicenseInfo +from providers.provider_api_scripts.saint import SaintDataIngester + + +@pytest.fixture +def ingester(): + with patch("providers.provider_api_scripts.saint.Variable") as mock_var: + mock_var.get.side_effect = lambda key, default_var=None, **kwargs: { + "INGESTION_LIMIT": 0, + "SKIPPED_INGESTION_ERRORS": {}, + "ENVIRONMENT": "local", + "SHOULD_VERBOSE_LOG": [], + "API_KEY_SAINT": "test_key", + }.get(key, default_var) + yield SaintDataIngester() + + +@pytest.mark.parametrize( + "previous, expected_result", + [ + pytest.param( + None, + { + "page": 1, + "pageSize": 100, + "api_key": "test_key", + }, + id="default_response", + ), + pytest.param( + {"page": 42, "pageSize": 100, "api_key": "dummy"}, + {"page": 43, "pageSize": 100, "api_key": "dummy"}, + id="basic_increment", + ), + ], +) +def test_get_next_query_params(previous, expected_result, ingester): + actual_result = ingester.get_next_query_params(previous) + assert actual_result == expected_result + + +@pytest.mark.parametrize( + "response_json, expected", + [ + pytest.param( + {"data": [{"id": 1}, {"id": 2}]}, + [{"id": 1}, {"id": 2}], + id="happy_path", + ), + pytest.param({}, None, id="empty_dict"), + pytest.param(None, None, id="None"), + ], +) +def test_get_batch_data(response_json, expected, ingester): + actual = ingester.get_batch_data(response_json) + assert actual == expected + + +@pytest.mark.parametrize( + "record, expected_data", + [ + pytest.param({}, None, id="empty_dict"), + pytest.param( + { + "id": 123, + "title": "A nice POI", + "PrimaryImage": { + "url": "https://saint.tech/images/123.jpg", + "width": 800, + "height": 600, + "license": {"url": "https://creativecommons.org/licenses/by/4.0/"}, + }, + }, + { + "foreign_landing_url": "https://saint.tech/poi/123", + "url": "https://saint.tech/images/123.jpg", + "license_info": LicenseInfo( + license="by", + version="4.0", + url="https://creativecommons.org/licenses/by/4.0/", + raw_url="https://creativecommons.org/licenses/by/4.0/", + ), + "foreign_identifier": "123", + "title": "A nice POI", + "creator": "IMG Saxony-Anhalt", + "creator_url": "https://saint.tech/en", + "width": 800, + "height": 600, + }, + id="happy_path", + ), + pytest.param( + {"id": 123, "title": "No image POI"}, + None, + id="no_image", + ), + pytest.param( + { + "id": 123, + "PrimaryImage": { + "url": "https://saint.tech/images/123.jpg", + }, + }, + None, + id="no_license", + ), + pytest.param( + { + "id": 123, + "PrimaryImage": { + "url": "https://saint.tech/images/123.jpg", + "license": None, + }, + }, + None, + id="null_license", + ), + ], +) +def test_get_record_data(record, expected_data, ingester): + actual_data = ingester.get_record_data(record) + assert actual_data == expected_data diff --git a/documentation/catalog/reference/DAGs.md b/documentation/catalog/reference/DAGs.md index 4b5c0332e39..9edbb104923 100644 --- a/documentation/catalog/reference/DAGs.md +++ b/documentation/catalog/reference/DAGs.md @@ -123,6 +123,7 @@ The following are DAGs grouped by their primary tag: | `nypl_workflow` | `@monthly` | `False` | image | | [`phylopic_workflow`](#phylopic_workflow) | `@weekly` | `False` | image | | [`rawpixel_workflow`](#rawpixel_workflow) | `@monthly` | `False` | image | +| [`saint_workflow`](#saint_workflow) | `@monthly` | `False` | image | | [`science_museum_workflow`](#science_museum_workflow) | `@monthly` | `False` | image | | [`smithsonian_workflow`](#smithsonian_workflow) | `@weekly` | `False` | image | | [`smk_workflow`](#smk_workflow) | `@monthly` | `False` | image | @@ -185,6 +186,7 @@ The following is documentation associated with each DAG (where available): 1. [`report_pending_reported_media`](#report_pending_reported_media) 1. [`rotate_db_snapshots`](#rotate_db_snapshots) 1. [`rotate_envfiles`](#rotate_envfiles) +1. [`saint_workflow`](#saint_workflow) 1. [`science_museum_workflow`](#science_museum_workflow) 1. [`smithsonian_workflow`](#smithsonian_workflow) 1. [`smk_workflow`](#smk_workflow) @@ -1097,6 +1099,18 @@ template and "task def" as an abbreviation for task definition. ---- +### `saint_workflow` + +Content Provider: SAiNT (IMG Saxony-Anhalt) + +ETL Process: Use the API to identify all CC licensed media. + +Output: TSV file containing the media and the respective meta-data. + +Notes: https://saint.tech/api/docs + +---- + ### `science_museum_workflow` Content Provider: Science Museum diff --git a/ingestion_server/Dockerfile b/ingestion_server/Dockerfile index 2d4630954bf..33aa169ac4a 100644 --- a/ingestion_server/Dockerfile +++ b/ingestion_server/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update \ COPY Pipfile Pipfile.lock / # Install Python dependencies system-wide (uses the active virtualenv) -RUN pipenv install --system --deploy --dev +RUN pipenv install --system --dev #################### # Ingestion server # diff --git a/justfile b/justfile index b96e718a9e2..8f463f594ea 100644 --- a/justfile +++ b/justfile @@ -134,7 +134,12 @@ precommit: # Run pre-commit to lint and reformat files lint hook="" *files="": precommit - python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files" } }} {{ files }} + #!/usr/bin/env bash + if ! command -v docker &> /dev/null; then + echo "Docker not found, skipping Docker-based hooks..." + SKIP="${SKIP:+$SKIP,}actionlint-docker,shfmt-docker,hadolint-docker" + fi + SKIP="$SKIP" python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files" } }} {{ files }} # Run codeowners validator locally. Only enable experimental hooks if there are no uncommitted changes. lint-codeowners checks="stable": diff --git a/load_sample_data.sh b/load_sample_data.sh index c692562c2d4..8525d1e9bdb 100755 --- a/load_sample_data.sh +++ b/load_sample_data.sh @@ -142,7 +142,8 @@ VALUES (now(), 'freesound', 'Freesound', 'https://freesound.org/', false, 'audio'), (now(), 'jamendo', 'Jamendo', 'https://www.jamendo.com', false, 'audio'), (now(), 'wikimedia_audio', 'Wikimedia', 'https://commons.wikimedia.org', false, 'audio'), - (now(), 'ccmixter', 'CCMixter', 'https://ccmixter.org', false, 'audio'); + (now(), 'ccmixter', 'CCMixter', 'https://ccmixter.org', false, 'audio'), + (now(), 'saint_tech', 'SAiNT (IMG Saxony-Anhalt)', 'https://saint.tech/en', false, 'image'); " #############