diff --git a/catalog/dags/providers/provider_api_scripts/wordpress.py b/catalog/dags/providers/provider_api_scripts/wordpress.py index ad8dd2c7bfc..826b6474ee0 100644 --- a/catalog/dags/providers/provider_api_scripts/wordpress.py +++ b/catalog/dags/providers/provider_api_scripts/wordpress.py @@ -11,7 +11,7 @@ """ import logging - +import re import lxml.html as html from common import constants @@ -172,9 +172,12 @@ def _get_title(image): if title := image.get("content", {}).get("rendered"): try: title = html.fromstring(title).text_content() - except UnicodeDecodeError as e: - logger.warning(f"Can't save the image's title ('{title}') due to {e}") - return None + except UnicodeDecodeError: + # lxml's HTML parser can raise UnicodeDecodeError on titles + # containing certain emoji. Fall back to a regex-based tag + # strip, which operates on the original str and never + # re-encodes it, so emoji are preserved correctly. + title = re.sub(r"<[^<]+?>", "", title).strip() return title @staticmethod diff --git a/catalog/tests/dags/providers/provider_api_scripts/test_wordpress.py b/catalog/tests/dags/providers/provider_api_scripts/test_wordpress.py index 1ceebc4ae62..9a2d2e04769 100644 --- a/catalog/tests/dags/providers/provider_api_scripts/test_wordpress.py +++ b/catalog/tests/dags/providers/provider_api_scripts/test_wordpress.py @@ -93,6 +93,11 @@ def test_get_title(ingester): expected_result = "Coffee Bean with bags" assert actual_result == expected_result +def test_get_title_handles_emoji(ingester): + image_data = {"content": {"rendered": "

Tomato Basil \U0001F33F Soup

\n"}} + actual_result = ingester._get_title(image_data) + expected_result = "Tomato Basil \U0001F33F Soup" + assert actual_result == expected_result def test_get_file_info(ingester): image_details = (