From c5988b500af5c81d56399573134efd3b31714dd0 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Sun, 21 Jun 2026 17:31:58 +0200 Subject: [PATCH 1/2] feat!: migrate to `pyproject.toml` - require python3.10 - add py3.13 and py3.14 to tests - move configs and dependencies into pyproject - update pre-commit; add lint for pyproject - enable more mypy checks --- .coveragerc | 4 - .github/workflows/python-package.yml | 18 +-- .github/workflows/python-publish.yml | 11 +- .gitignore | 1 + .isort.cfg | 2 - .pre-commit-config.yaml | 12 +- README.rst | 4 +- extruct/rdfa.py | 1 + extruct/xmldom.py | 2 +- pyproject.toml | 168 ++++++++++++++++++++++++--- pytest.ini | 8 -- requirements-dev.txt | 14 --- requirements.txt | 11 -- setup.cfg | 9 -- setup.py | 71 ----------- tox.ini | 26 ----- 16 files changed, 179 insertions(+), 183 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .isort.cfg delete mode 100644 pytest.ini delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index fa6e44bb..00000000 --- a/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -branch = true -source = extruct -omit = extruct/service.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 387622fa..318d5277 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,21 +16,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements-dev.txt + python -m pip install --group dev - name: tox run: | - tox -e `python -c "import sys; print('py' + ''.join(sys.version.split('.')[:2]))"` + tox run -e ${{ matrix.python-version }} - name: Upload coverage report uses: codecov/codecov-action@v5 with: @@ -41,19 +41,19 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.12'] + python-version: ["3.14"] tox-job: ["linters"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements-dev.txt + python -m pip install --group dev - name: tox run: | tox -e ${{ matrix.tox-job }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 9694ff03..92f26ec2 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,24 +9,25 @@ on: - "v*" jobs: + deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: '3.14' - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install build twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel + python -m build twine upload dist/* diff --git a/.gitignore b/.gitignore index 9e040fff..f553933a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ .tox/ .pytest_cache/ .coverage +coverage.xml diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index b9fb3f3e..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf9d5336..e5c6ab7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,17 +2,21 @@ repos: - hooks: - id: black repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 26.5.1 - hooks: - id: isort repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 8.0.1 - hooks: - id: mypy additional_dependencies: [types-requests, types-mock, lxml-stubs] repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v2.1.0 - hooks: - id: pyupgrade repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.21.2 + - hooks: + - id: pyproject-fmt + repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.25.0 diff --git a/README.rst b/README.rst index 01595119..470afb46 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ extruct ======= -.. image:: https://github.com/scrapinghub/extruct/workflows/build/badge.svg?branch=master +.. image:: https://github.com/scrapinghub/extruct/actions/workflows/python-package.yml/badge.svg?branch=master :target: https://github.com/scrapinghub/extruct/actions :alt: Build Status @@ -727,7 +727,7 @@ Development version :: mkvirtualenv extruct - pip install -r requirements-dev.txt + pip install --group dev Tests diff --git a/extruct/rdfa.py b/extruct/rdfa.py index d84bd595..30669954 100644 --- a/extruct/rdfa.py +++ b/extruct/rdfa.py @@ -4,6 +4,7 @@ Based on pyrdfa3 and rdflib """ + import json import logging import re diff --git a/extruct/xmldom.py b/extruct/xmldom.py index 91361a86..e9bbbc81 100644 --- a/extruct/xmldom.py +++ b/extruct/xmldom.py @@ -109,7 +109,7 @@ def attributes(self): a = Attr(name) a.value = value attrs[name] = a - return NamedNodeMap(attrs, {}, self) + return NamedNodeMap(attrs, {}, self) # type: ignore[arg-type] @property def parentNode(self): diff --git a/pyproject.toml b/pyproject.toml index b7087de0..5292ebe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,157 @@ -[tool.mypy] -show_column_numbers = true +[build-system] +build-backend = "setuptools.build_meta" +requires = [ "setuptools>=77" ] + +[project] +name = "extruct" +description = "Extract embedded metadata from HTML markup" +readme = "README.rst" +keywords = [ "extruct" ] +license-files = [ "LICENSE" ] +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dynamic = [ "version" ] +dependencies = [ + "html-text>=0.5.1", + "jstyleson", + "lxml", + "lxml-html-clean", + "mf2py>=1.1.0", + "pyrdfa3", + "rdflib>=6.0.0", + "w3lib", +] +[[project.authors]] +name = "Scrapinghub" +email = "info@scrapinghub.com" +[[project.maintainers]] +name = "Scrapinghub" +email = "info@scrapinghub.com" +[project.optional-dependencies] +cli = [ "requests" ] +[project.scripts] +extruct = "extruct.tool:main" +[project.urls] +Homepage = "https://github.com/scrapinghub/extruct" -disallow_untyped_defs=true -disallow_incomplete_defs=true -check_untyped_defs=true +[dependency-groups] +dev = [ + "pre-commit", + "pytest>=9.1.1", + "pytest-cov", + "readme-renderer", + "readme-renderer[md]", + "tox>=4.21", +] -warn_redundant_casts=true -warn_unused_ignores=true -warn_return_any=true +[tool.setuptools] +include-package-data = true +[tool.setuptools.dynamic] +version = { file = "extruct/VERSION" } +[tool.setuptools.package-data] +extruct = [ "VERSION" ] +[tool.setuptools.packages.find] +exclude = [ "tests" ] -strict_equality=true +[tool.isort] +profile = "black" +[tool.pyproject-fmt] +indent = 4 +keep_full_version = true +table_format = "long" + +[tool.mypy] +disallow_any_generics = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +warn_unreachable = true +extra_checks = true +strict_equality = true +strict_equality_for_none = true +show_column_numbers = true +warn_unused_configs = true [[tool.mypy.overrides]] module = [ - 'mf2py.*', - 'pyRdfa.*', - 'rdflib.*', - 'jstyleson.*', - 'urlparse.*', - 'html_text.*', -] -ignore_missing_imports=true + "html_text.*", + "jstyleson.*", + "mf2py.*", + "pyRdfa.*", + "rdflib.*", + "urlparse.*", +] +ignore_missing_imports = true + +[tool.pytest] +filterwarnings = [ + # mf2py + "ignore::DeprecationWarning:mf2py.*", + # pyRdfa + "ignore::DeprecationWarning:pyRdfa.*", +] +testpaths = [ + "tests", +] + +[tool.coverage.run] +branch = true +source = [ "extruct" ] +omit = [ "extruct/service.py" ] + +[tool.tox] +env_list = [ "py314", "py313", "py312", "py311", "py310", "linters", "twinecheck" ] +[tool.tox.env.linters] +package = "skip" +commands = [ + [ "pre-commit", "run", "--all-files", "--show-diff-on-failure" ], +] + +[tool.tox.env.py314] +commands = [ + [ + "pytest", + "--cov-report=term", + "--cov-report=", + "--cov-report=xml", + "--cov=extruct", + "{posargs}" + ], + [ "python", "-m", "readme_renderer", "README.rst", "-o", "/tmp/README.html" ], +] + +[tool.tox.env.twinecheck] +deps = [ "build==1.2.1", "twine==5.1" ] +commands = [ + [ "python", "-m", "build", "--sdist" ], + [ "twine", "check", "dist/*" ], +] + +[tool.tox.env_run_base] +dependency_groups = [ "dev" ] +commands = [ + [ + "pytest", + "--cov-report=term", + "--cov-report=", + "--cov-report=xml", + "--cov=extruct", + "{posargs}" + ], +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 708407bd..00000000 --- a/pytest.ini +++ /dev/null @@ -1,8 +0,0 @@ -[pytest] -filterwarnings = - ; https://github.com/RDFLib/pyrdfa3/issues/31 - ignore:the imp module is deprecated:DeprecationWarning - ignore:the imp module is deprecated:PendingDeprecationWarning - ; https://github.com/html5lib/html5lib-python/issues/402 - ; https://bugs.launchpad.net/beautifulsoup/+bug/1847592 - ignore:Using or importing the ABCs:DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 3f185ba4..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -# project requirements for development, install them using following command: -# pip install -r requirements-dev.txt - --r requirements.txt - -tox -bumpversion - -pytest -pytest-cov -readme_renderer -mock -black -pre-commit diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 92e1b91d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -# project requirements, install them using following command: -# pip install -r requirements.txt -lxml -lxml-html-clean -requests -rdflib>=6.0.0 -pyrdfa3 -mf2py>=1.1.0 -w3lib -html-text -jstyleson diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9c145abe..00000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[bumpversion] -current_version = 0.18.0 -commit = True -tag = True - -[bumpversion:file:extruct/VERSION] - -[wheel] -universal = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index bc4b0101..00000000 --- a/setup.py +++ /dev/null @@ -1,71 +0,0 @@ -# mypy: disallow_untyped_defs=False -import os - -from setuptools import find_packages, setup - - -def get_readme(): - path = os.path.join(os.path.dirname(__file__), "README.rst") - with open(path) as f: - return f.read().strip() - - -def get_version(): - path = os.path.join(os.path.dirname(__file__), "extruct", "VERSION") - with open(path) as f: - return f.read().strip() - - -setup( - name="extruct", - version=get_version(), - description="Extract embedded metadata from HTML markup", - long_description=get_readme(), - long_description_content_type="text/x-rst", - author="Scrapinghub", - author_email="info@scrapinghub.com", - maintainer="Scrapinghub", - maintainer_email="info@scrapinghub.com", - url="https://github.com/scrapinghub/extruct", - entry_points={ - "console_scripts": { - "extruct = extruct.tool:main", - } - }, - packages=find_packages( - exclude=[ - "tests", - ] - ), - package_data={"extruct": ["VERSION"]}, - python_requires=">=3.8", - install_requires=[ - "lxml", - "lxml-html-clean", - "rdflib>=6.0.0", - "pyrdfa3", - "mf2py", - "w3lib", - "html-text>=0.5.1", - "jstyleson", - ], - extras_require={ - "cli": [ - "requests", - ], - }, - keywords="extruct", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b9f8c12f..00000000 --- a/tox.ini +++ /dev/null @@ -1,26 +0,0 @@ -[tox] -envlist = py38, py39, py310, py311, py312 - - -[testenv] -deps = - -rrequirements-dev.txt -commands = - py.test --cov-report=term --cov-report= --cov-report=xml --cov=extruct {posargs:extruct tests} - -[testenv:py39] -commands = - py.test --cov-report=term --cov-report= --cov-report=xml --cov=extruct {posargs:extruct tests} - python -m readme_renderer README.rst -o /tmp/README.html - -[testenv:linters] -deps = -rrequirements-dev.txt -commands = pre-commit run --all-files --show-diff-on-failure - -[testenv:twinecheck] -deps = - twine==5.1.0 - build==1.2.1 -commands = - python -m build --sdist - twine check dist/* From 36aee6951bf5a548186197a761bfdc553ed2abc5 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Sat, 18 Jul 2026 20:13:50 +0200 Subject: [PATCH 2/2] refactor(tests): pytest style --- tests/test_dublincore.py | 9 ++-- tests/test_extruct.py | 32 +++++--------- tests/test_extruct_uniform.py | 20 ++++----- tests/test_jsonld.py | 29 +++++++------ tests/test_microdata.py | 81 ++++++++++++----------------------- tests/test_microformat.py | 13 +++--- tests/test_opengraph.py | 14 +++--- tests/test_rdfa.py | 9 +--- tests/test_tool.py | 20 ++++----- tests/test_uniform.py | 30 ++++++------- 10 files changed, 99 insertions(+), 158 deletions(-) diff --git a/tests/test_dublincore.py b/tests/test_dublincore.py index 696aa244..a36b50de 100644 --- a/tests/test_dublincore.py +++ b/tests/test_dublincore.py @@ -1,15 +1,11 @@ # mypy: disallow_untyped_defs=False import json -import unittest from extruct.dublincore import DublinCoreExtractor from tests import get_testdata, jsonize_dict -class TestDublincore(unittest.TestCase): - - maxDiff = None - +class TestDublincore: def test_dublincore(self): body = get_testdata("misc", "dublincore_test.html") expected = json.loads( @@ -18,4 +14,5 @@ def test_dublincore(self): dublincorext = DublinCoreExtractor() data = dublincorext.extract(body) - self.assertEqual(jsonize_dict(data), expected) + + assert jsonize_dict(data) == expected diff --git a/tests/test_extruct.py b/tests/test_extruct.py index f7ef6fc3..d65c289f 100644 --- a/tests/test_extruct.py +++ b/tests/test_extruct.py @@ -1,6 +1,5 @@ # mypy: disallow_untyped_defs=False import json -import unittest import pytest @@ -9,10 +8,7 @@ from tests import get_testdata, jsonize_dict, replace_node_ref_with_node_id -class TestGeneric(unittest.TestCase): - - maxDiff = None - +class TestGeneric: def test_all(self): body = get_testdata("songkick", "elysianfields.html") expected = json.loads( @@ -21,8 +17,7 @@ def test_all(self): data = extruct.extract( body, base_url="http://www.songkick.com/artists/236156-elysian-fields" ) - - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected def test_rdfa_is_preserving_order(self): # See https://github.com/scrapinghub/extruct/issues/116 @@ -33,7 +28,7 @@ def test_rdfa_is_preserving_order(self): data = extruct.extract( body, base_url="http://www.songkick.com/artists/236156-elysian-fields" ) - self.assertEqual(jsonize_dict(data)["rdfa"], expected["rdfa"]) + assert jsonize_dict(data)["rdfa"] == expected["rdfa"] def test_microdata_custom_url(self): body, expected = self._microdata_custom_url("product_custom_url.json") @@ -41,7 +36,7 @@ def test_microdata_custom_url(self): data = extruct.extract( tree, base_url="http://some-example.com", syntaxes=["microdata"] ) - self.assertEqual(data, expected) + assert data == expected def test_microdata_with_returning_node(self): body, expected = self._microdata_custom_url( @@ -54,7 +49,7 @@ def test_microdata_with_returning_node(self): return_html_node=True, ) replace_node_ref_with_node_id(data) - self.assertEqual(data, expected) + assert data == expected def test_deprecated_url(self): body, expected = self._microdata_custom_url("product_custom_url.json") @@ -62,11 +57,11 @@ def test_deprecated_url(self): data = extruct.extract( body, url="http://some-example.com", syntaxes=["microdata"] ) - self.assertEqual(data, expected) + assert data == expected def test_extra_kwargs(self): body, _ = self._microdata_custom_url("product_custom_url.json") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): extruct.extract(body, foo="bar") # type: ignore[call-arg] def _microdata_custom_url(self, test_file): @@ -81,14 +76,9 @@ def _microdata_custom_url(self, test_file): def test_errors(self): body = "" - # raise exceptions - with self.assertRaises(Exception): - data = extruct.extract(body) - - # ignore exceptions - data = extruct.extract(body, errors="ignore") - assert data == {} + with pytest.raises(Exception): + extruct.extract(body) # ignore exceptions - data = extruct.extract(body, errors="log") - assert data == {} + assert extruct.extract(body, errors="ignore") == {} + assert extruct.extract(body, errors="log") == {} diff --git a/tests/test_extruct_uniform.py b/tests/test_extruct_uniform.py index 38cf5eca..8e29c025 100644 --- a/tests/test_extruct_uniform.py +++ b/tests/test_extruct_uniform.py @@ -1,16 +1,12 @@ # mypy: disallow_untyped_defs=False import json -import unittest import extruct from extruct.utils import parse_html from tests import get_testdata, jsonize_dict, replace_node_ref_with_node_id -class TestFlatten(unittest.TestCase): - - maxDiff = None - +class TestFlatten: def test_microdata(self): body, tree, expected = self._testdata_html_and_tree( "schema.org", @@ -19,10 +15,10 @@ def test_microdata(self): ) syntax = "microdata" data = extruct.extract(body, uniform=True, syntaxes=[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected[syntax]) + assert jsonize_dict(data[syntax]) == expected[syntax] data = extruct.extract(tree, uniform=True, syntaxes=[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected[syntax]) + assert jsonize_dict(data[syntax]) == expected[syntax] def test_opengraph(self): body, tree, expected = self._testdata_html_and_tree( @@ -32,10 +28,10 @@ def test_opengraph(self): ) syntax = "opengraph" data = extruct.extract(body, uniform=True, syntaxes=[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected) + assert jsonize_dict(data[syntax]) == expected data = extruct.extract(tree, uniform=True, syntaxes=[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected) + assert jsonize_dict(data[syntax]) == expected def test_microdata_with_returning_node(self): body, tree, expected = self._testdata_html_and_tree( @@ -48,13 +44,13 @@ def test_microdata_with_returning_node(self): body, uniform=True, return_html_node=True, syntaxes=[syntax] ) replace_node_ref_with_node_id(data[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected[syntax]) + assert jsonize_dict(data[syntax]) == expected[syntax] data = extruct.extract( tree, uniform=True, return_html_node=True, syntaxes=[syntax] ) replace_node_ref_with_node_id(data[syntax]) - self.assertEqual(jsonize_dict(data[syntax]), expected[syntax]) + assert jsonize_dict(data[syntax]) == expected[syntax] def test_microformat(self): body = get_testdata("misc", "microformat_test.html") @@ -62,7 +58,7 @@ def test_microformat(self): get_testdata("misc", "microformat_flat_test.json").decode("UTF-8") ) data = extruct.extract(body, uniform=True, syntaxes=["microformat"]) - self.assertEqual(jsonize_dict(data["microformat"]), expected) + assert jsonize_dict(data["microformat"]) == expected def _testdata_html_and_tree(self, root, path1, path2): body = get_testdata(root, path1) diff --git a/tests/test_jsonld.py b/tests/test_jsonld.py index 178b3229..9fd4bb71 100644 --- a/tests/test_jsonld.py +++ b/tests/test_jsonld.py @@ -1,45 +1,46 @@ # mypy: disallow_untyped_defs=False import json -import unittest from extruct.jsonld import JsonLdExtractor from tests import get_testdata -class TestJsonLD(unittest.TestCase): +class TestJsonLD: def test_schemaorg_CreativeWork(self): - self.assertJsonLdCorrect(folder="schema.org", page="CreativeWork.001") + self._assert_jsonld_correct(folder="schema.org", page="CreativeWork.001") def test_songkick(self): - self.assertJsonLdCorrect( + self._assert_jsonld_correct( folder="songkick", page="Elysian Fields Brooklyn Tickets, The Owl Music Parlor, 31 Oct 2015", ) def test_jsonld_empty_item(self): - self.assertJsonLdCorrect(folder="songkick", page="jsonld_empty_item_test") + self._assert_jsonld_correct(folder="songkick", page="jsonld_empty_item_test") def test_jsonld_with_comments(self): for page in ["JoinAction.001", "AllocateAction.001"]: - self.assertJsonLdCorrect(folder="schema.org.invalid", page=page) + self._assert_jsonld_correct(folder="schema.org.invalid", page=page) for page in ["JoinAction.001", "AllocateAction.001"]: - self.assertJsonLdCorrect(folder="custom.invalid", page=page) + self._assert_jsonld_correct(folder="custom.invalid", page=page) def test_jsonld_with_control_characters(self): - self.assertJsonLdCorrect( + self._assert_jsonld_correct( folder="custom.invalid", page="JSONLD_with_control_characters" ) def test_jsonld_with_control_characters_comment(self): - self.assertJsonLdCorrect( + self._assert_jsonld_correct( folder="custom.invalid", page="JSONLD_with_control_characters_comment" ) def test_jsonld_with_json_including_js_comment(self): - self.assertJsonLdCorrect(folder="custom.invalid", page="JSONLD_with_JS_comment") + self._assert_jsonld_correct( + folder="custom.invalid", page="JSONLD_with_JS_comment" + ) - def assertJsonLdCorrect(self, folder, page): + def _assert_jsonld_correct(self, folder, page): body, expected = self._get_body_expected(folder, page) self._check_jsonld(body, expected) @@ -51,7 +52,7 @@ def _get_body_expected(self, folder, page): def _check_jsonld(self, body, expected): jsonlde = JsonLdExtractor() data = jsonlde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_null(self): page = "null_ld_mock" @@ -62,10 +63,10 @@ def test_null(self): jsonlde = JsonLdExtractor() data = jsonlde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_empty_jsonld_script(self): jsonlde = JsonLdExtractor() body = '' data = jsonlde.extract(body) - self.assertEqual(data, []) + assert data == [] diff --git a/tests/test_microdata.py b/tests/test_microdata.py index eda2d26c..31b6e2a3 100644 --- a/tests/test_microdata.py +++ b/tests/test_microdata.py @@ -1,15 +1,11 @@ # mypy: disallow_untyped_defs=False import json -import unittest from extruct.w3cmicrodata import MicrodataExtractor from tests import get_testdata -class TestMicrodata(unittest.TestCase): - - maxDiff = None - +class TestMicrodata: def _test_schemaorg(self, schema, indexes=None): indexes = indexes or [1] for i in indexes: @@ -19,7 +15,7 @@ def _test_schemaorg(self, schema, indexes=None): ) mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_schemaorg_CreativeWork(self): for i in [1]: @@ -29,10 +25,9 @@ def test_schemaorg_CreativeWork(self): "UTF-8" ) ) - mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_schemaorg_LocalBusiness(self): for i in [2, 3]: @@ -42,10 +37,9 @@ def test_schemaorg_LocalBusiness(self): "schema.org", "LocalBusiness.{:03d}.json".format(i) ).decode("UTF-8") ) - mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_schemaorg_MusicRecording(self): for i in [1]: @@ -55,10 +49,9 @@ def test_schemaorg_MusicRecording(self): "schema.org", "MusicRecording.{:03d}.json".format(i) ).decode("UTF-8") ) - mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_schemaorg_Event(self): for i in [1, 2, 3, 4, 8]: @@ -68,11 +61,9 @@ def test_schemaorg_Event(self): "UTF-8" ) ) - mde = MicrodataExtractor() data = mde.extract(body) - - self.assertEqual(data, expected) + assert data == expected def test_schemaorg_SearchAction(self): self._test_schemaorg("SearchAction") @@ -82,20 +73,18 @@ def test_w3c_textContent_values(self): expected = json.loads( get_testdata("w3c", "microdata.4.2.strings.json").decode("UTF-8") ) - mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_textContent_values_unclean(self): body = get_testdata("w3c", "microdata.4.2.strings.unclean.html") expected = json.loads( get_testdata("w3c", "microdata.4.2.strings.unclean.json").decode("UTF-8") ) - mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_5_2(self): body = get_testdata("w3c", "microdata.5.2.html") @@ -103,7 +92,7 @@ def test_w3c_5_2(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_5_3(self): body = get_testdata("w3c", "microdata.5.3.html") @@ -111,7 +100,7 @@ def test_w3c_5_3(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_5_5(self): body = get_testdata("w3c", "microdata.5.5.html") @@ -119,15 +108,14 @@ def test_w3c_5_5(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_7_1(self): body = get_testdata("w3c", "microdata.7.1.html") expected = json.loads(get_testdata("w3c", "microdata.7.1.json").decode("UTF-8")) - mde = MicrodataExtractor(strict=True) data = mde.extract(body, "http://blog.example.com/progress-report") - self.assertEqual(data, expected) + assert data == expected def test_w3c_meter_element(self): body = get_testdata("w3c", "microdata.4.2.meter.html") @@ -137,7 +125,7 @@ def test_w3c_meter_element(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_data_element(self): body = get_testdata("w3c", "microdata.4.2.data.html") @@ -147,7 +135,7 @@ def test_w3c_data_element(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_object_element(self): body = get_testdata("w3c", "microdata.object.html") @@ -157,13 +145,10 @@ def test_w3c_object_element(self): mde = MicrodataExtractor(strict=True) data = mde.extract(body, "http://www.example.com/microdata/test") - self.assertEqual(data, expected) + assert data == expected -class TestMicrodataFlat(unittest.TestCase): - - maxDiff = None - +class TestMicrodataFlat: def test_w3c_5_2(self): body = get_testdata("w3c", "microdata.5.2.html") expected = json.loads( @@ -172,7 +157,7 @@ def test_w3c_5_2(self): mde = MicrodataExtractor(nested=False, strict=True) data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_w3c_7_1(self): body = get_testdata("w3c", "microdata.7.1.html") @@ -182,13 +167,10 @@ def test_w3c_7_1(self): mde = MicrodataExtractor(nested=False, strict=True) data = mde.extract(body, "http://blog.example.com/progress-report") - self.assertEqual(data, expected) - + assert data == expected -class TestMicrodataWithText(unittest.TestCase): - - maxDiff = None +class TestMicrodataWithText: def test_w3c_5_2(self): body = get_testdata("w3c", "microdata.5.2.html") expected = json.loads( @@ -197,13 +179,10 @@ def test_w3c_5_2(self): mde = MicrodataExtractor(add_text_content=True) data = mde.extract(body) - self.assertEqual(data, expected) - + assert data == expected -class TestUrlJoin(unittest.TestCase): - - maxDiff = None +class TestUrlJoin: def test_join_none(self): body = get_testdata("schema.org", "product.html") expected = json.loads( @@ -212,7 +191,7 @@ def test_join_none(self): mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected def test_join_custom_url(self): body = get_testdata("schema.org", "product.html") @@ -222,13 +201,10 @@ def test_join_custom_url(self): mde = MicrodataExtractor() data = mde.extract(body, base_url="http://some-example.com") - self.assertEqual(data, expected) - - -class TestItemref(unittest.TestCase): + assert data == expected - maxDiff = None +class TestItemref: def test_join_none(self): body = get_testdata("schema.org", "product-ref.html") expected = json.loads( @@ -237,12 +213,10 @@ def test_join_none(self): mde = MicrodataExtractor() data = mde.extract(body) - self.assertEqual(data, expected) + assert data == expected -class TestMicrodataWithDescription(unittest.TestCase): - maxDiff = None - +class TestMicrodataWithDescription: def test_if_punctuations_in_description_are_correctly_formatted(self): body = get_testdata("websites", "microdata-with-description.html") expected = json.loads( @@ -251,5 +225,4 @@ def test_if_punctuations_in_description_are_correctly_formatted(self): mde = MicrodataExtractor() data = mde.extract(body) - - self.assertEqual(data, expected) + assert data == expected diff --git a/tests/test_microformat.py b/tests/test_microformat.py index b2908c15..ae0be1f3 100644 --- a/tests/test_microformat.py +++ b/tests/test_microformat.py @@ -1,21 +1,18 @@ # mypy: disallow_untyped_defs=False import json -import unittest from extruct.microformat import MicroformatExtractor from tests import get_testdata, jsonize_dict -class TestMicroformat(unittest.TestCase): - - maxDiff = None - +class TestMicroformat: def test_microformat(self): body = get_testdata("misc", "microformat_test.html") expected = json.loads( get_testdata("misc", "microformat_test.json").decode("UTF-8") ) - opengraphe = MicroformatExtractor() - data = opengraphe.extract(body) - self.assertEqual(jsonize_dict(data), expected) + microformat_extractor = MicroformatExtractor() + data = microformat_extractor.extract(body) + + assert jsonize_dict(data) == expected diff --git a/tests/test_opengraph.py b/tests/test_opengraph.py index 4bdcc010..6bc41450 100644 --- a/tests/test_opengraph.py +++ b/tests/test_opengraph.py @@ -1,22 +1,18 @@ # mypy: disallow_untyped_defs=False import json -import unittest from extruct.opengraph import OpenGraphExtractor from tests import get_testdata, jsonize_dict -class TestOpengraph(unittest.TestCase): - - maxDiff = None - - def _test_opengraph(self, name): +class TestOpengraph: + def _test_opengraph(self, name: str) -> None: body = get_testdata("misc", name + ".html") expected = json.loads(get_testdata("misc", name + ".json").decode("UTF-8")) - opengraphe = OpenGraphExtractor() - data = opengraphe.extract(body) - self.assertEqual(jsonize_dict(data), expected) + opengraph = OpenGraphExtractor() + data = opengraph.extract(body) + assert jsonize_dict(data) == expected def test_opengraph(self): self._test_opengraph("opengraph_test") diff --git a/tests/test_rdfa.py b/tests/test_rdfa.py index b3d81a7e..c2a0a65e 100644 --- a/tests/test_rdfa.py +++ b/tests/test_rdfa.py @@ -1,7 +1,5 @@ # mypy: disallow_untyped_defs=False import json -import unittest -from pprint import pformat from lxml.etree import XML, canonicalize @@ -21,10 +19,7 @@ def tupleize(d): return d -class TestRDFa(unittest.TestCase): - - maxDiff = None - +class TestRDFa: def assertJsonLDEqual(self, a, b, normalize_bnode_ids=True): sa = json.dumps( a, indent=2, separators=(",", ": "), sort_keys=True, ensure_ascii=True @@ -35,7 +30,7 @@ def assertJsonLDEqual(self, a, b, normalize_bnode_ids=True): if normalize_bnode_ids: sa = self.normalize_bnode_ids(sa) sb = self.normalize_bnode_ids(sb) - self.assertEqual(tupleize(json.loads(sa)), tupleize(json.loads(sb))) + assert tupleize(json.loads(sa)) == tupleize(json.loads(sb)) def normalize_bnode_ids(self, jsld): import re diff --git a/tests/test_tool.py b/tests/test_tool.py index f7043d8c..1c4f9309 100644 --- a/tests/test_tool.py +++ b/tests/test_tool.py @@ -28,7 +28,7 @@ def test_metadata_from_url_all_types(self, mock_get): mock_get.return_value = mock_response data = metadata_from_url(self.url) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_jsonld_only(self, mock_get): @@ -44,7 +44,7 @@ def test_metadata_from_url_jsonld_only(self, mock_get): mock_get.return_value = mock_response data = metadata_from_url(self.url, syntaxes=["json-ld"]) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_microdata_only(self, mock_get): @@ -61,7 +61,7 @@ def test_metadata_from_url_microdata_only(self, mock_get): data = metadata_from_url(self.url, syntaxes=["microdata"]) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_rdfa_only(self, mock_get): @@ -77,7 +77,7 @@ def test_metadata_from_url_rdfa_only(self, mock_get): mock_get.return_value = mock_response data = metadata_from_url(self.url, syntaxes=["rdfa"]) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_opengraph_only(self, mock_get): @@ -93,7 +93,7 @@ def test_metadata_from_url_opengraph_only(self, mock_get): mock_get.return_value = mock_response data = metadata_from_url(self.url, syntaxes=["opengraph"]) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_microformat_only(self, mock_get): @@ -109,7 +109,7 @@ def test_metadata_from_url_microformat_only(self, mock_get): mock_get.return_value = mock_response data = metadata_from_url(self.url, syntaxes=["microformat"]) - self.assertEqual(jsonize_dict(data), expected) + assert jsonize_dict(data) == expected @mock.patch("extruct.tool.requests.get") def test_metadata_from_url_unauthorized_page(self, mock_get): @@ -127,7 +127,7 @@ def test_metadata_from_url_unauthorized_page(self, mock_get): mock_response.raise_for_status.side_effect = http_error data = metadata_from_url(url) - self.assertEqual(data, expected) + assert data == expected @mock.patch("extruct.tool.requests.get") def test_main_all(self, mock_get): @@ -142,7 +142,7 @@ def test_main_all(self, mock_get): mock_get.return_value = mock_response data = main([self.url]) - self.assertEqual(data, expected) + assert data == expected @mock.patch("extruct.tool.requests.get") def test_main_single_syntax(self, mock_get): @@ -159,7 +159,7 @@ def test_main_single_syntax(self, mock_get): mock_get.return_value = mock_response data = main([self.url, "--syntax", "opengraph"]) - self.assertEqual(data, expected) + assert data == expected @mock.patch("extruct.tool.requests.get") def test_main_multiple_syntaxes(self, mock_get): @@ -177,7 +177,7 @@ def test_main_multiple_syntaxes(self, mock_get): mock_get.return_value = mock_response data = main([self.url, "--syntax", "opengraph", "microdata"]) - self.assertEqual(data, expected) + assert data == expected def build_mock_response(url, encoding="utf-8", content="", reason="OK", status=200): diff --git a/tests/test_uniform.py b/tests/test_uniform.py index 81d92849..b784135c 100644 --- a/tests/test_uniform.py +++ b/tests/test_uniform.py @@ -1,15 +1,11 @@ # mypy: disallow_untyped_defs=False -import unittest import extruct from extruct.uniform import _flatten, _uopengraph, flatten_dict, infer_context from tests import get_testdata -class TestUniform(unittest.TestCase): - - maxDiff = None - +class TestUniform: def test_uopengraph(self): expected = [ { @@ -29,7 +25,7 @@ def test_uopengraph(self): ] body = get_testdata("songkick", "elysianfields.html") data = extruct.extract(body, syntaxes=["opengraph"], uniform=True) - self.assertEqual(data["opengraph"], expected) + assert data["opengraph"] == expected def test_uopengraph_with_og_array(self): expected = [ @@ -55,7 +51,7 @@ def test_uopengraph_with_og_array(self): data = extruct.extract( body, syntaxes=["opengraph"], uniform=True, with_og_array=True ) - self.assertEqual(data["opengraph"], expected) + assert data["opengraph"] == expected def test_uopengraph_duplicated_priorities(self): # Ensures that first seen property is kept when flattening @@ -196,7 +192,7 @@ def test_umicroformat(self): ] body = get_testdata("misc", "microformat_test.html") data = extruct.extract(body, syntaxes=["microformat"], uniform=True) - self.assertEqual(data["microformat"], expected) + assert data["microformat"] == expected def test_umicrodata(self): expected = [ @@ -226,7 +222,7 @@ def test_umicrodata(self): ] body = get_testdata("misc", "product_microdata.html") data = extruct.extract(body, syntaxes=["microdata"], uniform=True) - self.assertEqual(data["microdata"], expected) + assert data["microdata"] == expected def test_udublincore(self): expected = [ @@ -294,17 +290,17 @@ def test_udublincore(self): ] body = get_testdata("misc", "dublincore_test.html") data = extruct.extract(body, syntaxes=["dublincore"], uniform=True) - self.assertEqual(data["dublincore"], expected) + assert data["dublincore"] == expected def test_infer_context(self): context = "http://schema.org/UsedCondition" - self.assertEqual(infer_context(context), ("http://schema.org", "UsedCondition")) + assert infer_context(context) == ("http://schema.org", "UsedCondition") context = "http://ogp.me/ns#description" - self.assertEqual(infer_context(context), ("http://ogp.me/ns", "description")) + assert infer_context(context) == ("http://ogp.me/ns", "description") context = "http://ogp.me/ns/fb#app_id" - self.assertEqual(infer_context(context), ("http://ogp.me/ns/fb", "app_id")) + assert infer_context(context) == ("http://ogp.me/ns/fb", "app_id") def test_flatten_dict(self): d = { @@ -322,9 +318,9 @@ def test_flatten_dict(self): "extra_weapon": "fear", "another_one": "ruthless efficiency", } - self.assertEqual( - flatten_dict(d, schema_context="http://schema.org", add_context=True), - expected, + assert ( + flatten_dict(d, schema_context="http://schema.org", add_context=True) + == expected ) def test_flatten(self): @@ -369,4 +365,4 @@ def test_flatten(self): "name": [""], "@type": ["h-hidden-phone"], } - self.assertEqual(_flatten(d, schema_context="http://schema.org"), expected) + assert _flatten(d, schema_context="http://schema.org") == expected