From cc9007782ace0e67c5b0ef2b42085ff37a64a45e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Tue, 21 Feb 2023 14:52:44 +0100 Subject: [PATCH 01/30] Testcase for #52: Support remote images --- tests/example/workspaces/remote/.gitignore | 1 + tests/example/workspaces/remote/mets.xml | 115 +++++++++++++++++++++ tests/model/test_document.py | 12 +++ 3 files changed, 128 insertions(+) create mode 100644 tests/example/workspaces/remote/.gitignore create mode 100644 tests/example/workspaces/remote/mets.xml diff --git a/tests/example/workspaces/remote/.gitignore b/tests/example/workspaces/remote/.gitignore new file mode 100644 index 0000000..821f5dd --- /dev/null +++ b/tests/example/workspaces/remote/.gitignore @@ -0,0 +1 @@ +/OCR-D-* diff --git a/tests/example/workspaces/remote/mets.xml b/tests/example/workspaces/remote/mets.xml new file mode 100644 index 0000000..1ed3c4a --- /dev/null +++ b/tests/example/workspaces/remote/mets.xml @@ -0,0 +1,115 @@ + + + + DFG-Koordinierungsprojekt zur Weiterentwicklung von Verfahren der Optical Character Recognition (OCR-D) + OCR-D + + + + + + + + DE-1 + Gq 14350;Beil.3-1839 + + + 1839 + + + + Berlin + + 2013 + Staatsbibliothek zu Berlin – Preußischer Kulturbesitz, Germany + [Electronic ed.] + + Rechtswissenschaft + Historische Drucke + + + PPN767122410 + + + + PPN767137728 + + http://resolver.staatsbibliothek-berlin.de/SBB0000F29300010000 + + Der Herold + + P_Drucke_Territorialrecht + + + 1839 + + + + ger + + + + Deutsches Territorialrecht des 19. Jahrhunderts + + + + reformatted digital + + CC BY-NC-SA 4.0 International + text + + + + + + + + + + 4. Januar-30. November = No. 1-20 + + + + + + + + + + + Staatsbibliothek zu Berlin - Preußischer Kulturbesitz + http://resolver.staatsbibliothek-berlin.de/SBB0000000100000000 + http://www.staatsbibliothek-berlin.de + mailto:info@sbb.spk-berlin.de + + + + + + + + + http://www.stabikat.de/DB=1/PPN?PPN=767137728 + http://digital.staatsbibliothek-berlin.de/dms/werkansicht/?PPN=PPN767137728 + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 0596d6a..38d8c2b 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -1,3 +1,5 @@ +import shutil + from pathlib import Path from tempfile import TemporaryDirectory @@ -187,3 +189,13 @@ def test_missing_image(self): image, info, exif = page.get_image(feature_selector='', feature_filter='binarized') # Assert no exceptions happened and no image returned self.assertIsNone(image) + + def test_remote_image(self): + path = TEST_BASE_PATH / 'example/workspaces/remote/mets.xml' + for file_group_dir in path.parent.glob('OCR-D-*'): + shutil.rmtree(file_group_dir) + doc = Document.load(path.as_uri()) + paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN','image/tiff')) + self.assertGreater(len(paths), 0) + self.assertIsInstance(list(paths.values())[0], Path) + From c5b5fef66369b90a8cc815ced2137c0a5ef4b2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Tue, 21 Feb 2023 14:58:02 +0100 Subject: [PATCH 02/30] Testcase for #52: cleanup --- tests/example/workspaces/remote/mets.xml | 103 +++-------------------- tests/model/test_document.py | 3 +- 2 files changed, 14 insertions(+), 92 deletions(-) diff --git a/tests/example/workspaces/remote/mets.xml b/tests/example/workspaces/remote/mets.xml index 1ed3c4a..a9c7028 100644 --- a/tests/example/workspaces/remote/mets.xml +++ b/tests/example/workspaces/remote/mets.xml @@ -1,113 +1,36 @@ - - - - DFG-Koordinierungsprojekt zur Weiterentwicklung von Verfahren der Optical Character Recognition (OCR-D) - OCR-D + + + + + ocrd/core v1.0.0 + - - DE-1 - Gq 14350;Beil.3-1839 - - - 1839 - - - - Berlin - - 2013 - Staatsbibliothek zu Berlin – Preußischer Kulturbesitz, Germany - [Electronic ed.] - - Rechtswissenschaft - Historische Drucke - - - PPN767122410 - - - - PPN767137728 - - http://resolver.staatsbibliothek-berlin.de/SBB0000F29300010000 - - Der Herold - - P_Drucke_Territorialrecht - - - 1839 - - - - ger - - - - Deutsches Territorialrecht des 19. Jahrhunderts - - - - reformatted digital - - CC BY-NC-SA 4.0 International - text - - - - - - - - - - 4. Januar-30. November = No. 1-20 - + remote - - - - - - Staatsbibliothek zu Berlin - Preußischer Kulturbesitz - http://resolver.staatsbibliothek-berlin.de/SBB0000000100000000 - http://www.staatsbibliothek-berlin.de - mailto:info@sbb.spk-berlin.de - - - - - - - - - http://www.stabikat.de/DB=1/PPN?PPN=767137728 - http://digital.staatsbibliothek-berlin.de/dms/werkansicht/?PPN=PPN767137728 - - - - - - + + - + diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 38d8c2b..d92dc4f 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -195,7 +195,6 @@ def test_remote_image(self): for file_group_dir in path.parent.glob('OCR-D-*'): shutil.rmtree(file_group_dir) doc = Document.load(path.as_uri()) - paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN','image/tiff')) + paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) self.assertGreater(len(paths), 0) self.assertIsInstance(list(paths.values())[0], Path) - From 94a5bb528f0437cb359629dd0a3cd4c0e7dd5b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Tue, 21 Feb 2023 15:15:35 +0100 Subject: [PATCH 03/30] Testcase for #52: cleanup --- tests/model/test_document.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/model/test_document.py b/tests/model/test_document.py index d92dc4f..922d18f 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -55,6 +55,16 @@ def test_get_image_paths_only_returns_matching_groups(self): self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_paths['PHYS_0017'].name) self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_paths['PHYS_0020'].name) + def test_get_image_path_with_remote_image(self): + path = TEST_BASE_PATH / 'example/workspaces/remote/mets.xml' + for file_group_dir in path.parent.glob('OCR-D-*'): + shutil.rmtree(file_group_dir) + doc = Document.load(path.as_uri()) + paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) + self.assertGreater(len(paths), 0) + self.assertIsInstance(list(paths.values())[0], Path) + + def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') file_group = doc.get_default_image_group(['OCR-D-IMG-BIN', 'OCR-D-IMG.*']) @@ -189,12 +199,3 @@ def test_missing_image(self): image, info, exif = page.get_image(feature_selector='', feature_filter='binarized') # Assert no exceptions happened and no image returned self.assertIsNone(image) - - def test_remote_image(self): - path = TEST_BASE_PATH / 'example/workspaces/remote/mets.xml' - for file_group_dir in path.parent.glob('OCR-D-*'): - shutil.rmtree(file_group_dir) - doc = Document.load(path.as_uri()) - paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) - self.assertGreater(len(paths), 0) - self.assertIsInstance(list(paths.values())[0], Path) From 8d8dcb329337c09d40590756371f44d2686ee854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Tue, 21 Feb 2023 17:24:16 +0100 Subject: [PATCH 04/30] Remove get_file_index and use OcrdMets new builtin caching --- ocrd_browser/model/document.py | 90 ++++++++++++++++------------------ tests/model/test_document.py | 9 ---- 2 files changed, 43 insertions(+), 56 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index 534fd4e..c116857 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -1,5 +1,8 @@ from __future__ import annotations -from typing import Optional, Tuple, List, Union, cast, Callable, Any, Dict, Sequence, TYPE_CHECKING + +from contextlib import contextmanager + +from typing import Optional, Tuple, List, Union, cast, Callable, Any, Dict, Sequence, Generator, TYPE_CHECKING import atexit import errno @@ -41,6 +44,17 @@ EventCallBack = Optional[Callable[[str, Any], None]] +@contextmanager +def caching_environment(value: str = '1') -> Generator[None, None, None]: + backup_caching = os.environ.get('OCRD_METS_CACHING', None) + os.environ['OCRD_METS_CACHING'] = value + try: + yield + finally: + if backup_caching is not None: + os.environ['OCRD_METS_CACHING'] = backup_caching + + def check_editable(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) def guard(self: 'Document', *args: List[Any], **kwargs: Dict[Any, Any]) -> Any: @@ -81,7 +95,9 @@ def load(cls, mets_url: Union[Path, str] = None, emitter: EventCallBack = None) return cls.create(emitter=emitter) mets_path = cls._to_path(mets_url) - workspace = Resolver().workspace_from_url(str(mets_path), download=False) + with caching_environment('1'): + workspace = Resolver().workspace_from_url(str(mets_path), download=False) + doc = cls(workspace, emitter=emitter, original_url=str(mets_url)) doc._empty = False return doc @@ -106,7 +122,10 @@ def _clone_workspace(cls, mets_url: Union[Path, str]) -> Workspace: cls.temporary_workspaces.append(temporary_workspace) # TODO download = False and lazy loading would be nice for responsiveness log.info("Cloning '%s' to '%s'", mets_url, temporary_workspace) - workspace = Resolver().workspace_from_url(mets_url=mets_url, dst_dir=temporary_workspace, download=True) + + with caching_environment('1'): + workspace = Resolver().workspace_from_url(mets_url=mets_url, dst_dir=temporary_workspace, download=True) + return workspace @check_editable @@ -219,49 +238,25 @@ def file_groups(self) -> List[FileGroupHandle]: def title(self) -> str: return str(self.workspace.mets.unique_identifier) if self.workspace and self.workspace.mets.unique_identifier else '' - def get_file_index(self) -> Dict[str, OcrdFile]: - """ - Return all OcrdFiles by file id and additionally augments the OcrdFile with static_page_id for fast(er) lookup - - Example: - page17 = [file for file in file_index.values() if file.static_page_id == 'PHYS_0017'] - - """ - log = getLogger('ocrd_browser.model.document.Document.get_file_index') - file_index = {} - if self.workspace: - for file in self.workspace.mets.find_files(): - file.static_page_id = None - file_index[file.ID] = file - - file_pointers: List[Element] = self.xpath( - 'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]/mets:fptr') - for file_pointer in file_pointers: - file_id = file_pointer.get('FILEID') - page_id = file_pointer.getparent().get('ID') - if file_id in file_index: - file_index[file_id].static_page_id = page_id - else: - log.warning("FILEID '%s' for PAGE '%s' not in mets:fileSec", file_id, page_id) - - return file_index - - def get_image_paths(self, file_group: FileGroupHandle) -> Dict[str, Path]: + def get_image_paths(self, file_group: FileGroupHandle) -> Dict[str, Optional[Path]]: """ Builds a Dict ID->Path for all page_ids fast More precisely: fast = Faster than iterating over page_ids and using mets.get_physical_page_for_file for each entry """ log = getLogger('ocrd_browser.model.document.Document.get_image_paths') - image_paths = {} - file_index = self.get_file_index() - for page_id in self.page_ids: - images = [image for image in file_index.values() if image.static_page_id == page_id and file_group.match(image)] - if len(images) > 0: - image_paths[page_id] = self.directory.joinpath(images[0].local_filename) - else: - log.warning('Found no images for PAGE %s and fileGrp %s', page_id, file_group) - image_paths[page_id] = None + image_paths: Dict[str, Optional[Path]] = {} + if self.workspace: + for page_id in self.page_ids: + for file in self.workspace.mets.find_files(pageId=page_id, fileGrp=file_group.group, mimetype=file_group.mime): + if page_id in image_paths: + log.warning('Multiple images for PAGE %s and fileGrp %s, using first %s', page_id, file_group, image_paths[page_id]) + else: + image_paths[page_id] = self.directory.joinpath(file.local_filename) + if page_id not in image_paths: + log.warning('Found no images for PAGE %s and fileGrp %s', page_id, file_group) + image_paths[page_id] = None + return image_paths def get_default_image_group(self, preferred_image_file_groups: Optional[List[str]] = None) -> Optional[FileGroupHandle]: @@ -418,14 +413,15 @@ def editable(self) -> bool: @editable.setter def editable(self, editable: bool) -> None: - if editable: - if self._original_url: - self.workspace = self._clone_workspace(self._original_url) + with caching_environment('1'): + if editable: + if self._original_url: + self.workspace = self._clone_workspace(self._original_url) + else: + # noinspection PyTypeChecker + self.workspace = Resolver().workspace_from_nothing(directory=None, mets_basename='mets.xml') else: - # noinspection PyTypeChecker - self.workspace = Resolver().workspace_from_nothing(directory=None, mets_basename='mets.xml') - else: - self.workspace = Resolver().workspace_from_url(self.baseurl_mets) + self.workspace = Resolver().workspace_from_url(self.baseurl_mets) self._editable = editable # self._empty = False # self._modified = False diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 922d18f..df7117c 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -30,14 +30,6 @@ def test_get_file_groups(self): ] self.assertEqual(expected, doc.file_groups) - def test_get_page_index(self): - doc = Document.load(self.path) - file_index = doc.get_file_index() - page17 = [file for file in file_index.values() if file.static_page_id == 'PHYS_0017'] - alto = [file for file in file_index.values() if file.mimetype == 'application/alto+xml'] - self.assertEqual(3, len(page17)) - self.assertEqual(2, len(alto)) - def test_get_image_paths(self): doc = Document.load(self.path) image_paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG', 'image/tiff')) @@ -64,7 +56,6 @@ def test_get_image_path_with_remote_image(self): self.assertGreater(len(paths), 0) self.assertIsInstance(list(paths.values())[0], Path) - def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') file_group = doc.get_default_image_group(['OCR-D-IMG-BIN', 'OCR-D-IMG.*']) From fba5419b21682dbd4a779883bf087f866715c2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Tue, 21 Feb 2023 18:09:15 +0100 Subject: [PATCH 05/30] Moved the problem closer to solution (doc.get_image_paths -> PageListStore.__init__) --- ocrd_browser/model/document.py | 10 +++++----- ocrd_browser/ui/page_store.py | 8 ++++++-- tests/model/test_document.py | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index c116857..f5c60a3 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -238,23 +238,23 @@ def file_groups(self) -> List[FileGroupHandle]: def title(self) -> str: return str(self.workspace.mets.unique_identifier) if self.workspace and self.workspace.mets.unique_identifier else '' - def get_image_paths(self, file_group: FileGroupHandle) -> Dict[str, Optional[Path]]: + def get_image_files(self, file_group: FileGroupHandle) -> Dict[str, Optional[OcrdFile]]: """ Builds a Dict ID->Path for all page_ids fast More precisely: fast = Faster than iterating over page_ids and using mets.get_physical_page_for_file for each entry """ log = getLogger('ocrd_browser.model.document.Document.get_image_paths') - image_paths: Dict[str, Optional[Path]] = {} + image_paths: Dict[str, Optional[OcrdFile]] = {} if self.workspace: for page_id in self.page_ids: for file in self.workspace.mets.find_files(pageId=page_id, fileGrp=file_group.group, mimetype=file_group.mime): if page_id in image_paths: - log.warning('Multiple images for PAGE %s and fileGrp %s, using first %s', page_id, file_group, image_paths[page_id]) + log.warning('Multiple files for PAGE %s and fileGrp %s, using first %s', page_id, file_group, image_paths[page_id]) else: - image_paths[page_id] = self.directory.joinpath(file.local_filename) + image_paths[page_id] = file if page_id not in image_paths: - log.warning('Found no images for PAGE %s and fileGrp %s', page_id, file_group) + log.warning('Found no files for PAGE %s and fileGrp %s', page_id, file_group) image_paths[page_id] = None return image_paths diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index a452b4d..6cb9f38 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -52,13 +52,16 @@ def __init__(self, document: Document): ) for icon_name in ['page-loading', 'page-missing'] } + # TODO end constructor here and add a new method to fill the store for testability and developer sanity # TODO: make file_group selectable, see https://github.com/hnesk/browse-ocrd/issues/7#issuecomment-707851109 self.file_group = document.get_default_image_group(Settings.get().file_groups.preferred_images) - file_lookup = document.get_image_paths(self.file_group) + file_lookup = document.get_image_files(self.file_group) order = count(start=1) for page_id in self.document.page_ids: file = file_lookup[page_id] - self.append((page_id, '', str(file) if file else None, None, next(order))) + # TODO: self.document.path(file) works only for local files + path = document.path(file) + self.append((page_id, '', str(path) if file else None, None, next(order))) GLib.timeout_add(10, self.start_loading) @@ -103,6 +106,7 @@ def _page_added(page_ids: List[str]) -> None: for page_id in page_ids: try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) + # TODO: self.document.path(file) works only for local files file_name = str(self.document.path(file)) self.append((page_id, '', file_name, None, len(self))) except StopIteration as e: diff --git a/tests/model/test_document.py b/tests/model/test_document.py index df7117c..c194873 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -1,4 +1,5 @@ import shutil +from ocrd_models import OcrdFile from pathlib import Path from tempfile import TemporaryDirectory @@ -32,29 +33,29 @@ def test_get_file_groups(self): def test_get_image_paths(self): doc = Document.load(self.path) - image_paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG', 'image/tiff')) - self.assertEqual(2, len(image_paths)) - self.assertEqual('INPUT_0017.tif', image_paths['PHYS_0017'].name) - self.assertEqual('INPUT_0020.tif', image_paths['PHYS_0020'].name) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG', 'image/tiff')) + self.assertEqual(2, len(image_files)) + self.assertEqual('INPUT_0017.tif', image_files['PHYS_0017'].basename) + self.assertEqual('INPUT_0020.tif', image_files['PHYS_0020'].basename) def test_get_image_paths_only_returns_matching_groups(self): """ Testcase for https://github.com/hnesk/browse-ocrd/issues/51 """ doc = Document.load(ASSETS_PATH / '../example/workspaces/kant_aufklaerung_1784_bin/mets.xml') - image_paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/png')) - self.assertEqual(2, len(image_paths)) - self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_paths['PHYS_0017'].name) - self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_paths['PHYS_0020'].name) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/png')) + self.assertEqual(2, len(image_files)) + self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_files['PHYS_0017'].basename) + self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_files['PHYS_0020'].basename) def test_get_image_path_with_remote_image(self): path = TEST_BASE_PATH / 'example/workspaces/remote/mets.xml' for file_group_dir in path.parent.glob('OCR-D-*'): shutil.rmtree(file_group_dir) doc = Document.load(path.as_uri()) - paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) - self.assertGreater(len(paths), 0) - self.assertIsInstance(list(paths.values())[0], Path) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) + self.assertGreater(len(image_files), 0) + self.assertIsInstance(list(image_files.values())[0], OcrdFile) def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') From 02a04a2b6febf6f3631aa7246057df40538ef654 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 27 Feb 2023 19:41:42 +0100 Subject: [PATCH 06/30] download remote files on demand --- ocrd_browser/model/document.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index 534fd4e..ed6b770 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -173,6 +173,8 @@ def path(self, other: Union[OcrdFile, Path, str]) -> Optional[Path]: if not self.directory: return None if isinstance(other, OcrdFile): + if not other.local_filename: + other = self.workspace.download_file(other) return self.directory.joinpath(other.local_filename) elif isinstance(other, Path): return self.directory.joinpath(other) @@ -258,7 +260,7 @@ def get_image_paths(self, file_group: FileGroupHandle) -> Dict[str, Path]: for page_id in self.page_ids: images = [image for image in file_index.values() if image.static_page_id == page_id and file_group.match(image)] if len(images) > 0: - image_paths[page_id] = self.directory.joinpath(images[0].local_filename) + image_paths[page_id] = self.path(images[0]) else: log.warning('Found no images for PAGE %s and fileGrp %s', page_id, file_group) image_paths[page_id] = None @@ -319,7 +321,7 @@ def page_for_file(self, page_file: OcrdFile) -> PcGtsType: def resolve_image(self, image_file: OcrdFile) -> Image: with pushd_popd(self.workspace.directory): - pil_image = Image.open(self.workspace.download_file(image_file).local_filename) + pil_image = Image.open(self.path(image_file)) # pil_image.load() return pil_image From 9c01137ab55023aeb276947cae33911c1eb5486e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Mon, 27 Feb 2023 21:49:38 +0100 Subject: [PATCH 07/30] Two testcases: One with single small remote resource, and one with several resources (~12MB) --- .../{remote => remote-many}/.gitignore | 0 tests/example/workspaces/remote-many/mets.xml | 65 +++++++++++++++++++ .../workspaces/remote-single/.gitignore | 1 + .../{remote => remote-single}/mets.xml | 0 tests/model/test_document.py | 5 +- 5 files changed, 70 insertions(+), 1 deletion(-) rename tests/example/workspaces/{remote => remote-many}/.gitignore (100%) create mode 100644 tests/example/workspaces/remote-many/mets.xml create mode 100644 tests/example/workspaces/remote-single/.gitignore rename tests/example/workspaces/{remote => remote-single}/mets.xml (100%) diff --git a/tests/example/workspaces/remote/.gitignore b/tests/example/workspaces/remote-many/.gitignore similarity index 100% rename from tests/example/workspaces/remote/.gitignore rename to tests/example/workspaces/remote-many/.gitignore diff --git a/tests/example/workspaces/remote-many/mets.xml b/tests/example/workspaces/remote-many/mets.xml new file mode 100644 index 0000000..321aeff --- /dev/null +++ b/tests/example/workspaces/remote-many/mets.xml @@ -0,0 +1,65 @@ + + + + + ocrd/core v1.0.0 + + + + + + + + remote + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/example/workspaces/remote-single/.gitignore b/tests/example/workspaces/remote-single/.gitignore new file mode 100644 index 0000000..821f5dd --- /dev/null +++ b/tests/example/workspaces/remote-single/.gitignore @@ -0,0 +1 @@ +/OCR-D-* diff --git a/tests/example/workspaces/remote/mets.xml b/tests/example/workspaces/remote-single/mets.xml similarity index 100% rename from tests/example/workspaces/remote/mets.xml rename to tests/example/workspaces/remote-single/mets.xml diff --git a/tests/model/test_document.py b/tests/model/test_document.py index c194873..3c6c4d6 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -49,7 +49,7 @@ def test_get_image_paths_only_returns_matching_groups(self): self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_files['PHYS_0020'].basename) def test_get_image_path_with_remote_image(self): - path = TEST_BASE_PATH / 'example/workspaces/remote/mets.xml' + path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' for file_group_dir in path.parent.glob('OCR-D-*'): shutil.rmtree(file_group_dir) doc = Document.load(path.as_uri()) @@ -57,6 +57,9 @@ def test_get_image_path_with_remote_image(self): self.assertGreater(len(image_files), 0) self.assertIsInstance(list(image_files.values())[0], OcrdFile) + + + def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') file_group = doc.get_default_image_group(['OCR-D-IMG-BIN', 'OCR-D-IMG.*']) From 834b073b67057299e2d85369d09ba72ffa5ea325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Mon, 27 Feb 2023 21:59:26 +0100 Subject: [PATCH 08/30] allow_download=True will download files --- ocrd_browser/model/document.py | 20 ++++++++++++-------- ocrd_browser/ui/page_store.py | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index 5a4f914..b93eca1 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -185,7 +185,7 @@ def baseurl_mets(self) -> str: """ return str(self.workspace.baseurl) + '/' + self.mets_filename if self.workspace else None - def path(self, other: Union[OcrdFile, Path, str]) -> Optional[Path]: + def path(self, other: Union[OcrdFile, Path, str], allow_download: bool = False) -> Optional[Path]: """ Resolves other relative to current workspace """ @@ -193,7 +193,10 @@ def path(self, other: Union[OcrdFile, Path, str]) -> Optional[Path]: return None if isinstance(other, OcrdFile): if not other.local_filename: - other = self.workspace.download_file(other) + if allow_download: + other = self.workspace.download_file(other) + else: + raise ValueError('path with allow_download=False for non local file called: #{s}: {}'.format(other.ID, other.url)) return self.directory.joinpath(other.local_filename) elif isinstance(other, Path): return self.directory.joinpath(other) @@ -240,13 +243,11 @@ def file_groups(self) -> List[FileGroupHandle]: def title(self) -> str: return str(self.workspace.mets.unique_identifier) if self.workspace and self.workspace.mets.unique_identifier else '' - def get_image_files(self, file_group: FileGroupHandle) -> Dict[str, Optional[OcrdFile]]: + def get_image_files(self, file_group: FileGroupHandle, allow_download=False) -> Dict[str, Optional[OcrdFile]]: """ - Builds a Dict ID->Path for all page_ids fast - - More precisely: fast = Faster than iterating over page_ids and using mets.get_physical_page_for_file for each entry + Builds a Dict PageID->OcrdFile|None for all page_ids """ - log = getLogger('ocrd_browser.model.document.Document.get_image_paths') + log = getLogger('ocrd_browser.model.document.Document.get_image_files') image_paths: Dict[str, Optional[OcrdFile]] = {} if self.workspace: for page_id in self.page_ids: @@ -254,6 +255,9 @@ def get_image_files(self, file_group: FileGroupHandle) -> Dict[str, Optional[Ocr if page_id in image_paths: log.warning('Multiple files for PAGE %s and fileGrp %s, using first %s', page_id, file_group, image_paths[page_id]) else: + if not file.local_filename: + if allow_download: + file = self.workspace.download_file(file) image_paths[page_id] = file if page_id not in image_paths: log.warning('Found no files for PAGE %s and fileGrp %s', page_id, file_group) @@ -316,7 +320,7 @@ def page_for_file(self, page_file: OcrdFile) -> PcGtsType: def resolve_image(self, image_file: OcrdFile) -> Image: with pushd_popd(self.workspace.directory): - pil_image = Image.open(self.path(image_file)) + pil_image = Image.open(self.path(image_file, allow_download=True)) # pil_image.load() return pil_image diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index 6cb9f38..f2be7e8 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -55,7 +55,7 @@ def __init__(self, document: Document): # TODO end constructor here and add a new method to fill the store for testability and developer sanity # TODO: make file_group selectable, see https://github.com/hnesk/browse-ocrd/issues/7#issuecomment-707851109 self.file_group = document.get_default_image_group(Settings.get().file_groups.preferred_images) - file_lookup = document.get_image_files(self.file_group) + file_lookup = document.get_image_files(self.file_group, allow_download=True) order = count(start=1) for page_id in self.document.page_ids: file = file_lookup[page_id] @@ -107,7 +107,7 @@ def _page_added(page_ids: List[str]) -> None: try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) # TODO: self.document.path(file) works only for local files - file_name = str(self.document.path(file)) + file_name = str(self.document.path(file, allow_download=True)) self.append((page_id, '', file_name, None, len(self))) except StopIteration as e: raise ValueError('Page {} in group {} not in workspace'.format(page_id, self.file_group)) from e @@ -122,7 +122,7 @@ def _page_changed(page_ids: List[str]) -> None: n, row = self.get_row_by_page_id(page_id) try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) - file_name = str(self.document.path(file)) + file_name = str(self.document.path(file, allow_download=True)) row[self.COLUMN_FILENAME] = file_name except StopIteration: pass From 88da0ce0fb9e43b99f221b57541b21d0c88db7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Mon, 27 Feb 2023 23:02:55 +0100 Subject: [PATCH 09/30] directly use document.get_image_files --- ocrd_browser/ui/page_store.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index f2be7e8..ccb66f1 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -55,13 +55,14 @@ def __init__(self, document: Document): # TODO end constructor here and add a new method to fill the store for testability and developer sanity # TODO: make file_group selectable, see https://github.com/hnesk/browse-ocrd/issues/7#issuecomment-707851109 self.file_group = document.get_default_image_group(Settings.get().file_groups.preferred_images) - file_lookup = document.get_image_files(self.file_group, allow_download=True) + files = document.get_image_files(self.file_group, allow_download=True) order = count(start=1) - for page_id in self.document.page_ids: - file = file_lookup[page_id] - # TODO: self.document.path(file) works only for local files - path = document.path(file) - self.append((page_id, '', str(path) if file else None, None, next(order))) + for page_id, file in files.items(): + if file: + path = str(document.path(file)) + self.append((page_id, '', path, None, next(order))) + else: + self.append((page_id, '', None, None, next(order))) GLib.timeout_add(10, self.start_loading) From 4be9a6168cfebb294927578cdb052115e8d29509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Mon, 27 Feb 2023 23:40:13 +0100 Subject: [PATCH 10/30] small refactorings --- ocrd_browser/model/document.py | 4 ++-- ocrd_browser/ui/page_store.py | 18 ++++++++++-------- tests/model/test_document.py | 19 ++++++++++++++++--- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index b93eca1..ea1fb5f 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -196,7 +196,7 @@ def path(self, other: Union[OcrdFile, Path, str], allow_download: bool = False) if allow_download: other = self.workspace.download_file(other) else: - raise ValueError('path with allow_download=False for non local file called: #{s}: {}'.format(other.ID, other.url)) + raise ValueError('path with allow_download=False for non local file called: #{}: {}'.format(other.ID, other.url)) return self.directory.joinpath(other.local_filename) elif isinstance(other, Path): return self.directory.joinpath(other) @@ -243,7 +243,7 @@ def file_groups(self) -> List[FileGroupHandle]: def title(self) -> str: return str(self.workspace.mets.unique_identifier) if self.workspace and self.workspace.mets.unique_identifier else '' - def get_image_files(self, file_group: FileGroupHandle, allow_download=False) -> Dict[str, Optional[OcrdFile]]: + def get_image_files(self, file_group: FileGroupHandle, allow_download: bool = False) -> Dict[str, Optional[OcrdFile]]: """ Builds a Dict PageID->OcrdFile|None for all page_ids """ diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index ccb66f1..6e4aa35 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -155,19 +155,21 @@ def _reordered(old_to_new_ids: Dict[str, str]) -> None: def _init_row(self, row: Gtk.TreeModelRow) -> None: if row[self.COLUMN_FILENAME] is not None: - row[1] = 'Loading {}'.format(row[self.COLUMN_FILENAME]) - row[3] = self.pixbufs['page-loading'] + row[PageListStore.COLUMN_TOOLTIP] = 'Loading {}'.format(row[self.COLUMN_FILENAME]) + row[PageListStore.COLUMN_THUMB] = self.pixbufs['page-loading'] else: - row[1] = 'No image for {}'.format(row[self.COLUMN_PAGE_ID]) - row[3] = self.pixbufs['page-missing'] + row[PageListStore.COLUMN_TOOLTIP] = 'No image for {}'.format(row[self.COLUMN_PAGE_ID]) + row[PageListStore.COLUMN_THUMB] = self.pixbufs['page-missing'] @staticmethod def _load_row(row: Gtk.TreeModelRow) -> Gtk.TreeModelRow: filename = row[PageListStore.COLUMN_FILENAME] - if filename is not None: - image = cv2.imread(filename) - row[1] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) - row[3] = cv_to_pixbuf(cv_scale(image, 100, None)) + if filename is None: + return row + + image = cv2.imread(filename) + row[PageListStore.COLUMN_TOOLTIP] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) + row[PageListStore.COLUMN_THUMB] = cv_to_pixbuf(cv_scale(image, 100, None)) return row @staticmethod diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 3c6c4d6..51d75a0 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -48,16 +48,29 @@ def test_get_image_paths_only_returns_matching_groups(self): self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_files['PHYS_0017'].basename) self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_files['PHYS_0020'].basename) - def test_get_image_path_with_remote_image(self): + def test_get_image_files_with_remote_image(self): path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' for file_group_dir in path.parent.glob('OCR-D-*'): shutil.rmtree(file_group_dir) doc = Document.load(path.as_uri()) - image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff')) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff'), allow_download=True) self.assertGreater(len(image_files), 0) - self.assertIsInstance(list(image_files.values())[0], OcrdFile) + file = image_files['PHYS_0002'] + self.assertIsInstance(file, OcrdFile) + self.assertIsNotNone(file.local_filename) + def test_get_image_files_with_remote_image_and_disallowed_download(self): + path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' + for file_group_dir in path.parent.glob('OCR-D-*'): + shutil.rmtree(file_group_dir) + doc = Document.load(path.as_uri()) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff'), allow_download=False) + self.assertGreater(len(image_files), 0) + file = image_files['PHYS_0002'] + self.assertIsInstance(file, OcrdFile) + self.assertIsNone(file.local_filename) + def test_get_default_image_group(self): From bcbcbb8c8932eba2c1f664ba4be58565dfbc99b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 2 Mar 2023 15:36:38 +0100 Subject: [PATCH 11/30] new icon "downloading" and renaming "loading" to "generating-thumb" --- gresources/icons/page-downloading.png | Bin 0 -> 7677 bytes ...-loading.png => page-generating-thumb.png} | Bin gresources/icons/page-icons.xcf | Bin 20470 -> 23145 bytes gresources/ocrd-browser.gresource.xml | 3 ++- 4 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 gresources/icons/page-downloading.png rename gresources/icons/{page-loading.png => page-generating-thumb.png} (100%) diff --git a/gresources/icons/page-downloading.png b/gresources/icons/page-downloading.png new file mode 100644 index 0000000000000000000000000000000000000000..207122ff8e7579c0d01c7092f6ef96d63742471a GIT binary patch literal 7677 zcmZX2WmFtNv-aYy!2$#b5Zv7*xCVC!?gV!y1oy?=oy7t{g1fsri)&zUz5AYfzH`5O z?vL*2u9=#ds;=khu8LGqmcc+JMg;%>7;>_bYS61Ww1p!hLi=rw7cT$+Gsj0m$4$+| zlfucx(bC4=g2K()$%4Yd%f=D_@LHY7&~>2_#thx^j^c-vbLw+$ksil5cz_iA*4cb= z{C3}`W0GP}xeH6p@xkKHJLD4lHt=n zujr*-M<5RW7t;MZtMmI)!9&OzMm&G7SKwyTGjII{66#cNEqQ&$ImE}1`Vf5X*HSxu z+{vn|yEdjaaY@%Z;3)x*{tqPX!{twd+#%Mf@RyyxLR#ka zff>37t_PlnYY%l(Y1?^?nSx8DD^lNnsiZxZlIgdKIPXxfxol{8X$ElSo!+V$O*~%E z>gWVK_1Ga|B)a7!^r8x9*VTg2fnQ2n%=U@EF)lfFYDI(en~B}#wN9q^DN+JO!fr2Xu2foNI`OXmalCKD3*UyxlfmxQXx^^Rzsjw9-HK$ zq9V8;_ltx_>t3e*;(Bzlh6{ZO0ace#Nol5CMN@NS6R%ChzLuWD1h<1>5$axgll4Mr zWe2B?)!yVr4w?_S^RO&cJjaQ(giKzyRcVG*hC2_h`yGB7(}P)xrnY|D6DA4W`ELkI z`$e<+J=}a;w%av*S+2*;jn!gE`zVW_&uhS9FO}WZXtJSBo-TM%k6Z%22x!kwWyX%c zVZ;mv5YU_SL!+0ns%Xsi=6)JW)U;HXL``MqNcV_%47Ix0)IE8V zKFBV)$Qt*bTqD%gFAbcFZs#z!uol1l`uIs}-b_ICbh3jC-xgTYir#S9!oZQpCNo!X2Sb}3*{O+x=lV@j?m(l#BT3AAcn(a5Bucp z&bRpZ;3u6%7J{TF3PB%c|l7 z_z$v&cO>at(az^L3t$Xg50upe2F_)eA?PHw)d>w5RWCb7xwIVnu88%jmG587(=)D< z3X=BDl%;a-i&{_N5Mtz#7WEU^^@=;yXA`7_n9K$umu6Hw7PV51uX*ZuIy{$eEoFYePMoF~ z)L6a4CV*c?jY-q(kriROYBw@ySd_K7B>UW^`UGO1zX%$n&$5`bjWqQo8tXz9T3-J|s#{Wfa zOr%iXh_p8VC<`a_)wY*d8BPd)?FG~(o$(bPAeX8cMZnwE*JTN~=B_ffH7}WaS)$HO z4aE7}6w!Y~MdKDYSkIUUf$b%sF>_=vV@|e%AFVlK*Y+6HbXLg(TC05OHzQ@Z85XFq z_>G5m*dPb5mXFYIi6mEUmCEi}otu%aXw3go-@f-9C%?ydwh4_D|JW>1hr?)w*%bCIqr z{X~Wu$DGe9aucJnS0+m8vo{EudX$~(yyd$U#%yHSW^3A3rWEr9uyW|qYMXwlYv;3h z+2nD3*!C zErJ@%2W8cbm*FlQ>%OAW0<01Fp0-R06i5HY*7~H2f`M(Nf zWF`szPS+q};Su_Gp-^-Ca>&x}#lR*{d-My+#H{vjYwP+eDlU$?BdrRkU{;-R<(E$k zAf3yxV5=iPoH;#HhS9QiuTatdl+81AI!JZZ$6MrQmyJ)!KV$j4#yWUSaj=*w ze66ax>@GP7LB0*CE~WUO;X)OF^+a9mWow+V^Jc4yd>fiwJzxd5g2K5QXr~{h_O%O% zK_j_xjZLT4dS;N_mbCGJea|zF3?7B7VbiDj|1*7_ljB(iK zFgWQ_nTf2dSBmLGuw2dM@uz`ELCwaje|YnOAjv-8Hy-@8wFo=|*?py-#+Em$pVQ_O zh2Ri^Jw>>II8I4&l!`D9W0s6(7H8?g=kVBlV>r6cu^@_#i!>C1UK% zn`+xL((T)WVW$5KRPk@a_{_J)iyiGgu^)De42L>HG$zjPtMd+wR%2Se&-BH5(GvU` zZYh(Acjye~$s%KlKmv3Z4z{mb8`H?{uyq)(>QvpOqq!a}rmYS8>odxRa$Bz5j%RU{ z5^%I0%|7ds`-IP#e;0gJoSgGemNtm)J&_5Os+rvN90ird_fO~Li()1b;Twu_rTc>s zIcC{*XNZ|E-Gzw*+3>6~{EDm8blF=@yzWUCISxq}B zj2E*>oN4^jb2f;!e6v??vM29ilo~f*er3%*+G!W}`>aYhmfkNI`}d3!jDiW_d@VhO zy|$ZaQ*PZWNeo3pbs#J4(~26sDN$-Tc)pN|3u(%t&kiS%Pm9|)cvb=K7G;u%M8bi` z+@w!T_-0Vbk}(m)zl>iKxt+aGEjbX!Q%M0b?d~t+mf~tI3QcUcDU7#Kx39-PVL~Wd zwAZ)hwdR_WmQIKWG3gl&<)J_{z$it~uQRkZrVFl;U<#i3qICc>KB^@L$NiP@(w_1-o3T~?aNB~WG0{x0_SKYdL#vEXyecS3TD_6uEJI?m z3Vy)4ft`VuYfNVATXs*wH6mBzQu<%8O03b*M_e>SgS!xmo<8Yj6T;PxUSU_;#O%3P zL{H|4Jt`4R8ugi?+`ie7@XHZb+kp|Z5C@+b%Ci*9BV$Z!NnDE1LvaRbSs9px`Aj4Z zuL8s{KPlWk>k`akGdiYmwvfme5=VRDO_&7XbwDxWu9$?v8|aW_LP_F-%}|$H0jol| zkm{;!+kx(m%}oIe&S%~_cg@*B=TH4Oa^~Jukiw z<48V)MD%d7vR|}2!;?CZqWRG|s#0|~4(t)eRln6Sj!8(zp{aZ*lE7=FfESromh90%^Mmx$AKfjh50E+uuUS{o>DJcT}dAhH(QAT2}aaQhGRAxNVJTo#ro00-G)yr zunZJ22_frQNR1XnA7VmAEne!4Mid%ALqB?0M(-C5g1LoRSht2-RedUC{(Wj6(?+#V zo~Ux?OKtrmMY6ql3-i5v7LGKJB=}zDo>GVRmSQJClh*h1&X@z;k(c>%m&D){L=zs{ ztfb6#=l*Zj(gi}EO5Nw;jwKf-zXW({rADofyQYIn4uN4^&zL=;O+gngDKhS?WY_Uz zD?7EHOjn)KJ1OZCH}0&lW^9?~3&YW(gq@~mpQwwO-HF0uSIoHl{*hN5nrEU^4klOcQfw6e7%7bX@WUH;v(FssIV1KEQPx^d-~N zE~w!n!{a{R=9aX}^@8l1jF}--D|4Wg(#2UEtDv3B=QlScuNz6YVBxQu z*09yl-H%kB!FOx_J_HrFB9qeTt#l4}zl_4PlPv5}55f@n>0xoEm*Hi)^^i7Ty*P<7 zPVa!dY*6r;!h{b@?LJ4pR`rrmZ2u)-d0YNK=BpUGnMXy-(l%P6{aYYMq+Z+$F)Rg9 z9kxnm2`>NH>4IkyE6mKb#oaf8p&ctk;gcNyMSIqs!$NJNu@=3tclEd^;lZZEOSG_oxqD+T8NQH7Pmh@S!a{zSdJUci;bFY6duwf&MSDzfqBw zX%}O+LP_E9?>ACYDSib%Wg**Xgh=vMP|M3;>vml?r-U>E|2I-f0yKx}-|URU3|G|7 z)K#|0ti^_tE}Og6sa4h9-`;wSmGyNG1SZ&ab^39sBUDtRP%LyJW-Aj-DdnEovEK8+ zX~QK6Rq)br2paicJzlm0|F!FLin3qOAef0AMmq$e!vq`Vt2UlrnyvlF2E18q%iBFN zx*I&+T+??YFT)ax zYLj0FAcc231ve}SZ*DAXu60>uzP1sO@e3t$6)eP&We4V9oc z$?CcS0B9fn(_jFZ*+ftwlAD~86w)CqDGn*RX4mc+v=*TvCn>JswR)a2;!3jO-5uDG zPsoL=x=tMn@BBdyhxB^Sr7|hmPMM!p^R8c8yVX`m`RpfHo0VT65$pG$Uu&v!;a@CoK}WazFngc+Awrl~ zSSwpwR^HwjJ{@rmYhJlBO?oA26)H7a&W33e$jE)zL(X(5@}y*B*SAaGUA?@BF54qM zDoV_cVfH(dE-o%2_a#sumoJ^3U0k@_Ul{W6@?HsH>85@pAt_R!+iqi7u}?Lvm#tbT zT(GULuNUXZ8fd5nWl{ziOqL>j)<=L&byB6U;Q>c&%2f-d{{A^o znT77WEjON&cb{UDjA`1%yRfh@2GADHhg^mj9wrXl4+lCGXkg0ot+vS=ZcxOAUv9<- zvdN^2nuQ3k`~=o&FpO9aKd-0ESefh6WKB@Y;Z|1W3I;VW&28S$%Hg&efT|XA4!_NY z0@Kwm@jE*e7S|bJ{MK3pUZwt!>=kZp@QV2iyl*JVup;NIQPEy{lQrqZ+BV5!!ygJ< z+(A6Y*8AA{@BL41ea`ZpP-VFQsTmaCi z8bbx+d8Ryu=vN z(BmU@q)&%(htUulI<;KkjCKDQTjm4}=oB~XGZht;vsaGWLzWKN>_mWITwh-}7D`A^ zYb!sOl9`oNxB*Lesg1pZgKC8a6bqV~BZrXk%Ba&eA^#`5t^4m)k5#W>CjBToMKbXK zA|j%0E+vB{3#i`X*L~A@|f|o(>5q< zIJmh79v=9@O#btPImh_a6mY~m!Giw|Ad1&( z!wOTW%`PA2zDovl*2`uNh0F1=&Gn-Y6mW_(ef|CLSy@?DNrPNL)JEsAz29|HNA_LN zF)>|E7nz}iq_3~f7x(k>dA-Az_l~>OfF&+jLv#P>_LMFvq>%2RbE>BYCik%5Q~(SH zKXiVB0<_O!3IdI&qc*be*{H&8K4qbYHV`K}dobwqs&$rbidIkHnCtt5C9tT7s=2w@ zROXBi6Z#cJMa6z){;Y{vdmdFhgeCVY|2(ydP^K}iEG1|--kn1!XJ~Pe;6Llo;&-_z zJi6~v?as-`38l+i(Lm)UJt$waxRAobv>IS)YN|8lQR^Ald;Sy0c5-v;KRBR)I@i2W z7rifEgTWaZKOn<)X>53?or9j$V7_V)h(lEL8#a6kNPz}euwu`_#f2y6J_X?u7Pj#8 zO#AfYcBe5d-g7;@cX4dWb9~{=l*^7qKp!1HH zbfG+;9%RAW0`Uz@-z1F|WVc99+eDKD&x_ayRmXl9I0SABI;$iSB=mFMDvPP|Rh-52aLEq8+b1hYM-tm0B-#X`gz%s=1z03dY$@hIX9M_DBY7g?Lk<>HK zaeLPqOM#r8o*toaniudypG19?N3td+d?iK&9O+*On)FWTjBcRp!kKU<|MaQ8t!)(Q zMp|0h>Y5spHJ&7DXD0)N7m)qm%37(Clp}#pL&uI7>`I(0IZP}pgFB@by4IPgqX0>Z zWciY>B6;zyxu*5$QIGO&A@WJoOkkDoxbp^rW^m{g&Mp98)66q zVx39wO{?Uy=f(Q&V4XBenu74Cyd0Ii7fH{=~u>09YQ&FeOkviV0XOW^OQ4_02(oiTAjgAoFxXf zh5el&%Gp(#;ZNI4Q^Gvgb>y1D<|fvtT4<@@Kbe#7Gk_2HCk;zdqjV-najshn0tfzo`Iu`ZTW7#N21MG3%&<4Po>sq?F-~y%;0+MWhSU!zk z|15VEiCxt)-M_bP)oV4d^z+MRLkADM<{mn~KDjfE?e^R?_6P5|)E-Vzx#}^b=;-Kp znMrAFQHRYWHS5(3LTN3YOqdcG_U``P`{{;WMnxuK9dIe*A)*yNd?+c z8xDT{5hyU!)z!-@M@L3bau1>Bm*C1uF8w97Dk4eCCSl^U=Q>YoYr5R#Uun@IMP+w@ zmwO<%AE`jAMtv;IIy0=)rb5eKRQ}$VsJH7yKO^w9HV&J|zERfUQdo4a-v%e_Um?tp zQNvHR6a>TI7)tj1s^RN36Z9jFz-J);kF6Ic?>U*%fHdldoxB)Vui3rJPJz9KvC@rz zq2HT6FAfq1c>&z-;wFA>Wgt3^?|veEjv^$5dDJ1!jxXScws>NTj!=#Rf?DXSVyVg_js5_II2K&{~;WzP@c33qOYv{w;zA_a7&J5Bu_NwIag8f zWfgIL#i)pZL-hTMsq5uuumX&k?60i={+^Djp0_|`OTmb{a7qQ-mzycnmlxj^4U_fR zF9G*y>V5)(sVDBmrJv6gipfO-`mv%gyPk^N#{2(c>U-tz?90SF2}s@dhvU3$(WwA* zr(;8Ze7jfrJ+A&H(i38?H=|K0*47hA=0h5`lM((Y@T$lc5GmGb&!#paA&@7ra4ZC5 zXe|D?S_-WpsFzF$N+*`Y>siVP{_fyX?og-l5mwpW2ZEF1VfULX(F1q`P0BVT-SsK` zH?PQ7{R}N9WCy~t@B0ipUZN_8U(VpfpoQMC2UnSj^lGEY7a>T&-;itGOO$M_%AR4qfhLhQ!LYXqh$Ds(qnISQ z3KI_0eM!{Gh^GX6IP>KNaiPoZsDi)xD3qwRuF_PIO9B4BJmG(35YU9;e=>;wzQYpx z($%iauuw`gz>-8hJqy$D@AMs~XiLiD?F#GH2rdoUL%3+i7wIdwDC^{QNlM~<$z{g> w^Bx>Ac7>3U-y9}`WEX!{>XW?{NwhNS>1(szkE!rxH!KT(~16VIMbcZ5>wUi61 z=FqDq_QEEc7>zZJiHY$yH8^9uF-`2yE0f-MY1A8o7badAOqJvN?VPpRXw(LsWX|)> zcjkQa&G544{CLIp-ABGl2YsC%9(vh-IARoA%?p~XZ)zsK(M*1(S-Q^DmlQaOK9BB0 z5B_lFjSGbr+b;#f7&R5HmhG!-CxzP^1+&g9%Mk0XO7f?d&)Tv3^H1iEuiD3Q z%PaO{Pn|uJ>w9Qn{?yshgua!9b9sBLa6T}fO=r@n!QrvYST-|~O^u|}_F(@%iWTa@ z?d;^U%eke6xx!>H=X{9`mRjjB0yhC4a;)PKf5}Ab>n}EJ9pI8V7OMb8MQ*@w$&Ci-1sOaFmME{1c$&7I0)8&{a_uq z7pw<+KqQGL!6q;Pwt#VP7Z?Lg&<3N1>%`k3IuOtscH?arcVgTMc7e@cxOkFLzT0r~ zCU;Nn9G#s`_%HK0ckC!)xNBqF%?FL`rTi#ogb`jz7~&zqAg>|x^E$#_UQg)Z4TL0b zA|!YVA9HTtl(6cQt+u9P;ja2Q}C$lRdA^6R`91I_bAv?b}5)sb}Cp??p83S z>`-u}j4F6dnhK7TyA=E=Tatf=QX^Ck3A}v6mFA%0v$-xRr?{DHxWCghLXh zWg<=rzGWgt3f5)9CI$C0(M}2mX2K!`4>Qq53N~hfx}=0mW~jpdZ|4{1UfJCwRh`upQ%? zg>E#{KT{*`CPa8IA9S6ta9N{JwhSp0Eh`mDmQ0~w8B{1&mMLVE*Htm&4P;ak&W2Yn z6A?Nvl*~k!6sl&TmJ|wSqM8(HXQGM}%4ed26gI#_IVtRci2x~Vg9$&WtZ|D8cWVj@ zvk#_9r?loyT&8s$#53m0mJXlef zthEahW$&#=>ehV6@$$RfN=dwh;GJSid~tiq?-w#@K3eBV&8{`ggj(MWpR4xo`Kqwk z(H2RDbU690$Bzq_qRl6NdA#WHwe_3PSAG8OZ;RP_ywAnypY-jSAY~is zc|;HIeNc1qea&NAn$zB)O~0kbIOy~JN7?$|o@6pH^EraFi3{l7SJRA_fA*+;VD^b( Z)w5Y|xZmS_9v|@Ntv`FX@ad7UUjYG6jynJV delta 537 zcmaF4h4I^b#tBZ0(i@$pG3jzKfB~N-gqF#J&~gC~8bJH9m*Ut&$ClVFGg?s=y@7G^j}e2Ox4XNm$eXg#msQi5h*Yp~Mwy}cHo~;nt_YQ>iGuiB=)u^xL9|h#60C77IL&MJ>YNY=? fh+6+wV59wyLAj@(^f@Sf2}(n)^uIOPFv icons/logo.png icons/icon.png - icons/page-loading.png icons/page-missing.png + icons/page-downloading.png + icons/page-generating-thumb.png icons/split-horizontal.svg icons/split-vertical.svg icons/icon-feature-border.svg From 694a6feeba3c0f20283b80b5d02ce573f720d8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 2 Mar 2023 15:54:06 +0100 Subject: [PATCH 12/30] code formatting --- tests/model/test_document.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 51d75a0..345f0af 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -59,7 +59,6 @@ def test_get_image_files_with_remote_image(self): self.assertIsInstance(file, OcrdFile) self.assertIsNotNone(file.local_filename) - def test_get_image_files_with_remote_image_and_disallowed_download(self): path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' for file_group_dir in path.parent.glob('OCR-D-*'): @@ -71,8 +70,6 @@ def test_get_image_files_with_remote_image_and_disallowed_download(self): self.assertIsInstance(file, OcrdFile) self.assertIsNone(file.local_filename) - - def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') file_group = doc.get_default_image_group(['OCR-D-IMG-BIN', 'OCR-D-IMG.*']) From 197555c645557ddc147286e32dcbc44777c6e9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=BCnsebeck?= Date: Thu, 2 Mar 2023 16:41:03 +0100 Subject: [PATCH 13/30] refactored LazyLoading to support remote image downloading --- ocrd_browser/ui/icon_store.py | 41 ++++++-- ocrd_browser/ui/page_browser.py | 12 +-- ocrd_browser/ui/page_store.py | 178 +++++++++++++++++++------------- 3 files changed, 145 insertions(+), 86 deletions(-) diff --git a/ocrd_browser/ui/icon_store.py b/ocrd_browser/ui/icon_store.py index 9651101..ed335b1 100644 --- a/ocrd_browser/ui/icon_store.py +++ b/ocrd_browser/ui/icon_store.py @@ -1,21 +1,19 @@ from gi.repository import Gtk, GLib -from typing import Callable, Sequence, Dict, Optional, Any, Iterator +from typing import Callable, Sequence, Dict, Optional, Any, Iterator, Tuple from concurrent.futures import Future, ThreadPoolExecutor, as_completed -RowInitCallback = Callable[[Gtk.TreeModelRow], None] RowLoadCallback = Callable[[Gtk.TreeModelRow], Gtk.TreeModelRow] RowHashCallback = Callable[[Gtk.TreeModelRow], str] +RowResult = Tuple[Optional[int], Optional[Gtk.TreeModelRow]] class LazyLoadingListStore(Gtk.ListStore): - def __init__(self, *column_types: type, init_row: RowInitCallback, load_row: RowLoadCallback, - hash_row: RowHashCallback): + def __init__(self, *column_types: type, load_row: RowLoadCallback, hash_row: RowHashCallback): column_type_list = list(column_types) column_type_list.append(str) super().__init__(*column_type_list) - self.init_row = init_row self.load_row = load_row self.hash_row = hash_row self.futures: Optional[Dict[Future[Gtk.TreeModelRow], Gtk.TreeModelRow]] = None @@ -43,7 +41,7 @@ def _submit_future(self, row: Gtk.TreeModelRow, pool: Optional[ThreadPoolExecuto row_hash = self.hash_row(row) - if row[-1] != row_hash and row not in self.futures.values(): + if row[-1] != row_hash or row not in self.futures.values(): future = pool.submit(self.load_row, row[:]) self.futures[future] = row @@ -57,8 +55,6 @@ def _do_insert(self, position: int, row: Sequence[Any]) -> None: def _on_row_inserted(self, list_store: Gtk.ListStore, _path: Gtk.TreePath, it: Gtk.TreeIter) -> None: row = list_store[it] - with self.handler_block(self.row_changed_handler): - self.init_row(row) self._submit_future(row) def _on_row_changed(self, list_store: Gtk.ListStore, _path: Gtk.TreePath, it: Gtk.TreeIter) -> None: @@ -70,15 +66,38 @@ def _collect_workers(self) -> Iterator[bool]: row = self.futures.pop(future) try: new_row_data = future.result() - except Exception as exc: - print('{} generated an exception: {}'.format(row[0], exc)) + except Exception as e: + import traceback + tb = "".join(traceback.format_exception(type(e), e, e.__traceback__)) + print('#{} generated an exception: {}'.format(row[0], tb)) else: + # Dont trigger event 'row-changed' for every single value with self.handler_block(self.row_changed_handler): + changed = False for i, (old, new) in enumerate(zip(row[:], new_row_data)): if old != new: row[i] = new_row_data[i] - row[-1] = self.hash_row(row) + changed = True + hsh = self.hash_row(row) + if hsh != row[-1]: + row[-1] = hsh + changed = True + # Trigger event 'row-changed' for the whole changed row + if changed: + n, r = self.get_row_by_column_value(0, row[0]) + path = Gtk.TreePath(n) + self.emit('row-changed', path, self.get_iter(path)) + yield True # Futures are finished for now, check back every 50ms if there is something new GLib.timeout_add(50, self._collect_workers().__next__, priority=GLib.PRIORITY_LOW) yield False + + def get_row_by_column_value(self, column: int, value: str) -> RowResult: + """ + Find index and row by column value + """ + for n, row in enumerate(self): + if row[column] == value: + return n, row + return None, None diff --git a/ocrd_browser/ui/page_browser.py b/ocrd_browser/ui/page_browser.py index 0c29b5e..c56f0e1 100644 --- a/ocrd_browser/ui/page_browser.py +++ b/ocrd_browser/ui/page_browser.py @@ -4,7 +4,7 @@ from ocrd_browser.util.gtk import resource_string from ocrd_browser.model import Document -from .page_store import PageListStore, ChangeList +from .page_store import PageListStore, ChangeList, Column @Gtk.Template(string=resource_string('page-list.ui')) @@ -45,9 +45,9 @@ def document_changed(self, subtype: str, page_ids: ChangeList) -> None: self.scroll_to_id(cast(List[str], page_ids)[-1]) def setup_ui(self) -> None: - self.set_text_column(PageListStore.COLUMN_PAGE_ID) - self.set_tooltip_column(PageListStore.COLUMN_TOOLTIP) - self.set_pixbuf_column(PageListStore.COLUMN_THUMB) + self.set_text_column(Column.PAGE_ID) + self.set_tooltip_column(Column.TOOLTIP) + self.set_pixbuf_column(Column.THUMB) text_renderers = [cell for cell in self.get_cells() if isinstance(cell, Gtk.CellRendererText)] text_renderer: Gtk.CellRendererText = text_renderers[0] text_renderer.props.ellipsize = Pango.EllipsizeMode.MIDDLE @@ -67,7 +67,7 @@ def on_context_menu(self, event: Gdk.EventButton, path: Gtk.TreePath, _renderer: self.context_menu.popup_at_pointer(event) def get_selected_ids(self) -> List[str]: - return [self.model[path][PageListStore.COLUMN_PAGE_ID] for path in self.get_selected_items()] + return [self.model[path][Column.PAGE_ID] for path in self.get_selected_items()] def goto_index(self, index: int) -> None: index = index if index >= 0 else len(self.model) + index @@ -99,7 +99,7 @@ def do_selection_changed(self) -> None: def do_item_activated(self, path: Gtk.TreePath) -> None: self.current = self.model.get_iter(path) - self.emit('page_activated', self.model[path][PageListStore.COLUMN_PAGE_ID]) + self.emit('page_activated', self.model[path][Column.PAGE_ID]) @GObject.Signal() def page_activated(self, page_id: str) -> None: diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index 6e4aa35..7897255 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -1,19 +1,38 @@ from gi.repository import Gtk, GLib, GdkPixbuf +from ocrd_models import OcrdFile -from typing import Tuple, Optional, Dict, List, Union, NewType, Callable, Any +from typing import Optional, Dict, List, Union, Callable, Any, Tuple from itertools import count +from enum import IntEnum, auto -from ocrd_browser.util.image import cv_to_pixbuf, cv_scale -from ocrd_browser.model import Document -from .icon_store import LazyLoadingListStore +from ..util.image import cv_to_pixbuf, cv_scale +from ..model import Document from ..util.config import Settings +from .icon_store import LazyLoadingListStore, RowResult import cv2 import os -RowResult = Tuple[Optional[int], Optional[Gtk.TreeModelRow]] ChangeList = Union[List[str], Dict[str, str]] -Column = NewType('Column', int) + + +class Column(IntEnum): + PAGE_ID = 0 + TOOLTIP = auto() + FILENAME = auto() + URL = auto() + THUMB = auto() + STATE = auto() + ORDER = auto() + HASH = auto() + + +class State(IntEnum): + MISSING = -1 + INIT = 0 + DOWNLOADING = 1 + GENERATING_THUMB = 2 + READY = 3 class PageListStore(LazyLoadingListStore): @@ -23,12 +42,6 @@ class PageListStore(LazyLoadingListStore): It utilizes LazyLoadingListStore for lazy thumbnail generation and contains the domain specific logic for handling Document events """ - COLUMN_PAGE_ID = Column(0) - COLUMN_TOOLTIP = Column(1) - COLUMN_FILENAME = Column(2) - COLUMN_THUMB = Column(3) - COLUMN_ORDER = Column(4) - COLUMN_HASH = Column(5) def __init__(self, document: Document): """ @@ -37,49 +50,72 @@ def __init__(self, document: Document): The actual image and data loading happens in _load_row """ columns = { - self.COLUMN_PAGE_ID: str, - self.COLUMN_TOOLTIP: str, - self.COLUMN_FILENAME: str, - self.COLUMN_THUMB: GdkPixbuf.Pixbuf, - self.COLUMN_ORDER: int - # self.COLUMN_HASH: str file hash = filename + modified_time (gets added by LazyLoadingListStore) + Column.PAGE_ID: str, + Column.TOOLTIP: str, + Column.FILENAME: str, + Column.URL: str, + Column.THUMB: GdkPixbuf.Pixbuf, + Column.STATE: int, + Column.ORDER: int + # Column.HASH: str (gets added by LazyLoadingListStore) } - super().__init__(*(columns.values()), init_row=self._init_row, load_row=self._load_row, hash_row=self._hash_row) - self.document = document - self.pixbufs: Dict[str, GdkPixbuf.Pixbuf] = { + super().__init__(*(columns.values()), load_row=self._load_row, hash_row=self._hash_row) + self.page_icons: Dict[str, GdkPixbuf.Pixbuf] = { icon_name: GdkPixbuf.Pixbuf.new_from_resource( - '/org/readmachine/ocrd-browser/icons/{}.png'.format(icon_name) - ) for icon_name in ['page-loading', 'page-missing'] + '/org/readmachine/ocrd-browser/icons/page-{}.png'.format(icon_name) + ) for icon_name in ['missing', 'downloading', 'generating-thumb'] } # TODO end constructor here and add a new method to fill the store for testability and developer sanity # TODO: make file_group selectable, see https://github.com/hnesk/browse-ocrd/issues/7#issuecomment-707851109 + self.clear() + self.document = document self.file_group = document.get_default_image_group(Settings.get().file_groups.preferred_images) - files = document.get_image_files(self.file_group, allow_download=True) - order = count(start=1) - for page_id, file in files.items(): - if file: - path = str(document.path(file)) - self.append((page_id, '', path, None, next(order))) - else: - self.append((page_id, '', None, None, next(order))) - + self.files = document.get_image_files(self.file_group, allow_download=False) + for page_id, file in self.files.items(): + self.add_file(page_id, file) GLib.timeout_add(10, self.start_loading) + def add_file(self, page_id: str, file: Optional[OcrdFile]) -> None: + row = [page_id, None, None, None, None, int(State.INIT), len(self)] + row, _ = self.file_to_row(row, file) + self.append(row) + + def file_to_row(self, row: List[Any], file: Optional[OcrdFile]) -> Tuple[List[Any], bool]: + changed = False + page_id = row[Column.PAGE_ID] + if file: + if file.local_filename: + path = str(self.document.path(file)) + if path != row[Column.FILENAME] or row[Column.HASH] != self._hash_row(row): + changed = True + row[Column.FILENAME] = path + row[Column.THUMB] = self.page_icons['generating-thumb'] + row[Column.TOOLTIP] = 'Generating Thumb {}'.format(file.local_filename) + row[Column.STATE] = int(State.GENERATING_THUMB) + elif file.url: + if file.url != row[Column.URL]: + changed = True + row[Column.URL] = file.url + row[Column.THUMB] = self.page_icons['downloading'] + row[Column.TOOLTIP] = 'Downloading {}'.format(file.url) + row[Column.STATE] = int(State.DOWNLOADING) + else: + if row[Column.FILENAME] is not None or row[Column.URL] is not None: + changed = True + row[Column.FILENAME] = None + row[Column.URL] = None + row[Column.TOOLTIP] = 'No image for {}'.format(page_id) + row[Column.THUMB] = self.page_icons['missing'] + row[Column.STATE] = int(State.MISSING) + + return row, changed + def get_row_by_page_id(self, page_id: str) -> RowResult: """ Find index and row by page_id """ - return self.get_row_by_column_value(self.COLUMN_PAGE_ID, page_id) - - def get_row_by_column_value(self, column: int, value: str) -> RowResult: - """ - Find index and row by column value - """ - for n, row in enumerate(self): - if row[column] == value: - return n, row - return None, None + return self.get_row_by_column_value(Column.PAGE_ID, page_id) def iter_for_id(self, page_id: str) -> Optional[Gtk.TreeIter]: """ @@ -108,8 +144,7 @@ def _page_added(page_ids: List[str]) -> None: try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) # TODO: self.document.path(file) works only for local files - file_name = str(self.document.path(file, allow_download=True)) - self.append((page_id, '', file_name, None, len(self))) + self.add_file(page_id, file) except StopIteration as e: raise ValueError('Page {} in group {} not in workspace'.format(page_id, self.file_group)) from e @@ -120,18 +155,22 @@ def _page_deleted(page_ids: List[str]) -> None: def _page_changed(page_ids: List[str]) -> None: for page_id in page_ids: + file = None n, row = self.get_row_by_page_id(page_id) try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) - file_name = str(self.document.path(file, allow_download=True)) - row[self.COLUMN_FILENAME] = file_name except StopIteration: pass + with self.handler_block(self.row_changed_handler): + row, changed = self.file_to_row(row, file) + if changed: + path = self.path_for_id(page_id) + self.emit('row-changed', path, self.get_iter(path)) def _reordered(old_to_new_ids: Dict[str, str]) -> None: id_to_position: Dict[str, int] = {} for n, row in enumerate(self): - id_to_position[row[self.COLUMN_PAGE_ID]] = n + id_to_position[row[Column.PAGE_ID]] = n positions: List[int] = list(range(0, len(old_to_new_ids))) for old, new in old_to_new_ids.items(): @@ -140,10 +179,10 @@ def _reordered(old_to_new_ids: Dict[str, str]) -> None: self.reorder(positions) # Update the order in the ListStore data, not needed for now, but might help if we have sorting - order = count(start=1) + order = count(start=0) for page_id in self.document.page_ids: n, row = self.get_row_by_page_id(page_id) - row[self.COLUMN_ORDER] = next(order) + row[Column.ORDER] = next(order) handler: Dict[str, Callable[[Any], None]] = { 'page_added': _page_added, @@ -153,30 +192,31 @@ def _reordered(old_to_new_ids: Dict[str, str]) -> None: } handler[subtype](changes) - def _init_row(self, row: Gtk.TreeModelRow) -> None: - if row[self.COLUMN_FILENAME] is not None: - row[PageListStore.COLUMN_TOOLTIP] = 'Loading {}'.format(row[self.COLUMN_FILENAME]) - row[PageListStore.COLUMN_THUMB] = self.pixbufs['page-loading'] - else: - row[PageListStore.COLUMN_TOOLTIP] = 'No image for {}'.format(row[self.COLUMN_PAGE_ID]) - row[PageListStore.COLUMN_THUMB] = self.pixbufs['page-missing'] - - @staticmethod - def _load_row(row: Gtk.TreeModelRow) -> Gtk.TreeModelRow: - filename = row[PageListStore.COLUMN_FILENAME] - if filename is None: - return row - - image = cv2.imread(filename) - row[PageListStore.COLUMN_TOOLTIP] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) - row[PageListStore.COLUMN_THUMB] = cv_to_pixbuf(cv_scale(image, 100, None)) + def _load_row(self, row: Gtk.TreeModelRow) -> Gtk.TreeModelRow: + page_id = row[Column.PAGE_ID] + file = self.files[page_id] + state = row[Column.STATE] + if state == State.DOWNLOADING: + if file: + file = self.document.workspace.download_file(file) + row[Column.FILENAME] = file.local_filename + row[Column.THUMB] = self.page_icons['generating-thumb'] + row[Column.TOOLTIP] = 'Thumbing {}'.format(file.local_filename) + row[Column.STATE] = int(State.GENERATING_THUMB) + elif state == State.GENERATING_THUMB: + filename = file.local_filename + image = cv2.imread(filename) + row[Column.TOOLTIP] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) + row[Column.THUMB] = cv_to_pixbuf(cv_scale(image, 100, None)) + row[Column.STATE] = int(State.READY) return row @staticmethod def _hash_row(row: Gtk.TreeModelRow) -> str: - file = row[PageListStore.COLUMN_FILENAME] + file = row[Column.FILENAME] + url = row[Column.URL] if file is not None: modified_time = os.path.getmtime(file) - return '{}:{}'.format(file, modified_time) + return '{}:{}:{}'.format(file, modified_time, url) else: - return '' + return '{}:0:{}'.format(file, url) From 68f2907739913977c077e5b5fa4630a78ba3a860 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 17 Nov 2025 11:54:03 +0100 Subject: [PATCH 14/30] PAGE view order arrows: avoid zero division when region is zero --- ocrd_browser/model/page_xml_renderer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ocrd_browser/model/page_xml_renderer.py b/ocrd_browser/model/page_xml_renderer.py index ef5cbcf..44bc033 100644 --- a/ocrd_browser/model/page_xml_renderer.py +++ b/ocrd_browser/model/page_xml_renderer.py @@ -333,6 +333,8 @@ def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: d = self.p1[0] - self.p0[0], self.p1[1] - self.p0[1] left = d[0] * c - d[1] * s, d[0] * s + d[1] * c right = d[0] * c + d[1] * s, -d[0] * s + d[1] * c + if d[0] == 0 and d[1] == 0: + return lf = self.size / (d[0] ** 2 + d[1] ** 2) ** 0.5 # Draw arrow shaft From d4a9f0fcb44fcd81fe0daddf0ae3152a667cebd4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 17 Nov 2025 11:54:38 +0100 Subject: [PATCH 15/30] enable debug logging --- ocrd_browser/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ocrd_browser/main.py b/ocrd_browser/main.py index f448d33..c3b82df 100755 --- a/ocrd_browser/main.py +++ b/ocrd_browser/main.py @@ -57,7 +57,9 @@ def main() -> None: if PROFILER: GLib.idle_add(startup_time) from ocrd_utils import initLogging + import logging initLogging() + logging.getLogger('ocrd').setLevel(logging.DEBUG) from ocrd_browser.application import OcrdBrowserApplication install_excepthook() app = OcrdBrowserApplication() From 46985850e3878046ff5c1510538637f90babf375 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 17 Nov 2025 11:55:08 +0100 Subject: [PATCH 16/30] XML view: add pretty-print toggle --- ocrd_browser/view/base.py | 24 ++++++++++++++++++++++++ ocrd_browser/view/xml.py | 7 ++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ocrd_browser/view/base.py b/ocrd_browser/view/base.py index b71af9c..a08bc90 100644 --- a/ocrd_browser/view/base.py +++ b/ocrd_browser/view/base.py @@ -108,6 +108,30 @@ def update_ui(self) -> None: pass +class BooleanSelector(Gtk.Box, Configurator): + + def __init__(self, label : str = "feature", tooltip : str = "enable feature?") -> None: + super().__init__(visible=True, spacing=3) + self.value = False + + # default label + self.button = Gtk.CheckButton(visible=True, label=label) # default label + self.button.set_tooltip_text(tooltip) + self.pack_start(self.button, False, True, 0) + + self.button.connect('toggled', self.value_changed) + + def set_value(self, value: bool) -> None: + self.value = value + self.button.set_active(value) + + def value_changed(self, button: Gtk.CheckButton) -> None: + self.emit('changed', button.get_active()) + + @GObject.Signal(arg_types=[bool]) + def changed(self, enabled: bool) -> None: + self.value = enabled + class PageQtySelector(Gtk.Box, Configurator): def __init__(self) -> None: diff --git a/ocrd_browser/view/xml.py b/ocrd_browser/view/xml.py index 90832ab..0c282c7 100644 --- a/ocrd_browser/view/xml.py +++ b/ocrd_browser/view/xml.py @@ -4,11 +4,12 @@ from ocrd_utils.constants import MIMETYPE_PAGE from ocrd_models.ocrd_page import to_xml +from ocrd_models.utils import xmllint_format from ocrd_browser.util.file_groups import FileGroupHandle from ocrd_browser.util.launcher import Launcher from ocrd_browser.view import View -from ocrd_browser.view.base import FileGroupSelector, FileGroupFilter +from ocrd_browser.view.base import FileGroupSelector, FileGroupFilter, BooleanSelector GObject.type_register(GtkSource.View) @@ -23,6 +24,7 @@ class ViewXml(View): def __init__(self, name: str, window: Gtk.Window): super().__init__(name, window) self.file_group = FileGroupHandle(None, MIMETYPE_PAGE) + self.indent = False self.font_size: Optional[int] = None # noinspection PyTypeChecker self.text_view: GtkSource.View = None @@ -32,6 +34,7 @@ def __init__(self, name: str, window: Gtk.Window): def build(self) -> None: super().build() self.add_configurator('file_group', FileGroupSelector(FileGroupFilter.XML)) + self.add_configurator('indent', BooleanSelector(label="indent", tooltip="pretty-print XML?")) button = Gtk.Button.new_with_label('PageViewer') button.connect('clicked', self.open_jpageviewer) button.set_visible(True) @@ -97,6 +100,8 @@ def redraw(self) -> None: text = f.read() else: text = to_xml(self.current.pc_gts) + if self.indent: + text = xmllint_format(text.encode('utf-8')).decode('utf-8') self.buffer.set_text(text) else: self.buffer.set_text('') From 7cadd4a5969c2020ab6cd89e053e6e9f83abc260 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 23 May 2024 11:37:31 +0200 Subject: [PATCH 17/30] serve: start maximised and without window decorations --- serve.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/serve.py b/serve.py index f237802..30a1d9b 100644 --- a/serve.py +++ b/serve.py @@ -93,8 +93,10 @@ def _browse_workspace(self, path): path = os.path.join(path, 'mets.xml') ## run app Popen([which('browse-ocrd'), - '--display', ':' + str(self.bwport - 8080), - path]) + '--maximize', + '--fullscreen', + '--display', ':' + str(self.bwport - 8080), + path]) ## proxy does not work, because the follow-up requests would need to be forwarded, too: # response = urllib.request.urlopen('http://' + self.bwhost + ':' + str(self.bwport)) # self.send_response(response.status) From 2483813ac7051c878ca915e2622732f626768349 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 23 May 2024 11:41:03 +0200 Subject: [PATCH 18/30] dockerfile: py38 instead of py37 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6f019a0..e0510f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.8 WORKDIR / COPY Makefile . From 83a08f3ca1be0ccd100287a838158e6490906c2c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 23 May 2024 11:56:09 +0200 Subject: [PATCH 19/30] tests: adapt to changes in core:26967052 (loglevel name) --- tests/__init__.py | 2 +- tests/util/test_launcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 1b5de2a..0b5850b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,7 +26,7 @@ gi.require_version('WebKit2', '4.0') initLogging() -setOverrideLogLevel('OFF', True) +setOverrideLogLevel('CRITICAL', True) TEST_BASE_PATH = (Path(__file__).parent).absolute() ASSETS_PATH = (TEST_BASE_PATH / 'assets').absolute() diff --git a/tests/util/test_launcher.py b/tests/util/test_launcher.py index 494e0ef..00fbdf7 100644 --- a/tests/util/test_launcher.py +++ b/tests/util/test_launcher.py @@ -55,7 +55,7 @@ def test_launch_missing(self): 'Tool "missingtool" not found in your config, to fix place the following section in your ocrd-browser.conf', log_watch.records[0].getMessage() ) - setOverrideLogLevel('OFF', True) + setOverrideLogLevel('CRITICAL', True) def test_launch_tool(self): with self.launcher.launch_tool(self.launcher.tools['echo'], self.doc, self.page_file, 'echo', stdout=subprocess.PIPE) as process: From 44c9b0aa671121790606001aaae4c5b2ac66716c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 23 May 2024 12:11:46 +0200 Subject: [PATCH 20/30] document: adapt to core@cfd1c915 (refresh_caches now hidden) --- ocrd_browser/model/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index e071a9f..065fff6 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -355,7 +355,7 @@ def reorder(self, ordered_page_ids: List[str]) -> None: page_sequence.append(div) old_to_new = dict(zip(old_page_ids, self.page_ids)) - self.workspace.mets.refresh_caches() + self.workspace.mets._refresh_caches() self.save_mets() self._emit('document_changed', 'reordered', old_to_new) From 0b4230d523d33778d3fcfd7923cc95868a65c24e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 24 May 2024 12:04:12 +0200 Subject: [PATCH 21/30] test_document test_save: workaround for core#1149 --- tests/model/test_document.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 345f0af..1be9e6d 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -139,6 +139,8 @@ def test_save(self): for page_id in doc.page_ids: for fg in doc.file_groups: original_file = doc.files_for_page_id(page_id, fg.group, fg.mime)[0] + # workaround for core#1149: remove .url absolute path from cloning + original_file.url = None saved_file = saved.files_for_page_id(page_id, fg.group, fg.mime)[0] self.assertEqual(original_file, saved_file) From 6566be27f764ba07d2404871a949a46609d3604c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 24 May 2024 13:01:24 +0200 Subject: [PATCH 22/30] test_document test_save: workaround for core#1226 --- tests/model/test_document.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 1be9e6d..f53efbc 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -141,6 +141,10 @@ def test_save(self): original_file = doc.files_for_page_id(page_id, fg.group, fg.mime)[0] # workaround for core#1149: remove .url absolute path from cloning original_file.url = None + # workaround for core#1226: remove URL FLocat directly + flocat = original_file._el.find('{http://www.loc.gov/METS/}FLocat[@LOCTYPE="URL"]') + if flocat is not None: + original_file._el.remove(flocat) saved_file = saved.files_for_page_id(page_id, fg.group, fg.mime)[0] self.assertEqual(original_file, saved_file) From 5ce98b35424a101855258fb4c4c0aa6009e4ff20 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 27 May 2024 00:49:46 +0200 Subject: [PATCH 23/30] tests/example METS files: use file paths instead of URLs --- tests/example/workspaces/heavy quoting/mets.xml | 8 ++++---- .../kant_aufklaerung_1784_missing_image/mets.xml | 8 ++++---- .../kant_aufklaerung_1784_missing_xml/mets.xml | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/example/workspaces/heavy quoting/mets.xml b/tests/example/workspaces/heavy quoting/mets.xml index 5acca98..14a667f 100644 --- a/tests/example/workspaces/heavy quoting/mets.xml +++ b/tests/example/workspaces/heavy quoting/mets.xml @@ -18,18 +18,18 @@ - + - + - + - + diff --git a/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml b/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml index 3619d38..eae8800 100644 --- a/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml +++ b/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml @@ -18,18 +18,18 @@ - + - + - + - + diff --git a/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml b/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml index ab31016..3d5d839 100644 --- a/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml +++ b/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml @@ -18,23 +18,23 @@ - + - + - + - + - + From 0dcfb3ab6e599e01026f26940bae9b11d2ae245e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Nov 2025 14:40:43 +0100 Subject: [PATCH 24/30] Page view renderer: recursive RO with star for unordered and recursive colors (as in PRImA) --- ocrd_browser/model/page_xml_renderer.py | 124 ++++++++++++++++++++---- 1 file changed, 104 insertions(+), 20 deletions(-) diff --git a/ocrd_browser/model/page_xml_renderer.py b/ocrd_browser/model/page_xml_renderer.py index 44bc033..4dde06d 100644 --- a/ocrd_browser/model/page_xml_renderer.py +++ b/ocrd_browser/model/page_xml_renderer.py @@ -9,19 +9,37 @@ import numpy as np from math import sin, cos, radians, inf from enum import IntFlag -from collections import defaultdict +from collections import defaultdict, OrderedDict as odict from logging import Logger from functools import lru_cache as memoized from PIL import ImageDraw, Image, ImageFont -from ocrd_models.ocrd_page import PcGtsType, PageType, BorderType, PrintSpaceType, RegionType, TextRegionType, TextLineType, WordType, GlyphType, GraphemeType, ChartRegionType, GraphicRegionType, SeparatorRegionType +from ocrd_models.ocrd_page import ( + PcGtsType, + PageType, + BorderType, + PrintSpaceType, + RegionType, + TextRegionType, + TextLineType, + WordType, + GlyphType, + GraphemeType, + ChartRegionType, + GraphicRegionType, + SeparatorRegionType, + OrderedGroupType, + OrderedGroupIndexedType, + UnorderedGroupType, + UnorderedGroupIndexedType, +) from ocrd_utils import coordinates_of_segment, getLogger, polygon_from_points, transform_coordinates -from shapely.geometry import Polygon, Point, LineString +from shapely.geometry import Polygon, Point, LineString, MultiPoint from shapely.validation import explain_validity -from shapely import prepared +from shapely import prepared, centroid RegionWithCoords = Union[RegionType, TextLineType, WordType, GlyphType, GraphemeType, PrintSpaceType, BorderType] __all__ = ['PageXmlRenderer', 'RegionMap', 'Feature', 'Region'] @@ -320,7 +338,7 @@ def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: class ArrowOperation(Operation): - def __init__(self, p0: Point, p1: Point, size: float = 30.0, width: int = 3, color: str = '#FF0000FF'): + def __init__(self, p0: Point, p1: Point, size: float = 20.0, width: int = 3, color: str = '#FF0000FF'): super().__init__(color, 200) # Depth 200 is on top of everything self.p0 = p0.coords[0] self.p1 = p1.coords[0] @@ -337,14 +355,36 @@ def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: return lf = self.size / (d[0] ** 2 + d[1] ** 2) ** 0.5 - # Draw arrow shaft + # Draw straight line draw.line([self.p0, self.p1], fill=self.color, width=self.width) - # Draw dot + # Draw destination dot draw.ellipse((self.p1[0] - 5, self.p1[1] - 5, self.p1[0] + 5, self.p1[1] + 5), fill=self.color) - # Draw left arrow wing - draw.line([(self.p1[0] + lf * left[0], self.p1[1] + lf * left[1]), self.p1], fill=self.color, width=self.width) - # Draw right arrow wing - draw.line([(self.p1[0] + lf * right[0], self.p1[1] + lf * right[1]), self.p1], fill=self.color, width=self.width) + # Draw destination arrow + draw.polygon([self.p1[0] + lf * left[0], self.p1[1] + lf * left[1], + self.p1[0] + lf * right[0], self.p1[1] + lf * right[1], + self.p1], fill=self.color, width=self.width) + + +class StarOperation(Operation): + def __init__(self, center: Point, points: List[Point], size: float = 30.0, width: int = 3, color: str = '#FF0000FF'): + super().__init__(color, 200) # Depth 200 is on top of everything + self.lines = [list(LineString((center, point)).coords) for point in points] + self.center = center.coords[0] + self.size = size + self.width = width + + def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: + # Draw lines + for line in self.lines: + draw.line(line, fill=self.color, width=self.width) + # Draw tangent dots + for line in self.lines: + point = line[-1] + draw.ellipse((point[0] - 5, point[1] - 5, point[0] + 5, point[1] + 5), fill=self.color) + # Draw central dot + draw.ellipse((self.center[0] - 7, self.center[1] - 7, + self.center[0] + 7, self.center[1] + 7), + outline=self.color, fill='#FFFFFFFF') class TextOperation(Operation): @@ -534,17 +574,61 @@ def region_priority(region: RegionType) -> int: if isinstance(region, TextRegionType): return -1 return -2 + id2region = dict() for region_ds in sorted(page.get_AllRegions(), key=region_priority): self.render_type(region_ds) - - if self.features & Feature.ORDER: - last_point: Optional[Point] = None - for region_ds in page.get_AllRegions(order='reading-order-only'): - region = self.region_factory.create(region_ds) - new_point = region.poly.representative_point() - if last_point: - self.operations.append(ArrowOperation(last_point, new_point, color='#FF0000CF')) - last_point = new_point + id2region[region_ds.id] = region_ds + + if self.features & Feature.ORDER and ( + group := (page.ReadingOrder and + (page.ReadingOrder.OrderedGroup or + page.ReadingOrder.UnorderedGroup))): + def add_refs(group, level: int = 0): + color: str = {0: '#DC143CFF', # '#FF0000CF' + 1: '#9400D3FF', + 2: '#8B0000FF', + }.get(level, '#8B0000FF') + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + iterator = group.get_AllIndexed + else: + iterator = group.get_UnorderedGroupChildren + points = [] + for ref in iterator(): + # ordered recursive case: arrow into first, arrow from last + if isinstance(ref, (OrderedGroupType, OrderedGroupIndexedType)): + first, last = add_refs(ref, level+1) + # unordered recursive case: arrow into center, arrow from center + elif isinstance(ref, (UnorderedGroupType, UnorderedGroupIndexedType)): + first = last = add_refs(ref, level+1) + elif ref.regionRef: + if region_ds := id2region.get(ref.regionRef, None): + region = self.region_factory.create(region_ds) + first = last = region.poly.representative_point() + else: + self.region_factory.logger.warning( + 'Page "%s" @ %s has unknown regionRef="%s"', self.region_factory.page_id, + group.id or 'ReadingOrder', ref.regionRef) + continue + else: + continue + # ordered iterative case: arrow from last + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + if len(points): + self.operations.append(ArrowOperation(points[-1], first, color=color)) + points.append(last) + else: + points.append(first) + if not len(points): + points = [Point(0, 0)] + # ordered iterative case: arrow from last + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + return points[0], points[-1] + # unordered iterative case: line to center + if isinstance(group, (UnorderedGroupType, UnorderedGroupIndexedType)): + center = centroid(MultiPoint(points)) + self.operations.append(StarOperation(center, points, color=color)) + return center + add_refs(group) def get_result(self) -> Tuple[Image.Image, RegionMap]: canvas, regions = self.operations.paint(self.canvas.copy()) From d769500283a95db5e3aee424a309ebff0a648e2f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 21 Nov 2025 15:27:12 +0100 Subject: [PATCH 25/30] get_files: warn+filter if download fails --- ocrd_browser/model/document.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index 065fff6..ace05c2 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -307,11 +307,17 @@ def page_for_id(self, page_id: str, file_group: str) -> Optional['Page']: return Page(self, page_id, file_group) def files_for_page_id(self, page_id: str, file_group: str = None, mimetype: str = None) -> List[OcrdFile]: + log = getLogger('ocrd_browser.model.document.Document.files_for_page_id') with pushd_popd(self.workspace.directory): files: List[OcrdFile] = self.workspace.mets.find_files(fileGrp=file_group, pageId=page_id, mimetype=mimetype) - files = [self.workspace.download_file(file) for file in files] - return files + def try_download(file): + try: + return self.workspace.download_file(file) + except FileNotFoundError as e: + log.warning(e) + return None + return list(filter(None, [try_download(file) for file in files])) def page_for_file(self, page_file: OcrdFile) -> PcGtsType: # cd and silence Warning: Value "ocrd-cis-word-alignment" ... does not match xsd enumeration restriction on TextDataTypeSimpleType From 79b0b86225a189488cd64b2b3e64b72e7756db01 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 30 May 2024 11:10:43 +0200 Subject: [PATCH 26/30] docker-build: depend on update of gresources --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 97f9e49..c095966 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ tests/assets: repo/assets mkdir -p $@ cp -r -t $@ repo/assets/data/* -docker-build: +docker-build: ocrd_browser/ui.gresource docker build --tag $(DOCKER_TAG) . docker-run: DATADIR ?= $(CURDIR) From e5ff6e3ef49cb4c66d51f3e8c1866f9a82de2b8a Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 14 Jul 2023 16:46:41 +0200 Subject: [PATCH 27/30] PageView tooltip: also show TableCellRole attributes, if any --- ocrd_browser/view/page.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ocrd_browser/view/page.py b/ocrd_browser/view/page.py index 20dd467..5969ea8 100644 --- a/ocrd_browser/view/page.py +++ b/ocrd_browser/view/page.py @@ -519,6 +519,19 @@ def _query_tooltip(self, _image: Gtk.Image, x: int, y: int, _keyboard_mode: bool ]: if hasattr(region.region, attribute) and getattr(region.region, attribute): content += '\n@{}={}'.format(attribute, getattr(region.region, attribute)) + if hasattr(region.region, 'Roles') and getattr(region.region, 'Roles'): + roles = getattr(region.region, 'Roles') + if hasattr(roles, 'TableCellRole') and getattr(roles, 'TableCellRole'): + cellrole = getattr(roles, 'TableCellRole') + for attribute in [ + 'rowIndex', + 'columnIndex', + 'rowSpan', + 'colSpan', + 'header', + ]: + if hasattr(cellrole, attribute): + content += '\n@{}={}'.format(attribute, getattr(cellrole, attribute)) if hasattr(region.region, 'TextStyle') and getattr(region.region, 'TextStyle'): style = getattr(region.region, 'TextStyle') for attribute in [ @@ -543,7 +556,7 @@ def _query_tooltip(self, _image: Gtk.Image, x: int, y: int, _keyboard_mode: bool 'smallCaps', 'letterSpaced', ]: - if getattr(style, attribute): + if hasattr(style, attribute): content += '\n@{}={}'.format(attribute, getattr(style, attribute)) if region.warnings: content += '\n\nWarnings:' + ('\n '.join(region.warnings)) From 7b96ecd5b013c83436294926a14a900fe508a350 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 28 Nov 2025 19:31:47 +0100 Subject: [PATCH 28/30] page view screenshot button: default to page id instead of 'untitled' --- ocrd_browser/ui.gresource | Bin 26787 -> 29219 bytes ocrd_browser/view/page.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_browser/ui.gresource b/ocrd_browser/ui.gresource index 684725f0691d91d062879aa76f09da82f3e78b2e..86a72b1b9570c3c0a7ceb4b920be43d6b0881274 100644 GIT binary patch delta 3147 zcmYjTc{r3^AAV3#Vr-$3$&zd{6Cz%Nw^R~Bn^DF#CNvmkWRl^jRA@q|@QV3bNEl19 zhor`k^{ea}k*%h&W*OV}JbmB$ex2*OulqjtIltdIbLRQue$pZISpqu$*95-NZpd`dgMMvoCIK#h zwF2b(KwPd50Efh~WnmxIaHH1{?{NVDC5;gy_?2J{`}-h%hX(-hop#F`ARpFSp$mP4 zxEq2Y{tyf918XS|Dj0_ZlMDb68hd!~)4_TZRFDlZ2f`_PMJaK}hqVOccSHOYg5kr8 zWyt=eH$#1dBlrMV%RqiK#A*}(AW*3o1msf#{d|K_K4gmT6><`mC{}!o zm?xwZ=;2L7c@qPO6pvuv0B=;V&n16PbYOrtR8ib*W+$bFfgcuXOYz3Q*YOS}Vc@UA zmp~5q-#(x?#lA!q{ME{fObhTMdw4;;{$7PPi+zCqO#ryu`y;)NEO$onHN$>QdQDt# zfj(OCP?r!ESd^+e;q0{d#a1CNr~ywgpuk zI1x(#8|Y~tCIZS(KOUy`OselSFdYM%dHe-a?g*WFVSy!)#?{iv1%n1EUnn#+b?!pz z==AM#TA2U&Gqt0`ktLn|=8(<)+kqXAu`e|dBae=RY9dfgmZ3h5%1xGxnfAxndr|Tu zkFakv+cM13$|Oorovu9kND+3^dw*KImEGFKg2yd;C9$^~$(aq0vvq5J{Q zc6QDhYHDipO1%vY4GQD8zBPa|;*`Vu^z`%xBJ{%+KGdI)U2~7{tvC1FU(Ig7dCcNs zbY0x07N-fzl{mMFdU?r@hs@QF*Qo0%>n!E##Nkn`Y?TD~UqojzpL^N`SK8!n)K%4yBvg`Z!IHyF9>24m? zOpMT^Brk4Ws{2dZ1@sEw&r`b+sgkeo4?HKY2c44%R89F~Vm%`@fC##_98?CwlyhlJ`olN+DZK__*Yd7{=SR_d&)9B2^Kq zmHw8qFwM0HcqcgEG`o1$7~>gv;F)pU3WrMt{KZqplwo<&yfslW?3 ziVEE35@QYX8-%F@@4CV0&njTPP}t}CT?p$kr<J)q4=w^6BIWZ>SturT$CJA9 zdlt`np7&yt*v2npL0<1J_KUZ5oJ4AeLD98`?+vF=yy4l#hT`~$KzxVAT;zfjlmE?1 zCRA>R_fT?zEMYsk`QW@#fj*LVyhkwYw7qz^9d+Kq^dzYixzgabR@v*fJet+>qJT{= zH}x+C`L!PAfA#(Q=)j9wb{@vQZ?-G>S#wtPr&8pZ-fpf5tpyWzp=FF(LJn;1HJC>fYFkNY6o} zH@vI;AU>m?nP?vzI*3tB@~mCdHNV{79P)L=N$Z`{61^%7Q8t|~x<1{Nne&QtJuw9X zYP}4p`CZ41@oKzLEK zFoHr`q_Q=pB?Ljdg{<~ammE%Wm8kRW1q&DEiTh#Fdt0LlkY#<7D4fpe+gMJ+V+9H~ zx@f6>sdH-9Nn}`K%bqRYpA+%X-J6*p_Bb(oh~il&E8QM{^5@vn=$)E5H8)9>iSITK z^er4$#mKAqRq_YQZ#j*snvWva=i|^E)<&kO?a(R$R~UDdRrSDuqVd%}?xF8i=wV5d z*q-EnuOddT9$a;I)g@)sEI%;`R<>h>SM+C9g{V#}SX>W6>$YzTJ$m=bV6q=LjzzoV zHM2x1)OG9DGEukLYUefwH#c%BM=(GJNn+^@%;X%NKxVB_e{^K^DJ#72dSZu`NA_588zytCEiL9b5B2%39F!Lgayi)itgn|Vp>XYY4dc5Zni zd_u`IqO|9|AopLWOI>jqhBaW~WTW7~46i@(qMzT(Kd!H`Lv~s84Vi%>>}%HXG$Ot> z4?|<#^jx2s*hF5E;hG#lu3;}Tu|l-m+xy8;e0||V}gNQ(;Hbtqez9xWK#jo^m%bJ z6MCgyqV92z@UWUm(Adz>f1a7g*uZcy%Csx3s8yedIf&*Lx7F9HU1*Chd-dY-B^$O^ zn!x7UQug2^s-n&O*~#tHsneE?i~Q`jRr&FVH6zVZ9o~}lY24zFBX40({X>JJF-PQv zbeI*}-1JyvWt*kyPxSE4llFA>OUL1a0@hu#^WqwgAI06)z$e}0a)nk}iTXJVeK|g} zS5M0z_qBSqf?(aT%_(Cng8p$RTYUy|P;NDH{qG4y{hXJ}sv{F>@`t{tg(wH~3ZG`IqQANyFu(fK|ANxKHP) zBq^Uj7XNv|L8y38$X`p`pBQk7ioWCvg{k7pg!@8L6rzWhzX!p`H-HEwH9)f1U#Cgh z5l&}RcrFHE69hQSi$Z|%_pcBSr!grAWCFzt!*94 zG~qaq7}38EZ5@aJ2f9!gHIiolTrmY zfVXRUPkv!=qG6#4_JG5+#U~7+P9W$SxOf!)-Dg|em^upo7WLqMu#viR@+&%zjy9&xiGwT%cwLcfch}VMncj8fK7HyWe8UzQHblawhgt{syOv+io%h{~f$|G~{?$0c0Ze5w zd{=wDBid=GmdaZ}F!=8Fl6L1pVf05m(^k2rYt{#!YpHNEay&7pEM(ASXfK@7499ZY zd9Q7UmGoL_4%JK?16n+o2IBiv6OD)x24f3l_IA6W3?PEDgwqb$Kh=Wfm k0`Ljl(lgAGFrz=#l7ZKBI=Qa$JC0>G^Fh}RuY32j-;x519{>OV diff --git a/ocrd_browser/view/page.py b/ocrd_browser/view/page.py index 5969ea8..8607e4e 100644 --- a/ocrd_browser/view/page.py +++ b/ocrd_browser/view/page.py @@ -632,7 +632,7 @@ def open_screenshotdialog(self, button: Gtk.Button) -> None: filter_png.set_name("PNG image files") filter_png.add_mime_type("image/png") dialog.add_filter(filter_png) - dialog.set_current_name("untitled.png") + dialog.set_current_name(self.current.id + '.png') response = dialog.run() if response == Gtk.ResponseType.OK: From 4ded593a93eea07a2f44752443636052f9c83eff Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 6 Dec 2025 00:45:53 +0100 Subject: [PATCH 29/30] adapt to ocrd v3 OcrdPage wrapper --- tests/model/test_document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/model/test_document.py b/tests/model/test_document.py index f53efbc..df26cdf 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -8,7 +8,7 @@ from tests import TestCase, ASSETS_PATH, TEST_BASE_PATH from ocrd_browser.model import Document, Page from datetime import datetime -from ocrd_models.ocrd_page import PcGtsType +from ocrd_models.ocrd_page import OcrdPage, PcGtsType # TODO: Later: from tests.assets import Assets, copy_of_directory @@ -182,7 +182,7 @@ def test_page_for_id_with_multiple_images_for_page_and_fileGrp(self): # with self.assertLogs('ocrd_browser.model.document', level='WARNING') as log_watch: page = doc.page_for_id('PHYS_0017', 'OCR-D-IMG-CLIP') self.assertIsInstance(page, Page) - self.assertIsInstance(page.pc_gts, PcGtsType) + self.assertIsInstance(page.pc_gts, (PcGtsType, OcrdPage)) # self.assertEqual(1, len(log_watch.records)) # self.assertEqual("No PAGE-XML but 2 images for page 'PHYS_0017' in fileGrp 'OCR-D-IMG-CLIP'", log_watch.records[0].msg) From 42eb522d1cab8125d676c40b5a9aab5bdcf9f754 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 6 Dec 2025 00:46:20 +0100 Subject: [PATCH 30/30] adapt to pydantic v2 --- ocrd_browser/util/config.py | 46 ++++++++++++++++++++++--------------- requirements.txt | 3 ++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/ocrd_browser/util/config.py b/ocrd_browser/util/config.py index 5d3c703..4a0970d 100644 --- a/ocrd_browser/util/config.py +++ b/ocrd_browser/util/config.py @@ -8,8 +8,8 @@ from configparser import ConfigParser from gi.repository import GLib from ocrd_utils import getLogger -from pydantic import BaseSettings, BaseModel as PydanticBaseModel, Field, validator, Extra -from pydantic.env_settings import SettingsSourceCallable +from pydantic import BaseModel as PydanticBaseModel, Field, field_validator as validator, Extra +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict from shutil import which from typing import List, Optional, Dict, Any, Tuple @@ -81,19 +81,24 @@ def _check_commandline(cls: Any, v: str, **placeholders: _DummyObject) -> str: class FileGroups(BaseModel): preferred_images: List[re.Pattern] # type: ignore[type-arg] - split_preferred_images = validator('preferred_images', pre=True, allow_reuse=True)(_split_regexes) + # FIXME: does not always work in pydantic v2 because of #7749 + @validator('preferred_images', mode="before") + @classmethod + def split_regexes(cls, v: str) -> List[re.Pattern[str]]: + return _split_regexes(cls, v) class Tool(BaseModel): commandline: str - shortcut: Optional[str] - name: Optional[str] + shortcut: Optional[str] = None + name: Optional[str] = None def named(self, name: str) -> Tool: self.name = name return self @validator('commandline') + @classmethod def check_commandline(cls, v: str) -> str: return _check_commandline(cls, v, file=DUMMY_FILE, workspace=DUMMY_WORKSPACE) @@ -102,24 +107,28 @@ class Settings(BaseSettings): file_groups: FileGroups = FileGroups(preferred_images='OCR-D-IMG,OCR-D-IMG.*') tool: Dict[str, Tool] = Field({}) + model_config = SettingsConfigDict( + env_nested_delimiter='__', + extra=Extra.forbid, + env_prefix='BROCRD__', + ) + @validator('tool') + @classmethod def check_tool(cls, tools: Dict[str, Tool]) -> Dict[str, Tool]: return {k.lower(): v.named(k) for k, v in tools.items()} - class Config: - env_nested_delimiter = '__' - extra = Extra.forbid - env_prefix = 'BROCRD__' - - @classmethod - def customise_sources( + @classmethod + def settings_customise_sources( cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - """Prioritize ENV over .conf files""" - return env_settings, init_settings, file_secret_settings + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + """Prioritize ENV over .conf files""" + return env_settings, init_settings, dotenv_settings, file_secret_settings class SettingsFactory: @@ -151,7 +160,6 @@ def build_from_files(cls, files: List[str]) -> Settings: log.info('Read %d config file(s): %s, tried %s', len(read_files), ', '.join(read_files), ', '.join(str(file) for file in files)) settings = Settings(**cls.config_to_dict(config)) - print(settings) return settings @staticmethod diff --git a/requirements.txt b/requirements.txt index 362d37e..747ce86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ Shapely Deprecated importlib_metadata>=3.6 importlib_resources;python_version<'3.8' -pydantic ~= 1.10 +pydantic >= 2.0 +pydantic_settings