diff --git a/Dockerfile b/Dockerfile index 6f019a0..e0510f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.8 WORKDIR / COPY Makefile . diff --git a/Makefile b/Makefile index 97f9e49..c095966 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ tests/assets: repo/assets mkdir -p $@ cp -r -t $@ repo/assets/data/* -docker-build: +docker-build: ocrd_browser/ui.gresource docker build --tag $(DOCKER_TAG) . docker-run: DATADIR ?= $(CURDIR) diff --git a/gresources/icons/page-downloading.png b/gresources/icons/page-downloading.png new file mode 100644 index 0000000..207122f Binary files /dev/null and b/gresources/icons/page-downloading.png differ diff --git a/gresources/icons/page-loading.png b/gresources/icons/page-generating-thumb.png similarity index 100% rename from gresources/icons/page-loading.png rename to gresources/icons/page-generating-thumb.png diff --git a/gresources/icons/page-icons.xcf b/gresources/icons/page-icons.xcf index 76ee385..76f9043 100644 Binary files a/gresources/icons/page-icons.xcf and b/gresources/icons/page-icons.xcf differ diff --git a/gresources/ocrd-browser.gresource.xml b/gresources/ocrd-browser.gresource.xml index c1e1070..02806e1 100644 --- a/gresources/ocrd-browser.gresource.xml +++ b/gresources/ocrd-browser.gresource.xml @@ -3,8 +3,9 @@ icons/logo.png icons/icon.png - icons/page-loading.png icons/page-missing.png + icons/page-downloading.png + icons/page-generating-thumb.png icons/split-horizontal.svg icons/split-vertical.svg icons/icon-feature-border.svg diff --git a/ocrd_browser/main.py b/ocrd_browser/main.py index f448d33..c3b82df 100755 --- a/ocrd_browser/main.py +++ b/ocrd_browser/main.py @@ -57,7 +57,9 @@ def main() -> None: if PROFILER: GLib.idle_add(startup_time) from ocrd_utils import initLogging + import logging initLogging() + logging.getLogger('ocrd').setLevel(logging.DEBUG) from ocrd_browser.application import OcrdBrowserApplication install_excepthook() app = OcrdBrowserApplication() diff --git a/ocrd_browser/model/document.py b/ocrd_browser/model/document.py index b27e916..ace05c2 100644 --- a/ocrd_browser/model/document.py +++ b/ocrd_browser/model/document.py @@ -1,5 +1,8 @@ from __future__ import annotations -from typing import Optional, Tuple, List, Union, cast, Callable, Any, Dict, Sequence, TYPE_CHECKING + +from contextlib import contextmanager + +from typing import Optional, Tuple, List, Union, cast, Callable, Any, Dict, Sequence, Generator, TYPE_CHECKING import atexit import errno @@ -41,6 +44,17 @@ EventCallBack = Optional[Callable[[str, Any], None]] +@contextmanager +def caching_environment(value: str = '1') -> Generator[None, None, None]: + backup_caching = os.environ.get('OCRD_METS_CACHING', None) + os.environ['OCRD_METS_CACHING'] = value + try: + yield + finally: + if backup_caching is not None: + os.environ['OCRD_METS_CACHING'] = backup_caching + + def check_editable(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) def guard(self: 'Document', *args: List[Any], **kwargs: Dict[Any, Any]) -> Any: @@ -81,7 +95,9 @@ def load(cls, mets_url: Union[Path, str] = None, emitter: EventCallBack = None) return cls.create(emitter=emitter) mets_path = cls._to_path(mets_url) - workspace = Resolver().workspace_from_url(str(mets_path), download=False) + with caching_environment('1'): + workspace = Resolver().workspace_from_url(str(mets_path), download=False) + doc = cls(workspace, emitter=emitter, original_url=str(mets_url)) doc._empty = False return doc @@ -106,7 +122,10 @@ def _clone_workspace(cls, mets_url: Union[Path, str]) -> Workspace: cls.temporary_workspaces.append(temporary_workspace) # TODO download = False and lazy loading would be nice for responsiveness log.info("Cloning '%s' to '%s'", mets_url, temporary_workspace) - workspace = Resolver().workspace_from_url(mets_url=mets_url, dst_dir=temporary_workspace, download=True) + + with caching_environment('1'): + workspace = Resolver().workspace_from_url(mets_url=mets_url, dst_dir=temporary_workspace, download=True) + return workspace @check_editable @@ -166,13 +185,18 @@ def baseurl_mets(self) -> str: """ return str(self.workspace.baseurl) + '/' + self.mets_filename if self.workspace else None - def path(self, other: Union[OcrdFile, Path, str]) -> Optional[Path]: + def path(self, other: Union[OcrdFile, Path, str], allow_download: bool = False) -> Optional[Path]: """ Resolves other relative to current workspace """ if not self.directory: return None if isinstance(other, OcrdFile): + if not other.local_filename: + if allow_download: + other = self.workspace.download_file(other) + else: + raise ValueError('path with allow_download=False for non local file called: #{}: {}'.format(other.ID, other.url)) return self.directory.joinpath(other.local_filename) elif isinstance(other, Path): return self.directory.joinpath(other) @@ -219,49 +243,26 @@ def file_groups(self) -> List[FileGroupHandle]: def title(self) -> str: return str(self.workspace.mets.unique_identifier) if self.workspace and self.workspace.mets.unique_identifier else '' - def get_file_index(self) -> Dict[str, OcrdFile]: + def get_image_files(self, file_group: FileGroupHandle, allow_download: bool = False) -> Dict[str, Optional[OcrdFile]]: """ - Return all OcrdFiles by file id and additionally augments the OcrdFile with static_page_id for fast(er) lookup - - Example: - page17 = [file for file in file_index.values() if file.static_page_id == 'PHYS_0017'] - + Builds a Dict PageID->OcrdFile|None for all page_ids """ - log = getLogger('ocrd_browser.model.document.Document.get_file_index') - file_index = {} + log = getLogger('ocrd_browser.model.document.Document.get_image_files') + image_paths: Dict[str, Optional[OcrdFile]] = {} if self.workspace: - for file in self.workspace.mets.find_files(): - file.static_page_id = None - file_index[file.ID] = file - - file_pointers: List[Element] = self.xpath( - 'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]/mets:fptr') - for file_pointer in file_pointers: - file_id = file_pointer.get('FILEID') - page_id = file_pointer.getparent().get('ID') - if file_id in file_index: - file_index[file_id].static_page_id = page_id - else: - log.warning("FILEID '%s' for PAGE '%s' not in mets:fileSec", file_id, page_id) - - return file_index - - def get_image_paths(self, file_group: FileGroupHandle) -> Dict[str, Path]: - """ - Builds a Dict ID->Path for all page_ids fast + for page_id in self.page_ids: + for file in self.workspace.mets.find_files(pageId=page_id, fileGrp=file_group.group, mimetype=file_group.mime): + if page_id in image_paths: + log.warning('Multiple files for PAGE %s and fileGrp %s, using first %s', page_id, file_group, image_paths[page_id]) + else: + if not file.local_filename: + if allow_download: + file = self.workspace.download_file(file) + image_paths[page_id] = file + if page_id not in image_paths: + log.warning('Found no files for PAGE %s and fileGrp %s', page_id, file_group) + image_paths[page_id] = None - More precisely: fast = Faster than iterating over page_ids and using mets.get_physical_page_for_file for each entry - """ - log = getLogger('ocrd_browser.model.document.Document.get_image_paths') - image_paths = {} - file_index = self.get_file_index() - for page_id in self.page_ids: - images = [image for image in file_index.values() if image.static_page_id == page_id and file_group.match(image)] - if len(images) > 0: - image_paths[page_id] = self.directory.joinpath(images[0].local_filename) - else: - log.warning('Found no images for PAGE %s and fileGrp %s', page_id, file_group) - image_paths[page_id] = None return image_paths def get_default_image_group(self, preferred_image_file_groups: PatternList = None) -> Optional[FileGroupHandle]: @@ -306,11 +307,17 @@ def page_for_id(self, page_id: str, file_group: str) -> Optional['Page']: return Page(self, page_id, file_group) def files_for_page_id(self, page_id: str, file_group: str = None, mimetype: str = None) -> List[OcrdFile]: + log = getLogger('ocrd_browser.model.document.Document.files_for_page_id') with pushd_popd(self.workspace.directory): files: List[OcrdFile] = self.workspace.mets.find_files(fileGrp=file_group, pageId=page_id, mimetype=mimetype) - files = [self.workspace.download_file(file) for file in files] - return files + def try_download(file): + try: + return self.workspace.download_file(file) + except FileNotFoundError as e: + log.warning(e) + return None + return list(filter(None, [try_download(file) for file in files])) def page_for_file(self, page_file: OcrdFile) -> PcGtsType: # cd and silence Warning: Value "ocrd-cis-word-alignment" ... does not match xsd enumeration restriction on TextDataTypeSimpleType @@ -319,7 +326,7 @@ def page_for_file(self, page_file: OcrdFile) -> PcGtsType: def resolve_image(self, image_file: OcrdFile) -> Image: with pushd_popd(self.workspace.directory): - pil_image = Image.open(self.workspace.download_file(image_file).local_filename) + pil_image = Image.open(self.path(image_file, allow_download=True)) # pil_image.load() return pil_image @@ -354,7 +361,7 @@ def reorder(self, ordered_page_ids: List[str]) -> None: page_sequence.append(div) old_to_new = dict(zip(old_page_ids, self.page_ids)) - self.workspace.mets.refresh_caches() + self.workspace.mets._refresh_caches() self.save_mets() self._emit('document_changed', 'reordered', old_to_new) @@ -418,14 +425,15 @@ def editable(self) -> bool: @editable.setter def editable(self, editable: bool) -> None: - if editable: - if self._original_url: - self.workspace = self._clone_workspace(self._original_url) + with caching_environment('1'): + if editable: + if self._original_url: + self.workspace = self._clone_workspace(self._original_url) + else: + # noinspection PyTypeChecker + self.workspace = Resolver().workspace_from_nothing(directory=None, mets_basename='mets.xml') else: - # noinspection PyTypeChecker - self.workspace = Resolver().workspace_from_nothing(directory=None, mets_basename='mets.xml') - else: - self.workspace = Resolver().workspace_from_url(self.baseurl_mets) + self.workspace = Resolver().workspace_from_url(self.baseurl_mets) self._editable = editable # self._empty = False # self._modified = False diff --git a/ocrd_browser/model/page_xml_renderer.py b/ocrd_browser/model/page_xml_renderer.py index ef5cbcf..4dde06d 100644 --- a/ocrd_browser/model/page_xml_renderer.py +++ b/ocrd_browser/model/page_xml_renderer.py @@ -9,19 +9,37 @@ import numpy as np from math import sin, cos, radians, inf from enum import IntFlag -from collections import defaultdict +from collections import defaultdict, OrderedDict as odict from logging import Logger from functools import lru_cache as memoized from PIL import ImageDraw, Image, ImageFont -from ocrd_models.ocrd_page import PcGtsType, PageType, BorderType, PrintSpaceType, RegionType, TextRegionType, TextLineType, WordType, GlyphType, GraphemeType, ChartRegionType, GraphicRegionType, SeparatorRegionType +from ocrd_models.ocrd_page import ( + PcGtsType, + PageType, + BorderType, + PrintSpaceType, + RegionType, + TextRegionType, + TextLineType, + WordType, + GlyphType, + GraphemeType, + ChartRegionType, + GraphicRegionType, + SeparatorRegionType, + OrderedGroupType, + OrderedGroupIndexedType, + UnorderedGroupType, + UnorderedGroupIndexedType, +) from ocrd_utils import coordinates_of_segment, getLogger, polygon_from_points, transform_coordinates -from shapely.geometry import Polygon, Point, LineString +from shapely.geometry import Polygon, Point, LineString, MultiPoint from shapely.validation import explain_validity -from shapely import prepared +from shapely import prepared, centroid RegionWithCoords = Union[RegionType, TextLineType, WordType, GlyphType, GraphemeType, PrintSpaceType, BorderType] __all__ = ['PageXmlRenderer', 'RegionMap', 'Feature', 'Region'] @@ -320,7 +338,7 @@ def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: class ArrowOperation(Operation): - def __init__(self, p0: Point, p1: Point, size: float = 30.0, width: int = 3, color: str = '#FF0000FF'): + def __init__(self, p0: Point, p1: Point, size: float = 20.0, width: int = 3, color: str = '#FF0000FF'): super().__init__(color, 200) # Depth 200 is on top of everything self.p0 = p0.coords[0] self.p1 = p1.coords[0] @@ -333,16 +351,40 @@ def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: d = self.p1[0] - self.p0[0], self.p1[1] - self.p0[1] left = d[0] * c - d[1] * s, d[0] * s + d[1] * c right = d[0] * c + d[1] * s, -d[0] * s + d[1] * c + if d[0] == 0 and d[1] == 0: + return lf = self.size / (d[0] ** 2 + d[1] ** 2) ** 0.5 - # Draw arrow shaft + # Draw straight line draw.line([self.p0, self.p1], fill=self.color, width=self.width) - # Draw dot + # Draw destination dot draw.ellipse((self.p1[0] - 5, self.p1[1] - 5, self.p1[0] + 5, self.p1[1] + 5), fill=self.color) - # Draw left arrow wing - draw.line([(self.p1[0] + lf * left[0], self.p1[1] + lf * left[1]), self.p1], fill=self.color, width=self.width) - # Draw right arrow wing - draw.line([(self.p1[0] + lf * right[0], self.p1[1] + lf * right[1]), self.p1], fill=self.color, width=self.width) + # Draw destination arrow + draw.polygon([self.p1[0] + lf * left[0], self.p1[1] + lf * left[1], + self.p1[0] + lf * right[0], self.p1[1] + lf * right[1], + self.p1], fill=self.color, width=self.width) + + +class StarOperation(Operation): + def __init__(self, center: Point, points: List[Point], size: float = 30.0, width: int = 3, color: str = '#FF0000FF'): + super().__init__(color, 200) # Depth 200 is on top of everything + self.lines = [list(LineString((center, point)).coords) for point in points] + self.center = center.coords[0] + self.size = size + self.width = width + + def paint(self, draw: ImageDraw.Draw, regions: RegionMap) -> None: + # Draw lines + for line in self.lines: + draw.line(line, fill=self.color, width=self.width) + # Draw tangent dots + for line in self.lines: + point = line[-1] + draw.ellipse((point[0] - 5, point[1] - 5, point[0] + 5, point[1] + 5), fill=self.color) + # Draw central dot + draw.ellipse((self.center[0] - 7, self.center[1] - 7, + self.center[0] + 7, self.center[1] + 7), + outline=self.color, fill='#FFFFFFFF') class TextOperation(Operation): @@ -532,17 +574,61 @@ def region_priority(region: RegionType) -> int: if isinstance(region, TextRegionType): return -1 return -2 + id2region = dict() for region_ds in sorted(page.get_AllRegions(), key=region_priority): self.render_type(region_ds) - - if self.features & Feature.ORDER: - last_point: Optional[Point] = None - for region_ds in page.get_AllRegions(order='reading-order-only'): - region = self.region_factory.create(region_ds) - new_point = region.poly.representative_point() - if last_point: - self.operations.append(ArrowOperation(last_point, new_point, color='#FF0000CF')) - last_point = new_point + id2region[region_ds.id] = region_ds + + if self.features & Feature.ORDER and ( + group := (page.ReadingOrder and + (page.ReadingOrder.OrderedGroup or + page.ReadingOrder.UnorderedGroup))): + def add_refs(group, level: int = 0): + color: str = {0: '#DC143CFF', # '#FF0000CF' + 1: '#9400D3FF', + 2: '#8B0000FF', + }.get(level, '#8B0000FF') + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + iterator = group.get_AllIndexed + else: + iterator = group.get_UnorderedGroupChildren + points = [] + for ref in iterator(): + # ordered recursive case: arrow into first, arrow from last + if isinstance(ref, (OrderedGroupType, OrderedGroupIndexedType)): + first, last = add_refs(ref, level+1) + # unordered recursive case: arrow into center, arrow from center + elif isinstance(ref, (UnorderedGroupType, UnorderedGroupIndexedType)): + first = last = add_refs(ref, level+1) + elif ref.regionRef: + if region_ds := id2region.get(ref.regionRef, None): + region = self.region_factory.create(region_ds) + first = last = region.poly.representative_point() + else: + self.region_factory.logger.warning( + 'Page "%s" @ %s has unknown regionRef="%s"', self.region_factory.page_id, + group.id or 'ReadingOrder', ref.regionRef) + continue + else: + continue + # ordered iterative case: arrow from last + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + if len(points): + self.operations.append(ArrowOperation(points[-1], first, color=color)) + points.append(last) + else: + points.append(first) + if not len(points): + points = [Point(0, 0)] + # ordered iterative case: arrow from last + if isinstance(group, (OrderedGroupType, OrderedGroupIndexedType)): + return points[0], points[-1] + # unordered iterative case: line to center + if isinstance(group, (UnorderedGroupType, UnorderedGroupIndexedType)): + center = centroid(MultiPoint(points)) + self.operations.append(StarOperation(center, points, color=color)) + return center + add_refs(group) def get_result(self) -> Tuple[Image.Image, RegionMap]: canvas, regions = self.operations.paint(self.canvas.copy()) diff --git a/ocrd_browser/ui.gresource b/ocrd_browser/ui.gresource index 684725f..86a72b1 100644 Binary files a/ocrd_browser/ui.gresource and b/ocrd_browser/ui.gresource differ diff --git a/ocrd_browser/ui/icon_store.py b/ocrd_browser/ui/icon_store.py index 9651101..ed335b1 100644 --- a/ocrd_browser/ui/icon_store.py +++ b/ocrd_browser/ui/icon_store.py @@ -1,21 +1,19 @@ from gi.repository import Gtk, GLib -from typing import Callable, Sequence, Dict, Optional, Any, Iterator +from typing import Callable, Sequence, Dict, Optional, Any, Iterator, Tuple from concurrent.futures import Future, ThreadPoolExecutor, as_completed -RowInitCallback = Callable[[Gtk.TreeModelRow], None] RowLoadCallback = Callable[[Gtk.TreeModelRow], Gtk.TreeModelRow] RowHashCallback = Callable[[Gtk.TreeModelRow], str] +RowResult = Tuple[Optional[int], Optional[Gtk.TreeModelRow]] class LazyLoadingListStore(Gtk.ListStore): - def __init__(self, *column_types: type, init_row: RowInitCallback, load_row: RowLoadCallback, - hash_row: RowHashCallback): + def __init__(self, *column_types: type, load_row: RowLoadCallback, hash_row: RowHashCallback): column_type_list = list(column_types) column_type_list.append(str) super().__init__(*column_type_list) - self.init_row = init_row self.load_row = load_row self.hash_row = hash_row self.futures: Optional[Dict[Future[Gtk.TreeModelRow], Gtk.TreeModelRow]] = None @@ -43,7 +41,7 @@ def _submit_future(self, row: Gtk.TreeModelRow, pool: Optional[ThreadPoolExecuto row_hash = self.hash_row(row) - if row[-1] != row_hash and row not in self.futures.values(): + if row[-1] != row_hash or row not in self.futures.values(): future = pool.submit(self.load_row, row[:]) self.futures[future] = row @@ -57,8 +55,6 @@ def _do_insert(self, position: int, row: Sequence[Any]) -> None: def _on_row_inserted(self, list_store: Gtk.ListStore, _path: Gtk.TreePath, it: Gtk.TreeIter) -> None: row = list_store[it] - with self.handler_block(self.row_changed_handler): - self.init_row(row) self._submit_future(row) def _on_row_changed(self, list_store: Gtk.ListStore, _path: Gtk.TreePath, it: Gtk.TreeIter) -> None: @@ -70,15 +66,38 @@ def _collect_workers(self) -> Iterator[bool]: row = self.futures.pop(future) try: new_row_data = future.result() - except Exception as exc: - print('{} generated an exception: {}'.format(row[0], exc)) + except Exception as e: + import traceback + tb = "".join(traceback.format_exception(type(e), e, e.__traceback__)) + print('#{} generated an exception: {}'.format(row[0], tb)) else: + # Dont trigger event 'row-changed' for every single value with self.handler_block(self.row_changed_handler): + changed = False for i, (old, new) in enumerate(zip(row[:], new_row_data)): if old != new: row[i] = new_row_data[i] - row[-1] = self.hash_row(row) + changed = True + hsh = self.hash_row(row) + if hsh != row[-1]: + row[-1] = hsh + changed = True + # Trigger event 'row-changed' for the whole changed row + if changed: + n, r = self.get_row_by_column_value(0, row[0]) + path = Gtk.TreePath(n) + self.emit('row-changed', path, self.get_iter(path)) + yield True # Futures are finished for now, check back every 50ms if there is something new GLib.timeout_add(50, self._collect_workers().__next__, priority=GLib.PRIORITY_LOW) yield False + + def get_row_by_column_value(self, column: int, value: str) -> RowResult: + """ + Find index and row by column value + """ + for n, row in enumerate(self): + if row[column] == value: + return n, row + return None, None diff --git a/ocrd_browser/ui/page_browser.py b/ocrd_browser/ui/page_browser.py index 0c29b5e..c56f0e1 100644 --- a/ocrd_browser/ui/page_browser.py +++ b/ocrd_browser/ui/page_browser.py @@ -4,7 +4,7 @@ from ocrd_browser.util.gtk import resource_string from ocrd_browser.model import Document -from .page_store import PageListStore, ChangeList +from .page_store import PageListStore, ChangeList, Column @Gtk.Template(string=resource_string('page-list.ui')) @@ -45,9 +45,9 @@ def document_changed(self, subtype: str, page_ids: ChangeList) -> None: self.scroll_to_id(cast(List[str], page_ids)[-1]) def setup_ui(self) -> None: - self.set_text_column(PageListStore.COLUMN_PAGE_ID) - self.set_tooltip_column(PageListStore.COLUMN_TOOLTIP) - self.set_pixbuf_column(PageListStore.COLUMN_THUMB) + self.set_text_column(Column.PAGE_ID) + self.set_tooltip_column(Column.TOOLTIP) + self.set_pixbuf_column(Column.THUMB) text_renderers = [cell for cell in self.get_cells() if isinstance(cell, Gtk.CellRendererText)] text_renderer: Gtk.CellRendererText = text_renderers[0] text_renderer.props.ellipsize = Pango.EllipsizeMode.MIDDLE @@ -67,7 +67,7 @@ def on_context_menu(self, event: Gdk.EventButton, path: Gtk.TreePath, _renderer: self.context_menu.popup_at_pointer(event) def get_selected_ids(self) -> List[str]: - return [self.model[path][PageListStore.COLUMN_PAGE_ID] for path in self.get_selected_items()] + return [self.model[path][Column.PAGE_ID] for path in self.get_selected_items()] def goto_index(self, index: int) -> None: index = index if index >= 0 else len(self.model) + index @@ -99,7 +99,7 @@ def do_selection_changed(self) -> None: def do_item_activated(self, path: Gtk.TreePath) -> None: self.current = self.model.get_iter(path) - self.emit('page_activated', self.model[path][PageListStore.COLUMN_PAGE_ID]) + self.emit('page_activated', self.model[path][Column.PAGE_ID]) @GObject.Signal() def page_activated(self, page_id: str) -> None: diff --git a/ocrd_browser/ui/page_store.py b/ocrd_browser/ui/page_store.py index 11b66c7..05c2386 100644 --- a/ocrd_browser/ui/page_store.py +++ b/ocrd_browser/ui/page_store.py @@ -1,19 +1,38 @@ from gi.repository import Gtk, GLib, GdkPixbuf +from ocrd_models import OcrdFile -from typing import Tuple, Optional, Dict, List, Union, NewType, Callable, Any +from typing import Optional, Dict, List, Union, Callable, Any, Tuple from itertools import count +from enum import IntEnum, auto -from ocrd_browser.util.image import cv_to_pixbuf, cv_scale -from ocrd_browser.model import Document -from .icon_store import LazyLoadingListStore +from ..util.image import cv_to_pixbuf, cv_scale +from ..model import Document from ..util.config import SettingsFactory +from .icon_store import LazyLoadingListStore, RowResult import cv2 import os -RowResult = Tuple[Optional[int], Optional[Gtk.TreeModelRow]] ChangeList = Union[List[str], Dict[str, str]] -Column = NewType('Column', int) + + +class Column(IntEnum): + PAGE_ID = 0 + TOOLTIP = auto() + FILENAME = auto() + URL = auto() + THUMB = auto() + STATE = auto() + ORDER = auto() + HASH = auto() + + +class State(IntEnum): + MISSING = -1 + INIT = 0 + DOWNLOADING = 1 + GENERATING_THUMB = 2 + READY = 3 class PageListStore(LazyLoadingListStore): @@ -23,12 +42,6 @@ class PageListStore(LazyLoadingListStore): It utilizes LazyLoadingListStore for lazy thumbnail generation and contains the domain specific logic for handling Document events """ - COLUMN_PAGE_ID = Column(0) - COLUMN_TOOLTIP = Column(1) - COLUMN_FILENAME = Column(2) - COLUMN_THUMB = Column(3) - COLUMN_ORDER = Column(4) - COLUMN_HASH = Column(5) def __init__(self, document: Document): """ @@ -37,45 +50,72 @@ def __init__(self, document: Document): The actual image and data loading happens in _load_row """ columns = { - self.COLUMN_PAGE_ID: str, - self.COLUMN_TOOLTIP: str, - self.COLUMN_FILENAME: str, - self.COLUMN_THUMB: GdkPixbuf.Pixbuf, - self.COLUMN_ORDER: int - # self.COLUMN_HASH: str file hash = filename + modified_time (gets added by LazyLoadingListStore) + Column.PAGE_ID: str, + Column.TOOLTIP: str, + Column.FILENAME: str, + Column.URL: str, + Column.THUMB: GdkPixbuf.Pixbuf, + Column.STATE: int, + Column.ORDER: int + # Column.HASH: str (gets added by LazyLoadingListStore) } - super().__init__(*(columns.values()), init_row=self._init_row, load_row=self._load_row, hash_row=self._hash_row) - self.document = document - self.pixbufs: Dict[str, GdkPixbuf.Pixbuf] = { + super().__init__(*(columns.values()), load_row=self._load_row, hash_row=self._hash_row) + self.page_icons: Dict[str, GdkPixbuf.Pixbuf] = { icon_name: GdkPixbuf.Pixbuf.new_from_resource( - '/org/readmachine/ocrd-browser/icons/{}.png'.format(icon_name) - ) for icon_name in ['page-loading', 'page-missing'] + '/org/readmachine/ocrd-browser/icons/page-{}.png'.format(icon_name) + ) for icon_name in ['missing', 'downloading', 'generating-thumb'] } + # TODO end constructor here and add a new method to fill the store for testability and developer sanity # TODO: make file_group selectable, see https://github.com/hnesk/browse-ocrd/issues/7#issuecomment-707851109 + self.clear() + self.document = document self.file_group = document.get_default_image_group(SettingsFactory.settings().file_groups.preferred_images) - file_lookup = document.get_image_paths(self.file_group) - order = count(start=1) - for page_id in self.document.page_ids: - file = file_lookup[page_id] - self.append((page_id, '', str(file) if file else None, None, next(order))) - + self.files = document.get_image_files(self.file_group, allow_download=False) + for page_id, file in self.files.items(): + self.add_file(page_id, file) GLib.timeout_add(10, self.start_loading) + def add_file(self, page_id: str, file: Optional[OcrdFile]) -> None: + row = [page_id, None, None, None, None, int(State.INIT), len(self)] + row, _ = self.file_to_row(row, file) + self.append(row) + + def file_to_row(self, row: List[Any], file: Optional[OcrdFile]) -> Tuple[List[Any], bool]: + changed = False + page_id = row[Column.PAGE_ID] + if file: + if file.local_filename: + path = str(self.document.path(file)) + if path != row[Column.FILENAME] or row[Column.HASH] != self._hash_row(row): + changed = True + row[Column.FILENAME] = path + row[Column.THUMB] = self.page_icons['generating-thumb'] + row[Column.TOOLTIP] = 'Generating Thumb {}'.format(file.local_filename) + row[Column.STATE] = int(State.GENERATING_THUMB) + elif file.url: + if file.url != row[Column.URL]: + changed = True + row[Column.URL] = file.url + row[Column.THUMB] = self.page_icons['downloading'] + row[Column.TOOLTIP] = 'Downloading {}'.format(file.url) + row[Column.STATE] = int(State.DOWNLOADING) + else: + if row[Column.FILENAME] is not None or row[Column.URL] is not None: + changed = True + row[Column.FILENAME] = None + row[Column.URL] = None + row[Column.TOOLTIP] = 'No image for {}'.format(page_id) + row[Column.THUMB] = self.page_icons['missing'] + row[Column.STATE] = int(State.MISSING) + + return row, changed + def get_row_by_page_id(self, page_id: str) -> RowResult: """ Find index and row by page_id """ - return self.get_row_by_column_value(self.COLUMN_PAGE_ID, page_id) - - def get_row_by_column_value(self, column: int, value: str) -> RowResult: - """ - Find index and row by column value - """ - for n, row in enumerate(self): - if row[column] == value: - return n, row - return None, None + return self.get_row_by_column_value(Column.PAGE_ID, page_id) def iter_for_id(self, page_id: str) -> Optional[Gtk.TreeIter]: """ @@ -103,8 +143,8 @@ def _page_added(page_ids: List[str]) -> None: for page_id in page_ids: try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) - file_name = str(self.document.path(file)) - self.append((page_id, '', file_name, None, len(self))) + # TODO: self.document.path(file) works only for local files + self.add_file(page_id, file) except StopIteration as e: raise ValueError('Page {} in group {} not in workspace'.format(page_id, self.file_group)) from e @@ -115,18 +155,22 @@ def _page_deleted(page_ids: List[str]) -> None: def _page_changed(page_ids: List[str]) -> None: for page_id in page_ids: + file = None n, row = self.get_row_by_page_id(page_id) try: file = next(iter(self.document.workspace.mets.find_files(pageId=page_id, fileGrp=self.file_group.group, mimetype=self.file_group.mime))) - file_name = str(self.document.path(file)) - row[self.COLUMN_FILENAME] = file_name except StopIteration: pass + with self.handler_block(self.row_changed_handler): + row, changed = self.file_to_row(row, file) + if changed: + path = self.path_for_id(page_id) + self.emit('row-changed', path, self.get_iter(path)) def _reordered(old_to_new_ids: Dict[str, str]) -> None: id_to_position: Dict[str, int] = {} for n, row in enumerate(self): - id_to_position[row[self.COLUMN_PAGE_ID]] = n + id_to_position[row[Column.PAGE_ID]] = n positions: List[int] = list(range(0, len(old_to_new_ids))) for old, new in old_to_new_ids.items(): @@ -135,10 +179,10 @@ def _reordered(old_to_new_ids: Dict[str, str]) -> None: self.reorder(positions) # Update the order in the ListStore data, not needed for now, but might help if we have sorting - order = count(start=1) + order = count(start=0) for page_id in self.document.page_ids: n, row = self.get_row_by_page_id(page_id) - row[self.COLUMN_ORDER] = next(order) + row[Column.ORDER] = next(order) handler: Dict[str, Callable[[Any], None]] = { 'page_added': _page_added, @@ -148,28 +192,31 @@ def _reordered(old_to_new_ids: Dict[str, str]) -> None: } handler[subtype](changes) - def _init_row(self, row: Gtk.TreeModelRow) -> None: - if row[self.COLUMN_FILENAME] is not None: - row[1] = 'Loading {}'.format(row[self.COLUMN_FILENAME]) - row[3] = self.pixbufs['page-loading'] - else: - row[1] = 'No image for {}'.format(row[self.COLUMN_PAGE_ID]) - row[3] = self.pixbufs['page-missing'] - - @staticmethod - def _load_row(row: Gtk.TreeModelRow) -> Gtk.TreeModelRow: - filename = row[PageListStore.COLUMN_FILENAME] - if filename is not None: + def _load_row(self, row: Gtk.TreeModelRow) -> Gtk.TreeModelRow: + page_id = row[Column.PAGE_ID] + file = self.files[page_id] + state = row[Column.STATE] + if state == State.DOWNLOADING: + if file: + file = self.document.workspace.download_file(file) + row[Column.FILENAME] = file.local_filename + row[Column.THUMB] = self.page_icons['generating-thumb'] + row[Column.TOOLTIP] = 'Thumbing {}'.format(file.local_filename) + row[Column.STATE] = int(State.GENERATING_THUMB) + elif state == State.GENERATING_THUMB: + filename = file.local_filename image = cv2.imread(filename) - row[1] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) - row[3] = cv_to_pixbuf(cv_scale(image, 100, None)) + row[Column.TOOLTIP] = '{} ({}x{})'.format(filename, image.shape[1], image.shape[0]) + row[Column.THUMB] = cv_to_pixbuf(cv_scale(image, 100, None)) + row[Column.STATE] = int(State.READY) return row @staticmethod def _hash_row(row: Gtk.TreeModelRow) -> str: - file = row[PageListStore.COLUMN_FILENAME] + file = row[Column.FILENAME] + url = row[Column.URL] if file is not None: modified_time = os.path.getmtime(file) - return '{}:{}'.format(file, modified_time) + return '{}:{}:{}'.format(file, modified_time, url) else: - return '' + return '{}:0:{}'.format(file, url) diff --git a/ocrd_browser/util/config.py b/ocrd_browser/util/config.py index 5d3c703..4a0970d 100644 --- a/ocrd_browser/util/config.py +++ b/ocrd_browser/util/config.py @@ -8,8 +8,8 @@ from configparser import ConfigParser from gi.repository import GLib from ocrd_utils import getLogger -from pydantic import BaseSettings, BaseModel as PydanticBaseModel, Field, validator, Extra -from pydantic.env_settings import SettingsSourceCallable +from pydantic import BaseModel as PydanticBaseModel, Field, field_validator as validator, Extra +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict from shutil import which from typing import List, Optional, Dict, Any, Tuple @@ -81,19 +81,24 @@ def _check_commandline(cls: Any, v: str, **placeholders: _DummyObject) -> str: class FileGroups(BaseModel): preferred_images: List[re.Pattern] # type: ignore[type-arg] - split_preferred_images = validator('preferred_images', pre=True, allow_reuse=True)(_split_regexes) + # FIXME: does not always work in pydantic v2 because of #7749 + @validator('preferred_images', mode="before") + @classmethod + def split_regexes(cls, v: str) -> List[re.Pattern[str]]: + return _split_regexes(cls, v) class Tool(BaseModel): commandline: str - shortcut: Optional[str] - name: Optional[str] + shortcut: Optional[str] = None + name: Optional[str] = None def named(self, name: str) -> Tool: self.name = name return self @validator('commandline') + @classmethod def check_commandline(cls, v: str) -> str: return _check_commandline(cls, v, file=DUMMY_FILE, workspace=DUMMY_WORKSPACE) @@ -102,24 +107,28 @@ class Settings(BaseSettings): file_groups: FileGroups = FileGroups(preferred_images='OCR-D-IMG,OCR-D-IMG.*') tool: Dict[str, Tool] = Field({}) + model_config = SettingsConfigDict( + env_nested_delimiter='__', + extra=Extra.forbid, + env_prefix='BROCRD__', + ) + @validator('tool') + @classmethod def check_tool(cls, tools: Dict[str, Tool]) -> Dict[str, Tool]: return {k.lower(): v.named(k) for k, v in tools.items()} - class Config: - env_nested_delimiter = '__' - extra = Extra.forbid - env_prefix = 'BROCRD__' - - @classmethod - def customise_sources( + @classmethod + def settings_customise_sources( cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - """Prioritize ENV over .conf files""" - return env_settings, init_settings, file_secret_settings + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + """Prioritize ENV over .conf files""" + return env_settings, init_settings, dotenv_settings, file_secret_settings class SettingsFactory: @@ -151,7 +160,6 @@ def build_from_files(cls, files: List[str]) -> Settings: log.info('Read %d config file(s): %s, tried %s', len(read_files), ', '.join(read_files), ', '.join(str(file) for file in files)) settings = Settings(**cls.config_to_dict(config)) - print(settings) return settings @staticmethod diff --git a/ocrd_browser/view/base.py b/ocrd_browser/view/base.py index 2732e25..ad93f98 100644 --- a/ocrd_browser/view/base.py +++ b/ocrd_browser/view/base.py @@ -108,6 +108,30 @@ def update_ui(self) -> None: pass +class BooleanSelector(Gtk.Box, Configurator): + + def __init__(self, label : str = "feature", tooltip : str = "enable feature?") -> None: + super().__init__(visible=True, spacing=3) + self.value = False + + # default label + self.button = Gtk.CheckButton(visible=True, label=label) # default label + self.button.set_tooltip_text(tooltip) + self.pack_start(self.button, False, True, 0) + + self.button.connect('toggled', self.value_changed) + + def set_value(self, value: bool) -> None: + self.value = value + self.button.set_active(value) + + def value_changed(self, button: Gtk.CheckButton) -> None: + self.emit('changed', button.get_active()) + + @GObject.Signal(arg_types=[bool]) + def changed(self, enabled: bool) -> None: + self.value = enabled + class PageQtySelector(Gtk.Box, Configurator): def __init__(self) -> None: diff --git a/ocrd_browser/view/page.py b/ocrd_browser/view/page.py index 20dd467..8607e4e 100644 --- a/ocrd_browser/view/page.py +++ b/ocrd_browser/view/page.py @@ -519,6 +519,19 @@ def _query_tooltip(self, _image: Gtk.Image, x: int, y: int, _keyboard_mode: bool ]: if hasattr(region.region, attribute) and getattr(region.region, attribute): content += '\n@{}={}'.format(attribute, getattr(region.region, attribute)) + if hasattr(region.region, 'Roles') and getattr(region.region, 'Roles'): + roles = getattr(region.region, 'Roles') + if hasattr(roles, 'TableCellRole') and getattr(roles, 'TableCellRole'): + cellrole = getattr(roles, 'TableCellRole') + for attribute in [ + 'rowIndex', + 'columnIndex', + 'rowSpan', + 'colSpan', + 'header', + ]: + if hasattr(cellrole, attribute): + content += '\n@{}={}'.format(attribute, getattr(cellrole, attribute)) if hasattr(region.region, 'TextStyle') and getattr(region.region, 'TextStyle'): style = getattr(region.region, 'TextStyle') for attribute in [ @@ -543,7 +556,7 @@ def _query_tooltip(self, _image: Gtk.Image, x: int, y: int, _keyboard_mode: bool 'smallCaps', 'letterSpaced', ]: - if getattr(style, attribute): + if hasattr(style, attribute): content += '\n@{}={}'.format(attribute, getattr(style, attribute)) if region.warnings: content += '\n\nWarnings:' + ('\n '.join(region.warnings)) @@ -619,7 +632,7 @@ def open_screenshotdialog(self, button: Gtk.Button) -> None: filter_png.set_name("PNG image files") filter_png.add_mime_type("image/png") dialog.add_filter(filter_png) - dialog.set_current_name("untitled.png") + dialog.set_current_name(self.current.id + '.png') response = dialog.run() if response == Gtk.ResponseType.OK: diff --git a/ocrd_browser/view/xml.py b/ocrd_browser/view/xml.py index 2378510..338bd1f 100644 --- a/ocrd_browser/view/xml.py +++ b/ocrd_browser/view/xml.py @@ -4,11 +4,12 @@ from ocrd_utils.constants import MIMETYPE_PAGE from ocrd_models.ocrd_page import to_xml +from ocrd_models.utils import xmllint_format from ocrd_browser.util.file_groups import FileGroupHandle from ocrd_browser.util.launcher import Launcher from ocrd_browser.view import View -from ocrd_browser.view.base import FileGroupSelector, FileGroupFilter +from ocrd_browser.view.base import FileGroupSelector, FileGroupFilter, BooleanSelector GObject.type_register(GtkSource.View) @@ -23,6 +24,7 @@ class ViewXml(View): def __init__(self, name: str, window: Gtk.Window): super().__init__(name, window) self.file_group = FileGroupHandle(None, MIMETYPE_PAGE) + self.indent = False self.font_size: Optional[int] = None # noinspection PyTypeChecker self.text_view: GtkSource.View = None @@ -32,6 +34,7 @@ def __init__(self, name: str, window: Gtk.Window): def build(self) -> None: super().build() self.add_configurator('file_group', FileGroupSelector(FileGroupFilter.XML)) + self.add_configurator('indent', BooleanSelector(label="indent", tooltip="pretty-print XML?")) button = Gtk.Button.new_with_label('PageViewer') button.connect('clicked', self.open_jpageviewer) button.set_visible(True) @@ -97,6 +100,8 @@ def redraw(self) -> None: text = f.read() else: text = to_xml(self.current.pc_gts) + if self.indent: + text = xmllint_format(text.encode('utf-8')).decode('utf-8') self.buffer.set_text(text) else: self.buffer.set_text('') diff --git a/requirements.txt b/requirements.txt index 362d37e..747ce86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ Shapely Deprecated importlib_metadata>=3.6 importlib_resources;python_version<'3.8' -pydantic ~= 1.10 +pydantic >= 2.0 +pydantic_settings diff --git a/serve.py b/serve.py index f237802..30a1d9b 100644 --- a/serve.py +++ b/serve.py @@ -93,8 +93,10 @@ def _browse_workspace(self, path): path = os.path.join(path, 'mets.xml') ## run app Popen([which('browse-ocrd'), - '--display', ':' + str(self.bwport - 8080), - path]) + '--maximize', + '--fullscreen', + '--display', ':' + str(self.bwport - 8080), + path]) ## proxy does not work, because the follow-up requests would need to be forwarded, too: # response = urllib.request.urlopen('http://' + self.bwhost + ':' + str(self.bwport)) # self.send_response(response.status) diff --git a/tests/__init__.py b/tests/__init__.py index 1b5de2a..0b5850b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,7 +26,7 @@ gi.require_version('WebKit2', '4.0') initLogging() -setOverrideLogLevel('OFF', True) +setOverrideLogLevel('CRITICAL', True) TEST_BASE_PATH = (Path(__file__).parent).absolute() ASSETS_PATH = (TEST_BASE_PATH / 'assets').absolute() diff --git a/tests/example/workspaces/heavy quoting/mets.xml b/tests/example/workspaces/heavy quoting/mets.xml index 5acca98..14a667f 100644 --- a/tests/example/workspaces/heavy quoting/mets.xml +++ b/tests/example/workspaces/heavy quoting/mets.xml @@ -18,18 +18,18 @@ - + - + - + - + diff --git a/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml b/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml index 3619d38..eae8800 100644 --- a/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml +++ b/tests/example/workspaces/kant_aufklaerung_1784_missing_image/mets.xml @@ -18,18 +18,18 @@ - + - + - + - + diff --git a/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml b/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml index ab31016..3d5d839 100644 --- a/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml +++ b/tests/example/workspaces/kant_aufklaerung_1784_missing_xml/mets.xml @@ -18,23 +18,23 @@ - + - + - + - + - + diff --git a/tests/example/workspaces/remote-many/.gitignore b/tests/example/workspaces/remote-many/.gitignore new file mode 100644 index 0000000..821f5dd --- /dev/null +++ b/tests/example/workspaces/remote-many/.gitignore @@ -0,0 +1 @@ +/OCR-D-* diff --git a/tests/example/workspaces/remote-many/mets.xml b/tests/example/workspaces/remote-many/mets.xml new file mode 100644 index 0000000..321aeff --- /dev/null +++ b/tests/example/workspaces/remote-many/mets.xml @@ -0,0 +1,65 @@ + + + + + ocrd/core v1.0.0 + + + + + + + + remote + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/example/workspaces/remote-single/.gitignore b/tests/example/workspaces/remote-single/.gitignore new file mode 100644 index 0000000..821f5dd --- /dev/null +++ b/tests/example/workspaces/remote-single/.gitignore @@ -0,0 +1 @@ +/OCR-D-* diff --git a/tests/example/workspaces/remote-single/mets.xml b/tests/example/workspaces/remote-single/mets.xml new file mode 100644 index 0000000..a9c7028 --- /dev/null +++ b/tests/example/workspaces/remote-single/mets.xml @@ -0,0 +1,38 @@ + + + + + ocrd/core v1.0.0 + + + + + + + + remote + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/model/test_document.py b/tests/model/test_document.py index 0596d6a..df26cdf 100644 --- a/tests/model/test_document.py +++ b/tests/model/test_document.py @@ -1,3 +1,6 @@ +import shutil +from ocrd_models import OcrdFile + from pathlib import Path from tempfile import TemporaryDirectory @@ -5,7 +8,7 @@ from tests import TestCase, ASSETS_PATH, TEST_BASE_PATH from ocrd_browser.model import Document, Page from datetime import datetime -from ocrd_models.ocrd_page import PcGtsType +from ocrd_models.ocrd_page import OcrdPage, PcGtsType # TODO: Later: from tests.assets import Assets, copy_of_directory @@ -28,30 +31,44 @@ def test_get_file_groups(self): ] self.assertEqual(expected, doc.file_groups) - def test_get_page_index(self): - doc = Document.load(self.path) - file_index = doc.get_file_index() - page17 = [file for file in file_index.values() if file.static_page_id == 'PHYS_0017'] - alto = [file for file in file_index.values() if file.mimetype == 'application/alto+xml'] - self.assertEqual(3, len(page17)) - self.assertEqual(2, len(alto)) - def test_get_image_paths(self): doc = Document.load(self.path) - image_paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG', 'image/tiff')) - self.assertEqual(2, len(image_paths)) - self.assertEqual('INPUT_0017.tif', image_paths['PHYS_0017'].name) - self.assertEqual('INPUT_0020.tif', image_paths['PHYS_0020'].name) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG', 'image/tiff')) + self.assertEqual(2, len(image_files)) + self.assertEqual('INPUT_0017.tif', image_files['PHYS_0017'].basename) + self.assertEqual('INPUT_0020.tif', image_files['PHYS_0020'].basename) def test_get_image_paths_only_returns_matching_groups(self): """ Testcase for https://github.com/hnesk/browse-ocrd/issues/51 """ doc = Document.load(ASSETS_PATH / '../example/workspaces/kant_aufklaerung_1784_bin/mets.xml') - image_paths = doc.get_image_paths(FileGroupHandle('OCR-D-IMG-BIN', 'image/png')) - self.assertEqual(2, len(image_paths)) - self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_paths['PHYS_0017'].name) - self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_paths['PHYS_0020'].name) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/png')) + self.assertEqual(2, len(image_files)) + self.assertEqual('OCR-D-IMG-BIN_0001.IMG-BIN.png', image_files['PHYS_0017'].basename) + self.assertEqual('OCR-D-IMG-BIN_0002.IMG-BIN.png', image_files['PHYS_0020'].basename) + + def test_get_image_files_with_remote_image(self): + path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' + for file_group_dir in path.parent.glob('OCR-D-*'): + shutil.rmtree(file_group_dir) + doc = Document.load(path.as_uri()) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff'), allow_download=True) + self.assertGreater(len(image_files), 0) + file = image_files['PHYS_0002'] + self.assertIsInstance(file, OcrdFile) + self.assertIsNotNone(file.local_filename) + + def test_get_image_files_with_remote_image_and_disallowed_download(self): + path = TEST_BASE_PATH / 'example/workspaces/remote-single/mets.xml' + for file_group_dir in path.parent.glob('OCR-D-*'): + shutil.rmtree(file_group_dir) + doc = Document.load(path.as_uri()) + image_files = doc.get_image_files(FileGroupHandle('OCR-D-IMG-BIN', 'image/tiff'), allow_download=False) + self.assertGreater(len(image_files), 0) + file = image_files['PHYS_0002'] + self.assertIsInstance(file, OcrdFile) + self.assertIsNone(file.local_filename) def test_get_default_image_group(self): doc = Document.load(ASSETS_PATH / 'kant_aufklaerung_1784-complex/data/mets.xml') @@ -122,6 +139,12 @@ def test_save(self): for page_id in doc.page_ids: for fg in doc.file_groups: original_file = doc.files_for_page_id(page_id, fg.group, fg.mime)[0] + # workaround for core#1149: remove .url absolute path from cloning + original_file.url = None + # workaround for core#1226: remove URL FLocat directly + flocat = original_file._el.find('{http://www.loc.gov/METS/}FLocat[@LOCTYPE="URL"]') + if flocat is not None: + original_file._el.remove(flocat) saved_file = saved.files_for_page_id(page_id, fg.group, fg.mime)[0] self.assertEqual(original_file, saved_file) @@ -159,7 +182,7 @@ def test_page_for_id_with_multiple_images_for_page_and_fileGrp(self): # with self.assertLogs('ocrd_browser.model.document', level='WARNING') as log_watch: page = doc.page_for_id('PHYS_0017', 'OCR-D-IMG-CLIP') self.assertIsInstance(page, Page) - self.assertIsInstance(page.pc_gts, PcGtsType) + self.assertIsInstance(page.pc_gts, (PcGtsType, OcrdPage)) # self.assertEqual(1, len(log_watch.records)) # self.assertEqual("No PAGE-XML but 2 images for page 'PHYS_0017' in fileGrp 'OCR-D-IMG-CLIP'", log_watch.records[0].msg) diff --git a/tests/util/test_launcher.py b/tests/util/test_launcher.py index 494e0ef..00fbdf7 100644 --- a/tests/util/test_launcher.py +++ b/tests/util/test_launcher.py @@ -55,7 +55,7 @@ def test_launch_missing(self): 'Tool "missingtool" not found in your config, to fix place the following section in your ocrd-browser.conf', log_watch.records[0].getMessage() ) - setOverrideLogLevel('OFF', True) + setOverrideLogLevel('CRITICAL', True) def test_launch_tool(self): with self.launcher.launch_tool(self.launcher.tools['echo'], self.doc, self.page_file, 'echo', stdout=subprocess.PIPE) as process: