From a30dafd7c4a97246ac07bb5ecac4a34be835093a Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 13:49:15 +0100 Subject: [PATCH 01/33] Introduced a pyproject.toml file. Moved several files believed to be unused into an Attic directory. --- .opf.yml => attic/.opf.yml | 0 setup.cfg => attic/setup.cfg | 0 setup.py => attic/setup.py | 0 todo.txt => attic/todo.txt | 0 pyproject.toml | 54 ++++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+) rename .opf.yml => attic/.opf.yml (100%) rename setup.cfg => attic/setup.cfg (100%) rename setup.py => attic/setup.py (100%) mode change 100755 => 100644 rename todo.txt => attic/todo.txt (100%) create mode 100644 pyproject.toml diff --git a/.opf.yml b/attic/.opf.yml similarity index 100% rename from .opf.yml rename to attic/.opf.yml diff --git a/setup.cfg b/attic/setup.cfg similarity index 100% rename from setup.cfg rename to attic/setup.cfg diff --git a/setup.py b/attic/setup.py old mode 100755 new mode 100644 similarity index 100% rename from setup.py rename to attic/setup.py diff --git a/todo.txt b/attic/todo.txt similarity index 100% rename from todo.txt rename to attic/todo.txt diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..5585c7bf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "opf-fido" +version = "0.1.0" # Replace with the actual version from find_version function +description = "Format Identification for Digital Objects (FIDO)." +readme = "A command-line tool to identify the file formats of digital objects. FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions." +authors = [ + { name="Adam Farquhar (BL)", email="" } # Add email if available +] +license = { text = "Apache License 2.0" } +homepage = "http://openpreservation.org/technology/products/fido/" +dependencies = [ + "olefile >= 0.46, < 1", + "six >= 1.10.0, < 2", + "win-unicode-console >= 0.5; python_version == '2.7' and platform_system == 'Windows'", + "importlib-resources", + "requests" +] + +[project.optional-dependencies] +testing = [ + "pytest", + "pytest-cov", +] +setup = [ + "pytest-runner" +] + +[project.scripts] +fido = "fido.fido:main" +fido-prepare = "fido.prepare:main" +fido-update-signatures = "fido.update_signatures:main" +fido-toxml = "fido.toxml:main" + +[project.classifiers] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7" +] + +[tool.setuptools.package-data] +"fido" = ["*.*", "conf/*.*", "signatures/*.*", "pronom/*.*"] + +[tool.pytest.ini_options] +addopts = "--maxfail=1 --strict-markers" + +[tool.flake8] +ignore = ["E501"] \ No newline at end of file From f4dd41a7280765b45e134653bc5d5a36875ef64e Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 14:49:27 +0100 Subject: [PATCH 02/33] Updated pyproject.toml. Fido now builds and the existing tests pass. --- {attic => .attic}/.opf.yml | 0 {attic => .attic}/setup.cfg | 0 {attic => .attic}/setup.py | 0 {attic => .attic}/todo.txt | 0 .gitignore | 4 +++ fido/__init__.py | 23 ++++++++--------- pyproject.toml | 50 ++++++++++++++++++++++--------------- requirements/packaging.txt | 2 -- 8 files changed, 45 insertions(+), 34 deletions(-) rename {attic => .attic}/.opf.yml (100%) rename {attic => .attic}/setup.cfg (100%) rename {attic => .attic}/setup.py (100%) rename {attic => .attic}/todo.txt (100%) delete mode 100644 requirements/packaging.txt diff --git a/attic/.opf.yml b/.attic/.opf.yml similarity index 100% rename from attic/.opf.yml rename to .attic/.opf.yml diff --git a/attic/setup.cfg b/.attic/setup.cfg similarity index 100% rename from attic/setup.cfg rename to .attic/setup.cfg diff --git a/attic/setup.py b/.attic/setup.py similarity index 100% rename from attic/setup.py rename to .attic/setup.py diff --git a/attic/todo.txt b/.attic/todo.txt similarity index 100% rename from attic/todo.txt rename to .attic/todo.txt diff --git a/.gitignore b/.gitignore index db422987..746de441 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.egg-info *.py[co] +.venv/ /MANIFEST /dist @@ -8,6 +9,9 @@ /.eggs .coverage .env/ +.pytest_cache/ +.ruff_cache +__pycache__/ /fmtinfo.csv /exp diff --git a/fido/__init__.py b/fido/__init__.py index a846cfca..b46a8941 100644 --- a/fido/__init__.py +++ b/fido/__init__.py @@ -13,14 +13,13 @@ from six.moves import input as rinput +__version__ = "1.8.0dev" -__version__ = '1.6.1' +CONFIG_DIR = join(abspath(dirname(__file__)), "conf") -CONFIG_DIR = join(abspath(dirname(__file__)), 'conf') - -def query_yes_no(question, default='yes'): +def query_yes_no(question, default="yes"): """ Ask a yes/no question via input() and return their answer. @@ -30,19 +29,19 @@ def query_yes_no(question, default='yes'): The "answer" return value is True for "yes" or False for "no". """ - valid = {'yes': True, 'y': True, 'no': False, 'n': False} + valid = {"yes": True, "y": True, "no": False, "n": False} if default is None: - prompt = ' [y/n] ' - elif default == 'yes': - prompt = ' [Y/n] ' - elif default == 'no': - prompt = ' [y/N] ' + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " else: raise ValueError('Invalid default answer: "%s"' % default) while True: - print(question + prompt, end='') + print(question + prompt, end="") choice = rinput().lower() - if default is not None and choice == '': + if default is not None and choice == "": return valid[default] if choice in valid: return valid[choice] diff --git a/pyproject.toml b/pyproject.toml index 5585c7bf..81d5b106 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,50 +2,60 @@ requires = ["setuptools>=42", "wheel"] build-backend = "setuptools.build_meta" +# These were in requirements/packing.txt +# twine>=1.8,<1.9 +# wheel==0.38.1 + [project] name = "opf-fido" -version = "0.1.0" # Replace with the actual version from find_version function -description = "Format Identification for Digital Objects (FIDO)." -readme = "A command-line tool to identify the file formats of digital objects. FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions." +dynamic = ["version"] +requires-python = ">= 3.8" +description = """ +Format Identification for Digital Objects (FIDO). +A command-line tool to identify the file formats of digital objects. +FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions. +""" +readme = "README.md" authors = [ - { name="Adam Farquhar (BL)", email="" } # Add email if available + { name="Adam Farquhar (BL)" } # Add email if available ] -license = { text = "Apache License 2.0" } -homepage = "http://openpreservation.org/technology/products/fido/" +license = { file = "LICENSE.txt" } + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] + dependencies = [ "olefile >= 0.46, < 1", "six >= 1.10.0, < 2", - "win-unicode-console >= 0.5; python_version == '2.7' and platform_system == 'Windows'", "importlib-resources", "requests" ] +[project.urls] +homepage = "http://openpreservation.org/technology/products/fido/" + [project.optional-dependencies] testing = [ "pytest", "pytest-cov", ] -setup = [ - "pytest-runner" -] [project.scripts] fido = "fido.fido:main" fido-prepare = "fido.prepare:main" -fido-update-signatures = "fido.update_signatures:main" fido-toxml = "fido.toxml:main" -[project.classifiers] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7" -] [tool.setuptools.package-data] -"fido" = ["*.*", "conf/*.*", "signatures/*.*", "pronom/*.*"] +"fido" = ["*.*", "conf/*.*", "pronom/*.*"] [tool.pytest.ini_options] addopts = "--maxfail=1 --strict-markers" diff --git a/requirements/packaging.txt b/requirements/packaging.txt deleted file mode 100644 index 6fad71b2..00000000 --- a/requirements/packaging.txt +++ /dev/null @@ -1,2 +0,0 @@ -twine>=1.8,<1.9 -wheel==0.38.1 From d3905f3964645dfde1b90db008588ff70e8508a8 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 15:46:42 +0100 Subject: [PATCH 03/33] Completed the core update to python 3 and move from setup to pyproject. Moved query_yes_or_no to update_signatures where it was used. Removed from __future__ import print_function Removed from __future__ import absolute_import Removed use of cStringIO and replaced with IO.StringIO Removed dependency `six`, including `six.moves`, `six.moves.range`, `six.moves.iteritems`, `six.PY2`, `six.urllib` Removed `importlib_resources` --- {fido/pronom => .attic}/http.py | 2 + fido/__init__.py | 37 +-- fido/fido.py | 418 ++++++++++++++---------- fido/package.py | 11 +- fido/prepare.py | 553 +++++++++++++++++--------------- fido/pronom/soap.py | 77 +++-- fido/toxml.py | 20 +- fido/update_signatures.py | 149 ++++++--- fido/versions.py | 105 +++--- pyproject.toml | 2 - 10 files changed, 766 insertions(+), 608 deletions(-) rename {fido/pronom => .attic}/http.py (97%) diff --git a/fido/pronom/http.py b/.attic/http.py similarity index 97% rename from fido/pronom/http.py rename to .attic/http.py index 8f87a02a..19633542 100644 --- a/fido/pronom/http.py +++ b/.attic/http.py @@ -21,6 +21,8 @@ """ from six.moves import urllib +# NOTE: from fido/pronom/ + def get_sig_xml_for_puid(puid): """Return the full PRONOM signature XML for the passed PUID.""" diff --git a/fido/__init__.py b/fido/__init__.py index b46a8941..04246684 100644 --- a/fido/__init__.py +++ b/fido/__init__.py @@ -7,42 +7,9 @@ It is designed for simple integration into automated work-flows. """ -from __future__ import print_function - -from os.path import abspath, dirname, join - -from six.moves import input as rinput - __version__ = "1.8.0dev" +# todo: move this to a conf/conf.py or something rather than init.py. Would require some cascading updates, though +from os.path import abspath, dirname, join CONFIG_DIR = join(abspath(dirname(__file__)), "conf") - - -def query_yes_no(question, default="yes"): - """ - Ask a yes/no question via input() and return their answer. - - `question` is a string that is presented to the user. `default` is the - presumed answer if the user just hits . It must be "yes" (the - default), "no" or None (meaning an answer is required of the user). - - The "answer" return value is True for "yes" or False for "no". - """ - valid = {"yes": True, "y": True, "no": False, "n": False} - if default is None: - prompt = " [y/n] " - elif default == "yes": - prompt = " [Y/n] " - elif default == "no": - prompt = " [y/N] " - else: - raise ValueError('Invalid default answer: "%s"' % default) - while True: - print(question + prompt, end="") - choice = rinput().lower() - if default is not None and choice == "": - return valid[default] - if choice in valid: - return valid[choice] - print('Please respond with "yes" or "no" (or "y" or "n").') diff --git a/fido/fido.py b/fido/fido.py index 3b5bc604..e0a85fe6 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -8,48 +8,40 @@ It is designed for simple integration into automated work-flows. """ -from __future__ import absolute_import - -from argparse import ArgumentParser, RawTextHelpFormatter -from contextlib import closing import os import platform import re import sys import tarfile import tempfile +from argparse import ArgumentParser, RawTextHelpFormatter +from contextlib import closing + try: from time import perf_counter except ImportError: from time import clock as perf_counter -from xml.etree import cElementTree as ET import zipfile +from xml.etree import cElementTree as ET -from six import PY2 -from six.moves import range - -from fido import __version__, CONFIG_DIR +from fido import CONFIG_DIR, __version__ +from fido.char_handler import escape from fido.package import OlePackage, ZipPackage from fido.versions import get_local_versions, sig_file_actions -from fido.char_handler import escape - defaults = { - 'bufsize': 128 * 1024, # (bytes) - 'regexcachesize': 2084, # (bytes) - 'printmatch': "OK,%(info.time)s,%(info.puid)s,\"%(info.formatname)s\",\"%(info.signaturename)s\",%(info.filesize)s,\"%(info.filename)s\",\"%(info.mimetype)s\",\"%(info.matchtype)s\"\n", - 'printnomatch': "KO,%(info.time)s,,,,%(info.filesize)s,\"%(info.filename)s\",,\"%(info.matchtype)s\"\n", - 'format_files': [ - 'formats-v116.xml', - 'format_extensions.xml' - ], - 'containersignature_file': 'container-signature-20231127.xml', - 'container_bufsize': 512 * 1024, # (bytes) - 'description': """Format Identification for Digital Objects (fido). + "bufsize": 128 * 1024, # (bytes) + "regexcachesize": 2084, # (bytes) + "printmatch": 'OK,%(info.time)s,%(info.puid)s,"%(info.formatname)s","%(info.signaturename)s",%(info.filesize)s,"%(info.filename)s","%(info.mimetype)s","%(info.matchtype)s"\n', + "printnomatch": 'KO,%(info.time)s,,,,%(info.filesize)s,"%(info.filename)s",,"%(info.matchtype)s"\n', + "format_files": ["formats-v116.xml", "format_extensions.xml"], + "containersignature_file": "container-signature-20231127.xml", + "container_bufsize": 512 * 1024, # (bytes) + "description": """Format Identification for Digital Objects (fido). FIDO is a command-line tool to identify the file formats of digital objects. It is designed for simple integration into automated work-flows.""", - 'epilog': """ + "epilog": """ Open Preservation Foundation (http://www.openpreservation.org) See License.txt for license information. Download from: https://github.com/openpreserve/fido/releases @@ -81,39 +73,52 @@ def duration(self): class Fido: """Main FIDO application class.""" - def __init__(self, quiet=False, bufsize=None, container_bufsize=None, printnomatch=None, printmatch=None, zip=False, nocontainer=False, handle_matches=None, conf_dir=CONFIG_DIR, format_files=None, containersignature_file=None): + def __init__( + self, + quiet=False, + bufsize=None, + container_bufsize=None, + printnomatch=None, + printmatch=None, + zip=False, + nocontainer=False, + handle_matches=None, + conf_dir=CONFIG_DIR, + format_files=None, + containersignature_file=None, + ): """Initialise a FIDO class instance.""" global defaults self.quiet = quiet - self.bufsize = defaults['bufsize'] if bufsize is None else bufsize - self.container_bufsize = defaults['container_bufsize'] if container_bufsize is None else container_bufsize - self.printmatch = defaults['printmatch'] if printmatch is None else printmatch - self.printnomatch = defaults['printnomatch'] if printnomatch is None else printnomatch + self.bufsize = defaults["bufsize"] if bufsize is None else bufsize + self.container_bufsize = defaults["container_bufsize"] if container_bufsize is None else container_bufsize + self.printmatch = defaults["printmatch"] if printmatch is None else printmatch + self.printnomatch = defaults["printnomatch"] if printnomatch is None else printnomatch self.handle_matches = self.print_matches if handle_matches is None else handle_matches self.zip = zip self.nocontainer = nocontainer self.conf_dir = conf_dir - self.format_files = defaults['format_files'] if format_files is None else format_files - self.containersignature_file = defaults['containersignature_file'] + self.format_files = defaults["format_files"] if format_files is None else format_files + self.containersignature_file = defaults["containersignature_file"] self.formats = [] self.puid_format_map = {} self.puid_has_priority_over_map = {} # load signatures for xml_file in self.format_files: self.load_fido_xml(os.path.join(os.path.abspath(self.conf_dir), xml_file)) - self.current_file = '' + self.current_file = "" self.current_filesize = 0 self.current_format = None self.current_sig = None self.current_pat = None self.current_count = 0 # Count of calls to match_formats - re._MAXCACHE = defaults['regexcachesize'] - self.externalsig = ET.XML('External') + re._MAXCACHE = defaults["regexcachesize"] + self.externalsig = ET.XML("External") def convert_container_sequence(self, sig): """Parse the PRONOM container sequences and convert to regular expressions.""" # The sequence is regex matching bytes from a file so the sequence must also be bytes - seq = b'(?s)' + seq = b"(?s)" inq = False byt = False rng = False @@ -130,18 +135,18 @@ def convert_container_sequence(self, sig): rng = True continue if not byt: - seq += b"\\x" + sig[i].lower().encode('utf8') + seq += b"\\x" + sig[i].lower().encode("utf8") byt = True continue if byt: - seq += sig[i].lower().encode('utf8') + seq += sig[i].lower().encode("utf8") byt = False continue if inq: if sig[i] == "'" and not rng: inq = False continue - seq += escape(sig[i]).encode('utf8') + seq += escape(sig[i]).encode("utf8") continue if rng: if sig[i] == "]": @@ -149,14 +154,14 @@ def convert_container_sequence(self, sig): rng = False continue if sig[i] != "-" and sig[i] != "'" and ror: - seq += escape(sig[i]).encode('utf8') + seq += escape(sig[i]).encode("utf8") continue if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and sig[i] != ":" and not ror and not byt: - seq += b"\\x" + sig[i].lower().encode('utf8') + seq += b"\\x" + sig[i].lower().encode("utf8") byt = True continue if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and not ror and byt: - seq += sig[i].lower().encode('utf8') + seq += sig[i].lower().encode("utf8") byt = False continue if sig[i] == "-" or sig[i] == " ": @@ -191,10 +196,14 @@ def format_signature_attributes(element): return { "path": element.findtext("Files/File/Path"), "id": element.attrib["Id"], - "signature": self.convert_container_sequence(element.findtext("Files/File/BinarySignatures/InternalSignatureCollection/InternalSignature/ByteSequence/SubSequence/Sequence")) + "signature": self.convert_container_sequence( + element.findtext( + "Files/File/BinarySignatures/InternalSignatureCollection/InternalSignature/ByteSequence/SubSequence/Sequence" + ) + ), } - elements = root.findall("ContainerSignatures/ContainerSignature[@ContainerType=\"{}\"]".format(signature_type)) + elements = root.findall('ContainerSignatures/ContainerSignature[@ContainerType="{}"]'.format(signature_type)) signatures = {} for el in elements: if el.find("Files/File/BinarySignatures") is None: @@ -229,10 +238,10 @@ def load_fido_xml(self, file): """ try: tree = ET.parse(file) - for element in tree.getroot().findall('./format'): + for element in tree.getroot().findall("./format"): self.process_format_element(element) except ET.ParseError as parse_excep: - sys.stderr.write('Failed to parse signature file {}, exception: {}\n'.format(file, parse_excep)) + sys.stderr.write("Failed to parse signature file {}, exception: {}\n".format(file, parse_excep)) sys.exit(1) return self.formats @@ -249,12 +258,14 @@ def process_format_element(self, element): self.formats.append(element) self.puid_format_map[puid] = element # Build some structures to speed things up - self.puid_has_priority_over_map[puid] = frozenset([puid_element.text for puid_element in element.findall('has_priority_over')]) + self.puid_has_priority_over_map[puid] = frozenset( + [puid_element.text for puid_element in element.findall("has_priority_over")] + ) # To delete a format: (1) remove from self.formats, (2) remove from puid_format_map, (3) remove from selt.puid_has_priority_over_map def get_signatures(self, format): """Return the signatures for the format element.""" - return format.findall('signature') + return format.findall("signature") def has_priority_over(self, format, possibly_inferior): """Return true if format has priority over possibly inferior.""" @@ -262,26 +273,26 @@ def has_priority_over(self, format, possibly_inferior): def get_puid(self, format): """Return the PUID for the format.""" - return format.find('puid').text + return format.find("puid").text def get_patterns(self, signature): """Return the patterns for a signature.""" - return signature.findall('pattern') + return signature.findall("pattern") def get_pos(self, pat): """Return the position from a pattern.""" - return pat.find('position').text + return pat.find("position").text def get_regex(self, pat): """Return the UTF-8 encoded regex from a pattern.""" # The regex is matching bytes from a file so regex must also be bytes - return pat.find('regex').text.encode('utf8') + return pat.find("regex").text.encode("utf8") def get_extension(self, format): """Return the extension for a format.""" - return format.find('extension').text + return format.find("extension").text - def print_matches(self, fullname, matches, delta_t, matchtype=''): + def print_matches(self, fullname, matches, delta_t, matchtype=""): """ The default match handler. Prints out information for each match in the list. @@ -290,8 +301,10 @@ def print_matches(self, fullname, matches, delta_t, matchtype=''): @param delta_t is the time taken for the match. @param matchtype is the type of match (signature, containersignature, extension, fail) """ + class Info: pass + obj = Info() obj.count = self.current_count obj.group_size = len(matches) @@ -300,53 +313,59 @@ class Info: obj.filesize = self.current_filesize obj.matchtype = matchtype if len(matches) == 0: - sys.stdout.write(self.printnomatch % { - "info.time": obj.time, - "info.filesize": obj.filesize, - "info.filename": obj.filename, - "info.count": obj.count, - "info.matchtype": "fail" - }) + sys.stdout.write( + self.printnomatch + % { + "info.time": obj.time, + "info.filesize": obj.filesize, + "info.filename": obj.filename, + "info.count": obj.count, + "info.matchtype": "fail", + } + ) return i = 0 - for (f, sig_name) in matches: + for f, sig_name in matches: i += 1 obj.group_index = i obj.puid = self.get_puid(f) - obj.formatname = f.find('name').text + obj.formatname = f.find("name").text obj.signaturename = sig_name - mime = f.find('mime') + mime = f.find("mime") obj.mimetype = mime.text if mime is not None else None - version = f.find('version') + version = f.find("version") obj.version = version.text if version is not None else None - alias = f.find('alias') + alias = f.find("alias") obj.alias = alias.text if alias is not None else None - apple_uti = f.find('apple_uid') + apple_uti = f.find("apple_uid") obj.apple_uti = apple_uti.text if apple_uti is not None else None - sys.stdout.write(self.printmatch % { - "info.time": obj.time, - "info.puid": obj.puid, - "info.formatname": obj.formatname, - "info.signaturename": obj.signaturename, - "info.filesize": obj.filesize, - "info.filename": obj.filename, - "info.mimetype": obj.mimetype, - "info.matchtype": obj.matchtype, - "info.version": obj.version, - "info.alias": obj.alias, - "info.apple_uti": obj.apple_uti, - "info.group_size": obj.group_size, - "info.group_index": obj.group_index, - "info.count": obj.count - }) + sys.stdout.write( + self.printmatch + % { + "info.time": obj.time, + "info.puid": obj.puid, + "info.formatname": obj.formatname, + "info.signaturename": obj.signaturename, + "info.filesize": obj.filesize, + "info.filename": obj.filename, + "info.mimetype": obj.mimetype, + "info.matchtype": obj.matchtype, + "info.version": obj.version, + "info.alias": obj.alias, + "info.apple_uti": obj.apple_uti, + "info.group_size": obj.group_size, + "info.group_index": obj.group_index, + "info.count": obj.count, + } + ) def print_summary(self, secs): """Print summary information on the number of matches and time taken.""" count = self.current_count if not self.quiet: - rate = (int(round(count / secs)) if secs != 0 else 9999) + rate = int(round(count / secs)) if secs != 0 else 9999 # print >> sys.stderr, 'FIDO: Processed %6d files in %6.2f msec, %2d files/sec' % (count, secs * 1000, rate) - sys.stderr.write('FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n' % (count, secs * 1000, rate)) + sys.stderr.write("FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n" % (count, secs * 1000, rate)) def identify_file(self, filename, extension=True): """ @@ -358,7 +377,7 @@ def identify_file(self, filename, extension=True): self.matchtype = "signature" try: timer = PerfTimer() - f = open(filename, 'rb') + f = open(filename, "rb") size = os.stat(filename)[6] self.current_filesize = size if self.current_filesize == 0: @@ -404,9 +423,9 @@ def identify_contents(self, filename, fileobj=None, type=False, extension=True): """ if not type: return - if type == 'zip': + if type == "zip": self.walk_zip(filename, fileobj, extension=extension) - elif type == 'tar': + elif type == "tar": self.walk_tar(filename, fileobj, extension=extension) else: # TODO: ouch! # TODO: Ouch indeed, this currently causes Fido to crash. @@ -435,18 +454,18 @@ def identify_multi_object_stream(self, stream, extension=True): content_length = -1 for line in stream: offset += len(line) - if line == '\n': + if line == "\n": if content_length < 0: raise EnvironmentError("No content-length provided.") else: break - pair = line.lower().split(':', 2) - if pair[0] == 'content-length': + pair = line.lower().split(":", 2) + if pair[0] == "content-length": content_length = int(pair[1]) if content_length == -1: return # Consume exactly content-length bytes - self.current_file = 'STDIN!(at ' + str(offset) + ' bytes)' + self.current_file = "STDIN!(at " + str(offset) + " bytes)" self.current_filesize = content_length bofbuffer, eofbuffer, _ = self.get_buffers(stream, content_length) matches = self.match_formats(bofbuffer, eofbuffer) @@ -467,7 +486,7 @@ def identify_stream(self, stream, filename, extension=True): timer = PerfTimer() bofbuffer, eofbuffer, bytes_read = self.get_buffers(stream, length=None) self.current_filesize = bytes_read - self.current_file = 'STDIN' + self.current_file = "STDIN" matches = self.match_formats(bofbuffer, eofbuffer) # MdR: this needs attention if len(matches) > 0: @@ -482,14 +501,14 @@ def identify_stream(self, stream, filename, extension=True): if filename is not None: self.current_file = filename else: - self.current_file = 'STDIN' + self.current_file = "STDIN" else: if filename is not None: self.current_file = filename matches = self.match_extensions(self.current_file) # we have to reset self.current_file if not on Windows if os.name != "nt": - self.current_file = 'STDIN' + self.current_file = "STDIN" self.handle_matches(self.current_file, matches, timer.duration(), "extension") def container_type(self, matches): @@ -500,16 +519,16 @@ def container_type(self, matches): that we can look inside of (e.g., zip, tar). @return False, zip, or tar. """ - for (format_, _) in matches: - container = format_.find('container') + for format_, _ in matches: + container = format_.find("container") if container is not None: return container.text # aside from checking elements, # check for fmt/111, which is OLE - puid = format_.find('puid') - if puid is not None and puid.text == 'fmt/111': - return 'ole' + puid = format_.find("puid") + if puid is not None and puid.text == "fmt/111": + return "ole" return False def can_recurse_into_container(self, container_type): @@ -524,18 +543,18 @@ def can_recurse_into_container(self, container_type): which are usually most interesting as compound objects rather than for their contents. """ - return container_type in ('zip', 'tar') + return container_type in ("zip", "tar") def blocking_read(self, file, bytes_to_read): """Perform a blocking read and return the buffer.""" bytes_read = 0 - buffer = b'' + buffer = b"" while bytes_read < bytes_to_read: readbuffer = file.read(bytes_to_read - bytes_read) buffer += readbuffer bytes_read = len(buffer) # break out if EOF is reached. - if readbuffer == '': + if readbuffer == "": break return buffer @@ -560,7 +579,7 @@ def get_buffers(self, stream, length=None, seekable=False): if len(buffer) == self.bufsize: prevbuffer = buffer else: - eofbuffer = prevbuffer if len(buffer) == 0 else prevbuffer[-(self.bufsize - len(buffer)):] + buffer + eofbuffer = prevbuffer if len(buffer) == 0 else prevbuffer[-(self.bufsize - len(buffer)) :] + buffer break return bofbuffer, eofbuffer, bytes_read bytes_unread = length - len(bofbuffer) @@ -596,13 +615,13 @@ def walk_zip(self, filename, fileobj=None, extension=True): Call self.handle_matches instead of returning a value. """ try: - with zipfile.ZipFile((fileobj if fileobj else filename), 'r') as zipstream: + with zipfile.ZipFile((fileobj if fileobj else filename), "r") as zipstream: for item in zipstream.infolist(): if item.file_size == 0: continue # TODO: Find a better test for isdir, Python 3.6 adds is_dir() test to ZipInfo class timer = PerfTimer() with zipstream.open(item) as f: - item_name = filename + '!' + item.filename + item_name = filename + "!" + item.filename self.current_file = item_name self.current_filesize = item.file_size if self.current_filesize == 0: @@ -615,7 +634,7 @@ def walk_zip(self, filename, fileobj=None, extension=True): matches = self.match_extensions(item_name) self.handle_matches(item_name, matches, timer.duration(), "extension") if self.container_type(matches): - target = tempfile.SpooledTemporaryFile(prefix='Fido') + target = tempfile.SpooledTemporaryFile(prefix="Fido") with zipstream.open(item) as source: self.copy_stream(source, target) # target.seek(0) @@ -634,13 +653,13 @@ def walk_tar(self, filename, fileobj, extension=True): Call self.handle_matches instead of returning a value. """ try: - with tarfile.TarFile(filename, fileobj=fileobj, mode='r') as tarstream: + with tarfile.TarFile(filename, fileobj=fileobj, mode="r") as tarstream: for item in tarstream.getmembers(): if not item.isfile(): continue timer = PerfTimer() with closing(tarstream.extractfile(item)) as f: - tar_item_name = filename + '!' + item.name + tar_item_name = filename + "!" + item.name self.current_file = tar_item_name self.current_filesize = item.size bofbuffer, eofbuffer, _ = self.get_buffers(f, item.size) @@ -660,7 +679,7 @@ def as_good_as_any(self, f1, match_list): """ if match_list != []: f1_puid = self.get_puid(f1) - for (f2, _) in match_list: + for f2, _ in match_list: if f1 == f2: continue if f1_puid in self.puid_has_priority_over_map[self.get_puid(f2)]: @@ -675,7 +694,7 @@ def buffered_read(self, file_pos, overlap): else: bufsize = self.container_bufsize + overlap file_end = self.current_filesize - with open(self.current_file, 'rb') as file_handle: + with open(self.current_file, "rb") as file_handle: file_handle.seek(file_pos) if file_end - file_pos < bufsize: file_read = file_end - file_pos @@ -705,19 +724,19 @@ def match_formats(self, bofbuffer, eofbuffer): pos = self.get_pos(pat) regex = self.get_regex(pat) # print 'trying ', regex - if pos == 'BOF': + if pos == "BOF": if not re.match(regex, bofbuffer): success = False break - elif pos == 'EOF': + elif pos == "EOF": if not re.search(regex, eofbuffer): success = False break - elif pos == 'VAR': + elif pos == "VAR": if not re.search(regex, bofbuffer): success = False break - elif pos == 'IFB': + elif pos == "IFB": if not re.search(regex, bofbuffer): success = False break @@ -740,7 +759,7 @@ def match_extensions(self, filename): if not myext: return result for element in self.formats: - for format_ in element.findall('extension'): + for format_ in element.findall("extension"): if myext == format_.text: result.append((element, self.externalsig.findtext("name"))) break @@ -759,7 +778,7 @@ def copy_stream(self, source, target): def list_files(roots, recurse=False): """Return the files one at a time. Roots could be a fileobj or a list.""" for root in roots: - root = (root if root[-1] != '\n' else root[:-1]) + root = root if root[-1] != "\n" else root[:-1] root = os.path.normpath(root) if os.path.isfile(root): yield root @@ -771,42 +790,105 @@ def list_files(roots, recurse=False): break -def set_up_platform(): - """Enable Unicode display when running Python from Windows console.""" - if platform.system() == 'Windows' and PY2: - import win_unicode_console # noqa: E402 - win_unicode_console.enable(use_unicode_argv=True) - - def main(args=None): """Main FIDO method.""" - set_up_platform() if not args: args = sys.argv[1:] - parser = ArgumentParser(description=defaults['description'], epilog=defaults['epilog'], fromfile_prefix_chars='@', formatter_class=RawTextHelpFormatter) - parser.add_argument('-v', default=False, action='store_true', help='show version information') - parser.add_argument('-q', default=False, action='store_true', help='run (more) quietly') - parser.add_argument('-recurse', default=False, action='store_true', help='recurse into subdirectories') - parser.add_argument('-zip', default=False, action='store_true', help='recurse into zip and tar files') - parser.add_argument('-noextension', default=False, action='store_true', help='disable extension matching, reduces number of matches but may reduce false positives') - parser.add_argument('-nocontainer', default=False, action='store_true', help='disable deep scan of container documents, increases speed but may reduce accuracy with big files') - parser.add_argument('-pronom_only', default=False, action='store_true', help='disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results') + parser = ArgumentParser( + description=defaults["description"], + epilog=defaults["epilog"], + fromfile_prefix_chars="@", + formatter_class=RawTextHelpFormatter, + ) + parser.add_argument("-v", default=False, action="store_true", help="show version information") + parser.add_argument("-q", default=False, action="store_true", help="run (more) quietly") + parser.add_argument("-recurse", default=False, action="store_true", help="recurse into subdirectories") + parser.add_argument("-zip", default=False, action="store_true", help="recurse into zip and tar files") + parser.add_argument( + "-noextension", + default=False, + action="store_true", + help="disable extension matching, reduces number of matches but may reduce false positives", + ) + parser.add_argument( + "-nocontainer", + default=False, + action="store_true", + help="disable deep scan of container documents, increases speed but may reduce accuracy with big files", + ) + parser.add_argument( + "-pronom_only", + default=False, + action="store_true", + help="disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results", + ) group = parser.add_mutually_exclusive_group() - group.add_argument('-input', default=False, help='file containing a list of files to check, one per line. - means stdin') - group.add_argument('files', nargs='*', default=[], metavar='FILE', help='files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.') - - parser.add_argument('-filename', default=None, help='filename if file contents passed through STDIN') - parser.add_argument('-useformats', metavar='INCLUDEPUIDS', default=None, help='comma separated string of formats to use in identification') - parser.add_argument('-nouseformats', metavar='EXCLUDEPUIDS', default=None, help='comma separated string of formats not to use in identification') - parser.add_argument('-matchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use on match. See nomatchprintf, README.txt.') - parser.add_argument('-nomatchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use if no match. See README.txt') - parser.add_argument('-bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default=' + str(defaults['bufsize']) + ' bytes)') - parser.add_argument('-sigs', default=None, metavar='SIG_ACT', help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.') - parser.add_argument('-container_bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default=' + str(defaults['container_bufsize']) + ' bytes)') - parser.add_argument('-loadformats', default=None, metavar='XML1,...,XMLn', help='comma separated string of XML format files to add.') - parser.add_argument('-confdir', default=CONFIG_DIR, help='configuration directory to load_fido_xml, for example, the format specifications from.') + group.add_argument( + "-input", default=False, help="file containing a list of files to check, one per line. - means stdin" + ) + group.add_argument( + "files", + nargs="*", + default=[], + metavar="FILE", + help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.", + ) + + parser.add_argument("-filename", default=None, help="filename if file contents passed through STDIN") + parser.add_argument( + "-useformats", + metavar="INCLUDEPUIDS", + default=None, + help="comma separated string of formats to use in identification", + ) + parser.add_argument( + "-nouseformats", + metavar="EXCLUDEPUIDS", + default=None, + help="comma separated string of formats not to use in identification", + ) + parser.add_argument( + "-matchprintf", + metavar="FORMATSTRING", + default=None, + help="format string (Python style) to use on match. See nomatchprintf, README.txt.", + ) + parser.add_argument( + "-nomatchprintf", + metavar="FORMATSTRING", + default=None, + help="format string (Python style) to use if no match. See README.txt", + ) + parser.add_argument( + "-bufsize", + type=int, + default=None, + help="size (in bytes) of the buffer to match against (default=" + str(defaults["bufsize"]) + " bytes)", + ) + parser.add_argument( + "-sigs", + default=None, + metavar="SIG_ACT", + help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.', + ) + parser.add_argument( + "-container_bufsize", + type=int, + default=None, + help="size (in bytes) of the buffer to match against (default=" + + str(defaults["container_bufsize"]) + + " bytes)", + ) + parser.add_argument( + "-loadformats", default=None, metavar="XML1,...,XMLn", help="comma separated string of XML format files to add." + ) + parser.add_argument( + "-confdir", + default=CONFIG_DIR, + help="configuration directory to load_fido_xml, for example, the format specifications from.", + ) if len(sys.argv) == 1: parser.print_help() @@ -817,16 +899,23 @@ def main(args=None): versions = get_local_versions(args.confdir) - defaults['xml_pronomSignature'] = versions.pronom_signature - defaults['containersignature_file'] = versions.pronom_container_signature - defaults['xml_fidoExtensionSignature'] = versions.fido_extension_signature - defaults['format_files'] = [defaults['xml_pronomSignature']] + defaults["xml_pronomSignature"] = versions.pronom_signature + defaults["containersignature_file"] = versions.pronom_container_signature + defaults["xml_fidoExtensionSignature"] = versions.fido_extension_signature + defaults["format_files"] = [defaults["xml_pronomSignature"]] if args.pronom_only: - versionHeader = "FIDO v{0} ({1}, {2})\n".format(__version__, defaults['xml_pronomSignature'], defaults['containersignature_file']) + versionHeader = "FIDO v{0} ({1}, {2})\n".format( + __version__, defaults["xml_pronomSignature"], defaults["containersignature_file"] + ) else: - versionHeader = "FIDO v{0} ({1}, {2}, {3})\n".format(__version__, defaults['xml_pronomSignature'], defaults['containersignature_file'], defaults['xml_fidoExtensionSignature']) - defaults['format_files'].append(defaults['xml_fidoExtensionSignature']) + versionHeader = "FIDO v{0} ({1}, {2}, {3})\n".format( + __version__, + defaults["xml_pronomSignature"], + defaults["containersignature_file"], + defaults["xml_fidoExtensionSignature"], + ) + defaults["format_files"].append(defaults["xml_fidoExtensionSignature"]) if args.v: sys.stdout.write(versionHeader) @@ -838,13 +927,13 @@ def main(args=None): if args.matchprintf: try: - args.matchprintf = args.matchprintf.decode('string_escape') + args.matchprintf = args.matchprintf.decode("string_escape") except AttributeError: args.matchprintf = args.matchprintf.replace(r"\n", "\n") args.matchprintf = args.matchprintf.replace(r"\t", "\t") if args.nomatchprintf: try: - args.nomatchprintf = args.nomatchprintf.decode('string_escape') + args.nomatchprintf = args.nomatchprintf.decode("string_escape") except AttributeError: args.matchprintf = args.matchprintf.replace(r"\n", "\n") args.matchprintf = args.matchprintf.replace(r"\t", "\t") @@ -857,33 +946,34 @@ def main(args=None): printnomatch=args.nomatchprintf, zip=args.zip, nocontainer=args.nocontainer, - conf_dir=args.confdir) + conf_dir=args.confdir, + ) # TODO: Allow conf options to be dis-included if args.loadformats: - for file in args.loadformats.split(','): + for file in args.loadformats.split(","): fido.load_fido_xml(file) # TODO: remove from maps if args.useformats: - args.useformats = args.useformats.split(',') - fido.formats = [f for f in fido.formats if f.find('puid').text in args.useformats] + args.useformats = args.useformats.split(",") + fido.formats = [f for f in fido.formats if f.find("puid").text in args.useformats] elif args.nouseformats: - args.nouseformats = args.nouseformats.split(',') - fido.formats = [f for f in fido.formats if f.find('puid').text not in args.nouseformats] + args.nouseformats = args.nouseformats.split(",") + fido.formats = [f for f in fido.formats if f.find("puid").text not in args.nouseformats] # Set up to use stdin, or open input files: - if args.input == '-': + if args.input == "-": args.files = sys.stdin elif args.input: - args.files = open(args.input, 'r') + args.files = open(args.input, "r") # RUN try: if not args.q: sys.stderr.write(versionHeader) sys.stderr.flush() - if (not args.input) and len(args.files) == 1 and args.files[0] == '-': + if (not args.input) and len(args.files) == 1 and args.files[0] == "-": if fido.zip: raise RuntimeError("Multiple content read from stdin not yet supported.") fido.identify_multi_object_stream(sys.stdin, extension=not args.noextension) @@ -895,7 +985,7 @@ def main(args=None): except KeyboardInterrupt: sys.stdout.flush() sys.stderr.flush() - sys.exit('FIDO: Interrupt while identifying file {0}'.format(fido.current_file)) + sys.exit("FIDO: Interrupt while identifying file {0}".format(fido.current_file)) if not args.q: sys.stdout.flush() @@ -903,5 +993,5 @@ def main(args=None): sys.stderr.flush() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fido/package.py b/fido/package.py index 165c87c4..af23f5bc 100644 --- a/fido/package.py +++ b/fido/package.py @@ -4,15 +4,14 @@ import zipfile import olefile -from six import iteritems -class Package(): +class Package: """Base class for container support.""" def _process_puid_map(self, data, puid_map): results = [] - for puid, signatures in iteritems(puid_map): + for puid, signatures in puid_map.items(): results.extend(self._process_matches(data, puid, signatures)) return results @@ -39,14 +38,14 @@ def detect_formats(self): try: with olefile.OleFileIO(self.ole) as ole: results = [] - for path, puid_map in iteritems(self.signatures): + for path, puid_map in self.signatures.items(): # Each OLE container signature lists the path of the file inside the OLE # on which it operates; if the file is missing, there can be no match. # This is not a precise match because the name of the stream may slightly # differ; for example, \x01CompObj instead of CompObj filepath = None for paths in ole.listdir(): - p = '/'.join(paths) + p = "/".join(paths) if p == path or p[1:] == path: filepath = p break @@ -77,7 +76,7 @@ def detect_formats(self): try: with zipfile.ZipFile(self.zip) as zip_: results = [] - for path, puid_map in iteritems(self.signatures): + for path, puid_map in self.signatures.items(): # Each ZIP container signature lists the path of the file inside the ZIP # on which it operates; if the file is missing, there can be no match. if path not in zip_.namelist(): diff --git a/fido/prepare.py b/fido/prepare.py index 965dbf5f..32a81d1a 100644 --- a/fido/prepare.py +++ b/fido/prepare.py @@ -3,24 +3,21 @@ """Format Identification for Digital Objects.""" -from __future__ import print_function - -from argparse import ArgumentParser import hashlib import sys +import zipfile +from argparse import ArgumentParser +from io import StringIO +from urllib.error import HTTPError +from urllib.parse import urlparse +from urllib.request import urlopen from xml.dom import minidom from xml.etree import ElementTree as ET -import zipfile - -from six.moves import cStringIO -from six.moves.urllib.request import urlopen -from six.moves.urllib.parse import urlparse -from six.moves.urllib.error import HTTPError -from .versions import get_local_versions from .char_handler import escape +from .versions import get_local_versions -FLG_INCOMPATIBLE = '__INCOMPATIBLE_SIG__' +FLG_INCOMPATIBLE = "__INCOMPATIBLE_SIG__" class NS: @@ -47,7 +44,7 @@ def __call__(self, path): TNA = NS("{http://pronom.nationalarchives.gov.uk}") # TNA namespace -def get_text_tna(element, tag, default=''): +def get_text_tna(element, tag, default=""): """Helper function to return the text for a tag or path using the TNA namespace.""" part = element.find(TNA(tag)) if part is None or part.text is None: @@ -57,7 +54,7 @@ def get_text_tna(element, tag, default=''): def prettify(elem): """Return a pretty-printed XML string for the Element.""" - rough_string = ET.tostring(elem, 'UTF-8') + rough_string = ET.tostring(elem, "UTF-8") reparsed = minidom.parseString(rough_string) return reparsed.toprettyxml(indent=" ") @@ -76,13 +73,18 @@ def __init__(self, pronom_files, format_list=None): def save(self, dst=sys.stdout): """Write the fido XML format definitions to @param dst.""" - tree = ET.ElementTree(ET.Element('formats', { - 'version': '0.3', - 'xmlns:xsi': "http://www.w3.org/2001/XMLSchema-instance", - 'xsi:noNamespaceSchemaLocation': "fido-formats.xsd", - 'xmlns:dc': "http://purl.org/dc/elements/1.1/", - 'xmlns:dcterms': "http://purl.org/dc/terms/" - })) + tree = ET.ElementTree( + ET.Element( + "formats", + { + "version": "0.3", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:noNamespaceSchemaLocation": "fido-formats.xsd", + "xmlns:dc": "http://purl.org/dc/elements/1.1/", + "xmlns:dcterms": "http://purl.org/dc/terms/", + }, + ) + ) root = tree.getroot() for f in self.formats: # MdR: this skipped puids without sig, but we want them ALL @@ -90,7 +92,7 @@ def save(self, dst=sys.stdout): # if f.find('signature'): root.append(f) self.indent(root) - with open(dst, 'wb') as file_: + with open(dst, "wb") as file_: file_.write(ET.tostring(root)) def indent(self, elem, level=0): @@ -124,7 +126,7 @@ def load_pronom_xml(self, puid_filter=None): """ formats = [] try: - zip = zipfile.ZipFile(self.pronom_files, 'r') + zip = zipfile.ZipFile(self.pronom_files, "r") for item in zip.infolist(): try: stream = zip.open(item) @@ -144,16 +146,21 @@ def load_pronom_xml(self, puid_filter=None): if puid_filter is None: id_map = {} for element in formats: - puid = element.find('puid').text + puid = element.find("puid").text # print('working on puid:{}'.format(puid)) - pronom_id = element.find('pronom_id').text + pronom_id = element.find("pronom_id").text id_map[pronom_id] = puid for element in formats: - for rel in element.findall('has_priority_over'): + for rel in element.findall("has_priority_over"): try: rel.text = id_map[rel.text] except KeyError: - print("Error looking up priority over PRONOM ID {0} for format {1}".format(rel.text, element.find('puid').text), file=sys.stderr) + print( + "Error looking up priority over PRONOM ID {0} for format {1}".format( + rel.text, element.find("puid").text + ), + file=sys.stderr, + ) self._sort_formats(formats) self.formats = formats @@ -167,124 +174,133 @@ def parse_pronom_xml(self, source, puid_filter=None): """ pronom_xml = ET.parse(source) pronom_root = pronom_xml.getroot() - pronom_format = pronom_root.find(TNA('report_format_detail/FileFormat')) - fido_format = ET.Element('format') + pronom_format = pronom_root.find(TNA("report_format_detail/FileFormat")) + fido_format = ET.Element("format") # Get the base Format information - for id in pronom_format.findall(TNA('FileFormatIdentifier')): - type = get_text_tna(id, 'IdentifierType') - if type == 'PUID': - puid = get_text_tna(id, 'Identifier') - ET.SubElement(fido_format, 'puid').text = puid + for id in pronom_format.findall(TNA("FileFormatIdentifier")): + type = get_text_tna(id, "IdentifierType") + if type == "PUID": + puid = get_text_tna(id, "Identifier") + ET.SubElement(fido_format, "puid").text = puid if puid_filter and puid != puid_filter: return None # A bit clumsy. I want to have puid first, then mime, then container. - for id in pronom_format.findall(TNA('FileFormatIdentifier')): - type = get_text_tna(id, 'IdentifierType') - if type == 'MIME': - ET.SubElement(fido_format, 'mime').text = get_text_tna(id, 'Identifier') - elif type == 'PUID': - puid = get_text_tna(id, 'Identifier') - if puid == 'x-fmt/263': - ET.SubElement(fido_format, 'container').text = 'zip' - elif puid == 'x-fmt/265': - ET.SubElement(fido_format, 'container').text = 'tar' - ET.SubElement(fido_format, 'name').text = get_text_tna(pronom_format, 'FormatName') - ET.SubElement(fido_format, 'version').text = get_text_tna(pronom_format, 'FormatVersion') - ET.SubElement(fido_format, 'alias').text = get_text_tna(pronom_format, 'FormatAliases') - ET.SubElement(fido_format, 'pronom_id').text = get_text_tna(pronom_format, 'FormatID') + for id in pronom_format.findall(TNA("FileFormatIdentifier")): + type = get_text_tna(id, "IdentifierType") + if type == "MIME": + ET.SubElement(fido_format, "mime").text = get_text_tna(id, "Identifier") + elif type == "PUID": + puid = get_text_tna(id, "Identifier") + if puid == "x-fmt/263": + ET.SubElement(fido_format, "container").text = "zip" + elif puid == "x-fmt/265": + ET.SubElement(fido_format, "container").text = "tar" + ET.SubElement(fido_format, "name").text = get_text_tna(pronom_format, "FormatName") + ET.SubElement(fido_format, "version").text = get_text_tna(pronom_format, "FormatVersion") + ET.SubElement(fido_format, "alias").text = get_text_tna(pronom_format, "FormatAliases") + ET.SubElement(fido_format, "pronom_id").text = get_text_tna(pronom_format, "FormatID") # Get the extensions from the ExternalSignature - for x in pronom_format.findall(TNA('ExternalSignature')): - ET.SubElement(fido_format, 'extension').text = get_text_tna(x, 'Signature') - for id in pronom_format.findall(TNA('FileFormatIdentifier')): - type = get_text_tna(id, 'IdentifierType') - if type == 'Apple Uniform Type Identifier': - ET.SubElement(fido_format, 'apple_uti').text = get_text_tna(id, 'Identifier') + for x in pronom_format.findall(TNA("ExternalSignature")): + ET.SubElement(fido_format, "extension").text = get_text_tna(x, "Signature") + for id in pronom_format.findall(TNA("FileFormatIdentifier")): + type = get_text_tna(id, "IdentifierType") + if type == "Apple Uniform Type Identifier": + ET.SubElement(fido_format, "apple_uti").text = get_text_tna(id, "Identifier") # Handle the relationships - for x in pronom_format.findall(TNA('RelatedFormat')): - rel = get_text_tna(x, 'RelationshipType') - if rel == 'Has priority over': - ET.SubElement(fido_format, 'has_priority_over').text = get_text_tna(x, 'RelatedFormatID') + for x in pronom_format.findall(TNA("RelatedFormat")): + rel = get_text_tna(x, "RelationshipType") + if rel == "Has priority over": + ET.SubElement(fido_format, "has_priority_over").text = get_text_tna(x, "RelatedFormatID") # Get the InternalSignature information - for pronom_sig in pronom_format.findall(TNA('InternalSignature')): - fido_sig = ET.SubElement(fido_format, 'signature') - ET.SubElement(fido_sig, 'name').text = get_text_tna(pronom_sig, 'SignatureName') + for pronom_sig in pronom_format.findall(TNA("InternalSignature")): + fido_sig = ET.SubElement(fido_format, "signature") + ET.SubElement(fido_sig, "name").text = get_text_tna(pronom_sig, "SignatureName") # There are some funny chars in the notes, which caused me trouble and it is a unicode string, - ET.SubElement(fido_sig, 'note').text = get_text_tna(pronom_sig, 'SignatureNote') - for pronom_pat in pronom_sig.findall(TNA('ByteSequence')): + ET.SubElement(fido_sig, "note").text = get_text_tna(pronom_sig, "SignatureNote") + for pronom_pat in pronom_sig.findall(TNA("ByteSequence")): # print('Parsing ID:{}'.format(puid)) - fido_pat = ET.SubElement(fido_sig, 'pattern') - pos = fido_position(get_text_tna(pronom_pat, 'PositionType')) - byte_seq = get_text_tna(pronom_pat, 'ByteSequenceValue') - offset = get_text_tna(pronom_pat, 'Offset') - max_offset = get_text_tna(pronom_pat, 'MaxOffset') + fido_pat = ET.SubElement(fido_sig, "pattern") + pos = fido_position(get_text_tna(pronom_pat, "PositionType")) + byte_seq = get_text_tna(pronom_pat, "ByteSequenceValue") + offset = get_text_tna(pronom_pat, "Offset") + max_offset = get_text_tna(pronom_pat, "MaxOffset") if not max_offset: pass # print "working on puid:", puid, ", position: ", pos, "with offset, maxoffset: ", offset, ",", max_offset try: - regex = convert_to_regex(byte_seq, 'Little', pos, offset, max_offset) + regex = convert_to_regex(byte_seq, "Little", pos, offset, max_offset) except ValueError as ve: - print('ValueError converting PUID {} signature to regex: {}'.format(puid, ve), file=sys.stderr) + print("ValueError converting PUID {} signature to regex: {}".format(puid, ve), file=sys.stderr) regex = FLG_INCOMPATIBLE # print "done puid", puid if regex == FLG_INCOMPATIBLE: - print("Error: incompatible PRONOM signature found for puid {} skipping...".format(puid), file=sys.stderr) + print( + "Error: incompatible PRONOM signature found for puid {} skipping...".format(puid), + file=sys.stderr, + ) # remove the empty 'signature' nodes # now that the signature is not compatible and thus "regex" is empty - remove = fido_format.findall('signature') + remove = fido_format.findall("signature") for r in remove: fido_format.remove(r) continue - ET.SubElement(fido_pat, 'position').text = pos - ET.SubElement(fido_pat, 'pronom_pattern').text = byte_seq - ET.SubElement(fido_pat, 'regex').text = regex + ET.SubElement(fido_pat, "position").text = pos + ET.SubElement(fido_pat, "pronom_pattern").text = byte_seq + ET.SubElement(fido_pat, "regex").text = regex # Get the format details - fido_details = ET.SubElement(fido_format, 'details') - ET.SubElement(fido_details, 'dc:description').text = get_text_tna(pronom_format, 'FormatDescription') - ET.SubElement(fido_details, 'dcterms:available').text = get_text_tna(pronom_format, 'ReleaseDate') - ET.SubElement(fido_details, 'dc:creator').text = get_text_tna(pronom_format, 'Developers/DeveloperCompoundName') - ET.SubElement(fido_details, 'dcterms:publisher').text = get_text_tna(pronom_format, 'Developers/OrganisationName') - for x in pronom_format.findall(TNA('RelatedFormat')): - rel = get_text_tna(x, 'RelationshipType') - if rel == 'Is supertype of': - ET.SubElement(fido_details, 'is_supertype_of').text = get_text_tna(x, 'RelatedFormatID') - for x in pronom_format.findall(TNA('RelatedFormat')): - rel = get_text_tna(x, 'RelationshipType') - if rel == 'Is subtype of': - ET.SubElement(fido_details, 'is_subtype_of').text = get_text_tna(x, 'RelatedFormatID') - ET.SubElement(fido_details, 'content_type').text = get_text_tna(pronom_format, 'FormatTypes') + fido_details = ET.SubElement(fido_format, "details") + ET.SubElement(fido_details, "dc:description").text = get_text_tna(pronom_format, "FormatDescription") + ET.SubElement(fido_details, "dcterms:available").text = get_text_tna(pronom_format, "ReleaseDate") + ET.SubElement(fido_details, "dc:creator").text = get_text_tna(pronom_format, "Developers/DeveloperCompoundName") + ET.SubElement(fido_details, "dcterms:publisher").text = get_text_tna( + pronom_format, "Developers/OrganisationName" + ) + for x in pronom_format.findall(TNA("RelatedFormat")): + rel = get_text_tna(x, "RelationshipType") + if rel == "Is supertype of": + ET.SubElement(fido_details, "is_supertype_of").text = get_text_tna(x, "RelatedFormatID") + for x in pronom_format.findall(TNA("RelatedFormat")): + rel = get_text_tna(x, "RelationshipType") + if rel == "Is subtype of": + ET.SubElement(fido_details, "is_subtype_of").text = get_text_tna(x, "RelatedFormatID") + ET.SubElement(fido_details, "content_type").text = get_text_tna(pronom_format, "FormatTypes") # References for x in pronom_format.findall(TNA("Document")): - r = ET.SubElement(fido_details, 'reference') - ET.SubElement(r, 'dc:title').text = get_text_tna(x, 'TitleText') - ET.SubElement(r, 'dc:creator').text = get_text_tna(x, 'Author/AuthorCompoundName') - ET.SubElement(r, 'dc:publisher').text = get_text_tna(x, 'Publisher/PublisherCompoundName') - ET.SubElement(r, 'dcterms:available').text = get_text_tna(x, 'PublicationDate') - for id in x.findall(TNA('DocumentIdentifier')): - type = get_text_tna(id, 'IdentifierType') - if type == 'URL': - ET.SubElement(r, 'dc:identifier').text = "http://" + get_text_tna(id, 'Identifier') + r = ET.SubElement(fido_details, "reference") + ET.SubElement(r, "dc:title").text = get_text_tna(x, "TitleText") + ET.SubElement(r, "dc:creator").text = get_text_tna(x, "Author/AuthorCompoundName") + ET.SubElement(r, "dc:publisher").text = get_text_tna(x, "Publisher/PublisherCompoundName") + ET.SubElement(r, "dcterms:available").text = get_text_tna(x, "PublicationDate") + for id in x.findall(TNA("DocumentIdentifier")): + type = get_text_tna(id, "IdentifierType") + if type == "URL": + ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna(id, "Identifier") else: - ET.SubElement(r, 'dc:identifier').text = get_text_tna(id, 'IdentifierType') + ":" + get_text_tna(id, 'Identifier') - ET.SubElement(r, 'dc:description').text = get_text_tna(x, 'DocumentNote') - ET.SubElement(r, 'dc:type').text = get_text_tna(x, 'DocumentType') - ET.SubElement(r, 'dcterms:license').text = get_text_tna(x, 'AvailabilityDescription') + " " + get_text_tna(x, 'AvailabilityNote') - ET.SubElement(r, 'dc:rights').text = get_text_tna(x, 'DocumentIPR') + ET.SubElement(r, "dc:identifier").text = ( + get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + ) + ET.SubElement(r, "dc:description").text = get_text_tna(x, "DocumentNote") + ET.SubElement(r, "dc:type").text = get_text_tna(x, "DocumentType") + ET.SubElement(r, "dcterms:license").text = ( + get_text_tna(x, "AvailabilityDescription") + " " + get_text_tna(x, "AvailabilityNote") + ) + ET.SubElement(r, "dc:rights").text = get_text_tna(x, "DocumentIPR") # Examples for x in pronom_format.findall(TNA("ReferenceFile")): - rf = ET.SubElement(fido_details, 'example_file') - ET.SubElement(rf, 'dc:title').text = get_text_tna(x, 'ReferenceFileName') - ET.SubElement(rf, 'dc:description').text = get_text_tna(x, 'ReferenceFileDescription') + rf = ET.SubElement(fido_details, "example_file") + ET.SubElement(rf, "dc:title").text = get_text_tna(x, "ReferenceFileName") + ET.SubElement(rf, "dc:description").text = get_text_tna(x, "ReferenceFileDescription") checksum = "" - for id in x.findall(TNA('ReferenceFileIdentifier')): - type = get_text_tna(id, 'IdentifierType') - if type == 'URL': + for id in x.findall(TNA("ReferenceFileIdentifier")): + type = get_text_tna(id, "IdentifierType") + if type == "URL": # Starting with PRONOM 89, some URLs contain http:// # and others do not. - url = get_text_tna(id, 'Identifier') + url = get_text_tna(id, "Identifier") if not urlparse(url).scheme: url = "http://" + url - ET.SubElement(rf, 'dc:identifier').text = url + ET.SubElement(rf, "dc:identifier").text = url # And calculate the checksum of this resource: m = hashlib.md5() try: @@ -292,49 +308,54 @@ def parse_pronom_xml(self, source, puid_filter=None): m.update(sock.read()) sock.close() except HTTPError as http_excep: - sys.stderr.write('HTTP {} error loading resource {}\n'.format(http_excep.code, url)) + sys.stderr.write("HTTP {} error loading resource {}\n".format(http_excep.code, url)) if http_excep.code == 404: continue checksum = m.hexdigest() else: - ET.SubElement(rf, 'dc:identifier').text = get_text_tna(id, 'IdentifierType') + ":" + get_text_tna(id, 'Identifier') - ET.SubElement(rf, 'dcterms:license').text = "" - ET.SubElement(rf, 'dc:rights').text = get_text_tna(x, 'ReferenceFileIPR') - checksumElement = ET.SubElement(rf, 'checksum') + ET.SubElement(rf, "dc:identifier").text = ( + get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + ) + ET.SubElement(rf, "dcterms:license").text = "" + ET.SubElement(rf, "dc:rights").text = get_text_tna(x, "ReferenceFileIPR") + checksumElement = ET.SubElement(rf, "checksum") checksumElement.text = checksum - checksumElement.attrib['type'] = "md5" + checksumElement.attrib["type"] = "md5" # Record Metadata - md = ET.SubElement(fido_details, 'record_metadata') - ET.SubElement(md, 'status').text = 'unknown' - ET.SubElement(md, 'dc:creator').text = get_text_tna(pronom_format, 'ProvenanceName') - ET.SubElement(md, 'dcterms:created').text = get_text_tna(pronom_format, 'ProvenanceSourceDate') - ET.SubElement(md, 'dcterms:modified').text = get_text_tna(pronom_format, 'LastUpdatedDate') - ET.SubElement(md, 'dc:description').text = get_text_tna(pronom_format, 'ProvenanceDescription') + md = ET.SubElement(fido_details, "record_metadata") + ET.SubElement(md, "status").text = "unknown" + ET.SubElement(md, "dc:creator").text = get_text_tna(pronom_format, "ProvenanceName") + ET.SubElement(md, "dcterms:created").text = get_text_tna(pronom_format, "ProvenanceSourceDate") + ET.SubElement(md, "dcterms:modified").text = get_text_tna(pronom_format, "LastUpdatedDate") + ET.SubElement(md, "dc:description").text = get_text_tna(pronom_format, "ProvenanceDescription") return fido_format # FIXME: I don't think that this quite works yet! def _sort_formats(self, formatlist): """Sort the format list based on their priority relationships so higher priority formats appear earlier in the list.""" + def compare_formats(f1, f2): - f1ID = f1.find('puid').text - f2ID = f2.find('puid').text - for worse in f1.findall('has_priority_over'): + f1ID = f1.find("puid").text + f2ID = f2.find("puid").text + for worse in f1.findall("has_priority_over"): if worse.text == f2ID: - return - 1 - for worse in f2.findall('has_priority_over'): + return -1 + for worse in f2.findall("has_priority_over"): if worse.text == f1ID: return 1 if f1ID < f2ID: - return - 1 + return -1 if f1ID == f2ID: return 0 return 1 + return sorted(formatlist, key=_cmp_to_key(compare_formats)) def _cmp_to_key(mycmp): """Convert a cmp= function into a key= function.""" + # From https://docs.python.org/3/howto/sorting.html#sortinghowto class K: """Wrapper class for comparator function.""" @@ -365,21 +386,23 @@ def __ne__(self, other): def fido_position(pronom_position): """Return BOF/EOF/VAR instead of the more verbose pronom position names.""" - if pronom_position == 'Absolute from BOF': - return 'BOF' - if pronom_position == 'Absolute from EOF': - return 'EOF' - if pronom_position == 'Variable': - return 'VAR' - if pronom_position == 'Indirect From BOF': - return 'IFB' + if pronom_position == "Absolute from BOF": + return "BOF" + if pronom_position == "Absolute from EOF": + return "EOF" + if pronom_position == "Variable": + return "VAR" + if pronom_position == "Indirect From BOF": + return "IFB" # to make sure FIDO does not crash (IFB aftermath) sys.stderr.write("Unknown pronom PositionType:" + pronom_position) - return 'VAR' + return "VAR" def _convert_err_msg(msg, c, i, chars, buf): - return "Conversion: {0}: char='{1}', at pos {2} in \n {3}\n {4}^\nBuffer = {5}".format(msg, c, i, chars, i * ' ', buf.getvalue()) + return "Conversion: {0}: char='{1}', at pos {2} in \n {3}\n {4}^\nBuffer = {5}".format( + msg, c, i, chars, i * " ", buf.getvalue() + ) def do_byte(chars, i, littleendian, esc=True): @@ -388,11 +411,11 @@ def do_byte(chars, i, littleendian, esc=True): @return a tuple (byte, 2) """ - c1 = '0123456789ABCDEF'.find(chars[i].upper()) - c2 = '0123456789ABCDEF'.find(chars[i + 1].upper()) - buf = cStringIO() - if (c1 < 0 or c2 < 0): - raise Exception(_convert_err_msg('bad byte sequence', chars[i:i + 2], i, chars, buf)) + c1 = "0123456789ABCDEF".find(chars[i].upper()) + c2 = "0123456789ABCDEF".find(chars[i + 1].upper()) + buf = StringIO() + if c1 < 0 or c2 < 0: + raise Exception(_convert_err_msg("bad byte sequence", chars[i : i + 2], i, chars, buf)) if littleendian: val = chr(16 * c1 + c2) else: @@ -413,7 +436,7 @@ def calculate_repetition(char, pos, offset, maxoffset): This function only has an effect when one or both offsets is greater than MAX_REGEX_REPS bytes (4GB). See: https://bugs.python.org/issue13169. """ - calcbuf = cStringIO() + calcbuf = StringIO() calcremain = False offsetremain = 0 @@ -429,24 +452,24 @@ def calculate_repetition(char, pos, offset, maxoffset): calcremain = True if pos == "BOF" or pos == "EOF": - if offset != '0': - calcbuf.write(char + '{' + str(offset)) + if offset != "0": + calcbuf.write(char + "{" + str(offset)) if maxoffset is not None: - calcbuf.write(',' + maxoffset) - calcbuf.write('}') + calcbuf.write("," + maxoffset) + calcbuf.write("}") elif maxoffset is not None: - calcbuf.write(char + '{0,' + maxoffset + '}') + calcbuf.write(char + "{0," + maxoffset + "}") if pos == "IFB": - if offset != '0': - calcbuf.write(char + '{' + str(offset)) + if offset != "0": + calcbuf.write(char + "{" + str(offset)) if maxoffset is not None: - calcbuf.write(',' + maxoffset) - calcbuf.write('}') + calcbuf.write("," + maxoffset) + calcbuf.write("}") if maxoffset is not None: - calcbuf.write(',}') + calcbuf.write(",}") elif maxoffset is not None: - calcbuf.write(char + '{0,' + maxoffset + '}') + calcbuf.write(char + "{0," + maxoffset + "}") if calcremain: # recursion happens here calcbuf.write(calculate_repetition(char, pos, offsetremain, maxoffsetremain)) @@ -458,16 +481,12 @@ def calculate_repetition(char, pos, offset, maxoffset): def do_all_bitmasks(chars, i, littleendian): """(byte & bitmask) == bitmask.""" - return do_any_all_bitmasks( - chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), - littleendian) + return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), littleendian) def do_any_bitmasks(chars, i, littleendian): """(byte & bitmask) != 0.""" - return do_any_all_bitmasks( - chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), - littleendian) + return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), littleendian) def do_any_all_bitmasks(chars, i, predicate, littleendian): @@ -485,13 +504,13 @@ def do_any_all_bitmasks(chars, i, predicate, littleendian): """ byt, inc = do_byte(chars, i + 1, littleendian, esc=False) bitmask = ord(byt) - regex = '({})'.format( - '|'.join(['\\x' + hex(byte)[2:].zfill(2) for byte in range(0x100) - if predicate(byte, bitmask)])) + regex = "({})".format( + "|".join(["\\x" + hex(byte)[2:].zfill(2) for byte in range(0x100) if predicate(byte, bitmask)]) + ) return regex, inc + 1 -def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''): +def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): """ Convert to regular expression. @@ -500,202 +519,210 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''): @param chars, a pronom bytesequence, into a @return regular expression. """ - if 'Big' in endianness: + if "Big" in endianness: littleendian = False else: littleendian = True if len(offset) == 0: - offset = '0' + offset = "0" if len(maxoffset) == 0: maxoffset = None - if maxoffset == '0': + if maxoffset == "0": maxoffset = None - buf = cStringIO() + buf = StringIO() buf.write("(?s)") # If a regex starts with (?s), it is equivalent to DOTALL. i = 0 - state = 'start' - if 'BOF' in pos: - buf.write('\\A') # start of regex - buf.write(calculate_repetition('.', pos, offset, maxoffset)) + state = "start" + if "BOF" in pos: + buf.write("\\A") # start of regex + buf.write(calculate_repetition(".", pos, offset, maxoffset)) - if 'IFB' in pos: - buf.write('\\A') - buf.write(calculate_repetition('.', pos, offset, maxoffset)) + if "IFB" in pos: + buf.write("\\A") + buf.write(calculate_repetition(".", pos, offset, maxoffset)) while True: if i == len(chars): break # print _convert_err_msg(state,chars[i],i,chars) - if state == 'start': + if state == "start": if chars[i].isalnum(): - state = 'bytes' - elif chars[i] == '&': - state = 'all-bitmask' - elif chars[i] == '~': - state = 'any-bitmask' - elif chars[i] == '[' and chars[i + 1] == '!': - state = 'non-match' - elif chars[i] == '[': - state = 'bracket' - elif chars[i] == '{': - state = 'curly' - elif chars[i] == '(': - state = 'paren' - elif chars[i] in '*+?': - state = 'specials' + state = "bytes" + elif chars[i] == "&": + state = "all-bitmask" + elif chars[i] == "~": + state = "any-bitmask" + elif chars[i] == "[" and chars[i + 1] == "!": + state = "non-match" + elif chars[i] == "[": + state = "bracket" + elif chars[i] == "{": + state = "curly" + elif chars[i] == "(": + state = "paren" + elif chars[i] in "*+?": + state = "specials" else: - raise ValueError(_convert_err_msg('Illegal character in start', chars[i], i, chars, buf)) - elif state == 'bytes': + raise ValueError(_convert_err_msg("Illegal character in start", chars[i], i, chars, buf)) + elif state == "bytes": (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc - state = 'start' - elif state == 'all-bitmask': + state = "start" + elif state == "all-bitmask": (byt, inc) = do_all_bitmasks(chars, i, littleendian) buf.write(byt) i += inc - state = 'start' - elif state == 'any-bitmask': + state = "start" + elif state == "any-bitmask": (byt, inc) = do_any_bitmasks(chars, i, littleendian) buf.write(byt) i += inc - state = 'start' - elif state == 'non-match': - buf.write('(?!') + state = "start" + elif state == "non-match": + buf.write("(?!") i += 2 while True: if chars[i].isalnum(): (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc - elif chars[i] == '&': + elif chars[i] == "&": (byt, inc) = do_all_bitmasks(chars, i, littleendian) buf.write(byt) i += inc - elif chars[i] == '~': + elif chars[i] == "~": (byt, inc) = do_any_bitmasks(chars, i, littleendian) buf.write(byt) i += inc - elif chars[i] == ']': + elif chars[i] == "]": break else: - raise Exception(_convert_err_msg('Illegal character in non-match', chars[i], i, chars, buf)) - buf.write(')') + raise Exception(_convert_err_msg("Illegal character in non-match", chars[i], i, chars, buf)) + buf.write(")") i += 1 - state = 'start' + state = "start" - elif state == 'bracket': + elif state == "bracket": try: - buf.write('[') + buf.write("[") i += 1 (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc # assert(chars[i] == ':') - if chars[i] != ':': + if chars[i] != ":": return "__INCOMPATIBLE_SIG__" - buf.write('-') + buf.write("-") i += 1 (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc # assert(chars[i] == ']') - if chars[i] != ']': + if chars[i] != "]": return "__INCOMPATIBLE_SIG__" - buf.write(']') + buf.write("]") i += 1 except Exception: - print(_convert_err_msg('Illegal character in bracket', chars[i], i, chars, buf)) + print(_convert_err_msg("Illegal character in bracket", chars[i], i, chars, buf)) raise - if i < len(chars) and chars[i] == '{': - state = 'curly-after-bracket' + if i < len(chars) and chars[i] == "{": + state = "curly-after-bracket" else: - state = 'start' - elif state == 'paren': - buf.write('(?:') + state = "start" + elif state == "paren": + buf.write("(?:") i += 1 while True: if chars[i].isalnum(): (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc - elif chars[i] == '|': - buf.write('|') + elif chars[i] == "|": + buf.write("|") i += 1 - elif chars[i] == ')': + elif chars[i] == ")": break # START fix FIDO-20 - elif chars[i] == '[': - buf.write('[') + elif chars[i] == "[": + buf.write("[") i += 1 (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc # assert(chars[i] == ':') - if chars[i] != ':': + if chars[i] != ":": return "__INCOMPATIBLE_SIG__" - buf.write('-') + buf.write("-") i += 1 (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) i += inc # assert(chars[i] == ']') - if chars[i] != ']': + if chars[i] != "]": return "__INCOMPATIBLE_SIG__" - buf.write(']') + buf.write("]") i += 1 else: - raise Exception(_convert_err_msg(('Current state = \'{0}\' : Illegal character in paren').format(state), chars[i], i, chars, buf)) - buf.write(')') + raise Exception( + _convert_err_msg( + ("Current state = '{0}' : Illegal character in paren").format(state), + chars[i], + i, + chars, + buf, + ) + ) + buf.write(")") i += 1 - state = 'start' + state = "start" # END fix FIDO-20 - elif state in ['curly', 'curly-after-bracket']: + elif state in ["curly", "curly-after-bracket"]: # {nnnn} or {nnn-nnn} or {nnn-*} # {nnn} or {nnn,nnn} or {nnn,} # when there is a curly-after-bracket, then the {m,n} applies to the bracketed item # The above, while sensible, appears to be incorrect. A '.' is always needed. # for droid equiv behavior # if state == 'curly': - buf.write('.') - buf.write('{') - i += 1 # skip the ( + buf.write(".") + buf.write("{") + i += 1 # skip the ( while True: if chars[i].isalnum(): buf.write(chars[i]) i += 1 - elif chars[i] == '-': - buf.write(',') + elif chars[i] == "-": + buf.write(",") i += 1 - elif chars[i] == '*': # skip the * + elif chars[i] == "*": # skip the * i += 1 - elif chars[i] == '}': + elif chars[i] == "}": break else: - raise Exception(_convert_err_msg('Illegal character in curly', chars[i], i, chars, buf)) - buf.write('}') - i += 1 # skip the ) - state = 'start' - elif state == 'specials': - if chars[i] == '*': - buf.write('.*') + raise Exception(_convert_err_msg("Illegal character in curly", chars[i], i, chars, buf)) + buf.write("}") + i += 1 # skip the ) + state = "start" + elif state == "specials": + if chars[i] == "*": + buf.write(".*") i += 1 - elif chars[i] == '+': - buf.write('.+') + elif chars[i] == "+": + buf.write(".+") i += 1 - elif chars[i] == '?': - if chars[i + 1] != '?': - raise Exception(_convert_err_msg('Illegal character after ?', chars[i + 1], i + 1, chars, buf)) - buf.write('.?') + elif chars[i] == "?": + if chars[i + 1] != "?": + raise Exception(_convert_err_msg("Illegal character after ?", chars[i + 1], i + 1, chars, buf)) + buf.write(".?") i += 2 - state = 'start' + state = "start" else: - raise Exception('Illegal state {0}'.format(state)) + raise Exception("Illegal state {0}".format(state)) - if 'EOF' in pos: - buf.write(calculate_repetition('.', pos, offset, maxoffset)) - buf.write('\\Z') + if "EOF" in pos: + buf.write(calculate_repetition(".", pos, offset, maxoffset)) + buf.write("\\Z") val = buf.getvalue() buf.close() @@ -714,7 +741,7 @@ def run(input=None, output=None, puid=None): info = FormatInfo(input) info.load_pronom_xml(puid) info.save(output) - print('Converted {0} PRONOM formats to FIDO signatures'.format(len(info.formats)), file=sys.stderr) + print("Converted {0} PRONOM formats to FIDO signatures".format(len(info.formats)), file=sys.stderr) def main(args=None): @@ -722,14 +749,14 @@ def main(args=None): if args is None: args = sys.argv[1:] - parser = ArgumentParser(description='Produce the FIDO format XML that is loaded at run-time') - parser.add_argument('-input', default=None, help='Input file, a Zip containing PRONOM XML files') - parser.add_argument('-output', default=None, help='Output file') - parser.add_argument('-puid', default=None, help='A particular PUID record to extract') + parser = ArgumentParser(description="Produce the FIDO format XML that is loaded at run-time") + parser.add_argument("-input", default=None, help="Input file, a Zip containing PRONOM XML files") + parser.add_argument("-output", default=None, help="Output file") + parser.add_argument("-puid", default=None, help="A particular PUID record to extract") args = parser.parse_args(args) run(input=args.input, output=args.output, puid=args.puid) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index d4ccb4f5..c8536123 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -20,33 +20,42 @@ PRONOM format signatures SOAP calls. """ import sys -from urllib.error import HTTPError, URLError +import urllib import xml.etree.ElementTree as ET -from six.moves import urllib +from urllib.error import HTTPError, URLError from fido import __version__ -ENCODING = 'utf-8' + +ENCODING = "utf-8" XML_PROC = ''.format(ENCODING) -TNA_DOMAIN = 'nationalarchives.gov.uk' -PRONOM_HOST = 'www.{}'.format(TNA_DOMAIN) -PRONOM_NS = 'http://pronom.{}'.format(TNA_DOMAIN) -SIG_NS = 'http://{}/pronom/SignatureFile'.format(PRONOM_HOST) +TNA_DOMAIN = "nationalarchives.gov.uk" +PRONOM_HOST = "www.{}".format(TNA_DOMAIN) +PRONOM_NS = "http://pronom.{}".format(TNA_DOMAIN) +SIG_NS = "http://{}/pronom/SignatureFile".format(PRONOM_HOST) NS = { - 'soap': 'http://schemas.xmlsoap.org/soap/envelope/', - 'xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsd': 'http://www.w3.org/2001/XMLSchema', - 'pronom': PRONOM_NS, - 'sig': SIG_NS + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsd": "http://www.w3.org/2001/XMLSchema", + "pronom": PRONOM_NS, + "sig": SIG_NS, } HEADERS = { - 'Host': PRONOM_HOST, - 'User-Agent': 'PRONOM UTILS v{0} (OPF)'.format(__version__), - 'Content-type': 'text/xml; charset="UTF-8"' + "Host": PRONOM_HOST, + "User-Agent": "PRONOM UTILS v{0} (OPF)".format(__version__), + "Content-type": 'text/xml; charset="UTF-8"', } +def get_sig_xml_for_puid(puid): + """Return the full PRONOM signature XML for the passed PUID.""" + req = urllib.request.Request("http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid)) + response = urllib.request.urlopen(req) + xml = response.read() + return xml + + def get_pronom_sig_version(): """ Get PRONOM signature version. @@ -54,8 +63,8 @@ def get_pronom_sig_version(): Return latest signature file version number as an int. Raises an HTTPError if there are problems. """ - tree = _get_soap_ele_tree('getSignatureFileVersionV1') - ver_ele = tree.find('.//pronom:Version/pronom:Version', NS) + tree = _get_soap_ele_tree("getSignatureFileVersionV1") + ver_ele = tree.find(".//pronom:Version/pronom:Version", NS) return int(ver_ele.text) @@ -70,18 +79,30 @@ def get_droid_signatures(version): xml = [] format_count = False try: - with urllib.request.urlopen('https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml'.format(version)) as f: - xml = f.read().decode('utf-8') + with urllib.request.urlopen( + "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format(version) + ) as f: + xml = f.read().decode("utf-8") root_ele = ET.fromstring(xml) - format_count = len(root_ele.findall('.//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat')) + format_count = len( + root_ele.findall(".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat") + ) except HTTPError as httpe: - sys.stderr.write("get_droid_signatures(): could not download signature file v{} due to exception: {}\n".format(version, httpe)) + sys.stderr.write( + "get_droid_signatures(): could not download signature file v{} due to exception: {}\n".format( + version, httpe + ) + ) return xml, format_count def _get_soap_ele_tree(soap_action): - soap_string = '{}<{} xmlns="{}" />'.format(XML_PROC, NS.get('xsi'), NS.get('xsd'), NS.get('soap'), soap_action, PRONOM_NS).encode(ENCODING) - soap_action = '\"{}:{}In\"'.format(PRONOM_NS, soap_action) + soap_string = '{}<{} xmlns="{}" />'.format( + XML_PROC, NS.get("xsi"), NS.get("xsd"), NS.get("soap"), soap_action, PRONOM_NS + ).encode( + ENCODING + ) + soap_action = '"{}:{}In"'.format(PRONOM_NS, soap_action) xml = _get_soap_response(soap_action, soap_string) for prefix, uri in NS.items(): ET.register_namespace(prefix, uri) @@ -90,14 +111,14 @@ def _get_soap_ele_tree(soap_action): def _get_soap_response(soap_action, soap_string): try: - req = urllib.request.Request('http://{}/pronom/service.asmx'.format(PRONOM_HOST), data=soap_string) + req = urllib.request.Request("http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string) except URLError: - print('There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.'.format(PRONOM_HOST)) - print('Please check your network connection and try again.') + print("There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format(PRONOM_HOST)) + print("Please check your network connection and try again.") sys.exit(1) for key, value in HEADERS.items(): req.add_header(key, value) - req.add_header('Content-length', '%d' % len(soap_string)) - req.add_header('SOAPAction', soap_action) + req.add_header("Content-length", "%d" % len(soap_string)) + req.add_header("SOAPAction", soap_action) response = urllib.request.urlopen(req) return response.read().decode(ENCODING) diff --git a/fido/toxml.py b/fido/toxml.py index 1a52b467..ca1905af 100644 --- a/fido/toxml.py +++ b/fido/toxml.py @@ -19,8 +19,6 @@ - http://support.microsoft.com/default.aspx?kbid=321788 """ -from __future__ import absolute_import - import csv import sys @@ -30,17 +28,22 @@ def main(): """Generate XML as read from CSV and send it to the standard output stream.""" - sys.stdout.write(""" + sys.stdout.write( + """ {0} {1} - """.format(__version__, get_local_versions().pronom_version)) + """.format( + __version__, get_local_versions().pronom_version + ) + ) reader = csv.reader(sys.stdin) for row in reader: - sys.stdout.write(""" + sys.stdout.write( + """ {0} {1} @@ -51,10 +54,13 @@ def main(): {6} {7} {8} - """.format(row[6], row[0], row[8], row[1], row[2], row[7], row[3], row[4], row[5])) + """.format( + row[6], row[0], row[8], row[1], row[2], row[7], row[3], row[4], row[5] + ) + ) sys.stdout.write("\n\n") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fido/update_signatures.py b/fido/update_signatures.py index 4a3f7710..db7b0419 100644 --- a/fido/update_signatures.py +++ b/fido/update_signatures.py @@ -13,39 +13,65 @@ PRONOM is available from http://www.nationalarchives.gov.uk/pronom/. """ -from __future__ import print_function - -from argparse import ArgumentParser import os -from shutil import rmtree import sys import time -from xml.etree import ElementTree as CET import zipfile +from argparse import ArgumentParser +from shutil import rmtree +from xml.etree import ElementTree as CET -from . import __version__, CONFIG_DIR, query_yes_no +from . import CONFIG_DIR, __version__, query_yes_no from .prepare import run as prepare_pronom_to_fido +from .pronom.soap import NS, get_droid_signatures, get_pronom_sig_version, get_sig_xml_for_puid from .versions import get_local_versions -from .pronom.soap import get_pronom_sig_version, get_droid_signatures, NS -from .pronom.http import get_sig_xml_for_puid -ABORT_MSG = 'Aborting update...' +ABORT_MSG = "Aborting update..." DEFAULTS = { - 'signatureFileName': 'DROID_SignatureFile-v{0}.xml', - 'pronomZipFileName': 'pronom-xml-v{0}.zip', - 'fidoSignatureVersion': 'format_extensions.xml', - 'containerVersion': 'container-signature-UPDATE-ME.xml', # container version is frozen and needs human attention before updating, + "signatureFileName": "DROID_SignatureFile-v{0}.xml", + "pronomZipFileName": "pronom-xml-v{0}.zip", + "fidoSignatureVersion": "format_extensions.xml", + "containerVersion": "container-signature-UPDATE-ME.xml", # container version is frozen and needs human attention before updating, } OPTIONS = { - 'http_throttle': 0.5, # in secs, to prevent DoS of PRONOM server - 'tmp_dir': os.path.join(CONFIG_DIR, 'tmp'), - 'deleteTempDirectory': True, - 'version': 'latest', + "http_throttle": 0.5, # in secs, to prevent DoS of PRONOM server + "tmp_dir": os.path.join(CONFIG_DIR, "tmp"), + "deleteTempDirectory": True, + "version": "latest", } +def query_yes_no(question, default="yes"): + """ + Ask a yes/no question via input() and return their answer. + + `question` is a string that is presented to the user. `default` is the + presumed answer if the user just hits . It must be "yes" (the + default), "no" or None (meaning an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + """ + valid = {"yes": True, "y": True, "no": False, "n": False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError('Invalid default answer: "%s"' % default) + while True: + print(question + prompt, end="") + choice = rinput().lower() + if default is not None and choice == "": + return valid[default] + if choice in valid: + return valid[choice] + print('Please respond with "yes" or "no" (or "y" or "n").') + + def run(defaults=None): """ Update PRONOM signatures. @@ -56,16 +82,16 @@ def run(defaults=None): defaults = defaults or DEFAULTS try: print("Contacting PRONOM...") - latest, sig_file = sig_version_check(defaults.get('version')) + latest, sig_file = sig_version_check(defaults.get("version")) download_sig_file(latest, sig_file) print("Extracting PRONOM PUID's from signature file...") tree = CET.parse(sig_file) - format_eles = tree.findall('.//sig:FileFormat', NS) + format_eles = tree.findall(".//sig:FileFormat", NS) print("Found {} PRONOM FileFormat elements".format(len(format_eles))) tmpdir, resume = init_sig_download(defaults) download_signatures(defaults, format_eles, resume, tmpdir) create_zip_file(defaults, format_eles, latest, tmpdir) - if defaults['deleteTempDirectory']: + if defaults["deleteTempDirectory"]: print("Deleting temporary folder and files...") rmtree(tmpdir, ignore_errors=True) update_versions_xml(latest) @@ -79,16 +105,16 @@ def run(defaults=None): sys.exit(ABORT_MSG) -def sig_version_check(version='latest'): +def sig_version_check(version="latest"): """Return a tuple consisting of current sig file version and the derived file name.""" - print('Sig version check for version:', version) - if version == 'latest': - print('Getting latest version number from PRONOM...') + print("Sig version check for version:", version) + if version == "latest": + print("Getting latest version number from PRONOM...") version = get_pronom_sig_version() if not version: - sys.exit('Failed to obtain PRONOM signature file version number, please try again.') + sys.exit("Failed to obtain PRONOM signature file version number, please try again.") - print('Querying PRONOM for signaturefile version {}.'.format(version)) + print("Querying PRONOM for signaturefile version {}.".format(version)) sig_file_name = _sig_file_name(version) if os.path.isfile(sig_file_name): print("You already have the PRONOM signature file, version", version) @@ -98,7 +124,7 @@ def sig_version_check(version='latest'): def _sig_file_name(version): - return os.path.join(CONFIG_DIR, DEFAULTS['signatureFileName'].format(version)) + return os.path.join(CONFIG_DIR, DEFAULTS["signatureFileName"].format(version)) def download_sig_file(version, sig_file): @@ -106,9 +132,9 @@ def download_sig_file(version, sig_file): print("Downloading signature file version {}...".format(version)) sig_xml, _ = get_droid_signatures(version) if not sig_xml: - sys.exit('Failed to obtain PRONOM signature file, please try again.') - print("Writing {0}...".format(DEFAULTS['signatureFileName'].format(version))) - with open(sig_file, 'w') as file_: + sys.exit("Failed to obtain PRONOM signature file, please try again.") + print("Writing {0}...".format(DEFAULTS["signatureFileName"].format(version))) + with open(sig_file, "w") as file_: file_.write(sig_xml) @@ -122,11 +148,11 @@ def init_sig_download(defaults): print("Downloading signatures can take a while") if not query_yes_no("Continue and download signatures?"): sys.exit(ABORT_MSG) - tmpdir = defaults['tmp_dir'] + tmpdir = defaults["tmp_dir"] resume = False if os.path.isdir(tmpdir): print("Found previously created temporary folder for download:", tmpdir) - resume = query_yes_no('Do you want to resume download (yes) or start over (no)?') + resume = query_yes_no("Do you want to resume download (yes) or start over (no)?") if resume: print("Resuming download...") else: @@ -144,12 +170,14 @@ def download_signatures(defaults, format_eles, resume, tmpdir): """Download PRONOM signatures and write to individual files.""" print("Downloading signatures, one moment please...") puid_count = len(format_eles) - one_percent = (float(puid_count) / 100) + one_percent = float(puid_count) / 100 numfiles = 0 for format_ele in format_eles: download_sig(format_ele, tmpdir, resume, defaults) numfiles += 1 - print(r"Downloaded {}/{} files [{}%]".format(numfiles, puid_count, int(float(numfiles) / one_percent)), end="\r") + print( + r"Downloaded {}/{} files [{}%]".format(numfiles, puid_count, int(float(numfiles) / one_percent)), end="\r" + ) print("100%") @@ -169,60 +197,77 @@ def download_sig(format_ele, tmpdir, resume, defaults): except Exception as e: sys.stderr.write("Failed to download signature file:" + puid) sys.stderr.write("Error:" + str(e)) - sys.exit('Please restart and resume download.') - with open(filename, 'wb') as file_: + sys.exit("Please restart and resume download.") + with open(filename, "wb") as file_: file_.write(xml) - time.sleep(defaults['http_throttle']) + time.sleep(defaults["http_throttle"]) def create_zip_file(defaults, format_eles, version, tmpdir): """Create zip file of signatures.""" print("Creating PRONOM zip...") - compression = zipfile.ZIP_DEFLATED if 'zlib' in sys.modules else zipfile.ZIP_STORED - modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'} - zf = zipfile.ZipFile(os.path.join(CONFIG_DIR, DEFAULTS['pronomZipFileName'].format(version)), mode='w') + compression = zipfile.ZIP_DEFLATED if "zlib" in sys.modules else zipfile.ZIP_STORED + modes = {zipfile.ZIP_DEFLATED: "deflated", zipfile.ZIP_STORED: "stored"} + zf = zipfile.ZipFile(os.path.join(CONFIG_DIR, DEFAULTS["pronomZipFileName"].format(version)), mode="w") print("Adding files with compression mode", modes[compression]) for format_ele in format_eles: _, puid_filename = get_puid_file_name(format_ele) filename = os.path.join(tmpdir, puid_filename) if os.path.isfile(filename): zf.write(filename, arcname=puid_filename, compress_type=compression) - if defaults['deleteTempDirectory']: + if defaults["deleteTempDirectory"]: os.unlink(filename) zf.close() def get_puid_file_name(format_ele): """Return a tupe of PUID and PUID file name derived from format_ele.""" - puid = format_ele.get('PUID') + puid = format_ele.get("PUID") type_part, num_part = puid.split("/") - return puid, 'puid.{}.{}.xml'.format(type_part, num_part) + return puid, "puid.{}.{}.xml".format(type_part, num_part) def update_versions_xml(version): """Create new versions identified sig XML file.""" - print('Updating versions.xml...') + print("Updating versions.xml...") versions = get_local_versions() versions.pronom_version = str(version) versions.pronom_signature = "formats-v" + str(version) + ".xml" - versions.pronom_container_signature = DEFAULTS['containerVersion'] - versions.fido_extension_signature = DEFAULTS['fidoSignatureVersion'] + versions.pronom_container_signature = DEFAULTS["containerVersion"] + versions.fido_extension_signature = DEFAULTS["fidoSignatureVersion"] versions.update_script = __version__ versions.write() def main(): """Main CLI entrypoint.""" - parser = ArgumentParser(description='Download and convert the latest PRONOM signatures') - parser.add_argument('-tmpdir', default=OPTIONS['tmp_dir'], help='Location to store temporary files', dest='tmp_dir') - parser.add_argument('-keep_tmp', default=OPTIONS['deleteTempDirectory'], help='Do not delete temporary files after completion', dest='deleteTempDirectory', action='store_false') - parser.add_argument('-http_throttle', default=OPTIONS['http_throttle'], help='Time (in seconds) to wait between downloads', type=float, dest='http_throttle') - parser.add_argument('-version', default=OPTIONS['version'], help='Download and convert a specific signature file by version', dest='version') + parser = ArgumentParser(description="Download and convert the latest PRONOM signatures") + parser.add_argument("-tmpdir", default=OPTIONS["tmp_dir"], help="Location to store temporary files", dest="tmp_dir") + parser.add_argument( + "-keep_tmp", + default=OPTIONS["deleteTempDirectory"], + help="Do not delete temporary files after completion", + dest="deleteTempDirectory", + action="store_false", + ) + parser.add_argument( + "-http_throttle", + default=OPTIONS["http_throttle"], + help="Time (in seconds) to wait between downloads", + type=float, + dest="http_throttle", + ) + parser.add_argument( + "-version", + default=OPTIONS["version"], + help="Download and convert a specific signature file by version", + dest="version", + ) args = parser.parse_args() opts = DEFAULTS.copy() opts.update(vars(args)) run(opts) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fido/versions.py b/fido/versions.py index 13af0cdb..94dae67b 100644 --- a/fido/versions.py +++ b/fido/versions.py @@ -17,16 +17,15 @@ PRONOM is available from http://www.nationalarchives.gov.uk/pronom/ """ -from __future__ import absolute_import +import importlib.resources import os import re -import importlib_resources import sys -import requests -import six from xml.etree import ElementTree as ET -from xml.etree.ElementTree import parse, ParseError +from xml.etree.ElementTree import ParseError, parse + +import requests from fido import CONFIG_DIR @@ -49,15 +48,15 @@ class LocalVersions(object): """ PROPS_MAPPING = { - 'pronom_version': 'pronomVersion', - 'pronom_signature': 'pronomSignature', - 'pronom_container_signature': 'pronomContainerSignature', - 'fido_extension_signature': 'fidoExtensionSignature', - 'update_script': 'updateScript', - 'update_site': 'updateSite', + "pronom_version": "pronomVersion", + "pronom_signature": "pronomSignature", + "pronom_container_signature": "pronomContainerSignature", + "fido_extension_signature": "fidoExtensionSignature", + "update_script": "updateScript", + "update_site": "updateSite", } - ROOT_ELEMENT = 'versions' + ROOT_ELEMENT = "versions" def __init__(self, versions_file): """Instantiate class based on the file indicated in `versions_file`.""" @@ -88,7 +87,7 @@ def __setattr__(self, name, value): def get_zip_file(self): """Obtain location to the PRONOM XML Zip file based on the current PRONOM version.""" - return os.path.join(self.conf_dir, 'pronom-xml-v{}.zip'.format(self.pronom_version)) + return os.path.join(self.conf_dir, "pronom-xml-v{}.zip".format(self.pronom_version)) def get_signature_file(self): """Obtain location to the current PRONOM signature file.""" @@ -97,15 +96,15 @@ def get_signature_file(self): def write(self): """Update versions.xml.""" # Check that all the fields are defined - for key, value in six.iteritems(self.PROPS_MAPPING): + for key, value in self.PROPS_MAPPING.items(): if self.root.find(value) is None: - raise ValueError('Field {} has not been defined!'.format(key)) - self.tree.write(self.versions_file, xml_declaration=True, method='xml', encoding='utf-8') + raise ValueError("Field {} has not been defined!".format(key)) + self.tree.write(self.versions_file, xml_declaration=True, method="xml", encoding="utf-8") def get_local_versions(config_dir=CONFIG_DIR): """Return an instance of LocalVersions loaded with `conf/versions.xml`.""" - return LocalVersions(os.path.join(config_dir, 'versions.xml')) + return LocalVersions(os.path.join(config_dir, "versions.xml")) def sig_file_actions(sig_act): @@ -115,16 +114,16 @@ def sig_file_actions(sig_act): # Get update URL, add trailing slash if missing update_url = versions.update_site - if not update_url.endswith('/'): - update_url += '/' + if not update_url.endswith("/"): + update_url += "/" # Parse parameter and take appropriate action - if sig_act == 'list': + if sig_act == "list": # List available signature files _list_available_versions(update_url) - elif sig_act in ['check', 'update']: + elif sig_act in ["check", "update"]: # Check or/and update signature file to latest - _check_update_signatures(sig_vers, update_url, versions, sig_act == 'update') + _check_update_signatures(sig_vers, update_url, versions, sig_act == "update") else: # Download a specific version of the signature file _download_sig_version(sig_act, update_url, versions) @@ -134,70 +133,74 @@ def sig_file_actions(sig_act): def _list_available_versions(update_url): """List available signature files.""" - resp = requests.get(update_url + 'format/') + resp = requests.get(update_url + "format/") tree = ET.fromstring(resp.content) - sys.stdout.write('Available signature versions:\n') - for child in tree.iter('signature'): - sys.stdout.write('{}\n'.format(child.get('version'))) + sys.stdout.write("Available signature versions:\n") + for child in tree.iter("signature"): + sys.stdout.write("{}\n".format(child.get("version"))) def _check_update_signatures(sig_vers, update_url, versions, is_update=False): is_new, latest = _version_check(sig_vers, update_url) if is_new: - sys.stdout.write('Updated signatures v{} are available, current version is v{}\n'.format(latest, sig_vers)) + sys.stdout.write("Updated signatures v{} are available, current version is v{}\n".format(latest, sig_vers)) if is_update: _output_details(latest, update_url, versions) else: - sys.stdout.write('Your signature files are up to date, current version is v{}\n'.format(sig_vers)) + sys.stdout.write("Your signature files are up to date, current version is v{}\n".format(sig_vers)) sys.exit(0) def _download_sig_version(sig_act, update_url, versions): - sys.stdout.write('Downloading signature files for version {}\n'.format(sig_act)) - match = re.search(r'^v?(\d+)$', sig_act, re.IGNORECASE) + sys.stdout.write("Downloading signature files for version {}\n".format(sig_act)) + match = re.search(r"^v?(\d+)$", sig_act, re.IGNORECASE) if not match: - sys.exit('{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(sig_act)) + sys.exit( + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(sig_act) + ) ver = sig_act - if not ver.startswith('v'): - ver = 'v' + sig_act - resp = requests.get(update_url + 'format/' + ver + '/') + if not ver.startswith("v"): + ver = "v" + sig_act + resp = requests.get(update_url + "format/" + ver + "/") if resp.status_code != 200: - sys.exit('No signature files found for {}, REST status {}'.format(sig_act, resp.status_code)) - _output_details(re.search(r'\d+|$', ver).group(), update_url, versions) # noqa: W605 + sys.exit("No signature files found for {}, REST status {}".format(sig_act, resp.status_code)) + _output_details(re.search(r"\d+|$", ver).group(), update_url, versions) # noqa: W605 def _get_version(ver_string): """Parse a PROMOM version number from a string.""" - match = re.search(r'^v?(\d+)$', ver_string, re.IGNORECASE) + match = re.search(r"^v?(\d+)$", ver_string, re.IGNORECASE) if not match: - sys.exit('{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(ver_string)) + sys.exit( + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(ver_string) + ) ver = ver_string - return ver_string if not ver.startswith('v') else ver_string[1:] + return ver_string if not ver.startswith("v") else ver_string[1:] def _output_details(version, update_url, versions): - sys.stdout.write('Updating signature file to {}.\n'.format(version)) - _write_sigs(version, update_url, 'fido', 'formats-v{}.xml') - _write_sigs(version, update_url, 'droid', 'DROID_SignatureFile-v{}.xml') - _write_sigs(version, update_url, 'pronom', 'pronom-xml-v{}.zip') - versions.pronom_version = '{}'.format(version) - versions.pronom_signature = 'formats-v{}.xml'.format(version) + sys.stdout.write("Updating signature file to {}.\n".format(version)) + _write_sigs(version, update_url, "fido", "formats-v{}.xml") + _write_sigs(version, update_url, "droid", "DROID_SignatureFile-v{}.xml") + _write_sigs(version, update_url, "pronom", "pronom-xml-v{}.zip") + versions.pronom_version = "{}".format(version) + versions.pronom_signature = "formats-v{}.xml".format(version) versions.write() def _version_check(sig_ver, update_url): - resp = requests.get(update_url + 'format/latest/') + resp = requests.get(update_url + "format/latest/") if resp.status_code != 200: - sys.exit('Error getting latest version info: HTTP Status {}'.format(resp.status_code)) + sys.exit("Error getting latest version info: HTTP Status {}".format(resp.status_code)) root_ele = ET.fromstring(resp.text) - latest = _get_version(root_ele.get('version')) + latest = _get_version(root_ele.get("version")) return int(latest) > int(sig_ver), latest def _write_sigs(latest, update_url, type, name_template): - sig_out = str(importlib_resources.files('fido').joinpath('conf', name_template.format(latest))) + sig_out = str(importlib.resources.files("fido").joinpath("conf", name_template.format(latest))) if os.path.exists(sig_out): return - resp = requests.get(update_url + 'format/{0}/{1}/'.format(latest, type)) - open(sig_out, 'wb').write(resp.content) + resp = requests.get(update_url + "format/{0}/{1}/".format(latest, type)) + open(sig_out, "wb").write(resp.content) diff --git a/pyproject.toml b/pyproject.toml index 81d5b106..614e62ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,6 @@ classifiers = [ dependencies = [ "olefile >= 0.46, < 1", - "six >= 1.10.0, < 2", - "importlib-resources", "requests" ] From 99467e8d560887108edf6ab32dd8ec0b5505dd68 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 16:59:54 +0100 Subject: [PATCH 04/33] Re-linted with flake8 and updated with fake8 config info. --- fido/fido.py | 1 - fido/update_signatures.py | 4 ++-- pyproject.toml | 16 ++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index e0a85fe6..cb2c1ef1 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -9,7 +9,6 @@ """ import os -import platform import re import sys import tarfile diff --git a/fido/update_signatures.py b/fido/update_signatures.py index db7b0419..43a2d681 100644 --- a/fido/update_signatures.py +++ b/fido/update_signatures.py @@ -21,7 +21,7 @@ from shutil import rmtree from xml.etree import ElementTree as CET -from . import CONFIG_DIR, __version__, query_yes_no +from . import CONFIG_DIR, __version__ from .prepare import run as prepare_pronom_to_fido from .pronom.soap import NS, get_droid_signatures, get_pronom_sig_version, get_sig_xml_for_puid from .versions import get_local_versions @@ -64,7 +64,7 @@ def query_yes_no(question, default="yes"): raise ValueError('Invalid default answer: "%s"' % default) while True: print(question + prompt, end="") - choice = rinput().lower() + choice = input().lower() if default is not None and choice == "": return valid[default] if choice in valid: diff --git a/pyproject.toml b/pyproject.toml index 614e62ee..e0f274f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descri readme = "README.md" authors = [ { name="Adam Farquhar (BL)" } # Add email if available -] + ] license = { file = "LICENSE.txt" } classifiers = [ @@ -30,12 +30,13 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11" -] + ] dependencies = [ "olefile >= 0.46, < 1", - "requests" -] + "requests", + "flake8" + ] [project.urls] homepage = "http://openpreservation.org/technology/products/fido/" @@ -44,7 +45,7 @@ homepage = "http://openpreservation.org/technology/products/fido/" testing = [ "pytest", "pytest-cov", -] + ] [project.scripts] fido = "fido.fido:main" @@ -59,4 +60,7 @@ fido-toxml = "fido.toxml:main" addopts = "--maxfail=1 --strict-markers" [tool.flake8] -ignore = ["E501"] \ No newline at end of file +exclude = ['.venv'] +ignore = ['E231', 'E241', 'E501', 'W503', 'E203'] +max-line-length = 130 +# count = true \ No newline at end of file From 15e4ec290fd5143bbd065e63090dcf21dfa71fb6 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 17:01:03 +0100 Subject: [PATCH 05/33] Moved unused Dockerfile to attic. --- Dockerfile => .attic/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfile => .attic/Dockerfile (100%) diff --git a/Dockerfile b/.attic/Dockerfile similarity index 100% rename from Dockerfile rename to .attic/Dockerfile From 97831146f0f76acd4fa1ac5bcee4a961f0181f7a Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 17:06:50 +0100 Subject: [PATCH 06/33] Added setuptools-git-versioning --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e0f274f7..7343340e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=42", "wheel"] +requires = ["setuptools>=42", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" # These were in requirements/packing.txt @@ -35,7 +35,7 @@ classifiers = [ dependencies = [ "olefile >= 0.46, < 1", "requests", - "flake8" + "flake8", ] [project.urls] @@ -59,6 +59,9 @@ fido-toxml = "fido.toxml:main" [tool.pytest.ini_options] addopts = "--maxfail=1 --strict-markers" +[tool.setuptools-git-versioning] +enabled = true + [tool.flake8] exclude = ['.venv'] ignore = ['E231', 'E241', 'E501', 'W503', 'E203'] From 7e7e2e4eb29f6897a3a0ddbcb636fa1cd78fe2df Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 10 Sep 2024 17:16:21 +0100 Subject: [PATCH 07/33] Added setuptools-git-versioning --- VERSION | 0 pyproject.toml | 9 +++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index 7343340e..dcc69997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,16 @@ fido-toxml = "fido.toxml:main" [tool.setuptools.package-data] "fido" = ["*.*", "conf/*.*", "pronom/*.*"] +[tool.setuptools-git-versioning] +enabled = true +# version_file = "VERSION" +# count_commits_from_version_file = true +# dev_template = "{tag}.{branch}{ccount}" # <--- note {branch} here +# dirty_template = "{tag}.{branch}{ccount}" + [tool.pytest.ini_options] addopts = "--maxfail=1 --strict-markers" -[tool.setuptools-git-versioning] -enabled = true [tool.flake8] exclude = ['.venv'] From abfe0182b43237138d310dea8c73e42fefb09f94 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 11 Sep 2024 13:10:54 +0100 Subject: [PATCH 08/33] Update test-pr.yml to ignore line-length and space before : --- .github/workflows/test-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 1c7d3b16..c6aab036 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -25,7 +25,7 @@ jobs: pip install -U flake8 pep257 pytest-cov codecov codacy-coverage pluggy pip install -e . - name: Lint code with flake8 - run: flake8 . --count --show-source --max-line-length=127 --statistics + run: flake8 . --count --show-source --ignore=E203,E501 --max-line-length=127 --statistics - name: Lint code with pep257 if: matrix.python-version == 2.7 run: pep257 --match="(?!fido).*\.py" ./fido From 21e05619e72a6c659ad7b36c85233149d50f11cf Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 11 Sep 2024 13:14:07 +0100 Subject: [PATCH 09/33] Update test-pr.yml added some additional Flake8 ignores. --- .github/workflows/test-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index c6aab036..4c129610 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -25,7 +25,7 @@ jobs: pip install -U flake8 pep257 pytest-cov codecov codacy-coverage pluggy pip install -e . - name: Lint code with flake8 - run: flake8 . --count --show-source --ignore=E203,E501 --max-line-length=127 --statistics + run: flake8 . --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=127 --statistics - name: Lint code with pep257 if: matrix.python-version == 2.7 run: pep257 --match="(?!fido).*\.py" ./fido From 295ba36db1bd082fc9188a28f36544c48c5b86bf Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 11 Sep 2024 13:26:26 +0100 Subject: [PATCH 10/33] Update test-pr.yml - Removed Python 3.6 and 3.7 from the matrix. --- .github/workflows/test-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 4c129610..beccf523 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 From 8c5ad5174bc52b4109dfaff464e34bfacb4e27fe Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 11 Sep 2024 18:11:21 +0100 Subject: [PATCH 11/33] Interim commit to help with rebasing after a PR was approved and merged into 1.8 --- fido/cli_args.py | 78 ++++++++++++++++++ pyproject.toml | 6 +- tests/test_cli_args.py | 61 ++++++++++++++ .../hard_packages}/bad.zip | Bin .../hard_packages}/foo.tar | Bin .../hard_packages}/foo.zip | Bin .../hard_packages}/unicode.zip | Bin .../hard_packages}/worse.zip | Bin tests/test_package.py | 16 ++-- 9 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 fido/cli_args.py create mode 100644 tests/test_cli_args.py rename tests/{fixtures => test_data/hard_packages}/bad.zip (100%) rename tests/{fixtures => test_data/hard_packages}/foo.tar (100%) rename tests/{fixtures => test_data/hard_packages}/foo.zip (100%) rename tests/{fixtures => test_data/hard_packages}/unicode.zip (100%) rename tests/{fixtures => test_data/hard_packages}/worse.zip (100%) diff --git a/fido/cli_args.py b/fido/cli_args.py new file mode 100644 index 00000000..260d7608 --- /dev/null +++ b/fido/cli_args.py @@ -0,0 +1,78 @@ +import argparse +import sys +from argparse import ArgumentParser, RawTextHelpFormatter + + +def build_parser() -> ArgumentParser: + defaults = { + "description": "FIDO - File Identification Tool", + "epilog": "For more information, visit the official documentation.", + } + + parser = ArgumentParser( + description=defaults["description"], + epilog=defaults["epilog"], + fromfile_prefix_chars="@", + formatter_class=RawTextHelpFormatter, + ) + parser.add_argument("-v", default=False, action="store_true", help="show version information") + parser.add_argument("-q", default=False, action="store_true", help="run (more) quietly") + parser.add_argument("-recurse", default=False, action="store_true", help="recurse into subdirectories") + parser.add_argument("-zip", default=False, action="store_true", help="recurse into zip and tar files") + parser.add_argument( + "-noextension", + default=False, + action="store_true", + help="disable extension matching, reduces number of matches but may reduce false positives", + ) + parser.add_argument( + "-nocontainer", + default=False, + action="store_true", + help="disable deep scan of container documents, increases speed but may reduce accuracy with big files", + ) + parser.add_argument( + "-pronom_only", + default=False, + action="store_true", + help="disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results", + ) + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-input", default=False, help="file containing a list of files to check, one per line. - means stdin" + ) + group.add_argument( + "files", + nargs="*", + default=[], + metavar="FILE", + help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.", + ) + + parser.add_argument("-filename", default=None, help="filename if file contents passed through STDIN") + parser.add_argument( + "-useformats", + metavar="INCLUDEPUIDS", + default=None, + help="comma separated string of formats to use in identification", + ) + parser.add_argument( + "-nouseformats", + metavar="EXCLUDEPUIDS", + default=None, + help="comma separated string of formats not to use in identification", + ) + + return parser + + +def parse_args(parser: ArgumentParser) -> argparse.Namespace: + try: + args = parser.parse_args() + except argparse.ArgumentError as e: + parser.print_help() + print(f"\nError: {e}\n") + sys.exit(1) + + return args diff --git a/pyproject.toml b/pyproject.toml index dcc69997..9aa09967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,7 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools-git-versioning>=2.0,<3"] +requires = ["setuptools>=42", "wheel", "twine", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" -# These were in requirements/packing.txt -# twine>=1.8,<1.9 -# wheel==0.38.1 - [project] name = "opf-fido" dynamic = ["version"] diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py new file mode 100644 index 00000000..4405a86f --- /dev/null +++ b/tests/test_cli_args.py @@ -0,0 +1,61 @@ +import argparse + +import pytest + +from fido.cli_args import build_parser, parse_args + +# Common argument string +ARG_STRING = ( + "-v -q -recurse -zip -noextension -nocontainer -pronom_only" + "-input files.txt" + "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" +) + +ARG_STRING = ( + "-v -q -recurse -zip -noextension -nocontainer -pronom_only" + "-input files.txt" + "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" +) + + +def test_build_parser(): + parser = build_parser() + assert isinstance(parser, argparse.ArgumentParser) + + # Check if all expected arguments are present + expected_args = ARG_STRING.split() + for arg in expected_args: + assert arg in parser._option_string_actions + + +def test_parse_args_valid(): + parser = build_parser() + + args = parse_args(parser.parse_args(ARG_STRING.split())) + + assert args.v is True + assert args.q is True + assert args.recurse is True + assert args.zip is True + assert args.noextension is True + assert args.nocontainer is True + assert args.pronom_only is True + assert args.input == "input_file" + assert args.files == ["file1", "file2"] + assert args.filename == "filename" + assert args.useformats == "fmt1,fmt2" + assert args.nouseformats == "fmt3,fmt4" + + +def test_parse_args_invalid(monkeypatch): + parser = build_parser() + + # Simulate invalid argument input + monkeypatch.setattr("sys.argv", ["prog", "--invalid"]) + with pytest.raises(SystemExit): + parse_args(parser) + + # Simulate missing required argument + monkeypatch.setattr("sys.argv", ["prog", "-input"]) + with pytest.raises(SystemExit): + parse_args(parser) diff --git a/tests/fixtures/bad.zip b/tests/test_data/hard_packages/bad.zip similarity index 100% rename from tests/fixtures/bad.zip rename to tests/test_data/hard_packages/bad.zip diff --git a/tests/fixtures/foo.tar b/tests/test_data/hard_packages/foo.tar similarity index 100% rename from tests/fixtures/foo.tar rename to tests/test_data/hard_packages/foo.tar diff --git a/tests/fixtures/foo.zip b/tests/test_data/hard_packages/foo.zip similarity index 100% rename from tests/fixtures/foo.zip rename to tests/test_data/hard_packages/foo.zip diff --git a/tests/fixtures/unicode.zip b/tests/test_data/hard_packages/unicode.zip similarity index 100% rename from tests/fixtures/unicode.zip rename to tests/test_data/hard_packages/unicode.zip diff --git a/tests/fixtures/worse.zip b/tests/test_data/hard_packages/worse.zip similarity index 100% rename from tests/fixtures/worse.zip rename to tests/test_data/hard_packages/worse.zip diff --git a/tests/test_package.py b/tests/test_package.py index f1565142..b4123cd2 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,13 +1,15 @@ import os -from fido.package import ZipPackage +import pytest +from fido.package import ZipPackage -FIXTURES_DIR = os.path.normpath(os.path.join(__file__, '..', 'fixtures')) +TEST_DATA_BAD_PACKAGES = os.path.normpath(os.path.join(__file__, "..", "test_data/hard_packages")) -def test_bad_zips(): - for filename in ('bad.zip', 'worse.zip', 'unicode.zip'): - p = ZipPackage(os.path.join(FIXTURES_DIR, filename), {}) - r = p.detect_formats() - assert isinstance(r, list) and len(r) == 0 +# None of these files should be identified as packages? +@pytest.mark.parametrize("filename", ["bad.zip", "worse.zip", "unicode.zip", "foo.zip", "foo.tar"]) +def test_bad_zip(filename): + p = ZipPackage(os.path.join(TEST_DATA_BAD_PACKAGES, filename), {}) + r = p.detect_formats() + assert isinstance(r, list) and len(r) == 0 From 0d5f11c44a6d1994db0db9497ad093b61d542495 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 11 Sep 2024 18:11:21 +0100 Subject: [PATCH 12/33] Interim commit to help with rebasing after a PR was approved and merged into 1.8 --- fido/cli_args.py | 78 ++++++++++++++++++ pyproject.toml | 6 +- tests/test_cli_args.py | 61 ++++++++++++++ .../hard_packages}/bad.zip | Bin .../hard_packages}/foo.tar | Bin .../hard_packages}/foo.zip | Bin .../hard_packages}/unicode.zip | Bin .../hard_packages}/worse.zip | Bin tests/test_package.py | 16 ++-- 9 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 fido/cli_args.py create mode 100644 tests/test_cli_args.py rename tests/{fixtures => test_data/hard_packages}/bad.zip (100%) rename tests/{fixtures => test_data/hard_packages}/foo.tar (100%) rename tests/{fixtures => test_data/hard_packages}/foo.zip (100%) rename tests/{fixtures => test_data/hard_packages}/unicode.zip (100%) rename tests/{fixtures => test_data/hard_packages}/worse.zip (100%) diff --git a/fido/cli_args.py b/fido/cli_args.py new file mode 100644 index 00000000..260d7608 --- /dev/null +++ b/fido/cli_args.py @@ -0,0 +1,78 @@ +import argparse +import sys +from argparse import ArgumentParser, RawTextHelpFormatter + + +def build_parser() -> ArgumentParser: + defaults = { + "description": "FIDO - File Identification Tool", + "epilog": "For more information, visit the official documentation.", + } + + parser = ArgumentParser( + description=defaults["description"], + epilog=defaults["epilog"], + fromfile_prefix_chars="@", + formatter_class=RawTextHelpFormatter, + ) + parser.add_argument("-v", default=False, action="store_true", help="show version information") + parser.add_argument("-q", default=False, action="store_true", help="run (more) quietly") + parser.add_argument("-recurse", default=False, action="store_true", help="recurse into subdirectories") + parser.add_argument("-zip", default=False, action="store_true", help="recurse into zip and tar files") + parser.add_argument( + "-noextension", + default=False, + action="store_true", + help="disable extension matching, reduces number of matches but may reduce false positives", + ) + parser.add_argument( + "-nocontainer", + default=False, + action="store_true", + help="disable deep scan of container documents, increases speed but may reduce accuracy with big files", + ) + parser.add_argument( + "-pronom_only", + default=False, + action="store_true", + help="disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results", + ) + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-input", default=False, help="file containing a list of files to check, one per line. - means stdin" + ) + group.add_argument( + "files", + nargs="*", + default=[], + metavar="FILE", + help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.", + ) + + parser.add_argument("-filename", default=None, help="filename if file contents passed through STDIN") + parser.add_argument( + "-useformats", + metavar="INCLUDEPUIDS", + default=None, + help="comma separated string of formats to use in identification", + ) + parser.add_argument( + "-nouseformats", + metavar="EXCLUDEPUIDS", + default=None, + help="comma separated string of formats not to use in identification", + ) + + return parser + + +def parse_args(parser: ArgumentParser) -> argparse.Namespace: + try: + args = parser.parse_args() + except argparse.ArgumentError as e: + parser.print_help() + print(f"\nError: {e}\n") + sys.exit(1) + + return args diff --git a/pyproject.toml b/pyproject.toml index dcc69997..9aa09967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,7 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools-git-versioning>=2.0,<3"] +requires = ["setuptools>=42", "wheel", "twine", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" -# These were in requirements/packing.txt -# twine>=1.8,<1.9 -# wheel==0.38.1 - [project] name = "opf-fido" dynamic = ["version"] diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py new file mode 100644 index 00000000..4405a86f --- /dev/null +++ b/tests/test_cli_args.py @@ -0,0 +1,61 @@ +import argparse + +import pytest + +from fido.cli_args import build_parser, parse_args + +# Common argument string +ARG_STRING = ( + "-v -q -recurse -zip -noextension -nocontainer -pronom_only" + "-input files.txt" + "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" +) + +ARG_STRING = ( + "-v -q -recurse -zip -noextension -nocontainer -pronom_only" + "-input files.txt" + "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" +) + + +def test_build_parser(): + parser = build_parser() + assert isinstance(parser, argparse.ArgumentParser) + + # Check if all expected arguments are present + expected_args = ARG_STRING.split() + for arg in expected_args: + assert arg in parser._option_string_actions + + +def test_parse_args_valid(): + parser = build_parser() + + args = parse_args(parser.parse_args(ARG_STRING.split())) + + assert args.v is True + assert args.q is True + assert args.recurse is True + assert args.zip is True + assert args.noextension is True + assert args.nocontainer is True + assert args.pronom_only is True + assert args.input == "input_file" + assert args.files == ["file1", "file2"] + assert args.filename == "filename" + assert args.useformats == "fmt1,fmt2" + assert args.nouseformats == "fmt3,fmt4" + + +def test_parse_args_invalid(monkeypatch): + parser = build_parser() + + # Simulate invalid argument input + monkeypatch.setattr("sys.argv", ["prog", "--invalid"]) + with pytest.raises(SystemExit): + parse_args(parser) + + # Simulate missing required argument + monkeypatch.setattr("sys.argv", ["prog", "-input"]) + with pytest.raises(SystemExit): + parse_args(parser) diff --git a/tests/fixtures/bad.zip b/tests/test_data/hard_packages/bad.zip similarity index 100% rename from tests/fixtures/bad.zip rename to tests/test_data/hard_packages/bad.zip diff --git a/tests/fixtures/foo.tar b/tests/test_data/hard_packages/foo.tar similarity index 100% rename from tests/fixtures/foo.tar rename to tests/test_data/hard_packages/foo.tar diff --git a/tests/fixtures/foo.zip b/tests/test_data/hard_packages/foo.zip similarity index 100% rename from tests/fixtures/foo.zip rename to tests/test_data/hard_packages/foo.zip diff --git a/tests/fixtures/unicode.zip b/tests/test_data/hard_packages/unicode.zip similarity index 100% rename from tests/fixtures/unicode.zip rename to tests/test_data/hard_packages/unicode.zip diff --git a/tests/fixtures/worse.zip b/tests/test_data/hard_packages/worse.zip similarity index 100% rename from tests/fixtures/worse.zip rename to tests/test_data/hard_packages/worse.zip diff --git a/tests/test_package.py b/tests/test_package.py index f1565142..b4123cd2 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,13 +1,15 @@ import os -from fido.package import ZipPackage +import pytest +from fido.package import ZipPackage -FIXTURES_DIR = os.path.normpath(os.path.join(__file__, '..', 'fixtures')) +TEST_DATA_BAD_PACKAGES = os.path.normpath(os.path.join(__file__, "..", "test_data/hard_packages")) -def test_bad_zips(): - for filename in ('bad.zip', 'worse.zip', 'unicode.zip'): - p = ZipPackage(os.path.join(FIXTURES_DIR, filename), {}) - r = p.detect_formats() - assert isinstance(r, list) and len(r) == 0 +# None of these files should be identified as packages? +@pytest.mark.parametrize("filename", ["bad.zip", "worse.zip", "unicode.zip", "foo.zip", "foo.tar"]) +def test_bad_zip(filename): + p = ZipPackage(os.path.join(TEST_DATA_BAD_PACKAGES, filename), {}) + r = p.detect_formats() + assert isinstance(r, list) and len(r) == 0 From 630be3a5b7bf784c087340e6e7a42b6af6d79f66 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Thu, 12 Sep 2024 17:37:56 +0100 Subject: [PATCH 13/33] Refactor `fido.py` moving CLI argument parsing into its own `cli_args.py`. Add some tests to the argument parsing. Add the CONFIG_DIR to the dictionary of defaults in fido.py. --- fido/cli_args.py | 65 ++++++++++++++------ fido/fido.py | 131 ++++------------------------------------- tests/test_cli_args.py | 91 +++++++++++++--------------- 3 files changed, 103 insertions(+), 184 deletions(-) diff --git a/fido/cli_args.py b/fido/cli_args.py index 260d7608..a6919853 100644 --- a/fido/cli_args.py +++ b/fido/cli_args.py @@ -3,11 +3,15 @@ from argparse import ArgumentParser, RawTextHelpFormatter -def build_parser() -> ArgumentParser: - defaults = { - "description": "FIDO - File Identification Tool", - "epilog": "For more information, visit the official documentation.", - } +def parse_cli_args(argv: list[str], defaults: dict) -> argparse.Namespace: + """ + Parse command-line arguments. + Args: + argv (list[str]): List of command-line arguments. Could be sys.argv + defaults (dict): Dictionary of default values. Expects to find configdir, bufsize and container_bufsize. + Returns: + argparse.Namespace: Parsed command-line arguments. Reference via name as in args.v or args.recurse. + """ parser = ArgumentParser( description=defaults["description"], @@ -63,16 +67,43 @@ def build_parser() -> ArgumentParser: default=None, help="comma separated string of formats not to use in identification", ) + parser.add_argument( + "-matchprintf", + metavar="FORMATSTRING", + default=None, + help="format string (Python style) to use on match. See nomatchprintf, README.txt.", + ) + parser.add_argument( + "-nomatchprintf", + metavar="FORMATSTRING", + default=None, + help="format string (Python style) to use if no match. See README.txt", + ) + parser.add_argument( + "-bufsize", + type=int, + default=None, + help=f"size (in bytes) of the buffer to match against (default={defaults['bufsize']})", + ) + parser.add_argument( + "-sigs", + default=None, + metavar="SIG_ACT", + help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.', + ) + parser.add_argument( + "-container_bufsize", + type=int, + default=None, + help=f"size (in bytes) of the buffer to match against (default={defaults['container_bufsize']}).", + ) + parser.add_argument( + "-loadformats", default=None, metavar="XML1,...,XMLn", help="comma separated string of XML format files to add." + ) + parser.add_argument( + "-confdir", + default=defaults["config_dir"], + help="configuration directory to load_fido_xml, for example, the format specifications from.", + ) - return parser - - -def parse_args(parser: ArgumentParser) -> argparse.Namespace: - try: - args = parser.parse_args() - except argparse.ArgumentError as e: - parser.print_help() - print(f"\nError: {e}\n") - sys.exit(1) - - return args + return parser.parse_args(argv) diff --git a/fido/fido.py b/fido/fido.py index cb2c1ef1..dcf333c2 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -13,23 +13,20 @@ import sys import tarfile import tempfile -from argparse import ArgumentParser, RawTextHelpFormatter -from contextlib import closing - -try: - from time import perf_counter -except ImportError: - from time import clock as perf_counter - import zipfile +from contextlib import closing +from time import perf_counter +from typing import Optional from xml.etree import cElementTree as ET from fido import CONFIG_DIR, __version__ from fido.char_handler import escape +from fido.cli_args import parse_cli_args from fido.package import OlePackage, ZipPackage from fido.versions import get_local_versions, sig_file_actions defaults = { + "config_dir": CONFIG_DIR, "bufsize": 128 * 1024, # (bytes) "regexcachesize": 2084, # (bytes) "printmatch": 'OK,%(info.time)s,%(info.puid)s,"%(info.formatname)s","%(info.signaturename)s",%(info.filesize)s,"%(info.filename)s","%(info.mimetype)s","%(info.matchtype)s"\n', @@ -74,8 +71,8 @@ class Fido: def __init__( self, - quiet=False, - bufsize=None, + quiet: bool = False, + bufsize: Optional[int] = None, container_bufsize=None, printnomatch=None, printmatch=None, @@ -793,107 +790,7 @@ def main(args=None): """Main FIDO method.""" if not args: args = sys.argv[1:] - - parser = ArgumentParser( - description=defaults["description"], - epilog=defaults["epilog"], - fromfile_prefix_chars="@", - formatter_class=RawTextHelpFormatter, - ) - parser.add_argument("-v", default=False, action="store_true", help="show version information") - parser.add_argument("-q", default=False, action="store_true", help="run (more) quietly") - parser.add_argument("-recurse", default=False, action="store_true", help="recurse into subdirectories") - parser.add_argument("-zip", default=False, action="store_true", help="recurse into zip and tar files") - parser.add_argument( - "-noextension", - default=False, - action="store_true", - help="disable extension matching, reduces number of matches but may reduce false positives", - ) - parser.add_argument( - "-nocontainer", - default=False, - action="store_true", - help="disable deep scan of container documents, increases speed but may reduce accuracy with big files", - ) - parser.add_argument( - "-pronom_only", - default=False, - action="store_true", - help="disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results", - ) - - group = parser.add_mutually_exclusive_group() - group.add_argument( - "-input", default=False, help="file containing a list of files to check, one per line. - means stdin" - ) - group.add_argument( - "files", - nargs="*", - default=[], - metavar="FILE", - help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.", - ) - - parser.add_argument("-filename", default=None, help="filename if file contents passed through STDIN") - parser.add_argument( - "-useformats", - metavar="INCLUDEPUIDS", - default=None, - help="comma separated string of formats to use in identification", - ) - parser.add_argument( - "-nouseformats", - metavar="EXCLUDEPUIDS", - default=None, - help="comma separated string of formats not to use in identification", - ) - parser.add_argument( - "-matchprintf", - metavar="FORMATSTRING", - default=None, - help="format string (Python style) to use on match. See nomatchprintf, README.txt.", - ) - parser.add_argument( - "-nomatchprintf", - metavar="FORMATSTRING", - default=None, - help="format string (Python style) to use if no match. See README.txt", - ) - parser.add_argument( - "-bufsize", - type=int, - default=None, - help="size (in bytes) of the buffer to match against (default=" + str(defaults["bufsize"]) + " bytes)", - ) - parser.add_argument( - "-sigs", - default=None, - metavar="SIG_ACT", - help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.', - ) - parser.add_argument( - "-container_bufsize", - type=int, - default=None, - help="size (in bytes) of the buffer to match against (default=" - + str(defaults["container_bufsize"]) - + " bytes)", - ) - parser.add_argument( - "-loadformats", default=None, metavar="XML1,...,XMLn", help="comma separated string of XML format files to add." - ) - parser.add_argument( - "-confdir", - default=CONFIG_DIR, - help="configuration directory to load_fido_xml, for example, the format specifications from.", - ) - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - args = parser.parse_args(args) - + args = parse_cli_args(args, defaults) timer = PerfTimer() versions = get_local_versions(args.confdir) @@ -904,15 +801,13 @@ def main(args=None): defaults["format_files"] = [defaults["xml_pronomSignature"]] if args.pronom_only: - versionHeader = "FIDO v{0} ({1}, {2})\n".format( - __version__, defaults["xml_pronomSignature"], defaults["containersignature_file"] + versionHeader = ( + f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']})\n" ) else: - versionHeader = "FIDO v{0} ({1}, {2}, {3})\n".format( - __version__, - defaults["xml_pronomSignature"], - defaults["containersignature_file"], - defaults["xml_fidoExtensionSignature"], + versionHeader = ( + f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']}, " + f"{defaults['xml_fidoExtensionSignature']})\n" ) defaults["format_files"].append(defaults["xml_fidoExtensionSignature"]) diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py index 4405a86f..a846bad0 100644 --- a/tests/test_cli_args.py +++ b/tests/test_cli_args.py @@ -1,61 +1,54 @@ -import argparse - import pytest -from fido.cli_args import build_parser, parse_args +from fido.cli_args import parse_cli_args +from fido.fido import defaults # Common argument string -ARG_STRING = ( - "-v -q -recurse -zip -noextension -nocontainer -pronom_only" - "-input files.txt" - "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" -) - -ARG_STRING = ( - "-v -q -recurse -zip -noextension -nocontainer -pronom_only" - "-input files.txt" - "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" -) - - -def test_build_parser(): - parser = build_parser() - assert isinstance(parser, argparse.ArgumentParser) - - # Check if all expected arguments are present - expected_args = ARG_STRING.split() - for arg in expected_args: - assert arg in parser._option_string_actions - - -def test_parse_args_valid(): - parser = build_parser() - - args = parse_args(parser.parse_args(ARG_STRING.split())) - - assert args.v is True - assert args.q is True - assert args.recurse is True - assert args.zip is True - assert args.noextension is True - assert args.nocontainer is True - assert args.pronom_only is True - assert args.input == "input_file" - assert args.files == ["file1", "file2"] - assert args.filename == "filename" + + +def test_parse_args_input_valid(): + arg_string = ( + "-v -q -recurse -zip -noextension -nocontainer -pronom_only " + "-input files.txt " + "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" + ) + args = parse_cli_args(arg_string.split(), defaults) + print(arg_string.split()) + print(args) + assert args.v + assert args.q + assert args.recurse + assert args.zip + assert args.noextension + assert args.nocontainer + assert args.pronom_only + assert args.input == "files.txt" assert args.useformats == "fmt1,fmt2" assert args.nouseformats == "fmt3,fmt4" -def test_parse_args_invalid(monkeypatch): - parser = build_parser() +def test_parse_args_files_valid(): + arg_string = "-q -zip file1.ext file2.ext" + args = parse_cli_args(arg_string.split(), defaults) + print(arg_string.split()) + print(args) + assert args.q + assert args.zip + assert args.noextension == False + assert args.nocontainer == False + assert args.pronom_only == False + assert args.files == ["file1.ext", "file2.ext"] + assert args.useformats is None + assert args.nouseformats is None - # Simulate invalid argument input - monkeypatch.setattr("sys.argv", ["prog", "--invalid"]) + +def test_parse_args_invalid(): + arg_string = "-q -zip -bad_arg file1.ext file2.ext" with pytest.raises(SystemExit): - parse_args(parser) + args = parse_cli_args(arg_string.split(), defaults) + - # Simulate missing required argument - monkeypatch.setattr("sys.argv", ["prog", "-input"]) +def test_parse_files_and_input_invalid(): + arg_string = "-q -zip -input files.txt file1.ext file2.ext" with pytest.raises(SystemExit): - parse_args(parser) + args = parse_cli_args(arg_string.split(), defaults) From eb1d7006580447cf6e42bc5285b0cd39cc099095 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Thu, 12 Sep 2024 17:46:23 +0100 Subject: [PATCH 14/33] Removed unused import. --- fido/cli_args.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fido/cli_args.py b/fido/cli_args.py index a6919853..ac28fa36 100644 --- a/fido/cli_args.py +++ b/fido/cli_args.py @@ -1,5 +1,4 @@ import argparse -import sys from argparse import ArgumentParser, RawTextHelpFormatter From 30d2d0dd0ea182511fcf7481f26ef95bb4933f8e Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Thu, 12 Sep 2024 17:51:18 +0100 Subject: [PATCH 15/33] Minor edits to make flake8 happier. --- fido/fido.py | 14 ++++++++++++-- tests/test_cli_args.py | 10 +++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index dcf333c2..e298b934 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -634,7 +634,12 @@ def walk_zip(self, filename, fileobj=None, extension=True): with zipstream.open(item) as source: self.copy_stream(source, target) # target.seek(0) - self.identify_contents(item_name, target, self.container_type(matches), extension=extension) + self.identify_contents( + item_name, + target, + self.container_type(matches), + extension=extension, + ) except IOError: sys.stderr.write("FIDO: ZipError {0}\n".format(filename)) except zipfile.BadZipfile: @@ -663,7 +668,12 @@ def walk_tar(self, filename, fileobj, extension=True): self.handle_matches(tar_item_name, matches, timer.duration()) if self.container_type(matches): f.seek(0) - self.identify_contents(tar_item_name, f, self.container_type(matches), extension=extension) + self.identify_contents( + tar_item_name, + f, + self.container_type(matches), + extension=extension, + ) except tarfile.TarError: sys.stderr.write("FIDO: Error: TarError {0}\n".format(filename)) diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py index a846bad0..0169a52b 100644 --- a/tests/test_cli_args.py +++ b/tests/test_cli_args.py @@ -34,9 +34,9 @@ def test_parse_args_files_valid(): print(args) assert args.q assert args.zip - assert args.noextension == False - assert args.nocontainer == False - assert args.pronom_only == False + assert args.noextension + assert args.nocontainer + assert args.pronom_only assert args.files == ["file1.ext", "file2.ext"] assert args.useformats is None assert args.nouseformats is None @@ -45,10 +45,10 @@ def test_parse_args_files_valid(): def test_parse_args_invalid(): arg_string = "-q -zip -bad_arg file1.ext file2.ext" with pytest.raises(SystemExit): - args = parse_cli_args(arg_string.split(), defaults) + parse_cli_args(arg_string.split(), defaults) def test_parse_files_and_input_invalid(): arg_string = "-q -zip -input files.txt file1.ext file2.ext" with pytest.raises(SystemExit): - args = parse_cli_args(arg_string.split(), defaults) + parse_cli_args(arg_string.split(), defaults) From bbbffbe408cf0cb48ab32ddc5cf5d88ae23b7fc9 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Thu, 12 Sep 2024 17:54:27 +0100 Subject: [PATCH 16/33] More minor edits to make flake8 happy. --- fido/cli_args.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fido/cli_args.py b/fido/cli_args.py index ac28fa36..fc74bfe9 100644 --- a/fido/cli_args.py +++ b/fido/cli_args.py @@ -1,8 +1,9 @@ import argparse from argparse import ArgumentParser, RawTextHelpFormatter +from typing import Any, Dict, List -def parse_cli_args(argv: list[str], defaults: dict) -> argparse.Namespace: +def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namespace: """ Parse command-line arguments. Args: From 529b4839b08644fe3a58b21d322c61da40679a82 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Thu, 12 Sep 2024 17:58:38 +0100 Subject: [PATCH 17/33] Minor edits to make pytest happy. --- tests/test_cli_args.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py index 0169a52b..3be8ae17 100644 --- a/tests/test_cli_args.py +++ b/tests/test_cli_args.py @@ -13,8 +13,6 @@ def test_parse_args_input_valid(): "-useformats=fmt1,fmt2 -nouseformats=fmt3,fmt4" ) args = parse_cli_args(arg_string.split(), defaults) - print(arg_string.split()) - print(args) assert args.v assert args.q assert args.recurse @@ -30,13 +28,11 @@ def test_parse_args_input_valid(): def test_parse_args_files_valid(): arg_string = "-q -zip file1.ext file2.ext" args = parse_cli_args(arg_string.split(), defaults) - print(arg_string.split()) - print(args) assert args.q assert args.zip - assert args.noextension - assert args.nocontainer - assert args.pronom_only + assert not args.noextension + assert not args.nocontainer + assert not args.pronom_only assert args.files == ["file1.ext", "file2.ext"] assert args.useformats is None assert args.nouseformats is None From b6489e2d77a715b0a14d8fe60ef4d537c6596594 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 12 Nov 2024 13:07:08 +0000 Subject: [PATCH 18/33] Moved flake8 from regular to optional dependencies. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9aa09967..1332e9bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ classifiers = [ dependencies = [ "olefile >= 0.46, < 1", "requests", - "flake8", ] [project.urls] @@ -41,6 +40,7 @@ homepage = "http://openpreservation.org/technology/products/fido/" testing = [ "pytest", "pytest-cov", + "flake8", ] [project.scripts] From f867afa2806632d6e961b4be7f5150330403c3fe Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 12 Nov 2024 13:18:14 +0000 Subject: [PATCH 19/33] Applied Black to all files. --- fido/char_handler.py | 26 ++--- fido/cli_args.py | 35 +++++-- fido/fido.py | 161 +++++++++++++++++++++++------- fido/prepare.py | 203 +++++++++++++++++++++++++++++--------- fido/pronom/soap.py | 22 ++++- fido/update_signatures.py | 40 ++++++-- fido/versions.py | 46 +++++++-- pyproject.toml | 7 +- tests/pronom/test_soap.py | 4 +- tests/test_package.py | 8 +- tests/test_prepare.py | 53 +++++----- 11 files changed, 447 insertions(+), 158 deletions(-) diff --git a/fido/char_handler.py b/fido/char_handler.py index bfa41cee..69d5c0c4 100644 --- a/fido/char_handler.py +++ b/fido/char_handler.py @@ -5,24 +5,28 @@ # \a\b\n\r\t\v # MdR: took out '<' and '>' out of _ordinary because they were converted to entities <> -# MdR: moved '!' from _ordinary to _special because it means "NOT" in the regex world. At this time no regex in any sig has a negate set, did this to be on the safe side -ORDINARY = frozenset(' "#%&\',-/0123456789:;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~') -SPECIAL = '$()*+.?![]^\\{|}' # Before: '$*+.?![]^\\{|}' -HEX = '0123456789abcdef' +# MdR: moved '!' from _ordinary to _special because it means "NOT" in the regex +# world. At this time no regex in any sig has a negate set, did this to be on +# the safe side +ORDINARY = frozenset( + " \"#%&',-/0123456789:;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~" +) +SPECIAL = "$()*+.?![]^\\{|}" # Before: '$*+.?![]^\\{|}' +HEX = "0123456789abcdef" def escape_char(c): """Add appropriate escape sequence to passed character c.""" - if c in '\n': - return '\\n' - if c == '\r': - return '\\r' + if c in "\n": + return "\\n" + if c == "\r": + return "\\r" if c in SPECIAL: - return '\\' + c + return "\\" + c (high, low) = divmod(ord(c), 16) - return '\\x' + HEX[high] + HEX[low] + return "\\x" + HEX[high] + HEX[low] def escape(string): """Escape characters in pattern that are non-printable, non-ascii, or special for regexes.""" - return ''.join(c if c in ORDINARY else escape_char(c) for c in string) + return "".join(c if c in ORDINARY else escape_char(c) for c in string) diff --git a/fido/cli_args.py b/fido/cli_args.py index fc74bfe9..2cbcd147 100644 --- a/fido/cli_args.py +++ b/fido/cli_args.py @@ -19,10 +19,24 @@ def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namesp fromfile_prefix_chars="@", formatter_class=RawTextHelpFormatter, ) - parser.add_argument("-v", default=False, action="store_true", help="show version information") - parser.add_argument("-q", default=False, action="store_true", help="run (more) quietly") - parser.add_argument("-recurse", default=False, action="store_true", help="recurse into subdirectories") - parser.add_argument("-zip", default=False, action="store_true", help="recurse into zip and tar files") + parser.add_argument( + "-v", default=False, action="store_true", help="show version information" + ) + parser.add_argument( + "-q", default=False, action="store_true", help="run (more) quietly" + ) + parser.add_argument( + "-recurse", + default=False, + action="store_true", + help="recurse into subdirectories", + ) + parser.add_argument( + "-zip", + default=False, + action="store_true", + help="recurse into zip and tar files", + ) parser.add_argument( "-noextension", default=False, @@ -44,7 +58,9 @@ def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namesp group = parser.add_mutually_exclusive_group() group.add_argument( - "-input", default=False, help="file containing a list of files to check, one per line. - means stdin" + "-input", + default=False, + help="file containing a list of files to check, one per line. - means stdin", ) group.add_argument( "files", @@ -54,7 +70,9 @@ def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namesp help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.", ) - parser.add_argument("-filename", default=None, help="filename if file contents passed through STDIN") + parser.add_argument( + "-filename", default=None, help="filename if file contents passed through STDIN" + ) parser.add_argument( "-useformats", metavar="INCLUDEPUIDS", @@ -98,7 +116,10 @@ def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namesp help=f"size (in bytes) of the buffer to match against (default={defaults['container_bufsize']}).", ) parser.add_argument( - "-loadformats", default=None, metavar="XML1,...,XMLn", help="comma separated string of XML format files to add." + "-loadformats", + default=None, + metavar="XML1,...,XMLn", + help="comma separated string of XML format files to add.", ) parser.add_argument( "-confdir", diff --git a/fido/fido.py b/fido/fido.py index e298b934..781e96f5 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -87,14 +87,24 @@ def __init__( global defaults self.quiet = quiet self.bufsize = defaults["bufsize"] if bufsize is None else bufsize - self.container_bufsize = defaults["container_bufsize"] if container_bufsize is None else container_bufsize + self.container_bufsize = ( + defaults["container_bufsize"] + if container_bufsize is None + else container_bufsize + ) self.printmatch = defaults["printmatch"] if printmatch is None else printmatch - self.printnomatch = defaults["printnomatch"] if printnomatch is None else printnomatch - self.handle_matches = self.print_matches if handle_matches is None else handle_matches + self.printnomatch = ( + defaults["printnomatch"] if printnomatch is None else printnomatch + ) + self.handle_matches = ( + self.print_matches if handle_matches is None else handle_matches + ) self.zip = zip self.nocontainer = nocontainer self.conf_dir = conf_dir - self.format_files = defaults["format_files"] if format_files is None else format_files + self.format_files = ( + defaults["format_files"] if format_files is None else format_files + ) self.containersignature_file = defaults["containersignature_file"] self.formats = [] self.puid_format_map = {} @@ -152,11 +162,24 @@ def convert_container_sequence(self, sig): if sig[i] != "-" and sig[i] != "'" and ror: seq += escape(sig[i]).encode("utf8") continue - if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and sig[i] != ":" and not ror and not byt: + if ( + sig[i] != "-" + and sig[i] != "'" + and sig[i] != " " + and sig[i] != ":" + and not ror + and not byt + ): seq += b"\\x" + sig[i].lower().encode("utf8") byt = True continue - if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and not ror and byt: + if ( + sig[i] != "-" + and sig[i] != "'" + and sig[i] != " " + and not ror + and byt + ): seq += sig[i].lower().encode("utf8") byt = False continue @@ -186,7 +209,9 @@ def extract_signatures(self, doc, signature_type="ZIP"): format_mappings = root.find("FileFormatMappings") def get_puid(doc, element_id): - return format_mappings.find('FileFormatMapping[@signatureId="{}"]'.format(element_id)).attrib["Puid"] + return format_mappings.find( + 'FileFormatMapping[@signatureId="{}"]'.format(element_id) + ).attrib["Puid"] def format_signature_attributes(element): return { @@ -199,7 +224,11 @@ def format_signature_attributes(element): ), } - elements = root.findall('ContainerSignatures/ContainerSignature[@ContainerType="{}"]'.format(signature_type)) + elements = root.findall( + 'ContainerSignatures/ContainerSignature[@ContainerType="{}"]'.format( + signature_type + ) + ) signatures = {} for el in elements: if el.find("Files/File/BinarySignatures") is None: @@ -217,7 +246,9 @@ def format_signature_attributes(element): def match_container(self, signature_type, klass, file, signature_file): """Return the signature matches for a container.""" - puids = klass(file, self.extract_signatures(signature_file, signature_type=signature_type)).detect_formats() + puids = klass( + file, self.extract_signatures(signature_file, signature_type=signature_type) + ).detect_formats() results = [] for puid in puids: format = self.puid_format_map[puid] @@ -237,7 +268,11 @@ def load_fido_xml(self, file): for element in tree.getroot().findall("./format"): self.process_format_element(element) except ET.ParseError as parse_excep: - sys.stderr.write("Failed to parse signature file {}, exception: {}\n".format(file, parse_excep)) + sys.stderr.write( + "Failed to parse signature file {}, exception: {}\n".format( + file, parse_excep + ) + ) sys.exit(1) return self.formats @@ -265,7 +300,10 @@ def get_signatures(self, format): def has_priority_over(self, format, possibly_inferior): """Return true if format has priority over possibly inferior.""" - return self.get_puid(possibly_inferior) in self.puid_has_priority_over_map[self.get_puid(format)] + return ( + self.get_puid(possibly_inferior) + in self.puid_has_priority_over_map[self.get_puid(format)] + ) def get_puid(self, format): """Return the PUID for the format.""" @@ -361,7 +399,10 @@ def print_summary(self, secs): if not self.quiet: rate = int(round(count / secs)) if secs != 0 else 9999 # print >> sys.stderr, 'FIDO: Processed %6d files in %6.2f msec, %2d files/sec' % (count, secs * 1000, rate) - sys.stderr.write("FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n" % (count, secs * 1000, rate)) + sys.stderr.write( + "FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n" + % (count, secs * 1000, rate) + ) def identify_file(self, filename, extension=True): """ @@ -377,18 +418,30 @@ def identify_file(self, filename, extension=True): size = os.stat(filename)[6] self.current_filesize = size if self.current_filesize == 0: - sys.stderr.write("FIDO: Zero byte file (empty): Path is: " + filename + "\n") + sys.stderr.write( + "FIDO: Zero byte file (empty): Path is: " + filename + "\n" + ) bofbuffer, eofbuffer, _ = self.get_buffers(f, size, seekable=True) matches = self.match_formats(bofbuffer, eofbuffer) container_type = self.container_type(matches) if not self.nocontainer and container_type in ("zip", "ole"): - container_file = ET.parse(os.path.join(os.path.abspath(self.conf_dir), self.containersignature_file)) + container_file = ET.parse( + os.path.join( + os.path.abspath(self.conf_dir), self.containersignature_file + ) + ) if container_type == "zip": - container_matches = self.match_container("ZIP", ZipPackage, filename, container_file) + container_matches = self.match_container( + "ZIP", ZipPackage, filename, container_file + ) else: - container_matches = self.match_container("OLE2", OlePackage, filename, container_file) + container_matches = self.match_container( + "OLE2", OlePackage, filename, container_file + ) if len(container_matches) > 0: - self.handle_matches(filename, container_matches, timer.duration(), "container") + self.handle_matches( + filename, container_matches, timer.duration(), "container" + ) return # from here is also repeated in walk_zip # we should make this uniform in a next version! @@ -408,7 +461,11 @@ def identify_file(self, filename, extension=True): self.identify_contents(filename, type=container, extension=extension) except IOError as io_excep: # print >> sys.stderr, "FIDO: Error in identify_file: Path is {0}".format(filename) - sys.stderr.write("FIDO: Error in identify_file: {}, exception: {}\n".format(filename, io_excep)) + sys.stderr.write( + "FIDO: Error in identify_file: {}, exception: {}\n".format( + filename, io_excep + ) + ) def identify_contents(self, filename, fileobj=None, type=False, extension=True): """ @@ -467,10 +524,14 @@ def identify_multi_object_stream(self, stream, extension=True): matches = self.match_formats(bofbuffer, eofbuffer) # MdR: this needs attention if len(matches) > 0: - self.handle_matches(self.current_file, matches, timer.duration(), "signature") + self.handle_matches( + self.current_file, matches, timer.duration(), "signature" + ) elif extension and (len(matches) == 0 or self.current_filesize == 0): matches = self.match_extensions(self.current_file) - self.handle_matches(self.current_file, matches, timer.duration(), "extension") + self.handle_matches( + self.current_file, matches, timer.duration(), "extension" + ) def identify_stream(self, stream, filename, extension=True): """ @@ -486,7 +547,9 @@ def identify_stream(self, stream, filename, extension=True): matches = self.match_formats(bofbuffer, eofbuffer) # MdR: this needs attention if len(matches) > 0: - self.handle_matches(self.current_file, matches, timer.duration(), "signature") + self.handle_matches( + self.current_file, matches, timer.duration(), "signature" + ) elif extension and (len(matches) == 0 or self.current_filesize == 0): # we can only determine the filename from the STDIN stream # on Linux, on Windows there is not a (simple) way to do that @@ -505,7 +568,9 @@ def identify_stream(self, stream, filename, extension=True): # we have to reset self.current_file if not on Windows if os.name != "nt": self.current_file = "STDIN" - self.handle_matches(self.current_file, matches, timer.duration(), "extension") + self.handle_matches( + self.current_file, matches, timer.duration(), "extension" + ) def container_type(self, matches): """ @@ -575,7 +640,11 @@ def get_buffers(self, stream, length=None, seekable=False): if len(buffer) == self.bufsize: prevbuffer = buffer else: - eofbuffer = prevbuffer if len(buffer) == 0 else prevbuffer[-(self.bufsize - len(buffer)) :] + buffer + eofbuffer = ( + prevbuffer + if len(buffer) == 0 + else prevbuffer[-(self.bufsize - len(buffer)) :] + buffer + ) break return bofbuffer, eofbuffer, bytes_read bytes_unread = length - len(bofbuffer) @@ -583,7 +652,9 @@ def get_buffers(self, stream, length=None, seekable=False): eofbuffer = bofbuffer elif bytes_unread < self.bufsize: # The buffs overlap - eofbuffer = bofbuffer[bytes_unread:] + self.blocking_read(stream, bytes_unread) + eofbuffer = bofbuffer[bytes_unread:] + self.blocking_read( + stream, bytes_unread + ) elif bytes_unread == self.bufsize: eofbuffer = self.blocking_read(stream, self.bufsize) elif seekable: # easy case when we can just seek! @@ -621,14 +692,24 @@ def walk_zip(self, filename, fileobj=None, extension=True): self.current_file = item_name self.current_filesize = item.file_size if self.current_filesize == 0: - sys.stderr.write("FIDO: Zero byte file (empty): Path is: " + item_name + "\n") + sys.stderr.write( + "FIDO: Zero byte file (empty): Path is: " + + item_name + + "\n" + ) bofbuffer, eofbuffer, _ = self.get_buffers(f, item.file_size) matches = self.match_formats(bofbuffer, eofbuffer) if len(matches) > 0 and self.current_filesize > 0: - self.handle_matches(item_name, matches, timer.duration(), "signature") - elif extension and (len(matches) == 0 or self.current_filesize == 0): + self.handle_matches( + item_name, matches, timer.duration(), "signature" + ) + elif extension and ( + len(matches) == 0 or self.current_filesize == 0 + ): matches = self.match_extensions(item_name) - self.handle_matches(item_name, matches, timer.duration(), "extension") + self.handle_matches( + item_name, matches, timer.duration(), "extension" + ) if self.container_type(matches): target = tempfile.SpooledTemporaryFile(prefix="Fido") with zipstream.open(item) as source: @@ -811,9 +892,7 @@ def main(args=None): defaults["format_files"] = [defaults["xml_pronomSignature"]] if args.pronom_only: - versionHeader = ( - f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']})\n" - ) + versionHeader = f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']})\n" else: versionHeader = ( f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']}, " @@ -861,10 +940,14 @@ def main(args=None): # TODO: remove from maps if args.useformats: args.useformats = args.useformats.split(",") - fido.formats = [f for f in fido.formats if f.find("puid").text in args.useformats] + fido.formats = [ + f for f in fido.formats if f.find("puid").text in args.useformats + ] elif args.nouseformats: args.nouseformats = args.nouseformats.split(",") - fido.formats = [f for f in fido.formats if f.find("puid").text not in args.nouseformats] + fido.formats = [ + f for f in fido.formats if f.find("puid").text not in args.nouseformats + ] # Set up to use stdin, or open input files: if args.input == "-": @@ -879,10 +962,16 @@ def main(args=None): sys.stderr.flush() if (not args.input) and len(args.files) == 1 and args.files[0] == "-": if fido.zip: - raise RuntimeError("Multiple content read from stdin not yet supported.") - fido.identify_multi_object_stream(sys.stdin, extension=not args.noextension) + raise RuntimeError( + "Multiple content read from stdin not yet supported." + ) + fido.identify_multi_object_stream( + sys.stdin, extension=not args.noextension + ) else: - fido.identify_stream(sys.stdin, args.filename, extension=not args.noextension) + fido.identify_stream( + sys.stdin, args.filename, extension=not args.noextension + ) else: for file in list_files(args.files, args.recurse): fido.identify_file(file, extension=not args.noextension) diff --git a/fido/prepare.py b/fido/prepare.py index 32a81d1a..7580bef9 100644 --- a/fido/prepare.py +++ b/fido/prepare.py @@ -140,7 +140,12 @@ def load_pronom_xml(self, puid_filter=None): try: zip.close() except Exception as e: - print("An error occured loading '{0}' (exception: {1})".format(self.pronom_files, e), file=sys.stderr) + print( + "An error occured loading '{0}' (exception: {1})".format( + self.pronom_files, e + ), + file=sys.stderr, + ) sys.exit() # Replace the formatID with puids in has_priority_over if puid_filter is None: @@ -195,28 +200,44 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(fido_format, "container").text = "zip" elif puid == "x-fmt/265": ET.SubElement(fido_format, "container").text = "tar" - ET.SubElement(fido_format, "name").text = get_text_tna(pronom_format, "FormatName") - ET.SubElement(fido_format, "version").text = get_text_tna(pronom_format, "FormatVersion") - ET.SubElement(fido_format, "alias").text = get_text_tna(pronom_format, "FormatAliases") - ET.SubElement(fido_format, "pronom_id").text = get_text_tna(pronom_format, "FormatID") + ET.SubElement(fido_format, "name").text = get_text_tna( + pronom_format, "FormatName" + ) + ET.SubElement(fido_format, "version").text = get_text_tna( + pronom_format, "FormatVersion" + ) + ET.SubElement(fido_format, "alias").text = get_text_tna( + pronom_format, "FormatAliases" + ) + ET.SubElement(fido_format, "pronom_id").text = get_text_tna( + pronom_format, "FormatID" + ) # Get the extensions from the ExternalSignature for x in pronom_format.findall(TNA("ExternalSignature")): ET.SubElement(fido_format, "extension").text = get_text_tna(x, "Signature") for id in pronom_format.findall(TNA("FileFormatIdentifier")): type = get_text_tna(id, "IdentifierType") if type == "Apple Uniform Type Identifier": - ET.SubElement(fido_format, "apple_uti").text = get_text_tna(id, "Identifier") + ET.SubElement(fido_format, "apple_uti").text = get_text_tna( + id, "Identifier" + ) # Handle the relationships for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Has priority over": - ET.SubElement(fido_format, "has_priority_over").text = get_text_tna(x, "RelatedFormatID") + ET.SubElement(fido_format, "has_priority_over").text = get_text_tna( + x, "RelatedFormatID" + ) # Get the InternalSignature information for pronom_sig in pronom_format.findall(TNA("InternalSignature")): fido_sig = ET.SubElement(fido_format, "signature") - ET.SubElement(fido_sig, "name").text = get_text_tna(pronom_sig, "SignatureName") + ET.SubElement(fido_sig, "name").text = get_text_tna( + pronom_sig, "SignatureName" + ) # There are some funny chars in the notes, which caused me trouble and it is a unicode string, - ET.SubElement(fido_sig, "note").text = get_text_tna(pronom_sig, "SignatureNote") + ET.SubElement(fido_sig, "note").text = get_text_tna( + pronom_sig, "SignatureNote" + ) for pronom_pat in pronom_sig.findall(TNA("ByteSequence")): # print('Parsing ID:{}'.format(puid)) fido_pat = ET.SubElement(fido_sig, "pattern") @@ -228,15 +249,24 @@ def parse_pronom_xml(self, source, puid_filter=None): pass # print "working on puid:", puid, ", position: ", pos, "with offset, maxoffset: ", offset, ",", max_offset try: - regex = convert_to_regex(byte_seq, "Little", pos, offset, max_offset) + regex = convert_to_regex( + byte_seq, "Little", pos, offset, max_offset + ) except ValueError as ve: - print("ValueError converting PUID {} signature to regex: {}".format(puid, ve), file=sys.stderr) + print( + "ValueError converting PUID {} signature to regex: {}".format( + puid, ve + ), + file=sys.stderr, + ) regex = FLG_INCOMPATIBLE # print "done puid", puid if regex == FLG_INCOMPATIBLE: print( - "Error: incompatible PRONOM signature found for puid {} skipping...".format(puid), + "Error: incompatible PRONOM signature found for puid {} skipping...".format( + puid + ), file=sys.stderr, ) # remove the empty 'signature' nodes @@ -250,47 +280,73 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(fido_pat, "regex").text = regex # Get the format details fido_details = ET.SubElement(fido_format, "details") - ET.SubElement(fido_details, "dc:description").text = get_text_tna(pronom_format, "FormatDescription") - ET.SubElement(fido_details, "dcterms:available").text = get_text_tna(pronom_format, "ReleaseDate") - ET.SubElement(fido_details, "dc:creator").text = get_text_tna(pronom_format, "Developers/DeveloperCompoundName") + ET.SubElement(fido_details, "dc:description").text = get_text_tna( + pronom_format, "FormatDescription" + ) + ET.SubElement(fido_details, "dcterms:available").text = get_text_tna( + pronom_format, "ReleaseDate" + ) + ET.SubElement(fido_details, "dc:creator").text = get_text_tna( + pronom_format, "Developers/DeveloperCompoundName" + ) ET.SubElement(fido_details, "dcterms:publisher").text = get_text_tna( pronom_format, "Developers/OrganisationName" ) for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Is supertype of": - ET.SubElement(fido_details, "is_supertype_of").text = get_text_tna(x, "RelatedFormatID") + ET.SubElement(fido_details, "is_supertype_of").text = get_text_tna( + x, "RelatedFormatID" + ) for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Is subtype of": - ET.SubElement(fido_details, "is_subtype_of").text = get_text_tna(x, "RelatedFormatID") - ET.SubElement(fido_details, "content_type").text = get_text_tna(pronom_format, "FormatTypes") + ET.SubElement(fido_details, "is_subtype_of").text = get_text_tna( + x, "RelatedFormatID" + ) + ET.SubElement(fido_details, "content_type").text = get_text_tna( + pronom_format, "FormatTypes" + ) # References for x in pronom_format.findall(TNA("Document")): r = ET.SubElement(fido_details, "reference") ET.SubElement(r, "dc:title").text = get_text_tna(x, "TitleText") - ET.SubElement(r, "dc:creator").text = get_text_tna(x, "Author/AuthorCompoundName") - ET.SubElement(r, "dc:publisher").text = get_text_tna(x, "Publisher/PublisherCompoundName") - ET.SubElement(r, "dcterms:available").text = get_text_tna(x, "PublicationDate") + ET.SubElement(r, "dc:creator").text = get_text_tna( + x, "Author/AuthorCompoundName" + ) + ET.SubElement(r, "dc:publisher").text = get_text_tna( + x, "Publisher/PublisherCompoundName" + ) + ET.SubElement(r, "dcterms:available").text = get_text_tna( + x, "PublicationDate" + ) for id in x.findall(TNA("DocumentIdentifier")): type = get_text_tna(id, "IdentifierType") if type == "URL": - ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna(id, "Identifier") + ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna( + id, "Identifier" + ) else: ET.SubElement(r, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + get_text_tna(id, "IdentifierType") + + ":" + + get_text_tna(id, "Identifier") ) ET.SubElement(r, "dc:description").text = get_text_tna(x, "DocumentNote") ET.SubElement(r, "dc:type").text = get_text_tna(x, "DocumentType") ET.SubElement(r, "dcterms:license").text = ( - get_text_tna(x, "AvailabilityDescription") + " " + get_text_tna(x, "AvailabilityNote") + get_text_tna(x, "AvailabilityDescription") + + " " + + get_text_tna(x, "AvailabilityNote") ) ET.SubElement(r, "dc:rights").text = get_text_tna(x, "DocumentIPR") # Examples for x in pronom_format.findall(TNA("ReferenceFile")): rf = ET.SubElement(fido_details, "example_file") ET.SubElement(rf, "dc:title").text = get_text_tna(x, "ReferenceFileName") - ET.SubElement(rf, "dc:description").text = get_text_tna(x, "ReferenceFileDescription") + ET.SubElement(rf, "dc:description").text = get_text_tna( + x, "ReferenceFileDescription" + ) checksum = "" for id in x.findall(TNA("ReferenceFileIdentifier")): type = get_text_tna(id, "IdentifierType") @@ -308,14 +364,20 @@ def parse_pronom_xml(self, source, puid_filter=None): m.update(sock.read()) sock.close() except HTTPError as http_excep: - sys.stderr.write("HTTP {} error loading resource {}\n".format(http_excep.code, url)) + sys.stderr.write( + "HTTP {} error loading resource {}\n".format( + http_excep.code, url + ) + ) if http_excep.code == 404: continue checksum = m.hexdigest() else: ET.SubElement(rf, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + get_text_tna(id, "IdentifierType") + + ":" + + get_text_tna(id, "Identifier") ) ET.SubElement(rf, "dcterms:license").text = "" ET.SubElement(rf, "dc:rights").text = get_text_tna(x, "ReferenceFileIPR") @@ -325,10 +387,18 @@ def parse_pronom_xml(self, source, puid_filter=None): # Record Metadata md = ET.SubElement(fido_details, "record_metadata") ET.SubElement(md, "status").text = "unknown" - ET.SubElement(md, "dc:creator").text = get_text_tna(pronom_format, "ProvenanceName") - ET.SubElement(md, "dcterms:created").text = get_text_tna(pronom_format, "ProvenanceSourceDate") - ET.SubElement(md, "dcterms:modified").text = get_text_tna(pronom_format, "LastUpdatedDate") - ET.SubElement(md, "dc:description").text = get_text_tna(pronom_format, "ProvenanceDescription") + ET.SubElement(md, "dc:creator").text = get_text_tna( + pronom_format, "ProvenanceName" + ) + ET.SubElement(md, "dcterms:created").text = get_text_tna( + pronom_format, "ProvenanceSourceDate" + ) + ET.SubElement(md, "dcterms:modified").text = get_text_tna( + pronom_format, "LastUpdatedDate" + ) + ET.SubElement(md, "dc:description").text = get_text_tna( + pronom_format, "ProvenanceDescription" + ) return fido_format # FIXME: I don't think that this quite works yet! @@ -415,7 +485,9 @@ def do_byte(chars, i, littleendian, esc=True): c2 = "0123456789ABCDEF".find(chars[i + 1].upper()) buf = StringIO() if c1 < 0 or c2 < 0: - raise Exception(_convert_err_msg("bad byte sequence", chars[i : i + 2], i, chars, buf)) + raise Exception( + _convert_err_msg("bad byte sequence", chars[i : i + 2], i, chars, buf) + ) if littleendian: val = chr(16 * c1 + c2) else: @@ -481,12 +553,16 @@ def calculate_repetition(char, pos, offset, maxoffset): def do_all_bitmasks(chars, i, littleendian): """(byte & bitmask) == bitmask.""" - return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), littleendian) + return do_any_all_bitmasks( + chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), littleendian + ) def do_any_bitmasks(chars, i, littleendian): """(byte & bitmask) != 0.""" - return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), littleendian) + return do_any_all_bitmasks( + chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), littleendian + ) def do_any_all_bitmasks(chars, i, predicate, littleendian): @@ -505,7 +581,13 @@ def do_any_all_bitmasks(chars, i, predicate, littleendian): byt, inc = do_byte(chars, i + 1, littleendian, esc=False) bitmask = ord(byt) regex = "({})".format( - "|".join(["\\x" + hex(byte)[2:].zfill(2) for byte in range(0x100) if predicate(byte, bitmask)]) + "|".join( + [ + "\\x" + hex(byte)[2:].zfill(2) + for byte in range(0x100) + if predicate(byte, bitmask) + ] + ) ) return regex, inc + 1 @@ -563,7 +645,11 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] in "*+?": state = "specials" else: - raise ValueError(_convert_err_msg("Illegal character in start", chars[i], i, chars, buf)) + raise ValueError( + _convert_err_msg( + "Illegal character in start", chars[i], i, chars, buf + ) + ) elif state == "bytes": (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) @@ -598,7 +684,11 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] == "]": break else: - raise Exception(_convert_err_msg("Illegal character in non-match", chars[i], i, chars, buf)) + raise Exception( + _convert_err_msg( + "Illegal character in non-match", chars[i], i, chars, buf + ) + ) buf.write(")") i += 1 state = "start" @@ -624,7 +714,11 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): buf.write("]") i += 1 except Exception: - print(_convert_err_msg("Illegal character in bracket", chars[i], i, chars, buf)) + print( + _convert_err_msg( + "Illegal character in bracket", chars[i], i, chars, buf + ) + ) raise if i < len(chars) and chars[i] == "{": state = "curly-after-bracket" @@ -667,7 +761,9 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): else: raise Exception( _convert_err_msg( - ("Current state = '{0}' : Illegal character in paren").format(state), + ( + "Current state = '{0}' : Illegal character in paren" + ).format(state), chars[i], i, chars, @@ -700,7 +796,11 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] == "}": break else: - raise Exception(_convert_err_msg("Illegal character in curly", chars[i], i, chars, buf)) + raise Exception( + _convert_err_msg( + "Illegal character in curly", chars[i], i, chars, buf + ) + ) buf.write("}") i += 1 # skip the ) state = "start" @@ -713,7 +813,11 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): i += 1 elif chars[i] == "?": if chars[i + 1] != "?": - raise Exception(_convert_err_msg("Illegal character after ?", chars[i + 1], i + 1, chars, buf)) + raise Exception( + _convert_err_msg( + "Illegal character after ?", chars[i + 1], i + 1, chars, buf + ) + ) buf.write(".?") i += 2 state = "start" @@ -741,7 +845,10 @@ def run(input=None, output=None, puid=None): info = FormatInfo(input) info.load_pronom_xml(puid) info.save(output) - print("Converted {0} PRONOM formats to FIDO signatures".format(len(info.formats)), file=sys.stderr) + print( + "Converted {0} PRONOM formats to FIDO signatures".format(len(info.formats)), + file=sys.stderr, + ) def main(args=None): @@ -749,10 +856,16 @@ def main(args=None): if args is None: args = sys.argv[1:] - parser = ArgumentParser(description="Produce the FIDO format XML that is loaded at run-time") - parser.add_argument("-input", default=None, help="Input file, a Zip containing PRONOM XML files") + parser = ArgumentParser( + description="Produce the FIDO format XML that is loaded at run-time" + ) + parser.add_argument( + "-input", default=None, help="Input file, a Zip containing PRONOM XML files" + ) parser.add_argument("-output", default=None, help="Output file") - parser.add_argument("-puid", default=None, help="A particular PUID record to extract") + parser.add_argument( + "-puid", default=None, help="A particular PUID record to extract" + ) args = parser.parse_args(args) run(input=args.input, output=args.output, puid=args.puid) diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index c8536123..67d2a734 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -50,7 +50,9 @@ def get_sig_xml_for_puid(puid): """Return the full PRONOM signature XML for the passed PUID.""" - req = urllib.request.Request("http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid)) + req = urllib.request.Request( + "http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid) + ) response = urllib.request.urlopen(req) xml = response.read() return xml @@ -80,12 +82,16 @@ def get_droid_signatures(version): format_count = False try: with urllib.request.urlopen( - "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format(version) + "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format( + version + ) ) as f: xml = f.read().decode("utf-8") root_ele = ET.fromstring(xml) format_count = len( - root_ele.findall(".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat") + root_ele.findall( + ".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat" + ) ) except HTTPError as httpe: sys.stderr.write( @@ -111,9 +117,15 @@ def _get_soap_ele_tree(soap_action): def _get_soap_response(soap_action, soap_string): try: - req = urllib.request.Request("http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string) + req = urllib.request.Request( + "http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string + ) except URLError: - print("There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format(PRONOM_HOST)) + print( + "There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format( + PRONOM_HOST + ) + ) print("Please check your network connection and try again.") sys.exit(1) for key, value in HEADERS.items(): diff --git a/fido/update_signatures.py b/fido/update_signatures.py index 43a2d681..919dfadf 100644 --- a/fido/update_signatures.py +++ b/fido/update_signatures.py @@ -23,7 +23,12 @@ from . import CONFIG_DIR, __version__ from .prepare import run as prepare_pronom_to_fido -from .pronom.soap import NS, get_droid_signatures, get_pronom_sig_version, get_sig_xml_for_puid +from .pronom.soap import ( + NS, + get_droid_signatures, + get_pronom_sig_version, + get_sig_xml_for_puid, +) from .versions import get_local_versions ABORT_MSG = "Aborting update..." @@ -112,7 +117,9 @@ def sig_version_check(version="latest"): print("Getting latest version number from PRONOM...") version = get_pronom_sig_version() if not version: - sys.exit("Failed to obtain PRONOM signature file version number, please try again.") + sys.exit( + "Failed to obtain PRONOM signature file version number, please try again." + ) print("Querying PRONOM for signaturefile version {}.".format(version)) sig_file_name = _sig_file_name(version) @@ -152,7 +159,9 @@ def init_sig_download(defaults): resume = False if os.path.isdir(tmpdir): print("Found previously created temporary folder for download:", tmpdir) - resume = query_yes_no("Do you want to resume download (yes) or start over (no)?") + resume = query_yes_no( + "Do you want to resume download (yes) or start over (no)?" + ) if resume: print("Resuming download...") else: @@ -162,7 +171,9 @@ def init_sig_download(defaults): except OSError: pass if not os.path.isdir(tmpdir): - sys.stderr.write("Failed to create temporary folder for PUID's, using: " + tmpdir) + sys.stderr.write( + "Failed to create temporary folder for PUID's, using: " + tmpdir + ) return tmpdir, resume @@ -176,7 +187,10 @@ def download_signatures(defaults, format_eles, resume, tmpdir): download_sig(format_ele, tmpdir, resume, defaults) numfiles += 1 print( - r"Downloaded {}/{} files [{}%]".format(numfiles, puid_count, int(float(numfiles) / one_percent)), end="\r" + r"Downloaded {}/{} files [{}%]".format( + numfiles, puid_count, int(float(numfiles) / one_percent) + ), + end="\r", ) print("100%") @@ -208,7 +222,10 @@ def create_zip_file(defaults, format_eles, version, tmpdir): print("Creating PRONOM zip...") compression = zipfile.ZIP_DEFLATED if "zlib" in sys.modules else zipfile.ZIP_STORED modes = {zipfile.ZIP_DEFLATED: "deflated", zipfile.ZIP_STORED: "stored"} - zf = zipfile.ZipFile(os.path.join(CONFIG_DIR, DEFAULTS["pronomZipFileName"].format(version)), mode="w") + zf = zipfile.ZipFile( + os.path.join(CONFIG_DIR, DEFAULTS["pronomZipFileName"].format(version)), + mode="w", + ) print("Adding files with compression mode", modes[compression]) for format_ele in format_eles: _, puid_filename = get_puid_file_name(format_ele) @@ -241,8 +258,15 @@ def update_versions_xml(version): def main(): """Main CLI entrypoint.""" - parser = ArgumentParser(description="Download and convert the latest PRONOM signatures") - parser.add_argument("-tmpdir", default=OPTIONS["tmp_dir"], help="Location to store temporary files", dest="tmp_dir") + parser = ArgumentParser( + description="Download and convert the latest PRONOM signatures" + ) + parser.add_argument( + "-tmpdir", + default=OPTIONS["tmp_dir"], + help="Location to store temporary files", + dest="tmp_dir", + ) parser.add_argument( "-keep_tmp", default=OPTIONS["deleteTempDirectory"], diff --git a/fido/versions.py b/fido/versions.py index 94dae67b..55fa2202 100644 --- a/fido/versions.py +++ b/fido/versions.py @@ -87,7 +87,9 @@ def __setattr__(self, name, value): def get_zip_file(self): """Obtain location to the PRONOM XML Zip file based on the current PRONOM version.""" - return os.path.join(self.conf_dir, "pronom-xml-v{}.zip".format(self.pronom_version)) + return os.path.join( + self.conf_dir, "pronom-xml-v{}.zip".format(self.pronom_version) + ) def get_signature_file(self): """Obtain location to the current PRONOM signature file.""" @@ -99,7 +101,9 @@ def write(self): for key, value in self.PROPS_MAPPING.items(): if self.root.find(value) is None: raise ValueError("Field {} has not been defined!".format(key)) - self.tree.write(self.versions_file, xml_declaration=True, method="xml", encoding="utf-8") + self.tree.write( + self.versions_file, xml_declaration=True, method="xml", encoding="utf-8" + ) def get_local_versions(config_dir=CONFIG_DIR): @@ -143,11 +147,19 @@ def _list_available_versions(update_url): def _check_update_signatures(sig_vers, update_url, versions, is_update=False): is_new, latest = _version_check(sig_vers, update_url) if is_new: - sys.stdout.write("Updated signatures v{} are available, current version is v{}\n".format(latest, sig_vers)) + sys.stdout.write( + "Updated signatures v{} are available, current version is v{}\n".format( + latest, sig_vers + ) + ) if is_update: _output_details(latest, update_url, versions) else: - sys.stdout.write("Your signature files are up to date, current version is v{}\n".format(sig_vers)) + sys.stdout.write( + "Your signature files are up to date, current version is v{}\n".format( + sig_vers + ) + ) sys.exit(0) @@ -157,15 +169,23 @@ def _download_sig_version(sig_act, update_url, versions): if not match: sys.exit( - '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(sig_act) + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format( + sig_act + ) ) ver = sig_act if not ver.startswith("v"): ver = "v" + sig_act resp = requests.get(update_url + "format/" + ver + "/") if resp.status_code != 200: - sys.exit("No signature files found for {}, REST status {}".format(sig_act, resp.status_code)) - _output_details(re.search(r"\d+|$", ver).group(), update_url, versions) # noqa: W605 + sys.exit( + "No signature files found for {}, REST status {}".format( + sig_act, resp.status_code + ) + ) + _output_details( + re.search(r"\d+|$", ver).group(), update_url, versions + ) # noqa: W605 def _get_version(ver_string): @@ -173,7 +193,9 @@ def _get_version(ver_string): match = re.search(r"^v?(\d+)$", ver_string, re.IGNORECASE) if not match: sys.exit( - '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(ver_string) + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format( + ver_string + ) ) ver = ver_string return ver_string if not ver.startswith("v") else ver_string[1:] @@ -192,14 +214,18 @@ def _output_details(version, update_url, versions): def _version_check(sig_ver, update_url): resp = requests.get(update_url + "format/latest/") if resp.status_code != 200: - sys.exit("Error getting latest version info: HTTP Status {}".format(resp.status_code)) + sys.exit( + "Error getting latest version info: HTTP Status {}".format(resp.status_code) + ) root_ele = ET.fromstring(resp.text) latest = _get_version(root_ele.get("version")) return int(latest) > int(sig_ver), latest def _write_sigs(latest, update_url, type, name_template): - sig_out = str(importlib.resources.files("fido").joinpath("conf", name_template.format(latest))) + sig_out = str( + importlib.resources.files("fido").joinpath("conf", name_template.format(latest)) + ) if os.path.exists(sig_out): return resp = requests.get(update_url + "format/{0}/{1}/".format(latest, type)) diff --git a/pyproject.toml b/pyproject.toml index 1332e9bb..07ef2c84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,5 +66,8 @@ addopts = "--maxfail=1 --strict-markers" [tool.flake8] exclude = ['.venv'] ignore = ['E231', 'E241', 'E501', 'W503', 'E203'] -max-line-length = 130 -# count = true \ No newline at end of file +max-line-length = 120 +# count = true + +[tool.ruff] +line-length = 120 \ No newline at end of file diff --git a/tests/pronom/test_soap.py b/tests/pronom/test_soap.py index daea45a5..cb6509db 100644 --- a/tests/pronom/test_soap.py +++ b/tests/pronom/test_soap.py @@ -32,5 +32,5 @@ def test_pronom_signature(): """Test that retrieving signatures gets something with length and no errors are thrown.""" version = soap.get_pronom_sig_version() xml, count = soap.get_droid_signatures(version) - assert len(xml) > 1000, 'Expected more than 1000 XML lines, got %s' % len(xml) - assert count > 1000, 'Expected more than 1000 signatures, got %s' % count + assert len(xml) > 1000, "Expected more than 1000 XML lines, got %s" % len(xml) + assert count > 1000, "Expected more than 1000 signatures, got %s" % count diff --git a/tests/test_package.py b/tests/test_package.py index b4123cd2..534a1f74 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -4,11 +4,15 @@ from fido.package import ZipPackage -TEST_DATA_BAD_PACKAGES = os.path.normpath(os.path.join(__file__, "..", "test_data/hard_packages")) +TEST_DATA_BAD_PACKAGES = os.path.normpath( + os.path.join(__file__, "..", "test_data/hard_packages") +) # None of these files should be identified as packages? -@pytest.mark.parametrize("filename", ["bad.zip", "worse.zip", "unicode.zip", "foo.zip", "foo.tar"]) +@pytest.mark.parametrize( + "filename", ["bad.zip", "worse.zip", "unicode.zip", "foo.zip", "foo.tar"] +) def test_bad_zip(filename): p = ZipPackage(os.path.join(TEST_DATA_BAD_PACKAGES, filename), {}) r = p.detect_formats() diff --git a/tests/test_prepare.py b/tests/test_prepare.py index f843f23e..752fcd38 100644 --- a/tests/test_prepare.py +++ b/tests/test_prepare.py @@ -15,43 +15,40 @@ def binrep_convert(byt): @pytest.mark.parametrize( - ('pronom_bytesequence', 'matches_predicate'), + ("pronom_bytesequence", "matches_predicate"), ( # ANY BITMASKS, e.g., ~FF # ~07 = 00000111. Match bytes with any of the first three bits set. - ('~07', lambda binrep: '1' in binrep[-3:]), + ("~07", lambda binrep: "1" in binrep[-3:]), # ~7f = 01111111. Match bytes with any of the first seven bits set. - ('~7f', lambda binrep: '1' in binrep[-7:]), + ("~7f", lambda binrep: "1" in binrep[-7:]), # ~00 = 00000000. Match no bytes. # TODO: is it possible to write a regular expression that matches no # bytes? The regex pattern returned here matches ANY byte... - ('~00', lambda binrep: True), - + ("~00", lambda binrep: True), # NEGATED ANY BITMASKS, e.g., [!~FF] # [!~80] = 10000000. Match bytes without the last bit set. - ('[!~80]', lambda binrep: binrep.startswith('0')), + ("[!~80]", lambda binrep: binrep.startswith("0")), # [!~ff] = 11111111. Match bytes without any of the bitmask bits set. - ('[!~ff]', lambda binrep: binrep == '00000000'), + ("[!~ff]", lambda binrep: binrep == "00000000"), # [!~87] = 10000111. - ('[!~87]', lambda br: br.startswith('0') and br.endswith('000')), - + ("[!~87]", lambda br: br.startswith("0") and br.endswith("000")), # ALL BITMASKS, e.g., &FF # &07 = 00000111. Match bytes with all first three bits set. - ('&07', lambda binrep: binrep.endswith('111')), + ("&07", lambda binrep: binrep.endswith("111")), # &7f = 01111111. Match bytes with all first seven bits set. - ('&7f', lambda binrep: binrep.endswith('1111111')), + ("&7f", lambda binrep: binrep.endswith("1111111")), # &00 = 00000000. Matches any byte. - ('&00', lambda binrep: True), - + ("&00", lambda binrep: True), # NEGATED ALL BITMASKS, e.g., [!&FF] # !&80 = 10000000. Match bytes without the last bit set. - ('[!&80]', lambda binrep: binrep.startswith('0')), + ("[!&80]", lambda binrep: binrep.startswith("0")), # !&87 = 10000111. Match all bytes that don't have the first three bits # set and the last bit set also. - ('[!&87]', lambda br: not (br.startswith('1') and br.endswith('111'))), + ("[!&87]", lambda br: not (br.startswith("1") and br.endswith("111"))), # !&ff = 11111111. Match all bytes except 255. - ('[!&ff]', lambda binrep: not binrep == '11111111'), - ) + ("[!&ff]", lambda binrep: not binrep == "11111111"), + ), ) def test_bitmasks(pronom_bytesequence, matches_predicate): patt = convert_to_regex(pronom_bytesequence) @@ -64,25 +61,21 @@ def test_bitmasks(pronom_bytesequence, matches_predicate): @pytest.mark.parametrize( - ('pronom_bytesequence', 'input_', 'matches_bool'), + ("pronom_bytesequence", "input_", "matches_bool"), ( # These are good: - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF', True), - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xDD\xCD\x03\x11\xFF', True), - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xDD\xCD\x02\xFE\xFF', True), - + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF", True), + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x03\x11\xFF", True), + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\xFE\xFF", True), # Bad because missing three anythings between AB and CD - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xCD\x02\x11\xFF', False), - + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xCD\x02\x11\xFF", False), # Bad because not at start of string - ('ab{3}cd(01|02|03)~07ff', '\xDA\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF', False), - + ("ab{3}cd(01|02|03)~07ff", "\xDA\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF", False), # Bad because 04 is not in (01|02|03) - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xDD\xCD\x04\x11\xFF', False), - + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x04\x11\xFF", False), # Bad because 18 is not in ~07 - ('ab{3}cd(01|02|03)~07ff', '\xAB\xDD\xDD\xDD\xCD\x02\x18\xFF', False), - ) + ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\x18\xFF", False), + ), ) def test_heterogenous_sequences(pronom_bytesequence, input_, matches_bool): """Tests potential PRONOM sequences in their fullness. From c6c1a5c2f5e14ad45e0f03e0bb30898fb1e5828e Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Tue, 12 Nov 2024 13:36:56 +0000 Subject: [PATCH 20/33] Incorporated fix to fido.blocking_read that should avoid hanging on some streams. --- fido/fido.py | 171 ++++++++++++--------------------------------- tests/test_fido.py | 70 ++++++++++++++++++- 2 files changed, 112 insertions(+), 129 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index 781e96f5..9e42a5a0 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -87,24 +87,14 @@ def __init__( global defaults self.quiet = quiet self.bufsize = defaults["bufsize"] if bufsize is None else bufsize - self.container_bufsize = ( - defaults["container_bufsize"] - if container_bufsize is None - else container_bufsize - ) + self.container_bufsize = defaults["container_bufsize"] if container_bufsize is None else container_bufsize self.printmatch = defaults["printmatch"] if printmatch is None else printmatch - self.printnomatch = ( - defaults["printnomatch"] if printnomatch is None else printnomatch - ) - self.handle_matches = ( - self.print_matches if handle_matches is None else handle_matches - ) + self.printnomatch = defaults["printnomatch"] if printnomatch is None else printnomatch + self.handle_matches = self.print_matches if handle_matches is None else handle_matches self.zip = zip self.nocontainer = nocontainer self.conf_dir = conf_dir - self.format_files = ( - defaults["format_files"] if format_files is None else format_files - ) + self.format_files = defaults["format_files"] if format_files is None else format_files self.containersignature_file = defaults["containersignature_file"] self.formats = [] self.puid_format_map = {} @@ -162,24 +152,11 @@ def convert_container_sequence(self, sig): if sig[i] != "-" and sig[i] != "'" and ror: seq += escape(sig[i]).encode("utf8") continue - if ( - sig[i] != "-" - and sig[i] != "'" - and sig[i] != " " - and sig[i] != ":" - and not ror - and not byt - ): + if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and sig[i] != ":" and not ror and not byt: seq += b"\\x" + sig[i].lower().encode("utf8") byt = True continue - if ( - sig[i] != "-" - and sig[i] != "'" - and sig[i] != " " - and not ror - and byt - ): + if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and not ror and byt: seq += sig[i].lower().encode("utf8") byt = False continue @@ -209,9 +186,7 @@ def extract_signatures(self, doc, signature_type="ZIP"): format_mappings = root.find("FileFormatMappings") def get_puid(doc, element_id): - return format_mappings.find( - 'FileFormatMapping[@signatureId="{}"]'.format(element_id) - ).attrib["Puid"] + return format_mappings.find('FileFormatMapping[@signatureId="{}"]'.format(element_id)).attrib["Puid"] def format_signature_attributes(element): return { @@ -224,11 +199,7 @@ def format_signature_attributes(element): ), } - elements = root.findall( - 'ContainerSignatures/ContainerSignature[@ContainerType="{}"]'.format( - signature_type - ) - ) + elements = root.findall('ContainerSignatures/ContainerSignature[@ContainerType="{}"]'.format(signature_type)) signatures = {} for el in elements: if el.find("Files/File/BinarySignatures") is None: @@ -246,9 +217,7 @@ def format_signature_attributes(element): def match_container(self, signature_type, klass, file, signature_file): """Return the signature matches for a container.""" - puids = klass( - file, self.extract_signatures(signature_file, signature_type=signature_type) - ).detect_formats() + puids = klass(file, self.extract_signatures(signature_file, signature_type=signature_type)).detect_formats() results = [] for puid in puids: format = self.puid_format_map[puid] @@ -268,11 +237,7 @@ def load_fido_xml(self, file): for element in tree.getroot().findall("./format"): self.process_format_element(element) except ET.ParseError as parse_excep: - sys.stderr.write( - "Failed to parse signature file {}, exception: {}\n".format( - file, parse_excep - ) - ) + sys.stderr.write("Failed to parse signature file {}, exception: {}\n".format(file, parse_excep)) sys.exit(1) return self.formats @@ -300,10 +265,7 @@ def get_signatures(self, format): def has_priority_over(self, format, possibly_inferior): """Return true if format has priority over possibly inferior.""" - return ( - self.get_puid(possibly_inferior) - in self.puid_has_priority_over_map[self.get_puid(format)] - ) + return self.get_puid(possibly_inferior) in self.puid_has_priority_over_map[self.get_puid(format)] def get_puid(self, format): """Return the PUID for the format.""" @@ -399,10 +361,7 @@ def print_summary(self, secs): if not self.quiet: rate = int(round(count / secs)) if secs != 0 else 9999 # print >> sys.stderr, 'FIDO: Processed %6d files in %6.2f msec, %2d files/sec' % (count, secs * 1000, rate) - sys.stderr.write( - "FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n" - % (count, secs * 1000, rate) - ) + sys.stderr.write("FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n" % (count, secs * 1000, rate)) def identify_file(self, filename, extension=True): """ @@ -418,30 +377,18 @@ def identify_file(self, filename, extension=True): size = os.stat(filename)[6] self.current_filesize = size if self.current_filesize == 0: - sys.stderr.write( - "FIDO: Zero byte file (empty): Path is: " + filename + "\n" - ) + sys.stderr.write("FIDO: Zero byte file (empty): Path is: " + filename + "\n") bofbuffer, eofbuffer, _ = self.get_buffers(f, size, seekable=True) matches = self.match_formats(bofbuffer, eofbuffer) container_type = self.container_type(matches) if not self.nocontainer and container_type in ("zip", "ole"): - container_file = ET.parse( - os.path.join( - os.path.abspath(self.conf_dir), self.containersignature_file - ) - ) + container_file = ET.parse(os.path.join(os.path.abspath(self.conf_dir), self.containersignature_file)) if container_type == "zip": - container_matches = self.match_container( - "ZIP", ZipPackage, filename, container_file - ) + container_matches = self.match_container("ZIP", ZipPackage, filename, container_file) else: - container_matches = self.match_container( - "OLE2", OlePackage, filename, container_file - ) + container_matches = self.match_container("OLE2", OlePackage, filename, container_file) if len(container_matches) > 0: - self.handle_matches( - filename, container_matches, timer.duration(), "container" - ) + self.handle_matches(filename, container_matches, timer.duration(), "container") return # from here is also repeated in walk_zip # we should make this uniform in a next version! @@ -461,11 +408,7 @@ def identify_file(self, filename, extension=True): self.identify_contents(filename, type=container, extension=extension) except IOError as io_excep: # print >> sys.stderr, "FIDO: Error in identify_file: Path is {0}".format(filename) - sys.stderr.write( - "FIDO: Error in identify_file: {}, exception: {}\n".format( - filename, io_excep - ) - ) + sys.stderr.write("FIDO: Error in identify_file: {}, exception: {}\n".format(filename, io_excep)) def identify_contents(self, filename, fileobj=None, type=False, extension=True): """ @@ -524,14 +467,10 @@ def identify_multi_object_stream(self, stream, extension=True): matches = self.match_formats(bofbuffer, eofbuffer) # MdR: this needs attention if len(matches) > 0: - self.handle_matches( - self.current_file, matches, timer.duration(), "signature" - ) + self.handle_matches(self.current_file, matches, timer.duration(), "signature") elif extension and (len(matches) == 0 or self.current_filesize == 0): matches = self.match_extensions(self.current_file) - self.handle_matches( - self.current_file, matches, timer.duration(), "extension" - ) + self.handle_matches(self.current_file, matches, timer.duration(), "extension") def identify_stream(self, stream, filename, extension=True): """ @@ -547,9 +486,7 @@ def identify_stream(self, stream, filename, extension=True): matches = self.match_formats(bofbuffer, eofbuffer) # MdR: this needs attention if len(matches) > 0: - self.handle_matches( - self.current_file, matches, timer.duration(), "signature" - ) + self.handle_matches(self.current_file, matches, timer.duration(), "signature") elif extension and (len(matches) == 0 or self.current_filesize == 0): # we can only determine the filename from the STDIN stream # on Linux, on Windows there is not a (simple) way to do that @@ -568,9 +505,7 @@ def identify_stream(self, stream, filename, extension=True): # we have to reset self.current_file if not on Windows if os.name != "nt": self.current_file = "STDIN" - self.handle_matches( - self.current_file, matches, timer.duration(), "extension" - ) + self.handle_matches(self.current_file, matches, timer.duration(), "extension") def container_type(self, matches): """ @@ -606,16 +541,20 @@ def can_recurse_into_container(self, container_type): """ return container_type in ("zip", "tar") + # This is updated following PR #191: FIX: Develop out FIDO tests with pytest + # It should fix a problem that streams (not files) would hang. + # Needs thorough testing, though. def blocking_read(self, file, bytes_to_read): """Perform a blocking read and return the buffer.""" bytes_read = 0 buffer = b"" while bytes_read < bytes_to_read: readbuffer = file.read(bytes_to_read - bytes_read) + last_read_len = len(readbuffer) buffer += readbuffer - bytes_read = len(buffer) - # break out if EOF is reached. - if readbuffer == "": + bytes_read += last_read_len + # break out if EOF is reached, that is zero bytes read. + if last_read_len < 1: break return buffer @@ -640,11 +579,7 @@ def get_buffers(self, stream, length=None, seekable=False): if len(buffer) == self.bufsize: prevbuffer = buffer else: - eofbuffer = ( - prevbuffer - if len(buffer) == 0 - else prevbuffer[-(self.bufsize - len(buffer)) :] + buffer - ) + eofbuffer = prevbuffer if len(buffer) == 0 else prevbuffer[-(self.bufsize - len(buffer)) :] + buffer break return bofbuffer, eofbuffer, bytes_read bytes_unread = length - len(bofbuffer) @@ -652,9 +587,7 @@ def get_buffers(self, stream, length=None, seekable=False): eofbuffer = bofbuffer elif bytes_unread < self.bufsize: # The buffs overlap - eofbuffer = bofbuffer[bytes_unread:] + self.blocking_read( - stream, bytes_unread - ) + eofbuffer = bofbuffer[bytes_unread:] + self.blocking_read(stream, bytes_unread) elif bytes_unread == self.bufsize: eofbuffer = self.blocking_read(stream, self.bufsize) elif seekable: # easy case when we can just seek! @@ -692,24 +625,14 @@ def walk_zip(self, filename, fileobj=None, extension=True): self.current_file = item_name self.current_filesize = item.file_size if self.current_filesize == 0: - sys.stderr.write( - "FIDO: Zero byte file (empty): Path is: " - + item_name - + "\n" - ) + sys.stderr.write("FIDO: Zero byte file (empty): Path is: " + item_name + "\n") bofbuffer, eofbuffer, _ = self.get_buffers(f, item.file_size) matches = self.match_formats(bofbuffer, eofbuffer) if len(matches) > 0 and self.current_filesize > 0: - self.handle_matches( - item_name, matches, timer.duration(), "signature" - ) - elif extension and ( - len(matches) == 0 or self.current_filesize == 0 - ): + self.handle_matches(item_name, matches, timer.duration(), "signature") + elif extension and (len(matches) == 0 or self.current_filesize == 0): matches = self.match_extensions(item_name) - self.handle_matches( - item_name, matches, timer.duration(), "extension" - ) + self.handle_matches(item_name, matches, timer.duration(), "extension") if self.container_type(matches): target = tempfile.SpooledTemporaryFile(prefix="Fido") with zipstream.open(item) as source: @@ -892,7 +815,9 @@ def main(args=None): defaults["format_files"] = [defaults["xml_pronomSignature"]] if args.pronom_only: - versionHeader = f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']})\n" + versionHeader = ( + f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']})\n" + ) else: versionHeader = ( f"FIDO v{__version__} ({defaults['xml_pronomSignature']}, {defaults['containersignature_file']}, " @@ -940,14 +865,10 @@ def main(args=None): # TODO: remove from maps if args.useformats: args.useformats = args.useformats.split(",") - fido.formats = [ - f for f in fido.formats if f.find("puid").text in args.useformats - ] + fido.formats = [f for f in fido.formats if f.find("puid").text in args.useformats] elif args.nouseformats: args.nouseformats = args.nouseformats.split(",") - fido.formats = [ - f for f in fido.formats if f.find("puid").text not in args.nouseformats - ] + fido.formats = [f for f in fido.formats if f.find("puid").text not in args.nouseformats] # Set up to use stdin, or open input files: if args.input == "-": @@ -962,16 +883,10 @@ def main(args=None): sys.stderr.flush() if (not args.input) and len(args.files) == 1 and args.files[0] == "-": if fido.zip: - raise RuntimeError( - "Multiple content read from stdin not yet supported." - ) - fido.identify_multi_object_stream( - sys.stdin, extension=not args.noextension - ) + raise RuntimeError("Multiple content read from stdin not yet supported.") + fido.identify_multi_object_stream(sys.stdin, extension=not args.noextension) else: - fido.identify_stream( - sys.stdin, args.filename, extension=not args.noextension - ) + fido.identify_stream(sys.stdin, args.filename, extension=not args.noextension) else: for file in list_files(args.files, args.recurse): fido.identify_file(file, extension=not args.noextension) diff --git a/tests/test_fido.py b/tests/test_fido.py index 952a588e..ae4bce62 100644 --- a/tests/test_fido.py +++ b/tests/test_fido.py @@ -1,9 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import csv +import io from time import sleep -from fido.fido import PerfTimer +from fido.fido import Fido, PerfTimer def test_perf_timer(): @@ -11,3 +13,69 @@ def test_perf_timer(): sleep(3.6) duration = timer.duration() assert duration > 0 + + +# Magic number for fmt/1000. +MAGIC = b"\x5a\x58\x54\x61\x70\x65\x21\x1a\x01" + +# Expected positive PUID. +PUID = "fmt/1000" + +# Expected result. +OK = "OK" + + +def test_file_identification(tmp_path, capsys): + """Reference for Fido-based format identification + 1. Create a byte-stream with a known magic number and serialize to tempfile. + 2. Call identify_file(...) to identify the file against Fido's known formats. + """ + # Create a temporary file and write our skeleton file out to it. + tmp_file = tmp_path / "tmp_file" + tmp_file.write_bytes(MAGIC) + + # Create a Fido instance and call identify_file. The identify_file function + # will create and manage a file for itself. + f = Fido() + f.identify_file(str(tmp_file)) + + # Capture the stdout returned by Fido and make assertions about its + # validity. + captured = capsys.readouterr() + # TODO: there is a signature that generates an error + # min repeat greater than max repeat at position 8 + # assert captured.err == "" + reader = csv.reader(io.StringIO(captured.out), delimiter=",") + assert reader is not None + row = next(reader) + assert row[0] == OK, "row hasn't returned a positive identification" + assert row[2] == PUID, "row doesn't contain expected PUID value" + assert int(row[5]) == len(MAGIC), "row doesn't contain stream length" + + +def test_stream_identification(capsys): + """Reference for Fido-based format identification + 1. Create a byte-stream with a known magic number. + 2. Call identify_stream(...) to identify the file against Fido's known formats. + """ + # Create the stream object with the known magic-number. + fstream = io.BytesIO(MAGIC) + + # Create a Fido instance and call identify_stream. The identify_stream function + # will work on the stream as-is. This could be an open file handle that the + # caller is managing for itself. + f = Fido() + f.identify_stream(fstream, "filename to display", extension=False) + + # Capture the stdout returned by Fido and make assertions about its + # validity. + captured = capsys.readouterr() + # TODO: as above, there is a signature that outputs an error + # min repeat greater than max repeat at position 8 + # assert captured.err == "" + reader = csv.reader(io.StringIO(captured.out), delimiter=",") + assert reader is not None + row = next(reader) + assert row[0] == OK, "row hasn't returned a positive identification" + assert row[2] == PUID, "row doesn't contain expected PUID value" + assert int(row[5]) == len(MAGIC), "row doesn't contain stream length" From 6710f9dadfcc5b9eab5867c34b1a4a8c2f7b3ea9 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 13 Nov 2024 09:58:55 +0000 Subject: [PATCH 21/33] Parameterized file and stream id tests to simply adding additional test cases in the future. --- tests/test_fido.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/test_fido.py b/tests/test_fido.py index ae4bce62..e0c256ec 100644 --- a/tests/test_fido.py +++ b/tests/test_fido.py @@ -5,34 +5,34 @@ import io from time import sleep +import pytest + from fido.fido import Fido, PerfTimer def test_perf_timer(): timer = PerfTimer() - sleep(3.6) + sleep(0.2) duration = timer.duration() assert duration > 0 -# Magic number for fmt/1000. -MAGIC = b"\x5a\x58\x54\x61\x70\x65\x21\x1a\x01" - -# Expected positive PUID. -PUID = "fmt/1000" - -# Expected result. -OK = "OK" +id_test_data = [(b"\x5a\x58\x54\x61\x70\x65\x21\x1a\x01", "fmt/1000", "OK")] -def test_file_identification(tmp_path, capsys): +@pytest.mark.parametrize( + "magic, expected_puid, expected_result", + id_test_data, + # Add additional test cases here +) +def test_file_identification(tmp_path, capsys, magic: bytes, expected_puid: str, expected_result: str): """Reference for Fido-based format identification 1. Create a byte-stream with a known magic number and serialize to tempfile. 2. Call identify_file(...) to identify the file against Fido's known formats. """ # Create a temporary file and write our skeleton file out to it. tmp_file = tmp_path / "tmp_file" - tmp_file.write_bytes(MAGIC) + tmp_file.write_bytes(magic) # Create a Fido instance and call identify_file. The identify_file function # will create and manage a file for itself. @@ -48,18 +48,23 @@ def test_file_identification(tmp_path, capsys): reader = csv.reader(io.StringIO(captured.out), delimiter=",") assert reader is not None row = next(reader) - assert row[0] == OK, "row hasn't returned a positive identification" - assert row[2] == PUID, "row doesn't contain expected PUID value" - assert int(row[5]) == len(MAGIC), "row doesn't contain stream length" + assert row[0] == expected_result, "row hasn't returned a positive identification" + assert row[2] == expected_puid, "row doesn't contain expected PUID value" + assert int(row[5]) == len(magic), "row doesn't contain stream length" -def test_stream_identification(capsys): +@pytest.mark.parametrize( + "magic, expected_puid, expected_result", + id_test_data, + # Add additional test cases here +) +def test_stream_identification(capsys, magic: bytes, expected_puid: str, expected_result: str): """Reference for Fido-based format identification 1. Create a byte-stream with a known magic number. 2. Call identify_stream(...) to identify the file against Fido's known formats. """ # Create the stream object with the known magic-number. - fstream = io.BytesIO(MAGIC) + fstream = io.BytesIO(magic) # Create a Fido instance and call identify_stream. The identify_stream function # will work on the stream as-is. This could be an open file handle that the @@ -76,6 +81,6 @@ def test_stream_identification(capsys): reader = csv.reader(io.StringIO(captured.out), delimiter=",") assert reader is not None row = next(reader) - assert row[0] == OK, "row hasn't returned a positive identification" - assert row[2] == PUID, "row doesn't contain expected PUID value" - assert int(row[5]) == len(MAGIC), "row doesn't contain stream length" + assert row[0] == expected_result, "row hasn't returned a positive identification" + assert row[2] == expected_puid, "row doesn't contain expected PUID value" + assert int(row[5]) == len(magic), "row doesn't contain stream length" From 3c58f13893ad266f584a449bac620406322e1c3c Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 13 Nov 2024 17:01:10 +0000 Subject: [PATCH 22/33] Created a utils package and moved PerfTimer and the char_handler to it. Moved the pronom related modules to the pronom package as a basis for future refactoring. --- fido/fido.py | 22 +-- fido/{ => pronom}/prepare.py | 196 ++++++------------------- fido/{ => pronom}/update_signatures.py | 29 ++-- fido/{ => pronom}/versions.py | 0 fido/toxml.py | 11 +- fido/utils/__init__.py | 0 fido/{ => utils}/char_handler.py | 0 fido/utils/timer.py | 17 +++ pyproject.toml | 4 +- tests/test_fido.py | 3 +- tests/test_prepare.py | 16 +- 11 files changed, 92 insertions(+), 206 deletions(-) rename fido/{ => pronom}/prepare.py (85%) rename fido/{ => pronom}/update_signatures.py (92%) rename fido/{ => pronom}/versions.py (100%) create mode 100644 fido/utils/__init__.py rename fido/{ => utils}/char_handler.py (100%) create mode 100644 fido/utils/timer.py diff --git a/fido/fido.py b/fido/fido.py index 9e42a5a0..d910da65 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -15,15 +15,15 @@ import tempfile import zipfile from contextlib import closing -from time import perf_counter from typing import Optional from xml.etree import cElementTree as ET from fido import CONFIG_DIR, __version__ -from fido.char_handler import escape from fido.cli_args import parse_cli_args from fido.package import OlePackage, ZipPackage -from fido.versions import get_local_versions, sig_file_actions +from fido.pronom.versions import get_local_versions, sig_file_actions +from fido.utils.char_handler import escape +from fido.utils.timer import PerfTimer defaults = { "config_dir": CONFIG_DIR, @@ -50,22 +50,6 @@ } -class PerfTimer: - """Utility class that carries out simple process timings.""" - - def __init__(self): - """New instance with start time running.""" - self.start_time = perf_counter() - - def start(self): - """Start new timer.""" - self.start_time = perf_counter() - - def duration(self): - """Return the duration since instantiation or start() was last called.""" - return perf_counter() - self.start_time - - class Fido: """Main FIDO application class.""" diff --git a/fido/prepare.py b/fido/pronom/prepare.py similarity index 85% rename from fido/prepare.py rename to fido/pronom/prepare.py index 7580bef9..929ad9a0 100644 --- a/fido/prepare.py +++ b/fido/pronom/prepare.py @@ -14,8 +14,8 @@ from xml.dom import minidom from xml.etree import ElementTree as ET -from .char_handler import escape -from .versions import get_local_versions +from fido.pronom.versions import get_local_versions +from fido.utils.char_handler import escape FLG_INCOMPATIBLE = "__INCOMPATIBLE_SIG__" @@ -141,9 +141,7 @@ def load_pronom_xml(self, puid_filter=None): zip.close() except Exception as e: print( - "An error occured loading '{0}' (exception: {1})".format( - self.pronom_files, e - ), + "An error occured loading '{0}' (exception: {1})".format(self.pronom_files, e), file=sys.stderr, ) sys.exit() @@ -200,44 +198,28 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(fido_format, "container").text = "zip" elif puid == "x-fmt/265": ET.SubElement(fido_format, "container").text = "tar" - ET.SubElement(fido_format, "name").text = get_text_tna( - pronom_format, "FormatName" - ) - ET.SubElement(fido_format, "version").text = get_text_tna( - pronom_format, "FormatVersion" - ) - ET.SubElement(fido_format, "alias").text = get_text_tna( - pronom_format, "FormatAliases" - ) - ET.SubElement(fido_format, "pronom_id").text = get_text_tna( - pronom_format, "FormatID" - ) + ET.SubElement(fido_format, "name").text = get_text_tna(pronom_format, "FormatName") + ET.SubElement(fido_format, "version").text = get_text_tna(pronom_format, "FormatVersion") + ET.SubElement(fido_format, "alias").text = get_text_tna(pronom_format, "FormatAliases") + ET.SubElement(fido_format, "pronom_id").text = get_text_tna(pronom_format, "FormatID") # Get the extensions from the ExternalSignature for x in pronom_format.findall(TNA("ExternalSignature")): ET.SubElement(fido_format, "extension").text = get_text_tna(x, "Signature") for id in pronom_format.findall(TNA("FileFormatIdentifier")): type = get_text_tna(id, "IdentifierType") if type == "Apple Uniform Type Identifier": - ET.SubElement(fido_format, "apple_uti").text = get_text_tna( - id, "Identifier" - ) + ET.SubElement(fido_format, "apple_uti").text = get_text_tna(id, "Identifier") # Handle the relationships for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Has priority over": - ET.SubElement(fido_format, "has_priority_over").text = get_text_tna( - x, "RelatedFormatID" - ) + ET.SubElement(fido_format, "has_priority_over").text = get_text_tna(x, "RelatedFormatID") # Get the InternalSignature information for pronom_sig in pronom_format.findall(TNA("InternalSignature")): fido_sig = ET.SubElement(fido_format, "signature") - ET.SubElement(fido_sig, "name").text = get_text_tna( - pronom_sig, "SignatureName" - ) + ET.SubElement(fido_sig, "name").text = get_text_tna(pronom_sig, "SignatureName") # There are some funny chars in the notes, which caused me trouble and it is a unicode string, - ET.SubElement(fido_sig, "note").text = get_text_tna( - pronom_sig, "SignatureNote" - ) + ET.SubElement(fido_sig, "note").text = get_text_tna(pronom_sig, "SignatureNote") for pronom_pat in pronom_sig.findall(TNA("ByteSequence")): # print('Parsing ID:{}'.format(puid)) fido_pat = ET.SubElement(fido_sig, "pattern") @@ -249,14 +231,10 @@ def parse_pronom_xml(self, source, puid_filter=None): pass # print "working on puid:", puid, ", position: ", pos, "with offset, maxoffset: ", offset, ",", max_offset try: - regex = convert_to_regex( - byte_seq, "Little", pos, offset, max_offset - ) + regex = convert_to_regex(byte_seq, "Little", pos, offset, max_offset) except ValueError as ve: print( - "ValueError converting PUID {} signature to regex: {}".format( - puid, ve - ), + "ValueError converting PUID {} signature to regex: {}".format(puid, ve), file=sys.stderr, ) regex = FLG_INCOMPATIBLE @@ -264,9 +242,7 @@ def parse_pronom_xml(self, source, puid_filter=None): # print "done puid", puid if regex == FLG_INCOMPATIBLE: print( - "Error: incompatible PRONOM signature found for puid {} skipping...".format( - puid - ), + "Error: incompatible PRONOM signature found for puid {} skipping...".format(puid), file=sys.stderr, ) # remove the empty 'signature' nodes @@ -280,73 +256,47 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(fido_pat, "regex").text = regex # Get the format details fido_details = ET.SubElement(fido_format, "details") - ET.SubElement(fido_details, "dc:description").text = get_text_tna( - pronom_format, "FormatDescription" - ) - ET.SubElement(fido_details, "dcterms:available").text = get_text_tna( - pronom_format, "ReleaseDate" - ) - ET.SubElement(fido_details, "dc:creator").text = get_text_tna( - pronom_format, "Developers/DeveloperCompoundName" - ) + ET.SubElement(fido_details, "dc:description").text = get_text_tna(pronom_format, "FormatDescription") + ET.SubElement(fido_details, "dcterms:available").text = get_text_tna(pronom_format, "ReleaseDate") + ET.SubElement(fido_details, "dc:creator").text = get_text_tna(pronom_format, "Developers/DeveloperCompoundName") ET.SubElement(fido_details, "dcterms:publisher").text = get_text_tna( pronom_format, "Developers/OrganisationName" ) for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Is supertype of": - ET.SubElement(fido_details, "is_supertype_of").text = get_text_tna( - x, "RelatedFormatID" - ) + ET.SubElement(fido_details, "is_supertype_of").text = get_text_tna(x, "RelatedFormatID") for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") if rel == "Is subtype of": - ET.SubElement(fido_details, "is_subtype_of").text = get_text_tna( - x, "RelatedFormatID" - ) - ET.SubElement(fido_details, "content_type").text = get_text_tna( - pronom_format, "FormatTypes" - ) + ET.SubElement(fido_details, "is_subtype_of").text = get_text_tna(x, "RelatedFormatID") + ET.SubElement(fido_details, "content_type").text = get_text_tna(pronom_format, "FormatTypes") # References for x in pronom_format.findall(TNA("Document")): r = ET.SubElement(fido_details, "reference") ET.SubElement(r, "dc:title").text = get_text_tna(x, "TitleText") - ET.SubElement(r, "dc:creator").text = get_text_tna( - x, "Author/AuthorCompoundName" - ) - ET.SubElement(r, "dc:publisher").text = get_text_tna( - x, "Publisher/PublisherCompoundName" - ) - ET.SubElement(r, "dcterms:available").text = get_text_tna( - x, "PublicationDate" - ) + ET.SubElement(r, "dc:creator").text = get_text_tna(x, "Author/AuthorCompoundName") + ET.SubElement(r, "dc:publisher").text = get_text_tna(x, "Publisher/PublisherCompoundName") + ET.SubElement(r, "dcterms:available").text = get_text_tna(x, "PublicationDate") for id in x.findall(TNA("DocumentIdentifier")): type = get_text_tna(id, "IdentifierType") if type == "URL": - ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna( - id, "Identifier" - ) + ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna(id, "Identifier") else: ET.SubElement(r, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") - + ":" - + get_text_tna(id, "Identifier") + get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") ) ET.SubElement(r, "dc:description").text = get_text_tna(x, "DocumentNote") ET.SubElement(r, "dc:type").text = get_text_tna(x, "DocumentType") ET.SubElement(r, "dcterms:license").text = ( - get_text_tna(x, "AvailabilityDescription") - + " " - + get_text_tna(x, "AvailabilityNote") + get_text_tna(x, "AvailabilityDescription") + " " + get_text_tna(x, "AvailabilityNote") ) ET.SubElement(r, "dc:rights").text = get_text_tna(x, "DocumentIPR") # Examples for x in pronom_format.findall(TNA("ReferenceFile")): rf = ET.SubElement(fido_details, "example_file") ET.SubElement(rf, "dc:title").text = get_text_tna(x, "ReferenceFileName") - ET.SubElement(rf, "dc:description").text = get_text_tna( - x, "ReferenceFileDescription" - ) + ET.SubElement(rf, "dc:description").text = get_text_tna(x, "ReferenceFileDescription") checksum = "" for id in x.findall(TNA("ReferenceFileIdentifier")): type = get_text_tna(id, "IdentifierType") @@ -364,20 +314,14 @@ def parse_pronom_xml(self, source, puid_filter=None): m.update(sock.read()) sock.close() except HTTPError as http_excep: - sys.stderr.write( - "HTTP {} error loading resource {}\n".format( - http_excep.code, url - ) - ) + sys.stderr.write("HTTP {} error loading resource {}\n".format(http_excep.code, url)) if http_excep.code == 404: continue checksum = m.hexdigest() else: ET.SubElement(rf, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") - + ":" - + get_text_tna(id, "Identifier") + get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") ) ET.SubElement(rf, "dcterms:license").text = "" ET.SubElement(rf, "dc:rights").text = get_text_tna(x, "ReferenceFileIPR") @@ -387,18 +331,10 @@ def parse_pronom_xml(self, source, puid_filter=None): # Record Metadata md = ET.SubElement(fido_details, "record_metadata") ET.SubElement(md, "status").text = "unknown" - ET.SubElement(md, "dc:creator").text = get_text_tna( - pronom_format, "ProvenanceName" - ) - ET.SubElement(md, "dcterms:created").text = get_text_tna( - pronom_format, "ProvenanceSourceDate" - ) - ET.SubElement(md, "dcterms:modified").text = get_text_tna( - pronom_format, "LastUpdatedDate" - ) - ET.SubElement(md, "dc:description").text = get_text_tna( - pronom_format, "ProvenanceDescription" - ) + ET.SubElement(md, "dc:creator").text = get_text_tna(pronom_format, "ProvenanceName") + ET.SubElement(md, "dcterms:created").text = get_text_tna(pronom_format, "ProvenanceSourceDate") + ET.SubElement(md, "dcterms:modified").text = get_text_tna(pronom_format, "LastUpdatedDate") + ET.SubElement(md, "dc:description").text = get_text_tna(pronom_format, "ProvenanceDescription") return fido_format # FIXME: I don't think that this quite works yet! @@ -485,9 +421,7 @@ def do_byte(chars, i, littleendian, esc=True): c2 = "0123456789ABCDEF".find(chars[i + 1].upper()) buf = StringIO() if c1 < 0 or c2 < 0: - raise Exception( - _convert_err_msg("bad byte sequence", chars[i : i + 2], i, chars, buf) - ) + raise Exception(_convert_err_msg("bad byte sequence", chars[i : i + 2], i, chars, buf)) if littleendian: val = chr(16 * c1 + c2) else: @@ -553,16 +487,12 @@ def calculate_repetition(char, pos, offset, maxoffset): def do_all_bitmasks(chars, i, littleendian): """(byte & bitmask) == bitmask.""" - return do_any_all_bitmasks( - chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), littleendian - ) + return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) == bitmask), littleendian) def do_any_bitmasks(chars, i, littleendian): """(byte & bitmask) != 0.""" - return do_any_all_bitmasks( - chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), littleendian - ) + return do_any_all_bitmasks(chars, i, lambda byt, bitmask: ((byt & bitmask) != 0), littleendian) def do_any_all_bitmasks(chars, i, predicate, littleendian): @@ -581,13 +511,7 @@ def do_any_all_bitmasks(chars, i, predicate, littleendian): byt, inc = do_byte(chars, i + 1, littleendian, esc=False) bitmask = ord(byt) regex = "({})".format( - "|".join( - [ - "\\x" + hex(byte)[2:].zfill(2) - for byte in range(0x100) - if predicate(byte, bitmask) - ] - ) + "|".join(["\\x" + hex(byte)[2:].zfill(2) for byte in range(0x100) if predicate(byte, bitmask)]) ) return regex, inc + 1 @@ -645,11 +569,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] in "*+?": state = "specials" else: - raise ValueError( - _convert_err_msg( - "Illegal character in start", chars[i], i, chars, buf - ) - ) + raise ValueError(_convert_err_msg("Illegal character in start", chars[i], i, chars, buf)) elif state == "bytes": (byt, inc) = do_byte(chars, i, littleendian) buf.write(byt) @@ -684,11 +604,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] == "]": break else: - raise Exception( - _convert_err_msg( - "Illegal character in non-match", chars[i], i, chars, buf - ) - ) + raise Exception(_convert_err_msg("Illegal character in non-match", chars[i], i, chars, buf)) buf.write(")") i += 1 state = "start" @@ -714,11 +630,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): buf.write("]") i += 1 except Exception: - print( - _convert_err_msg( - "Illegal character in bracket", chars[i], i, chars, buf - ) - ) + print(_convert_err_msg("Illegal character in bracket", chars[i], i, chars, buf)) raise if i < len(chars) and chars[i] == "{": state = "curly-after-bracket" @@ -761,9 +673,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): else: raise Exception( _convert_err_msg( - ( - "Current state = '{0}' : Illegal character in paren" - ).format(state), + ("Current state = '{0}' : Illegal character in paren").format(state), chars[i], i, chars, @@ -796,11 +706,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): elif chars[i] == "}": break else: - raise Exception( - _convert_err_msg( - "Illegal character in curly", chars[i], i, chars, buf - ) - ) + raise Exception(_convert_err_msg("Illegal character in curly", chars[i], i, chars, buf)) buf.write("}") i += 1 # skip the ) state = "start" @@ -813,11 +719,7 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): i += 1 elif chars[i] == "?": if chars[i + 1] != "?": - raise Exception( - _convert_err_msg( - "Illegal character after ?", chars[i + 1], i + 1, chars, buf - ) - ) + raise Exception(_convert_err_msg("Illegal character after ?", chars[i + 1], i + 1, chars, buf)) buf.write(".?") i += 2 state = "start" @@ -856,16 +758,10 @@ def main(args=None): if args is None: args = sys.argv[1:] - parser = ArgumentParser( - description="Produce the FIDO format XML that is loaded at run-time" - ) - parser.add_argument( - "-input", default=None, help="Input file, a Zip containing PRONOM XML files" - ) + parser = ArgumentParser(description="Produce the FIDO format XML that is loaded at run-time") + parser.add_argument("-input", default=None, help="Input file, a Zip containing PRONOM XML files") parser.add_argument("-output", default=None, help="Output file") - parser.add_argument( - "-puid", default=None, help="A particular PUID record to extract" - ) + parser.add_argument("-puid", default=None, help="A particular PUID record to extract") args = parser.parse_args(args) run(input=args.input, output=args.output, puid=args.puid) diff --git a/fido/update_signatures.py b/fido/pronom/update_signatures.py similarity index 92% rename from fido/update_signatures.py rename to fido/pronom/update_signatures.py index 919dfadf..93432b95 100644 --- a/fido/update_signatures.py +++ b/fido/pronom/update_signatures.py @@ -21,15 +21,16 @@ from shutil import rmtree from xml.etree import ElementTree as CET -from . import CONFIG_DIR, __version__ -from .prepare import run as prepare_pronom_to_fido -from .pronom.soap import ( +from pronom.prepare import run as prepare_pronom_to_fido + +from fido import CONFIG_DIR, __version__ +from fido.pronom.soap import ( NS, get_droid_signatures, get_pronom_sig_version, get_sig_xml_for_puid, ) -from .versions import get_local_versions +from fido.pronom.versions import get_local_versions ABORT_MSG = "Aborting update..." @@ -117,9 +118,7 @@ def sig_version_check(version="latest"): print("Getting latest version number from PRONOM...") version = get_pronom_sig_version() if not version: - sys.exit( - "Failed to obtain PRONOM signature file version number, please try again." - ) + sys.exit("Failed to obtain PRONOM signature file version number, please try again.") print("Querying PRONOM for signaturefile version {}.".format(version)) sig_file_name = _sig_file_name(version) @@ -159,9 +158,7 @@ def init_sig_download(defaults): resume = False if os.path.isdir(tmpdir): print("Found previously created temporary folder for download:", tmpdir) - resume = query_yes_no( - "Do you want to resume download (yes) or start over (no)?" - ) + resume = query_yes_no("Do you want to resume download (yes) or start over (no)?") if resume: print("Resuming download...") else: @@ -171,9 +168,7 @@ def init_sig_download(defaults): except OSError: pass if not os.path.isdir(tmpdir): - sys.stderr.write( - "Failed to create temporary folder for PUID's, using: " + tmpdir - ) + sys.stderr.write("Failed to create temporary folder for PUID's, using: " + tmpdir) return tmpdir, resume @@ -187,9 +182,7 @@ def download_signatures(defaults, format_eles, resume, tmpdir): download_sig(format_ele, tmpdir, resume, defaults) numfiles += 1 print( - r"Downloaded {}/{} files [{}%]".format( - numfiles, puid_count, int(float(numfiles) / one_percent) - ), + r"Downloaded {}/{} files [{}%]".format(numfiles, puid_count, int(float(numfiles) / one_percent)), end="\r", ) print("100%") @@ -258,9 +251,7 @@ def update_versions_xml(version): def main(): """Main CLI entrypoint.""" - parser = ArgumentParser( - description="Download and convert the latest PRONOM signatures" - ) + parser = ArgumentParser(description="Download and convert the latest PRONOM signatures") parser.add_argument( "-tmpdir", default=OPTIONS["tmp_dir"], diff --git a/fido/versions.py b/fido/pronom/versions.py similarity index 100% rename from fido/versions.py rename to fido/pronom/versions.py diff --git a/fido/toxml.py b/fido/toxml.py index ca1905af..9e240da0 100644 --- a/fido/toxml.py +++ b/fido/toxml.py @@ -22,8 +22,9 @@ import csv import sys +from fido.pronom.versions import get_local_versions + from . import __version__ -from .versions import get_local_versions def main(): @@ -34,9 +35,7 @@ def main(): {0} {1} - """.format( - __version__, get_local_versions().pronom_version - ) + """.format(__version__, get_local_versions().pronom_version) ) reader = csv.reader(sys.stdin) @@ -54,9 +53,7 @@ def main(): {6} {7} {8} - """.format( - row[6], row[0], row[8], row[1], row[2], row[7], row[3], row[4], row[5] - ) + """.format(row[6], row[0], row[8], row[1], row[2], row[7], row[3], row[4], row[5]) ) sys.stdout.write("\n\n") diff --git a/fido/utils/__init__.py b/fido/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fido/char_handler.py b/fido/utils/char_handler.py similarity index 100% rename from fido/char_handler.py rename to fido/utils/char_handler.py diff --git a/fido/utils/timer.py b/fido/utils/timer.py new file mode 100644 index 00000000..af1cbb19 --- /dev/null +++ b/fido/utils/timer.py @@ -0,0 +1,17 @@ +from time import perf_counter + + +class PerfTimer: + """Utility class that carries out simple process timings.""" + + def __init__(self): + """New instance with start time running.""" + self.start_time = perf_counter() + + def start(self): + """Start new timer.""" + self.start_time = perf_counter() + + def duration(self): + """Return the duration since instantiation or start() was last called.""" + return perf_counter() - self.start_time diff --git a/pyproject.toml b/pyproject.toml index 07ef2c84..90162d4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,9 +45,9 @@ testing = [ [project.scripts] fido = "fido.fido:main" -fido-prepare = "fido.prepare:main" +fido-prepare = "fido.pronom.prepare:main" fido-toxml = "fido.toxml:main" - +fido-update-signatures = "fido.pronom.update_signatures:run" [tool.setuptools.package-data] "fido" = ["*.*", "conf/*.*", "pronom/*.*"] diff --git a/tests/test_fido.py b/tests/test_fido.py index e0c256ec..420dbd58 100644 --- a/tests/test_fido.py +++ b/tests/test_fido.py @@ -7,7 +7,8 @@ import pytest -from fido.fido import Fido, PerfTimer +from fido.fido import Fido +from fido.utils.timer import PerfTimer def test_perf_timer(): diff --git a/tests/test_prepare.py b/tests/test_prepare.py index 752fcd38..10cbbadd 100644 --- a/tests/test_prepare.py +++ b/tests/test_prepare.py @@ -2,7 +2,7 @@ import pytest -from fido.prepare import convert_to_regex +from fido.pronom.prepare import convert_to_regex def binrep_convert(byt): @@ -64,17 +64,17 @@ def test_bitmasks(pronom_bytesequence, matches_predicate): ("pronom_bytesequence", "input_", "matches_bool"), ( # These are good: - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF", True), - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x03\x11\xFF", True), - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\xFE\xFF", True), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xdd\xcd\x02\x11\xff", True), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xdd\xcd\x03\x11\xff", True), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xdd\xcd\x02\xfe\xff", True), # Bad because missing three anythings between AB and CD - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xCD\x02\x11\xFF", False), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xcd\x02\x11\xff", False), # Bad because not at start of string - ("ab{3}cd(01|02|03)~07ff", "\xDA\xAB\xDD\xDD\xDD\xCD\x02\x11\xFF", False), + ("ab{3}cd(01|02|03)~07ff", "\xda\xab\xdd\xdd\xdd\xcd\x02\x11\xff", False), # Bad because 04 is not in (01|02|03) - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x04\x11\xFF", False), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xdd\xcd\x04\x11\xff", False), # Bad because 18 is not in ~07 - ("ab{3}cd(01|02|03)~07ff", "\xAB\xDD\xDD\xDD\xCD\x02\x18\xFF", False), + ("ab{3}cd(01|02|03)~07ff", "\xab\xdd\xdd\xdd\xcd\x02\x18\xff", False), ), ) def test_heterogenous_sequences(pronom_bytesequence, input_, matches_bool): From e0a2fd914a7a2ee842749c5a90713334545ce209 Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 13 Nov 2024 17:44:46 +0000 Subject: [PATCH 23/33] Security enhancements recomended from the Codacy review. The main was to use defusedxml rather than xml.etree. Also modifications in pronom.prepare to avoid using some python builtin identifiers like input, id, zip. --- fido/fido.py | 3 +- fido/pronom/prepare.py | 75 ++++++++++++++++---------------- fido/pronom/soap.py | 30 ++++--------- fido/pronom/update_signatures.py | 2 +- fido/pronom/versions.py | 51 +++++----------------- pyproject.toml | 3 +- 6 files changed, 64 insertions(+), 100 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index d910da65..cb36aea3 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -16,7 +16,8 @@ import zipfile from contextlib import closing from typing import Optional -from xml.etree import cElementTree as ET + +from defusedxml import cElementTree as ET from fido import CONFIG_DIR, __version__ from fido.cli_args import parse_cli_args diff --git a/fido/pronom/prepare.py b/fido/pronom/prepare.py index 929ad9a0..1409c09e 100644 --- a/fido/pronom/prepare.py +++ b/fido/pronom/prepare.py @@ -12,7 +12,8 @@ from urllib.parse import urlparse from urllib.request import urlopen from xml.dom import minidom -from xml.etree import ElementTree as ET + +from defusedxml import ElementTree as ET from fido.pronom.versions import get_local_versions from fido.utils.char_handler import escape @@ -126,10 +127,10 @@ def load_pronom_xml(self, puid_filter=None): """ formats = [] try: - zip = zipfile.ZipFile(self.pronom_files, "r") - for item in zip.infolist(): + pronom_collection = zipfile.ZipFile(self.pronom_files, "r") + for item in pronom_collection.infolist(): try: - stream = zip.open(item) + stream = pronom_collection.open(item) # Work is done here! format_ = self.parse_pronom_xml(stream, puid_filter) if format_ is not None: @@ -138,7 +139,7 @@ def load_pronom_xml(self, puid_filter=None): stream.close() finally: try: - zip.close() + pronom_collection.close() except Exception as e: print( "An error occured loading '{0}' (exception: {1})".format(self.pronom_files, e), @@ -180,20 +181,20 @@ def parse_pronom_xml(self, source, puid_filter=None): pronom_format = pronom_root.find(TNA("report_format_detail/FileFormat")) fido_format = ET.Element("format") # Get the base Format information - for id in pronom_format.findall(TNA("FileFormatIdentifier")): - type = get_text_tna(id, "IdentifierType") - if type == "PUID": - puid = get_text_tna(id, "Identifier") + for xml_id in pronom_format.findall(TNA("FileFormatIdentifier")): + xml_id_type = get_text_tna(xml_id, "IdentifierType") + if xml_id_type == "PUID": + puid = get_text_tna(xml_id, "Identifier") ET.SubElement(fido_format, "puid").text = puid if puid_filter and puid != puid_filter: return None # A bit clumsy. I want to have puid first, then mime, then container. - for id in pronom_format.findall(TNA("FileFormatIdentifier")): - type = get_text_tna(id, "IdentifierType") - if type == "MIME": - ET.SubElement(fido_format, "mime").text = get_text_tna(id, "Identifier") - elif type == "PUID": - puid = get_text_tna(id, "Identifier") + for xml_id in pronom_format.findall(TNA("FileFormatIdentifier")): + xml_id_type = get_text_tna(xml_id, "IdentifierType") + if xml_id_type == "MIME": + ET.SubElement(fido_format, "mime").text = get_text_tna(xml_id, "Identifier") + elif xml_id_type == "PUID": + puid = get_text_tna(xml_id, "Identifier") if puid == "x-fmt/263": ET.SubElement(fido_format, "container").text = "zip" elif puid == "x-fmt/265": @@ -205,10 +206,10 @@ def parse_pronom_xml(self, source, puid_filter=None): # Get the extensions from the ExternalSignature for x in pronom_format.findall(TNA("ExternalSignature")): ET.SubElement(fido_format, "extension").text = get_text_tna(x, "Signature") - for id in pronom_format.findall(TNA("FileFormatIdentifier")): - type = get_text_tna(id, "IdentifierType") - if type == "Apple Uniform Type Identifier": - ET.SubElement(fido_format, "apple_uti").text = get_text_tna(id, "Identifier") + for xml_id in pronom_format.findall(TNA("FileFormatIdentifier")): + xml_id_type = get_text_tna(xml_id, "IdentifierType") + if xml_id_type == "Apple Uniform Type Identifier": + ET.SubElement(fido_format, "apple_uti").text = get_text_tna(xml_id, "Identifier") # Handle the relationships for x in pronom_format.findall(TNA("RelatedFormat")): rel = get_text_tna(x, "RelationshipType") @@ -278,13 +279,13 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(r, "dc:creator").text = get_text_tna(x, "Author/AuthorCompoundName") ET.SubElement(r, "dc:publisher").text = get_text_tna(x, "Publisher/PublisherCompoundName") ET.SubElement(r, "dcterms:available").text = get_text_tna(x, "PublicationDate") - for id in x.findall(TNA("DocumentIdentifier")): - type = get_text_tna(id, "IdentifierType") - if type == "URL": - ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna(id, "Identifier") + for xml_id in x.findall(TNA("DocumentIdentifier")): + xml_id_type = get_text_tna(xml_id, "IdentifierType") + if xml_id_type == "URL": + ET.SubElement(r, "dc:identifier").text = "http://" + get_text_tna(xml_id, "Identifier") else: ET.SubElement(r, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + get_text_tna(xml_id, "IdentifierType") + ":" + get_text_tna(xml_id, "Identifier") ) ET.SubElement(r, "dc:description").text = get_text_tna(x, "DocumentNote") ET.SubElement(r, "dc:type").text = get_text_tna(x, "DocumentType") @@ -298,12 +299,12 @@ def parse_pronom_xml(self, source, puid_filter=None): ET.SubElement(rf, "dc:title").text = get_text_tna(x, "ReferenceFileName") ET.SubElement(rf, "dc:description").text = get_text_tna(x, "ReferenceFileDescription") checksum = "" - for id in x.findall(TNA("ReferenceFileIdentifier")): - type = get_text_tna(id, "IdentifierType") - if type == "URL": + for xml_id in x.findall(TNA("ReferenceFileIdentifier")): + xml_id_type = get_text_tna(xml_id, "IdentifierType") + if xml_id_type == "URL": # Starting with PRONOM 89, some URLs contain http:// # and others do not. - url = get_text_tna(id, "Identifier") + url = get_text_tna(xml_id, "Identifier") if not urlparse(url).scheme: url = "http://" + url ET.SubElement(rf, "dc:identifier").text = url @@ -321,7 +322,7 @@ def parse_pronom_xml(self, source, puid_filter=None): checksum = m.hexdigest() else: ET.SubElement(rf, "dc:identifier").text = ( - get_text_tna(id, "IdentifierType") + ":" + get_text_tna(id, "Identifier") + get_text_tna(xml_id, "IdentifierType") + ":" + get_text_tna(xml_id, "Identifier") ) ET.SubElement(rf, "dcterms:license").text = "" ET.SubElement(rf, "dc:rights").text = get_text_tna(x, "ReferenceFileIPR") @@ -735,18 +736,18 @@ def convert_to_regex(chars, endianness="", pos="BOF", offset="0", maxoffset=""): return val -def run(input=None, output=None, puid=None): +def run(input_file=None, output_file=None, puid=None): """Convert PRONOM formats into FIDO signatures.""" versions = get_local_versions() - if input is None: - input = versions.get_zip_file() - if output is None: - output = versions.get_signature_file() + if input_file is None: + input_file = versions.get_zip_file() + if output_file is None: + output_file = versions.get_signature_file() - info = FormatInfo(input) + info = FormatInfo(input_file) info.load_pronom_xml(puid) - info.save(output) + info.save(output_file) print( "Converted {0} PRONOM formats to FIDO signatures".format(len(info.formats)), file=sys.stderr, @@ -764,7 +765,7 @@ def main(args=None): parser.add_argument("-puid", default=None, help="A particular PUID record to extract") args = parser.parse_args(args) - run(input=args.input, output=args.output, puid=args.puid) + run(input_file=args.input, output_file=args.output, puid=args.puid) if __name__ == "__main__": diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index 67d2a734..af714e32 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -19,11 +19,13 @@ PRONOM format signatures SOAP calls. """ + import sys import urllib -import xml.etree.ElementTree as ET from urllib.error import HTTPError, URLError +import defusedxml.ElementTree as ET + from fido import __version__ ENCODING = "utf-8" @@ -50,9 +52,7 @@ def get_sig_xml_for_puid(puid): """Return the full PRONOM signature XML for the passed PUID.""" - req = urllib.request.Request( - "http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid) - ) + req = urllib.request.Request("http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid)) response = urllib.request.urlopen(req) xml = response.read() return xml @@ -82,16 +82,12 @@ def get_droid_signatures(version): format_count = False try: with urllib.request.urlopen( - "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format( - version - ) + "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format(version) ) as f: xml = f.read().decode("utf-8") root_ele = ET.fromstring(xml) format_count = len( - root_ele.findall( - ".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat" - ) + root_ele.findall(".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat") ) except HTTPError as httpe: sys.stderr.write( @@ -105,9 +101,7 @@ def get_droid_signatures(version): def _get_soap_ele_tree(soap_action): soap_string = '{}<{} xmlns="{}" />'.format( XML_PROC, NS.get("xsi"), NS.get("xsd"), NS.get("soap"), soap_action, PRONOM_NS - ).encode( - ENCODING - ) + ).encode(ENCODING) soap_action = '"{}:{}In"'.format(PRONOM_NS, soap_action) xml = _get_soap_response(soap_action, soap_string) for prefix, uri in NS.items(): @@ -117,15 +111,9 @@ def _get_soap_ele_tree(soap_action): def _get_soap_response(soap_action, soap_string): try: - req = urllib.request.Request( - "http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string - ) + req = urllib.request.Request("http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string) except URLError: - print( - "There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format( - PRONOM_HOST - ) - ) + print("There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format(PRONOM_HOST)) print("Please check your network connection and try again.") sys.exit(1) for key, value in HEADERS.items(): diff --git a/fido/pronom/update_signatures.py b/fido/pronom/update_signatures.py index 93432b95..a99a49a3 100644 --- a/fido/pronom/update_signatures.py +++ b/fido/pronom/update_signatures.py @@ -19,8 +19,8 @@ import zipfile from argparse import ArgumentParser from shutil import rmtree -from xml.etree import ElementTree as CET +from defusedxml import ElementTree as CET from pronom.prepare import run as prepare_pronom_to_fido from fido import CONFIG_DIR, __version__ diff --git a/fido/pronom/versions.py b/fido/pronom/versions.py index 55fa2202..1b3f529b 100644 --- a/fido/pronom/versions.py +++ b/fido/pronom/versions.py @@ -17,15 +17,14 @@ PRONOM is available from http://www.nationalarchives.gov.uk/pronom/ """ - import importlib.resources import os import re import sys -from xml.etree import ElementTree as ET -from xml.etree.ElementTree import ParseError, parse import requests +from defusedxml.ElementTree import ElementTree as ET +from defusedxml.ElementTree import ParseError, parse from fido import CONFIG_DIR @@ -87,9 +86,7 @@ def __setattr__(self, name, value): def get_zip_file(self): """Obtain location to the PRONOM XML Zip file based on the current PRONOM version.""" - return os.path.join( - self.conf_dir, "pronom-xml-v{}.zip".format(self.pronom_version) - ) + return os.path.join(self.conf_dir, "pronom-xml-v{}.zip".format(self.pronom_version)) def get_signature_file(self): """Obtain location to the current PRONOM signature file.""" @@ -101,9 +98,7 @@ def write(self): for key, value in self.PROPS_MAPPING.items(): if self.root.find(value) is None: raise ValueError("Field {} has not been defined!".format(key)) - self.tree.write( - self.versions_file, xml_declaration=True, method="xml", encoding="utf-8" - ) + self.tree.write(self.versions_file, xml_declaration=True, method="xml", encoding="utf-8") def get_local_versions(config_dir=CONFIG_DIR): @@ -147,19 +142,11 @@ def _list_available_versions(update_url): def _check_update_signatures(sig_vers, update_url, versions, is_update=False): is_new, latest = _version_check(sig_vers, update_url) if is_new: - sys.stdout.write( - "Updated signatures v{} are available, current version is v{}\n".format( - latest, sig_vers - ) - ) + sys.stdout.write("Updated signatures v{} are available, current version is v{}\n".format(latest, sig_vers)) if is_update: _output_details(latest, update_url, versions) else: - sys.stdout.write( - "Your signature files are up to date, current version is v{}\n".format( - sig_vers - ) - ) + sys.stdout.write("Your signature files are up to date, current version is v{}\n".format(sig_vers)) sys.exit(0) @@ -169,23 +156,15 @@ def _download_sig_version(sig_act, update_url, versions): if not match: sys.exit( - '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format( - sig_act - ) + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(sig_act) ) ver = sig_act if not ver.startswith("v"): ver = "v" + sig_act resp = requests.get(update_url + "format/" + ver + "/") if resp.status_code != 200: - sys.exit( - "No signature files found for {}, REST status {}".format( - sig_act, resp.status_code - ) - ) - _output_details( - re.search(r"\d+|$", ver).group(), update_url, versions - ) # noqa: W605 + sys.exit("No signature files found for {}, REST status {}".format(sig_act, resp.status_code)) + _output_details(re.search(r"\d+|$", ver).group(), update_url, versions) # noqa: W605 def _get_version(ver_string): @@ -193,9 +172,7 @@ def _get_version(ver_string): match = re.search(r"^v?(\d+)$", ver_string, re.IGNORECASE) if not match: sys.exit( - '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format( - ver_string - ) + '{} is not a valid version number, to download a sig file try "-sig v104" or "-sig 104".'.format(ver_string) ) ver = ver_string return ver_string if not ver.startswith("v") else ver_string[1:] @@ -214,18 +191,14 @@ def _output_details(version, update_url, versions): def _version_check(sig_ver, update_url): resp = requests.get(update_url + "format/latest/") if resp.status_code != 200: - sys.exit( - "Error getting latest version info: HTTP Status {}".format(resp.status_code) - ) + sys.exit("Error getting latest version info: HTTP Status {}".format(resp.status_code)) root_ele = ET.fromstring(resp.text) latest = _get_version(root_ele.get("version")) return int(latest) > int(sig_ver), latest def _write_sigs(latest, update_url, type, name_template): - sig_out = str( - importlib.resources.files("fido").joinpath("conf", name_template.format(latest)) - ) + sig_out = str(importlib.resources.files("fido").joinpath("conf", name_template.format(latest))) if os.path.exists(sig_out): return resp = requests.get(update_url + "format/{0}/{1}/".format(latest, type)) diff --git a/pyproject.toml b/pyproject.toml index 90162d4b..dc306e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,8 @@ classifiers = [ dependencies = [ "olefile >= 0.46, < 1", - "requests", + "requests >= 2", + "defusedxml >= 0.7" ] [project.urls] From f2517060739a62a0a19ea3c508fdead4027c03dd Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 13 Nov 2024 17:44:46 +0000 Subject: [PATCH 24/33] Security enhancements recommended from the Codacy review. The main was to use defusedxml rather than xml.etree. Also made modifications in pronom.prepare to avoid using python builtin identifiers input, id, zip. --- fido/fido.py | 2 +- fido/pronom/soap.py | 6 +++--- fido/pronom/versions.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index cb36aea3..6e2abace 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -17,7 +17,7 @@ from contextlib import closing from typing import Optional -from defusedxml import cElementTree as ET +from defusedxml import ElementTree as ET from fido import CONFIG_DIR, __version__ from fido.cli_args import parse_cli_args diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index af714e32..a9ec649b 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -24,7 +24,7 @@ import urllib from urllib.error import HTTPError, URLError -import defusedxml.ElementTree as ET +from defusedxml.ElementTree import fromstring from fido import __version__ @@ -85,7 +85,7 @@ def get_droid_signatures(version): "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format(version) ) as f: xml = f.read().decode("utf-8") - root_ele = ET.fromstring(xml) + root_ele = fromstring(xml) format_count = len( root_ele.findall(".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat") ) @@ -106,7 +106,7 @@ def _get_soap_ele_tree(soap_action): xml = _get_soap_response(soap_action, soap_string) for prefix, uri in NS.items(): ET.register_namespace(prefix, uri) - return ET.fromstring(xml) + return fromstring(xml) def _get_soap_response(soap_action, soap_string): diff --git a/fido/pronom/versions.py b/fido/pronom/versions.py index 1b3f529b..25843d23 100644 --- a/fido/pronom/versions.py +++ b/fido/pronom/versions.py @@ -23,7 +23,7 @@ import sys import requests -from defusedxml.ElementTree import ElementTree as ET +from defusedxml import ElementTree as ET from defusedxml.ElementTree import ParseError, parse from fido import CONFIG_DIR From f7631a367a382dc2ab2958740b8e455d3a12141f Mon Sep 17 00:00:00 2001 From: Adam Farquhar Date: Wed, 13 Nov 2024 17:44:46 +0000 Subject: [PATCH 25/33] Security enhancements recommended from the Codacy review. The main was to use defusedxml rather than xml.etree. Also made modifications in pronom.prepare to avoid using python builtin identifiers input, id, zip. --- fido/fido.py | 2 +- fido/pronom/soap.py | 7 ++++--- fido/pronom/versions.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fido/fido.py b/fido/fido.py index cb36aea3..6e2abace 100755 --- a/fido/fido.py +++ b/fido/fido.py @@ -17,7 +17,7 @@ from contextlib import closing from typing import Optional -from defusedxml import cElementTree as ET +from defusedxml import ElementTree as ET from fido import CONFIG_DIR, __version__ from fido.cli_args import parse_cli_args diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index af714e32..49ac7070 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -23,8 +23,9 @@ import sys import urllib from urllib.error import HTTPError, URLError +from xml.etree import ElementTree as ET -import defusedxml.ElementTree as ET +from defusedxml.ElementTree import fromstring from fido import __version__ @@ -85,7 +86,7 @@ def get_droid_signatures(version): "https://www.nationalarchives.gov.uk/documents/DROID_SignatureFile_V{}.xml".format(version) ) as f: xml = f.read().decode("utf-8") - root_ele = ET.fromstring(xml) + root_ele = fromstring(xml) format_count = len( root_ele.findall(".//{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat") ) @@ -106,7 +107,7 @@ def _get_soap_ele_tree(soap_action): xml = _get_soap_response(soap_action, soap_string) for prefix, uri in NS.items(): ET.register_namespace(prefix, uri) - return ET.fromstring(xml) + return fromstring(xml) def _get_soap_response(soap_action, soap_string): diff --git a/fido/pronom/versions.py b/fido/pronom/versions.py index 1b3f529b..25843d23 100644 --- a/fido/pronom/versions.py +++ b/fido/pronom/versions.py @@ -23,7 +23,7 @@ import sys import requests -from defusedxml.ElementTree import ElementTree as ET +from defusedxml import ElementTree as ET from defusedxml.ElementTree import ParseError, parse from fido import CONFIG_DIR From 12782a23f77a0587f617e96fef6558ab287af1e1 Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Thu, 11 Jun 2026 22:06:32 +0200 Subject: [PATCH 26/33] Move Makefile to attic --- Makefile => .attic/Makefile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Makefile => .attic/Makefile (100%) diff --git a/Makefile b/.attic/Makefile similarity index 100% rename from Makefile rename to .attic/Makefile From 5d8175ae144becdeb8af012978077a56aa71ef0b Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Thu, 11 Jun 2026 22:10:00 +0200 Subject: [PATCH 27/33] Modification of dev docs --- .github/workflows/test-pr.yml | 7 ++---- CONTRIBUTING.md | 5 +--- README.md | 25 +++++++++++--------- fido/__init__.py | 2 +- pyproject.toml | 44 +++++++++++++++++++---------------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index beccf523..7d729d49 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: [3.8, 3.9, 3.10] steps: - uses: actions/checkout@v3 @@ -25,10 +25,7 @@ jobs: pip install -U flake8 pep257 pytest-cov codecov codacy-coverage pluggy pip install -e . - name: Lint code with flake8 - run: flake8 . --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=127 --statistics - - name: Lint code with pep257 - if: matrix.python-version == 2.7 - run: pep257 --match="(?!fido).*\.py" ./fido + run: flake8 . --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=120 --statistics - name: Test using pytest run: pytest --cov=fido - name: Generate LCOV coverage report diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08f1bbbf..1e295f8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,6 @@ The checklist below contains some of our expectations, and will help you create - New code should contributions should adhere to the [PEP 8 -- Style Guide for Python Code](https://www.python.org/dev/peps/pep-0008/) and [PEP 257 -- Docstring Conventions](https://www.python.org/dev/peps/pep-0257/). - Linebreaks should be used to limit line length within reason; we do not strictly enforce the 80-character line limit of PEP 8. - Non-trivial changes should be accompanied by corresponding unit tests. -- FIDO runs on Python 2 and 3 (specifically versions 2.7, 3.4 and 3.5); changes must preserve this 2/3 compatibility. - A pull request should resolve an existing GitHub issue and the name of its git branch should reference that issue by using the following naming convention: `dev/issue--short-description`, e.g., `dev/issue-126-add-contributing-doc`. - Git commits should be of a manageable size and should introduce one logical change; git commit messages should adhere to the [seven rules of a great Git commit message](https://chris.beams.io/posts/git-commit/): - Separate subject from body with a blank line @@ -55,12 +54,10 @@ The checklist below contains some of our expectations, and will help you create - Wrap the body at 72 characters - Use the body to explain what and why vs. how -FIDO's Travis Continuous Integration configuration runs `pytest` to execute the tests, `flake8` to check PEP 8 conformance, and `pep257` to check PEP 257 (docstring) conformance. You should run these tools locally before pushing a commit by running the following commands: - $ python setup.py test + $ pytest $ flake8 --ignore=E501 ./fido - $ pep257 --match='(?!fido).*\.py' ./fido ## Code Review & Approval diff --git a/README.md b/README.md index 80f90a8b..b3c82086 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Format Identification for Digital Objects (fido) By [Open Preservation Foundation](http://www.openpreservation.org) -[![Build Status](https://travis-ci.org/openpreserve/fido.svg?branch=master)](https://travis-ci.org/openpreserve/fido) [![Code Coverage](https://codecov.io/gh/openpreserve/fido/branch/master/graph/badge.svg)](https://codecov.io/gh/openpreserve/fido) +[![Code Coverage](https://codecov.io/gh/openpreserve/fido/branch/master/graph/badge.svg)](https://codecov.io/gh/openpreserve/fido) FIDO is a command-line tool to identify the file formats of digital objects. It is designed for simple integration into automated work-flows. @@ -69,8 +69,9 @@ Any platform 1. Download the latest zip release from 2. Unzip into some directory 3. Open a command shell, cd to the directory that you placed the zip contents into -4. Run `python setup.py install` to install FIDO and dependencies. This may require sudo on Linux/OSX or admin privileges on Windows. -5. You should now be able to see the help text: +4. Create and activate local Python Environment in this directory. +5. Run `python -m pip install .` to install FIDO and dependencies. This may require sudo on Linux/OSX or admin privileges on Windows. +6. You should now be able to see the help text: `fido -h` Using pip @@ -137,6 +138,8 @@ FIDO 1.3.3 and later have experimental Python 3 support. FIDO 1.4 and later have Python 3 support. +FIDO 2.0 and later use pyproject Configuration for Installation and Development. + Format Definitions ------------------ @@ -185,10 +188,10 @@ Examples running FIDO Identify all files in the current directory and below, sending output into file-info.csv: - `python fido.py -recurse . > file-info.csv` + `fido -recurse . > file-info.csv` Do the same as above, but also look inside of zip or tar files: - `python fido.py -recurse -zip . > file-info.csv` + `fido -recurse -zip . > file-info.csv` Take input from a list of files: @@ -196,29 +199,29 @@ Linux: ```shell ls > files.txt -python fido.py -input files.txt +fido -input files.txt ``` Windows: ```shell dir /b > files.txt -python fido.py -input files.txt +fido -input files.txt ``` Take input from a pipe: Linux: - `find . -type f | python fido.py -input -` + `find . -type f | fido -input -` Windows: - `dir /b | python fido.py -input -` + `dir /b | fido -input -` Only show files that could not be identified: - `python fido.py -matchprintf "" .` + `fido -matchprintf "" .` Only show files that could be identified: - `python fido.py -nomatchprintf "" .` + `fido -nomatchprintf "" .` Deep scan of container objects ------------------------------ diff --git a/fido/__init__.py b/fido/__init__.py index 04246684..a4c1f9ef 100644 --- a/fido/__init__.py +++ b/fido/__init__.py @@ -7,7 +7,7 @@ It is designed for simple integration into automated work-flows. """ -__version__ = "1.8.0dev" +__version__ = "2.0.0-dev" # todo: move this to a conf/conf.py or something rather than init.py. Would require some cascading updates, though from os.path import abspath, dirname, join diff --git a/pyproject.toml b/pyproject.toml index dc306e9e..463e85e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,27 @@ [build-system] -requires = ["setuptools>=42", "wheel", "twine", "setuptools-git-versioning>=2.0,<3"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] name = "opf-fido" dynamic = ["version"] requires-python = ">= 3.8" -description = """ -Format Identification for Digital Objects (FIDO). -A command-line tool to identify the file formats of digital objects. -FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions. -""" +description = "Format Identification for Digital Objects (FIDO)." readme = "README.md" authors = [ - { name="Adam Farquhar (BL)" } # Add email if available + { name = "Adam Farquhar (BL)" } # Add email if available ] -license = { file = "LICENSE.txt" } +license = "Apache-2.0" +license-files = ["LICENSE.txt"] classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11" ] dependencies = [ @@ -35,14 +31,20 @@ dependencies = [ ] [project.urls] -homepage = "http://openpreservation.org/technology/products/fido/" +Homepage = "https://openpreservation.org/technology/products/fido/" +Repository = "https://github.com/openpreserve/fido" +Issues = "https://github.com/openpreserve/fido/issues" [project.optional-dependencies] -testing = [ - "pytest", +dev = [ "pytest-cov", "flake8", ] +build = [ + "build", + "twine", + "ruff", + ] [project.scripts] fido = "fido.fido:main" @@ -50,16 +52,18 @@ fido-prepare = "fido.pronom.prepare:main" fido-toxml = "fido.toxml:main" fido-update-signatures = "fido.pronom.update_signatures:run" +[tool.setuptools] +include-package-data = true + +[tool.setuptools.dynamic] +version = { attr = "fido.__version__" } + +[tool.setuptools.packages.find] +include = ["fido*"] + [tool.setuptools.package-data] "fido" = ["*.*", "conf/*.*", "pronom/*.*"] -[tool.setuptools-git-versioning] -enabled = true -# version_file = "VERSION" -# count_commits_from_version_file = true -# dev_template = "{tag}.{branch}{ccount}" # <--- note {branch} here -# dirty_template = "{tag}.{branch}{ccount}" - [tool.pytest.ini_options] addopts = "--maxfail=1 --strict-markers" From 897ee7dc0efec58214b394ca569d9640abadb036 Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Thu, 11 Jun 2026 22:27:10 +0200 Subject: [PATCH 28/33] Fix https protocoll --- fido/pronom/soap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fido/pronom/soap.py b/fido/pronom/soap.py index 49ac7070..aa1ce4a7 100644 --- a/fido/pronom/soap.py +++ b/fido/pronom/soap.py @@ -112,7 +112,7 @@ def _get_soap_ele_tree(soap_action): def _get_soap_response(soap_action, soap_string): try: - req = urllib.request.Request("http://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string) + req = urllib.request.Request("https://{}/pronom/service.asmx".format(PRONOM_HOST), data=soap_string) except URLError: print("There was a problem contacting the PRONOM service at http://{}/pronom/service.asmx.".format(PRONOM_HOST)) print("Please check your network connection and try again.") From 823e4ebba64d4523d8b0c60e2aab60ceb01ccc96 Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Thu, 11 Jun 2026 22:27:49 +0200 Subject: [PATCH 29/33] Mocking SOAP calls --- tests/pronom/test_soap.py | 128 ++++++++++++++++++++++++++++++-------- 1 file changed, 102 insertions(+), 26 deletions(-) diff --git a/tests/pronom/test_soap.py b/tests/pronom/test_soap.py index cb6509db..f28a6ecd 100644 --- a/tests/pronom/test_soap.py +++ b/tests/pronom/test_soap.py @@ -1,36 +1,112 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -FIDO: Format Identifier for Digital Objects +"""Offline contract tests for PRONOM SOAP helpers.""" + +import unittest.mock +import urllib.error + +import pytest + +import fido.pronom + + +class FakeResponse: + """Small response shim used to mock urllib responses.""" + + def __init__(self, payload): + self.payload = payload + + def read(self): + return self.payload + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +class FakeRequest: + """Simple urllib Request shim with header support.""" + + def __init__(self, url, data=None): + self.url = url + self.data = data + self.headers = {} -Copyright 2010 The Open Preservation Foundation + def add_header(self, key, value): + self.headers[key] = value -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +@unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) +def test_pronom_version_from_mocked_soap_response(mock_request): + """Parse signature version from a mocked SOAP payload.""" + xml = b""" + + + + + 116 + + + + +""" + + mock_request.Request.side_effect = lambda url, data=None: FakeRequest(url, data=data) + mock_request.urlopen.return_value = FakeResponse(xml) -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + assert fido.pronom.soap.get_pronom_sig_version() == 116 -PRONOM SOAP call test cases. + +@unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) +def test_get_droid_signatures_counts_file_formats(mock_request): + """Count FileFormat elements from a mocked DROID signature XML file.""" + sig_xml = b""" + + + + + + """ -from fido.pronom import soap + mock_request.urlopen.return_value = FakeResponse(sig_xml) + + xml, count = fido.pronom.soap.get_droid_signatures(116) + assert "SignatureFile" in xml + assert count == 2 + + +@unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) +def test_get_droid_signatures_handles_http_error(mock_request, capsys): + """Return fallback values and log an error when download fails.""" + + mock_request.urlopen.side_effect = urllib.error.HTTPError(url="http://example.test", code=500, msg="boom", hdrs=None, fp=None) + + xml, count = fido.pronom.soap.get_droid_signatures(116) + captured = capsys.readouterr() + assert xml == [] + assert count is False + assert "could not download signature file v116" in captured.err + + +@unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) +def test_get_sig_xml_for_puid_returns_raw_xml(mock_request): + """Return unmodified PRONOM XML bytes for a PUID.""" + payload = b"bar" + mock_request.urlopen.return_value = FakeResponse(payload) + + assert fido.pronom.soap.get_sig_xml_for_puid("fmt/18") == payload + + +@unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) +def test_get_soap_response_exits_on_request_error(mock_request, capsys): + """Exit with code 1 when creating the request fails.""" -def test_pronom_version(): - """Test that the returned PRONOM sig version is an integer > 90.""" - version = soap.get_pronom_sig_version() - assert version > 90 + mock_request.Request.side_effect = urllib.error.URLError("offline") + with pytest.raises(SystemExit) as exc: + fido.pronom.soap._get_soap_response('"action"', b"") -def test_pronom_signature(): - """Test that retrieving signatures gets something with length and no errors are thrown.""" - version = soap.get_pronom_sig_version() - xml, count = soap.get_droid_signatures(version) - assert len(xml) > 1000, "Expected more than 1000 XML lines, got %s" % len(xml) - assert count > 1000, "Expected more than 1000 signatures, got %s" % count + captured = capsys.readouterr() + assert exc.value.code == 1 + assert "There was a problem contacting the PRONOM service" in captured.out From e6825a5aea7abe8c76e72c020d494f390885123f Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Mon, 15 Jun 2026 21:37:50 +0200 Subject: [PATCH 30/33] Extend overall test coverage --- tests/pronom/test_update_signatures.py | 378 +++++++++++++++++ tests/pronom/test_versions.py | 365 ++++++++++++++++ tests/test_fido.py | 559 +++++++++++++++++++++++++ tests/test_package.py | 86 +++- tests/test_prepare.py | 458 ++++++++++++++++++++ tests/test_toxml.py | 43 ++ 6 files changed, 1888 insertions(+), 1 deletion(-) create mode 100644 tests/pronom/test_update_signatures.py create mode 100644 tests/pronom/test_versions.py create mode 100644 tests/test_toxml.py diff --git a/tests/pronom/test_update_signatures.py b/tests/pronom/test_update_signatures.py new file mode 100644 index 00000000..37408596 --- /dev/null +++ b/tests/pronom/test_update_signatures.py @@ -0,0 +1,378 @@ +import importlib +import os +import sys +from types import ModuleType, SimpleNamespace + +import pytest + + +@pytest.fixture +def update_signatures_module(monkeypatch): + fake_pronom = ModuleType("pronom") + fake_prepare = ModuleType("pronom.prepare") + fake_prepare.run = lambda: None + fake_pronom.prepare = fake_prepare + + monkeypatch.setitem(sys.modules, "pronom", fake_pronom) + monkeypatch.setitem(sys.modules, "pronom.prepare", fake_prepare) + module = importlib.import_module("fido.pronom.update_signatures") + return importlib.reload(module) + + +def test_get_puid_file_name(update_signatures_module): + element = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + + puid, filename = update_signatures_module.get_puid_file_name(element) + + assert puid == "fmt/18" + assert filename == "puid.fmt.18.xml" + + +def test_sig_version_check_latest_uses_service(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr(update_signatures_module, "get_pronom_sig_version", lambda: 116) + monkeypatch.setattr(update_signatures_module.os.path, "isfile", lambda _p: False) + + version, sig_file_name = update_signatures_module.sig_version_check("latest") + + assert version == 116 + assert sig_file_name.endswith("DROID_SignatureFile-v116.xml") + + +def test_sig_version_check_exits_when_existing_file_not_confirmed(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr(update_signatures_module.os.path, "isfile", lambda _p: True) + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: False) + + with pytest.raises(SystemExit) as exc: + update_signatures_module.sig_version_check("116") + + assert str(exc.value) == update_signatures_module.ABORT_MSG + + +def test_sig_version_check_allows_existing_file_when_confirmed(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr(update_signatures_module.os.path, "isfile", lambda _p: True) + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: True) + + version, sig_file_name = update_signatures_module.sig_version_check("116") + + assert version == "116" + assert sig_file_name.endswith("DROID_SignatureFile-v116.xml") + + +def test_download_sig_file_writes_xml(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "get_droid_signatures", lambda _v: ("", 1)) + out = tmp_path / "sig.xml" + + update_signatures_module.download_sig_file(116, str(out)) + + assert out.read_text(encoding="utf-8") == "" + + +def test_download_sig_file_exits_when_service_returns_empty(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "get_droid_signatures", lambda _v: ("", False)) + + with pytest.raises(SystemExit): + update_signatures_module.download_sig_file(116, str(tmp_path / "sig.xml")) + + +def test_update_versions_xml_assigns_expected_values(update_signatures_module, monkeypatch): + calls = {"write": 0} + versions = SimpleNamespace( + pronom_version=None, + pronom_signature=None, + pronom_container_signature=None, + fido_extension_signature=None, + update_script=None, + ) + versions.write = lambda: calls.__setitem__("write", calls["write"] + 1) + monkeypatch.setattr(update_signatures_module, "get_local_versions", lambda: versions) + + update_signatures_module.update_versions_xml(117) + + assert versions.pronom_version == "117" + assert versions.pronom_signature == "formats-v117.xml" + assert versions.pronom_container_signature == update_signatures_module.DEFAULTS["containerVersion"] + assert versions.fido_extension_signature == update_signatures_module.DEFAULTS["fidoSignatureVersion"] + assert versions.update_script == update_signatures_module.__version__ + assert calls["write"] == 1 + + +@pytest.mark.parametrize( + "default, user_inputs, expected", + [ + ("yes", [""], True), + ("no", [""], False), + (None, ["", "y"], True), + ], +) +def test_query_yes_no_handles_defaults(update_signatures_module, monkeypatch, default, user_inputs, expected): + answers = iter(user_inputs) + monkeypatch.setattr("builtins.input", lambda: next(answers)) + + assert update_signatures_module.query_yes_no("Proceed?", default=default) is expected + + +def test_query_yes_no_rejects_invalid_default(update_signatures_module): + with pytest.raises(ValueError): + update_signatures_module.query_yes_no("Proceed?", default="maybe") + + +def test_sig_version_check_exits_when_latest_lookup_fails(update_signatures_module, monkeypatch): + monkeypatch.setattr(update_signatures_module, "get_pronom_sig_version", lambda: None) + + with pytest.raises(SystemExit) as exc: + update_signatures_module.sig_version_check("latest") + + assert "Failed to obtain PRONOM signature file version number" in str(exc.value) + + +def test_init_sig_download_creates_directory(update_signatures_module, monkeypatch, tmp_path): + target = tmp_path / "tmp_download" + prompts = iter([True]) + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: next(prompts)) + + tmpdir, resume = update_signatures_module.init_sig_download({"tmp_dir": str(target)}) + + assert tmpdir == str(target) + assert resume is False + assert target.is_dir() + + +def test_init_sig_download_exits_when_user_declines(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: False) + + with pytest.raises(SystemExit) as exc: + update_signatures_module.init_sig_download({"tmp_dir": str(tmp_path / "nope")}) + + assert str(exc.value) == update_signatures_module.ABORT_MSG + + +def test_init_sig_download_resumes_existing_directory(update_signatures_module, monkeypatch, tmp_path): + target = tmp_path / "tmp_download" + target.mkdir() + prompts = iter([True, True]) + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: next(prompts)) + + tmpdir, resume = update_signatures_module.init_sig_download({"tmp_dir": str(target)}) + + assert tmpdir == str(target) + assert resume is True + + +def test_init_sig_download_start_over_existing_directory(update_signatures_module, monkeypatch, tmp_path): + target = tmp_path / "tmp_download" + target.mkdir() + prompts = iter([True, False]) + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: next(prompts)) + + tmpdir, resume = update_signatures_module.init_sig_download({"tmp_dir": str(target)}) + + assert tmpdir == str(target) + assert resume is False + + +def test_init_sig_download_reports_failed_mkdir(update_signatures_module, monkeypatch, capsys, tmp_path): + target = tmp_path / "tmp_download" + monkeypatch.setattr(update_signatures_module, "query_yes_no", lambda _q: True) + monkeypatch.setattr(update_signatures_module.os, "mkdir", lambda _p: (_ for _ in ()).throw(OSError("fail"))) + monkeypatch.setattr(update_signatures_module.os.path, "isdir", lambda _p: False) + + tmpdir, resume = update_signatures_module.init_sig_download({"tmp_dir": str(target)}) + + assert tmpdir == str(target) + assert resume is False + assert "Failed to create temporary folder for PUID's" in capsys.readouterr().err + + +def test_download_sig_skips_existing_when_resuming(update_signatures_module, monkeypatch, tmp_path): + called = {"download": 0} + format_ele = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + existing = tmp_path / "puid.fmt.18.xml" + existing.write_text("already", encoding="utf-8") + + monkeypatch.setattr(update_signatures_module, "get_sig_xml_for_puid", lambda _p: called.__setitem__("download", 1)) + + update_signatures_module.download_sig(format_ele, str(tmp_path), True, {"http_throttle": 0.0}) + + assert called["download"] == 0 + assert existing.read_text(encoding="utf-8") == "already" + + +def test_download_sig_exits_on_download_error(update_signatures_module, monkeypatch, tmp_path): + format_ele = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + monkeypatch.setattr(update_signatures_module, "get_sig_xml_for_puid", lambda _p: (_ for _ in ()).throw(RuntimeError("x"))) + + with pytest.raises(SystemExit) as exc: + update_signatures_module.download_sig(format_ele, str(tmp_path), False, {"http_throttle": 0.0}) + + assert "Please restart and resume download" in str(exc.value) + + +def test_download_sig_writes_xml_and_sleeps(update_signatures_module, monkeypatch, tmp_path): + format_ele = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + calls = {"sleep": 0} + monkeypatch.setattr(update_signatures_module, "get_sig_xml_for_puid", lambda _p: b"") + monkeypatch.setattr(update_signatures_module.time, "sleep", lambda _s: calls.__setitem__("sleep", calls["sleep"] + 1)) + + update_signatures_module.download_sig(format_ele, str(tmp_path), False, {"http_throttle": 0.0}) + + assert (tmp_path / "puid.fmt.18.xml").read_bytes() == b"" + assert calls["sleep"] == 1 + + +def test_create_zip_file_honors_keep_temp_option(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + + format_ele = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + source = tmp_path / "puid.fmt.18.xml" + source.write_bytes(b"") + + update_signatures_module.create_zip_file( + {"deleteTempDirectory": False}, + [format_ele], + "117", + str(tmp_path), + ) + + archive = tmp_path / "pronom-xml-v117.zip" + assert archive.exists() + assert source.exists() + + +def test_create_zip_file_deletes_temp_files_when_configured(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + + format_ele = SimpleNamespace(get=lambda key: "fmt/18" if key == "PUID" else None) + source = tmp_path / "puid.fmt.18.xml" + source.write_bytes(b"") + + update_signatures_module.create_zip_file( + {"deleteTempDirectory": True}, + [format_ele], + "117", + str(tmp_path), + ) + + assert (tmp_path / "pronom-xml-v117.zip").exists() + assert not source.exists() + + +def test_create_zip_file_skips_missing_temp_file(update_signatures_module, monkeypatch, tmp_path): + monkeypatch.setattr(update_signatures_module, "CONFIG_DIR", str(tmp_path)) + format_ele = SimpleNamespace(get=lambda key: "fmt/404" if key == "PUID" else None) + + update_signatures_module.create_zip_file( + {"deleteTempDirectory": True}, + [format_ele], + "117", + str(tmp_path), + ) + + assert (tmp_path / "pronom-xml-v117.zip").exists() + + +def test_run_calls_expected_pipeline(update_signatures_module, monkeypatch): + calls = [] + format_eles = [SimpleNamespace(get=lambda key: "fmt/1" if key == "PUID" else None)] + + monkeypatch.setattr(update_signatures_module, "sig_version_check", lambda _v: (calls.append("sig_version_check") or ("116", "sig.xml"))) + monkeypatch.setattr(update_signatures_module, "download_sig_file", lambda _v, _s: calls.append("download_sig_file")) + monkeypatch.setattr( + update_signatures_module.CET, + "parse", + lambda _sig: SimpleNamespace(findall=lambda _path, _ns: format_eles), + ) + monkeypatch.setattr(update_signatures_module, "init_sig_download", lambda _d: (calls.append("init_sig_download") or ("tmp", False))) + monkeypatch.setattr(update_signatures_module, "download_signatures", lambda *_a: calls.append("download_signatures")) + monkeypatch.setattr(update_signatures_module, "create_zip_file", lambda *_a: calls.append("create_zip_file")) + monkeypatch.setattr(update_signatures_module, "rmtree", lambda *_a, **_k: calls.append("rmtree")) + monkeypatch.setattr(update_signatures_module, "update_versions_xml", lambda _v: calls.append("update_versions_xml")) + monkeypatch.setattr(update_signatures_module, "prepare_pronom_to_fido", lambda: calls.append("prepare_pronom_to_fido")) + + update_signatures_module.run({"version": "latest", "deleteTempDirectory": True}) + + assert calls == [ + "sig_version_check", + "download_sig_file", + "init_sig_download", + "download_signatures", + "create_zip_file", + "rmtree", + "update_versions_xml", + "prepare_pronom_to_fido", + ] + + +def test_run_skips_rmtree_when_delete_disabled(update_signatures_module, monkeypatch): + calls = [] + format_eles = [SimpleNamespace(get=lambda key: "fmt/1" if key == "PUID" else None)] + + monkeypatch.setattr(update_signatures_module, "sig_version_check", lambda _v: (calls.append("sig_version_check") or ("116", "sig.xml"))) + monkeypatch.setattr(update_signatures_module, "download_sig_file", lambda _v, _s: calls.append("download_sig_file")) + monkeypatch.setattr( + update_signatures_module.CET, + "parse", + lambda _sig: SimpleNamespace(findall=lambda _path, _ns: format_eles), + ) + monkeypatch.setattr(update_signatures_module, "init_sig_download", lambda _d: (calls.append("init_sig_download") or ("tmp", False))) + monkeypatch.setattr(update_signatures_module, "download_signatures", lambda *_a: calls.append("download_signatures")) + monkeypatch.setattr(update_signatures_module, "create_zip_file", lambda *_a: calls.append("create_zip_file")) + monkeypatch.setattr(update_signatures_module, "rmtree", lambda *_a, **_k: calls.append("rmtree")) + monkeypatch.setattr(update_signatures_module, "update_versions_xml", lambda _v: calls.append("update_versions_xml")) + monkeypatch.setattr(update_signatures_module, "prepare_pronom_to_fido", lambda: calls.append("prepare_pronom_to_fido")) + + update_signatures_module.run({"version": "latest", "deleteTempDirectory": False}) + + assert "rmtree" not in calls + + +def test_run_exits_on_keyboard_interrupt(update_signatures_module, monkeypatch): + monkeypatch.setattr(update_signatures_module, "sig_version_check", lambda _v: (_ for _ in ()).throw(KeyboardInterrupt())) + + with pytest.raises(SystemExit) as exc: + update_signatures_module.run({"version": "latest", "deleteTempDirectory": True}) + + assert str(exc.value) == update_signatures_module.ABORT_MSG + + +def test_download_signatures_processes_all_entries(update_signatures_module, monkeypatch): + format_eles = [ + SimpleNamespace(get=lambda key, val=v: val if key == "PUID" else None) + for v in ["fmt/1", "fmt/2", "fmt/3"] + ] + calls = {"count": 0} + + monkeypatch.setattr(update_signatures_module, "download_sig", lambda *_a: calls.__setitem__("count", calls["count"] + 1)) + + update_signatures_module.download_signatures({"http_throttle": 0.0}, format_eles, False, os.getcwd()) + + assert calls["count"] == 3 + + +def test_main_parses_cli_args_and_calls_run(update_signatures_module, monkeypatch): + captured = {} + monkeypatch.setattr( + update_signatures_module.sys, + "argv", + [ + "prog", + "-tmpdir", + "/tmp/fido", + "-keep_tmp", + "-http_throttle", + "0.1", + "-version", + "116", + ], + ) + monkeypatch.setattr(update_signatures_module, "run", lambda opts: captured.__setitem__("opts", opts)) + + update_signatures_module.main() + + assert captured["opts"]["tmp_dir"] == "/tmp/fido" + assert captured["opts"]["deleteTempDirectory"] is False + assert captured["opts"]["http_throttle"] == 0.1 + assert captured["opts"]["version"] == "116" + assert captured["opts"]["signatureFileName"] == update_signatures_module.DEFAULTS["signatureFileName"] diff --git a/tests/pronom/test_versions.py b/tests/pronom/test_versions.py new file mode 100644 index 00000000..5a7e57cb --- /dev/null +++ b/tests/pronom/test_versions.py @@ -0,0 +1,365 @@ +from pathlib import Path +from types import SimpleNamespace +from xml.etree import ElementTree as XET + +import pytest + +from fido.pronom import versions + + +class DummyResponse: + def __init__(self, status_code=200, content=b"", text=""): + self.status_code = status_code + self.content = content + self.text = text + + +@pytest.mark.parametrize( + "value, expected", + [("104", "104"), ("v104", "104"), ("V205", "V205")], +) +def test_get_version_accepts_valid_values(value, expected): + assert versions._get_version(value) == expected + + +@pytest.mark.parametrize("value", ["", "foo", "v", "104a", "v-1"]) +def test_get_version_rejects_invalid_values(value): + with pytest.raises(SystemExit): + versions._get_version(value) + + +def test_local_versions_write_and_reload(tmp_path): + versions_file = tmp_path / "versions.xml" + versions_file.write_text( + """ + + 100 + formats-v100.xml + container-signature-20200101.xml + format_extensions.xml + 0.0.1 + https://example.test +""", + encoding="utf-8", + ) + local = versions.LocalVersions(str(versions_file)) + + local.pronom_version = "116" + local.pronom_signature = "formats-v116.xml" + local.pronom_container_signature = "container-signature-20231127.xml" + local.fido_extension_signature = "format_extensions.xml" + local.update_script = "1.0.0" + local.update_site = "https://example.test" + local.write() + + reloaded = versions.LocalVersions(str(versions_file)) + assert reloaded.pronom_version == "116" + assert reloaded.get_zip_file().endswith("pronom-xml-v116.zip") + assert reloaded.get_signature_file().endswith("formats-v116.xml") + + +def test_local_versions_write_requires_all_fields(tmp_path): + versions_file = tmp_path / "versions.xml" + versions_file.write_text("", encoding="utf-8") + local = versions.LocalVersions(str(versions_file)) + + with pytest.raises(ValueError): + local.write() + + +def test_version_check_returns_newer_version(monkeypatch): + monkeypatch.setattr( + versions.requests, + "get", + lambda _url: DummyResponse(status_code=200, text=''), + ) + + is_new, latest = versions._version_check("116", "https://example.test/") + assert is_new is True + assert latest == "117" + + +def test_version_check_returns_not_new(monkeypatch): + monkeypatch.setattr( + versions.requests, + "get", + lambda _url: DummyResponse(status_code=200, text=''), + ) + + is_new, latest = versions._version_check("116", "https://example.test/") + assert is_new is False + assert latest == "116" + + +def test_version_check_exits_on_http_error(monkeypatch): + monkeypatch.setattr( + versions.requests, + "get", + lambda _url: DummyResponse(status_code=500, text=""), + ) + + with pytest.raises(SystemExit): + versions._version_check("116", "https://example.test/") + + +def test_write_sigs_downloads_when_target_missing(monkeypatch, tmp_path): + conf_dir = tmp_path / "conf" + conf_dir.mkdir() + + class FakeFiles: + def __init__(self, base): + self.base = Path(base) + + def joinpath(self, *parts): + return self.base.joinpath(*parts) + + monkeypatch.setattr(versions.importlib.resources, "files", lambda _pkg: FakeFiles(tmp_path)) + monkeypatch.setattr( + versions.requests, + "get", + lambda _url: DummyResponse(status_code=200, content=b"payload"), + ) + + versions._write_sigs("116", "https://example.test/", "fido", "formats-v{}.xml") + + assert (conf_dir / "formats-v116.xml").read_bytes() == b"payload" + + +def test_write_sigs_skips_when_target_exists(monkeypatch, tmp_path): + conf_dir = tmp_path / "conf" + conf_dir.mkdir() + existing = conf_dir / "formats-v116.xml" + existing.write_text("present", encoding="utf-8") + + class FakeFiles: + def __init__(self, base): + self.base = Path(base) + + def joinpath(self, *parts): + return self.base.joinpath(*parts) + + called = {"http": 0} + monkeypatch.setattr(versions.importlib.resources, "files", lambda _pkg: FakeFiles(tmp_path)) + monkeypatch.setattr(versions.requests, "get", lambda _url: called.__setitem__("http", called["http"] + 1)) + + versions._write_sigs("116", "https://example.test/", "fido", "formats-v{}.xml") + + assert existing.read_text(encoding="utf-8") == "present" + assert called["http"] == 0 + + +def test_list_available_versions_prints_all(monkeypatch, capsys): + payload = b"" + monkeypatch.setattr(versions.requests, "get", lambda _url: DummyResponse(status_code=200, content=payload)) + + versions._list_available_versions("https://example.test/") + + output = capsys.readouterr().out + assert "Available signature versions:" in output + assert "116" in output + assert "117" in output + + +@pytest.mark.parametrize( + "sig_act, expected_call, expected_update_flag", + [ + ("list", "list", None), + ("check", "check", False), + ("update", "check", True), + ("116", "download", None), + ], +) +def test_sig_file_actions_dispatches(monkeypatch, sig_act, expected_call, expected_update_flag): + calls = {} + fake_versions = SimpleNamespace(pronom_version="116", update_site="https://example.test") + + monkeypatch.setattr(versions, "get_local_versions", lambda: fake_versions) + monkeypatch.setattr(versions, "_list_available_versions", lambda update_url: calls.__setitem__("list", update_url)) + monkeypatch.setattr( + versions, + "_check_update_signatures", + lambda sig_vers, update_url, vers, is_update: calls.__setitem__( + "check", (sig_vers, update_url, vers, is_update) + ), + ) + monkeypatch.setattr( + versions, + "_download_sig_version", + lambda action, update_url, vers: calls.__setitem__("download", (action, update_url, vers)), + ) + + with pytest.raises(SystemExit) as exc: + versions.sig_file_actions(sig_act) + + assert exc.value.code == 0 + if expected_call == "list": + assert calls["list"] == "https://example.test/" + elif expected_call == "check": + sig_vers, update_url, vers, is_update = calls["check"] + assert sig_vers == "116" + assert update_url == "https://example.test/" + assert vers is fake_versions + assert is_update is expected_update_flag + else: + action, update_url, vers = calls["download"] + assert action == "116" + assert update_url == "https://example.test/" + assert vers is fake_versions + + +def test_check_update_signatures_emits_update_message(monkeypatch, capsys): + calls = {"details": 0} + monkeypatch.setattr(versions, "_version_check", lambda _s, _u: (True, "117")) + monkeypatch.setattr(versions, "_output_details", lambda *_a: calls.__setitem__("details", calls["details"] + 1)) + + with pytest.raises(SystemExit) as exc: + versions._check_update_signatures("116", "https://example.test/", object(), is_update=False) + + assert exc.value.code == 0 + output = capsys.readouterr().out + assert "Updated signatures v117 are available" in output + assert calls["details"] == 0 + + +def test_check_update_signatures_updates_when_requested(monkeypatch): + calls = {"details": 0} + monkeypatch.setattr(versions, "_version_check", lambda _s, _u: (True, "117")) + monkeypatch.setattr(versions, "_output_details", lambda *_a: calls.__setitem__("details", calls["details"] + 1)) + + with pytest.raises(SystemExit): + versions._check_update_signatures("116", "https://example.test/", object(), is_update=True) + + assert calls["details"] == 1 + + +def test_check_update_signatures_emits_up_to_date_message(monkeypatch, capsys): + monkeypatch.setattr(versions, "_version_check", lambda _s, _u: (False, "116")) + + with pytest.raises(SystemExit) as exc: + versions._check_update_signatures("116", "https://example.test/", object(), is_update=False) + + assert exc.value.code == 0 + assert "up to date" in capsys.readouterr().out + + +def test_download_sig_version_normalizes_number(monkeypatch): + calls = {} + monkeypatch.setattr(versions.requests, "get", lambda _url: DummyResponse(status_code=200, content=b"", text="")) + monkeypatch.setattr( + versions, + "_output_details", + lambda version, update_url, vers: calls.__setitem__("out", (version, update_url, vers)), + ) + target = object() + + versions._download_sig_version("104", "https://example.test/", target) + + assert calls["out"] == ("104", "https://example.test/", target) + + +def test_download_sig_version_exits_on_http_error(monkeypatch): + monkeypatch.setattr(versions.requests, "get", lambda _url: DummyResponse(status_code=404, content=b"", text="")) + + with pytest.raises(SystemExit) as exc: + versions._download_sig_version("104", "https://example.test/", object()) + + assert "No signature files found" in str(exc.value) + + +def test_output_details_sets_versions_and_downloads(monkeypatch): + calls = [] + target = SimpleNamespace(pronom_version="", pronom_signature="") + target.write = lambda: calls.append("write") + monkeypatch.setattr(versions, "_write_sigs", lambda *args: calls.append(args)) + + versions._output_details("117", "https://example.test/", target) + + assert target.pronom_version == "117" + assert target.pronom_signature == "formats-v117.xml" + assert calls[0][2] == "fido" + assert calls[1][2] == "droid" + assert calls[2][2] == "pronom" + assert calls[3] == "write" + + +def test_local_versions_getattr_unknown_returns_none(tmp_path): + versions_file = tmp_path / "versions.xml" + versions_file.write_text("", encoding="utf-8") + local = versions.LocalVersions(str(versions_file)) + + assert local.unknown_attribute is None + + +def test_sig_file_actions_keeps_trailing_slash(monkeypatch): + calls = {} + fake_versions = SimpleNamespace(pronom_version="116", update_site="https://example.test/") + + monkeypatch.setattr(versions, "get_local_versions", lambda: fake_versions) + monkeypatch.setattr(versions, "_list_available_versions", lambda update_url: calls.__setitem__("list", update_url)) + + with pytest.raises(SystemExit): + versions.sig_file_actions("list") + + assert calls["list"] == "https://example.test/" + + +def test_download_sig_version_rejects_invalid_input(): + with pytest.raises(SystemExit) as exc: + versions._download_sig_version("v104beta", "https://example.test/", object()) + + assert "not a valid version number" in str(exc.value) + + +def test_download_sig_version_accepts_prefixed_version(monkeypatch): + calls = {} + monkeypatch.setattr(versions.requests, "get", lambda _url: DummyResponse(status_code=200, content=b"", text="")) + monkeypatch.setattr( + versions, + "_output_details", + lambda version, update_url, vers: calls.__setitem__("out", (version, update_url, vers)), + ) + target = object() + + versions._download_sig_version("v104", "https://example.test/", target) + + assert calls["out"] == ("104", "https://example.test/", target) + + +def test_get_local_versions_uses_config_dir(monkeypatch): + captured = {} + marker = object() + + def fake_local_versions(path): + captured["path"] = path + return marker + + monkeypatch.setattr(versions, "LocalVersions", fake_local_versions) + + result = versions.get_local_versions("/tmp/custom-conf") + + assert result is marker + assert captured["path"] == "/tmp/custom-conf/versions.xml" + + +def test_local_versions_init_fallback_uses_empty_root(monkeypatch, tmp_path): + versions_file = tmp_path / "missing-versions.xml" + + monkeypatch.setattr(versions, "parse", lambda _path: (_ for _ in ()).throw(IOError("missing"))) + monkeypatch.setattr(versions, "ET", XET) + + local = versions.LocalVersions(str(versions_file)) + + assert local.root.tag == versions.LocalVersions.ROOT_ELEMENT + + +def test_local_versions_setattr_creates_missing_field(monkeypatch, tmp_path): + versions_file = tmp_path / "versions.xml" + versions_file.write_text("", encoding="utf-8") + + monkeypatch.setattr(versions, "ET", XET) + local = versions.LocalVersions(str(versions_file)) + + local.pronom_version = "116" + + assert local.root.find("pronomVersion") is not None + assert local.root.find("pronomVersion").text == "116" diff --git a/tests/test_fido.py b/tests/test_fido.py index 420dbd58..40b7cbd0 100644 --- a/tests/test_fido.py +++ b/tests/test_fido.py @@ -3,10 +3,13 @@ import csv import io +from xml.etree import ElementTree as XET from time import sleep +from types import SimpleNamespace import pytest +import fido.fido as fido_mod from fido.fido import Fido from fido.utils.timer import PerfTimer @@ -85,3 +88,559 @@ def test_stream_identification(capsys, magic: bytes, expected_puid: str, expecte assert row[0] == expected_result, "row hasn't returned a positive identification" assert row[2] == expected_puid, "row doesn't contain expected PUID value" assert int(row[5]) == len(magic), "row doesn't contain stream length" + + +def _format_element(puid, container=None, extension=None, has_priority_over=None): + fmt = XET.Element("format") + XET.SubElement(fmt, "puid").text = puid + XET.SubElement(fmt, "name").text = puid + if container is not None: + XET.SubElement(fmt, "container").text = container + if extension is not None: + XET.SubElement(fmt, "extension").text = extension + if has_priority_over is not None: + XET.SubElement(fmt, "has_priority_over").text = has_priority_over + return fmt + + +def _make_fido(monkeypatch): + monkeypatch.setattr(Fido, "load_fido_xml", lambda self, _path: None) + return Fido(format_files=[]) + + +def test_container_type_detects_zip_ole_and_false(monkeypatch): + fido = _make_fido(monkeypatch) + + zip_fmt = _format_element("fmt/999", container="zip") + ole_fmt = _format_element("fmt/111") + plain_fmt = _format_element("fmt/1") + + assert fido.container_type([(zip_fmt, "sig")]) == "zip" + assert fido.container_type([(ole_fmt, "sig")]) == "ole" + assert fido.container_type([(plain_fmt, "sig")]) is False + + +def test_identify_contents_dispatches_supported_types(monkeypatch): + fido = _make_fido(monkeypatch) + calls = {"zip": 0, "tar": 0} + + monkeypatch.setattr(fido, "walk_zip", lambda *_args, **_kwargs: calls.__setitem__("zip", calls["zip"] + 1)) + monkeypatch.setattr(fido, "walk_tar", lambda *_args, **_kwargs: calls.__setitem__("tar", calls["tar"] + 1)) + + fido.identify_contents("a.zip", type="zip") + fido.identify_contents("a.tar", type="tar") + fido.identify_contents("a.bin", type="unknown") + fido.identify_contents("a.bin", type=False) + + assert calls == {"zip": 1, "tar": 1} + + +def test_get_buffers_seekable_branches(monkeypatch): + fido = _make_fido(monkeypatch) + fido.bufsize = 8 + data = b"0123456789abcdefghij" + + bof_1, eof_1, _ = fido.get_buffers(io.BytesIO(data), length=12) + assert bof_1 == data[:8] + assert eof_1 == data[4:12] + + bof_2, eof_2, _ = fido.get_buffers(io.BytesIO(data), length=16) + assert bof_2 == data[:8] + assert eof_2 == data[8:16] + + bof_3, eof_3, _ = fido.get_buffers(io.BytesIO(data), length=20, seekable=True) + assert bof_3 == data[:8] + assert eof_3 == data[-8:] + + +def test_match_extensions_honors_priority_over(monkeypatch): + fido = _make_fido(monkeypatch) + superior = _format_element("fmt/1", extension="txt", has_priority_over="fmt/2") + inferior = _format_element("fmt/2", extension="txt") + fido.formats = [superior, inferior] + fido.puid_has_priority_over_map = { + "fmt/1": frozenset(["fmt/2"]), + "fmt/2": frozenset(), + } + + matches = fido.match_extensions("sample.txt") + + assert len(matches) == 1 + assert matches[0][0].find("puid").text == "fmt/1" + assert matches[0][1] == "External" + + +def test_identify_file_reports_io_error(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + monkeypatch.setattr("builtins.open", lambda *_a, **_k: (_ for _ in ()).throw(IOError("boom"))) + + fido.identify_file("/does/not/exist") + + assert "FIDO: Error in identify_file" in capsys.readouterr().err + + +def test_identify_file_prefers_container_matches(monkeypatch, tmp_path): + fido = _make_fido(monkeypatch) + hits = {} + sample = tmp_path / "sample.bin" + sample.write_bytes(b"abc") + zip_fmt = _format_element("fmt/999", container="zip") + + monkeypatch.setattr(fido, "match_formats", lambda *_a: [(zip_fmt, "sig")]) + monkeypatch.setattr(fido, "match_container", lambda *_a: [(zip_fmt, "container sig")]) + monkeypatch.setattr(fido_mod.ET, "parse", lambda _p: XET.ElementTree(XET.Element("root"))) + monkeypatch.setattr( + fido, + "handle_matches", + lambda filename, matches, _duration, matchtype: hits.update( + {"filename": filename, "matches": matches, "matchtype": matchtype} + ), + ) + + fido.identify_file(str(sample)) + + assert hits["matchtype"] == "container" + assert hits["filename"] == str(sample) + assert len(hits["matches"]) == 1 + + +def test_identify_file_extension_fallback(monkeypatch, tmp_path): + fido = _make_fido(monkeypatch) + sample = tmp_path / "sample.txt" + sample.write_bytes(b"abc") + ext_fmt = _format_element("fmt/1", extension="txt") + hits = {} + + monkeypatch.setattr(fido, "match_formats", lambda *_a: []) + monkeypatch.setattr(fido, "match_extensions", lambda _name: [(ext_fmt, "External")]) + monkeypatch.setattr( + fido, + "handle_matches", + lambda _filename, _matches, _duration, matchtype: hits.update({"matchtype": matchtype}), + ) + + fido.identify_file(str(sample), extension=True) + + assert hits["matchtype"] == "extension" + + +def test_identify_file_recurses_into_zip_when_enabled(monkeypatch, tmp_path): + fido = _make_fido(monkeypatch) + fido.zip = True + sample = tmp_path / "sample.zip" + sample.write_bytes(b"abc") + zip_fmt = _format_element("fmt/999", container="zip") + calls = {"recurse": 0} + + monkeypatch.setattr(fido, "match_formats", lambda *_a: [(zip_fmt, "sig")]) + monkeypatch.setattr(fido, "handle_matches", lambda *_a: None) + monkeypatch.setattr( + fido, + "identify_contents", + lambda _filename, type=None, extension=True: calls.__setitem__("recurse", calls["recurse"] + 1), + ) + + fido.identify_file(str(sample), extension=False) + + assert calls["recurse"] == 1 + + +def test_identify_stream_non_windows_fallback_filename(monkeypatch): + fido = _make_fido(monkeypatch) + calls = {} + monkeypatch.setattr(fido, "match_formats", lambda *_a: []) + monkeypatch.setattr( + fido, + "match_extensions", + lambda filename: calls.__setitem__("ext_filename", filename) or [], + ) + monkeypatch.setattr( + fido, + "handle_matches", + lambda filename, _matches, _duration, matchtype: calls.update( + {"handled_filename": filename, "matchtype": matchtype} + ), + ) + fake_os = SimpleNamespace( + name="posix", + readlink=lambda _p: (_ for _ in ()).throw(OSError("no link")), + ) + monkeypatch.setattr(fido_mod, "os", fake_os) + + fido.identify_stream(io.BytesIO(b"abc"), "provided.name", extension=True) + + assert calls["ext_filename"] == "provided.name" + assert calls["handled_filename"] == "STDIN" + assert calls["matchtype"] == "extension" + + +def test_identify_stream_windows_uses_provided_filename(monkeypatch): + fido = _make_fido(monkeypatch) + calls = {} + monkeypatch.setattr(fido, "match_formats", lambda *_a: []) + monkeypatch.setattr( + fido, + "match_extensions", + lambda filename: calls.__setitem__("ext_filename", filename) or [], + ) + monkeypatch.setattr( + fido, + "handle_matches", + lambda filename, _matches, _duration, _matchtype: calls.update({"handled_filename": filename}), + ) + fake_os = SimpleNamespace(name="nt") + monkeypatch.setattr(fido_mod, "os", fake_os) + + fido.identify_stream(io.BytesIO(b"abc"), "provided.name", extension=True) + + assert calls["ext_filename"] == "provided.name" + assert calls["handled_filename"] == "provided.name" + + +def test_walk_zip_handles_bad_zip(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + monkeypatch.setattr(fido_mod.zipfile, "ZipFile", lambda *_a, **_k: (_ for _ in ()).throw(fido_mod.zipfile.BadZipfile())) + + fido.walk_zip("bad.zip") + + assert "FIDO: ZipError bad.zip" in capsys.readouterr().err + + +def test_walk_tar_handles_tar_error(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + monkeypatch.setattr(fido_mod.tarfile, "TarFile", lambda *_a, **_k: (_ for _ in ()).throw(fido_mod.tarfile.TarError())) + + fido.walk_tar("bad.tar", None) + + assert "FIDO: Error: TarError bad.tar" in capsys.readouterr().err + + +def _main_args(**overrides): + args = { + "confdir": "/tmp", + "pronom_only": False, + "v": False, + "sigs": None, + "matchprintf": None, + "nomatchprintf": None, + "q": True, + "bufsize": None, + "container_bufsize": None, + "zip": False, + "nocontainer": False, + "loadformats": None, + "useformats": None, + "nouseformats": None, + "input": None, + "files": ["-"], + "recurse": False, + "noextension": False, + "filename": None, + } + args.update(overrides) + return SimpleNamespace(**args) + + +def test_main_version_flag_exits(monkeypatch, capsys): + monkeypatch.setattr(fido_mod, "parse_cli_args", lambda *_a: _main_args(v=True)) + monkeypatch.setattr( + fido_mod, + "get_local_versions", + lambda _c: SimpleNamespace( + pronom_signature="formats-v116.xml", + pronom_container_signature="container-signature.xml", + fido_extension_signature="format_extensions.xml", + ), + ) + + with pytest.raises(SystemExit) as exc: + fido_mod.main(["-v"]) + + assert exc.value.code == 0 + assert "FIDO v" in capsys.readouterr().out + + +def test_main_sigs_calls_action(monkeypatch): + calls = {} + monkeypatch.setattr(fido_mod, "parse_cli_args", lambda *_a: _main_args(sigs="LiSt")) + monkeypatch.setattr( + fido_mod, + "get_local_versions", + lambda _c: SimpleNamespace( + pronom_signature="formats-v116.xml", + pronom_container_signature="container-signature.xml", + fido_extension_signature="format_extensions.xml", + ), + ) + monkeypatch.setattr(fido_mod, "sig_file_actions", lambda value: calls.__setitem__("sigs", value)) + + with pytest.raises(SystemExit) as exc: + fido_mod.main(["-sig", "list"]) + + assert exc.value.code == 0 + assert calls["sigs"] == "list" + + +def test_main_stdin_zip_raises_runtime(monkeypatch): + class DummyFido: + def __init__(self, **kwargs): + self.zip = kwargs["zip"] + self.current_file = "" + + def print_summary(self, _secs): + return None + + monkeypatch.setattr(fido_mod, "parse_cli_args", lambda *_a: _main_args(zip=True, files=["-"])) + monkeypatch.setattr( + fido_mod, + "get_local_versions", + lambda _c: SimpleNamespace( + pronom_signature="formats-v116.xml", + pronom_container_signature="container-signature.xml", + fido_extension_signature="format_extensions.xml", + ), + ) + monkeypatch.setattr(fido_mod, "Fido", DummyFido) + + with pytest.raises(RuntimeError): + fido_mod.main([]) + + +def test_print_matches_no_match_outputs_ko(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + fido.current_filesize = 12 + fido.current_count = 1 + + fido.print_matches("/tmp/file.bin", [], 0.01) + + assert "KO" in capsys.readouterr().out + + +def test_print_matches_with_match_outputs_ok(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + fmt = _format_element("fmt/1", extension="txt") + XET.SubElement(fmt, "mime").text = "text/plain" + XET.SubElement(fmt, "version").text = "1.0" + XET.SubElement(fmt, "alias").text = "Alias" + fido.current_filesize = 12 + fido.current_count = 1 + + fido.print_matches("/tmp/file.txt", [(fmt, "sig")], 0.01, "signature") + + output = capsys.readouterr().out + assert "OK" in output + assert "fmt/1" in output + + +def test_match_formats_covers_positions_and_exception_path(monkeypatch, capsys): + fido = _make_fido(monkeypatch) + + def add_signature(fmt, name, pos, regex): + sig = XET.SubElement(fmt, "signature") + XET.SubElement(sig, "name").text = name + pat = XET.SubElement(sig, "pattern") + XET.SubElement(pat, "position").text = pos + XET.SubElement(pat, "regex").text = regex + + bof_fmt = _format_element("fmt/1") + eof_fmt = _format_element("fmt/2") + var_fmt = _format_element("fmt/3") + ifb_fmt = _format_element("fmt/4") + bad_fmt = _format_element("fmt/5") + + add_signature(bof_fmt, "bof", "BOF", "abc") + add_signature(eof_fmt, "eof", "EOF", "xyz") + add_signature(var_fmt, "var", "VAR", "abc") + add_signature(ifb_fmt, "ifb", "IFB", "abc") + bad_sig = XET.SubElement(bad_fmt, "signature") + XET.SubElement(bad_sig, "name").text = "bad" + bad_pat = XET.SubElement(bad_sig, "pattern") + XET.SubElement(bad_pat, "position").text = "BOF" + + fido.formats = [bof_fmt, eof_fmt, var_fmt, ifb_fmt, bad_fmt] + fido.puid_has_priority_over_map = { + "fmt/1": frozenset(), + "fmt/2": frozenset(), + "fmt/3": frozenset(), + "fmt/4": frozenset(), + "fmt/5": frozenset(), + } + + matches = fido.match_formats(b"abc", b"xyz") + matched_puids = {m[0].find("puid").text for m in matches} + + assert {"fmt/1", "fmt/2", "fmt/3", "fmt/4"}.issubset(matched_puids) + assert "NoneType" in capsys.readouterr().err + + +def test_get_buffers_non_seekable_long_stream(monkeypatch): + fido = _make_fido(monkeypatch) + fido.bufsize = 8 + data = b"0123456789abcdefghijABCDEFGHIJ" + + bof, eof, _ = fido.get_buffers(io.BytesIO(data), length=len(data), seekable=False) + + assert bof == data[:8] + assert eof == data[-8:] + + +def test_buffered_read_overlap_and_non_overlap(monkeypatch, tmp_path): + fido = _make_fido(monkeypatch) + sample = tmp_path / "sample.bin" + sample.write_bytes(b"0123456789abcdefghij") + fido.current_file = str(sample) + fido.current_filesize = sample.stat().st_size + fido.bufsize = 8 + fido.container_bufsize = 4 + + no_overlap = fido.buffered_read(0, overlap=False) + with_overlap = fido.buffered_read(2, overlap=True) + + assert no_overlap == b"01234567" + assert with_overlap == b"23456789" + + +def test_walk_zip_recurses_for_nested_container(monkeypatch, tmp_path): + fido = _make_fido(monkeypatch) + sample = tmp_path / "sample.zip" + import zipfile + + with zipfile.ZipFile(sample, "w") as zf: + zf.writestr("inner.bin", b"abc") + + zip_fmt = _format_element("fmt/999", container="zip") + calls = {"recurse": 0} + monkeypatch.setattr(fido, "match_formats", lambda *_a: [(zip_fmt, "sig")]) + monkeypatch.setattr(fido, "handle_matches", lambda *_a: None) + monkeypatch.setattr( + fido, + "identify_contents", + lambda *_a, **_k: calls.__setitem__("recurse", calls["recurse"] + 1), + ) + + fido.walk_zip(str(sample), extension=False) + + assert calls["recurse"] == 1 + + +def test_walk_tar_processes_files_and_recurses(monkeypatch): + fido = _make_fido(monkeypatch) + zip_fmt = _format_element("fmt/999", container="zip") + calls = {"handled": 0, "recurse": 0} + + class DummyMember: + def __init__(self, name, is_file=True): + self.name = name + self.size = 3 + self._is_file = is_file + + def isfile(self): + return self._is_file + + class DummyTar: + def __init__(self, *_a, **_k): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def getmembers(self): + return [DummyMember("dir", is_file=False), DummyMember("file.bin", is_file=True)] + + def extractfile(self, _item): + return io.BytesIO(b"abc") + + monkeypatch.setattr(fido_mod.tarfile, "TarFile", DummyTar) + monkeypatch.setattr(fido, "get_buffers", lambda *_a, **_k: (b"abc", b"abc", 3)) + monkeypatch.setattr(fido, "match_formats", lambda *_a: [(zip_fmt, "sig")]) + monkeypatch.setattr(fido, "handle_matches", lambda *_a: calls.__setitem__("handled", calls["handled"] + 1)) + monkeypatch.setattr(fido, "identify_contents", lambda *_a, **_k: calls.__setitem__("recurse", calls["recurse"] + 1)) + + fido.walk_tar("archive.tar", None) + + assert calls["handled"] == 1 + assert calls["recurse"] == 1 + + +def test_main_loadformats_and_useformats(monkeypatch, tmp_path, capsys): + class DummyFido: + def __init__(self, **kwargs): + self.zip = kwargs["zip"] + self.current_file = "" + self.loaded = [] + self.formats = [_format_element("fmt/1"), _format_element("fmt/2")] + + def load_fido_xml(self, file): + self.loaded.append(file) + + def identify_file(self, *_a, **_k): + return None + + def print_summary(self, _secs): + return None + + listed = tmp_path / "files.txt" + listed.write_text("a.bin\n", encoding="utf-8") + created = {} + monkeypatch.setattr( + fido_mod, + "parse_cli_args", + lambda *_a: _main_args( + q=False, + loadformats="extra1.xml,extra2.xml", + useformats="fmt/1", + input=str(listed), + files=[], + ), + ) + monkeypatch.setattr( + fido_mod, + "get_local_versions", + lambda _c: SimpleNamespace( + pronom_signature="formats-v116.xml", + pronom_container_signature="container-signature.xml", + fido_extension_signature="format_extensions.xml", + ), + ) + monkeypatch.setattr(fido_mod, "list_files", lambda *_a, **_k: []) + monkeypatch.setattr(fido_mod, "Fido", lambda **kwargs: created.setdefault("fido", DummyFido(**kwargs))) + + fido_mod.main([]) + + assert created["fido"].loaded == ["extra1.xml", "extra2.xml"] + assert len(created["fido"].formats) == 1 + assert created["fido"].formats[0].find("puid").text == "fmt/1" + assert "FIDO v" in capsys.readouterr().err + + +def test_main_keyboard_interrupt_exits_with_context(monkeypatch): + class DummyFido: + def __init__(self, **kwargs): + self.zip = kwargs["zip"] + self.current_file = "STDIN" + + def identify_stream(self, *_a, **_k): + raise KeyboardInterrupt() + + def print_summary(self, _secs): + return None + + monkeypatch.setattr(fido_mod, "parse_cli_args", lambda *_a: _main_args(zip=False, files=["-"])) + monkeypatch.setattr( + fido_mod, + "get_local_versions", + lambda _c: SimpleNamespace( + pronom_signature="formats-v116.xml", + pronom_container_signature="container-signature.xml", + fido_extension_signature="format_extensions.xml", + ), + ) + monkeypatch.setattr(fido_mod, "Fido", DummyFido) + + with pytest.raises(SystemExit) as exc: + fido_mod.main([]) + + assert "Interrupt while identifying file STDIN" in str(exc.value) diff --git a/tests/test_package.py b/tests/test_package.py index 534a1f74..dc68bb3f 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,8 +1,10 @@ import os +import zipfile +from io import BytesIO import pytest -from fido.package import ZipPackage +from fido.package import OlePackage, Package, ZipPackage TEST_DATA_BAD_PACKAGES = os.path.normpath( os.path.join(__file__, "..", "test_data/hard_packages") @@ -17,3 +19,85 @@ def test_bad_zip(filename): p = ZipPackage(os.path.join(TEST_DATA_BAD_PACKAGES, filename), {}) r = p.detect_formats() assert isinstance(r, list) and len(r) == 0 + + +def test_zip_detect_formats_positive_match(tmp_path): + zip_path = tmp_path / "sample.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("content/file.bin", b"abcdef") + + signatures = { + "content/file.bin": { + "fmt/1": [{"signature": b"abc"}], + "fmt/2": [{"signature": b"xyz"}], + } + } + package = ZipPackage(str(zip_path), signatures) + + assert package.detect_formats() == ["fmt/1"] + + +def test_zip_detect_formats_missing_path(tmp_path): + zip_path = tmp_path / "sample.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("other/file.bin", b"abcdef") + + signatures = { + "content/file.bin": { + "fmt/1": [{"signature": b"abc"}], + } + } + package = ZipPackage(str(zip_path), signatures) + + assert package.detect_formats() == [] + + +def test_package_process_matches_all_matching_signatures(): + package = Package() + signatures = [{"signature": b"abc"}, {"signature": b"def"}, {"signature": b"xxx"}] + + matches = package._process_matches(b"abcdef", "fmt/1", signatures) + + assert matches == ["fmt/1", "fmt/1"] + + +def test_package_process_puid_map_aggregates_results(): + package = Package() + puid_map = { + "fmt/1": [{"signature": b"abc"}], + "fmt/2": [{"signature": b"def"}], + } + + matches = package._process_puid_map(b"abcdef", puid_map) + + assert matches == ["fmt/1", "fmt/2"] + + +def test_ole_detect_formats_positive(monkeypatch): + class DummyOle: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def listdir(self): + return [["Root", "Object"]] + + def openstream(self, _filepath): + return BytesIO(b"abcdef") + + monkeypatch.setattr("fido.package.olefile.OleFileIO", lambda _ole: DummyOle()) + + signatures = {"Root/Object": {"fmt/1": [{"signature": b"abc"}], "fmt/2": [{"signature": b"xyz"}]}} + package = OlePackage("dummy.ole", signatures) + + assert package.detect_formats() == ["fmt/1"] + + +def test_ole_detect_formats_handles_ioerror(monkeypatch): + monkeypatch.setattr("fido.package.olefile.OleFileIO", lambda _ole: (_ for _ in ()).throw(IOError("boom"))) + + package = OlePackage("dummy.ole", {}) + + assert package.detect_formats() == [] diff --git a/tests/test_prepare.py b/tests/test_prepare.py index 10cbbadd..13f2ac2e 100644 --- a/tests/test_prepare.py +++ b/tests/test_prepare.py @@ -1,7 +1,13 @@ import re +from types import SimpleNamespace +from xml.etree import ElementTree as XET +import hashlib +import io +import urllib.error import pytest +from fido.pronom import prepare from fido.pronom.prepare import convert_to_regex @@ -87,3 +93,455 @@ def test_heterogenous_sequences(pronom_bytesequence, input_, matches_bool): assert re.search(patt, input_) else: assert not re.search(patt, input_) + + +def _make_format_element(puid, pronom_id, priority_over=None): + element = XET.Element("format") + XET.SubElement(element, "puid").text = puid + XET.SubElement(element, "pronom_id").text = pronom_id + if priority_over is not None: + XET.SubElement(element, "has_priority_over").text = priority_over + return element + + +def test_ns_helper_supports_attr_and_path_calls(): + namespace = prepare.NS("{urn:test}") + + assert namespace.value == "{urn:test}value" + assert namespace("one/two") == "{urn:test}one/{urn:test}two" + + +def test_get_text_tna_returns_text_or_default(): + root = XET.Element("root") + parent = XET.SubElement(root, prepare.TNA("foo")) + tag = XET.SubElement(parent, prepare.TNA("bar")) + tag.text = " value " + + assert prepare.get_text_tna(root, "foo/bar") == "value" + assert prepare.get_text_tna(root, "missing", default="fallback") == "fallback" + + +def test_fido_position_unknown_value_writes_stderr(capsys): + result = prepare.fido_position("Unknown position") + + captured = capsys.readouterr() + assert result == "VAR" + assert "Unknown pronom PositionType" in captured.err + + +def test_calculate_repetition_handles_large_offsets(monkeypatch): + monkeypatch.setattr(prepare, "MAX_REGEX_REPS", 4) + + repetition = prepare.calculate_repetition(".", "BOF", "5", "6") + + assert repetition == ".{4,4}.{1,2}" + + +def test_convert_to_regex_supports_eof_position(): + regex = convert_to_regex("ab", pos="EOF", offset="1", maxoffset="2") + + assert regex.startswith("(?s)") + assert regex.endswith("\\Z") + assert ".{1,2}" in regex + + +def test_cmp_to_key_supports_custom_sorting(): + values = [1, 3, 2] + key = prepare._cmp_to_key(lambda a, b: b - a) + + assert sorted(values, key=key) == [3, 2, 1] + + +def test_run_uses_default_versions_and_calls_formatinfo(monkeypatch, capsys): + calls = {} + versions = SimpleNamespace( + get_zip_file=lambda: "input.zip", + get_signature_file=lambda: "output.xml", + ) + + class DummyFormatInfo: + def __init__(self, input_file): + calls["input_file"] = input_file + self.formats = ["fmt/1", "fmt/2"] + + def load_pronom_xml(self, puid): + calls["puid"] = puid + + def save(self, output_file): + calls["output_file"] = output_file + + monkeypatch.setattr(prepare, "get_local_versions", lambda: versions) + monkeypatch.setattr(prepare, "FormatInfo", DummyFormatInfo) + + prepare.run() + + captured = capsys.readouterr() + assert calls == {"input_file": "input.zip", "puid": None, "output_file": "output.xml"} + assert "Converted 2 PRONOM formats" in captured.err + + +def test_main_parses_args_and_dispatches(monkeypatch): + calls = {} + + def fake_run(input_file=None, output_file=None, puid=None): + calls["input_file"] = input_file + calls["output_file"] = output_file + calls["puid"] = puid + + monkeypatch.setattr(prepare, "run", fake_run) + + prepare.main(["-input", "a.zip", "-output", "b.xml", "-puid", "fmt/99"]) + + assert calls == {"input_file": "a.zip", "output_file": "b.xml", "puid": "fmt/99"} + + +def test_load_pronom_xml_rewrites_priority_ids(monkeypatch): + class DummyStream: + def close(self): + return None + + class DummyZip: + def __init__(self, *_args, **_kwargs): + self.items = ["one.xml", "two.xml"] + + def infolist(self): + return list(self.items) + + def open(self, _item): + return DummyStream() + + def close(self): + return None + + format_one = _make_format_element("fmt/1", "100", "200") + format_two = _make_format_element("fmt/2", "200") + parsed = [format_one, format_two] + + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare.zipfile, "ZipFile", DummyZip) + monkeypatch.setattr(info, "parse_pronom_xml", lambda _stream, _puid: parsed.pop(0)) + monkeypatch.setattr(info, "_sort_formats", lambda formats: formats) + + info.load_pronom_xml() + + assert info.formats[0].find("has_priority_over").text == "fmt/2" + assert len(info.formats) == 2 + + +def _minimal_pronom_xml(puid="x-fmt/263", include_reference_url=True): + ns = "http://pronom.nationalarchives.gov.uk" + + def n(tag): + return f"{{{ns}}}{tag}" + + root = XET.Element(n("root")) + report = XET.SubElement(root, n("report_format_detail")) + fmt = XET.SubElement(report, n("FileFormat")) + + for id_type, ident in [ + ("PUID", puid), + ("MIME", "application/test"), + ("Apple Uniform Type Identifier", "public.test"), + ]: + xml_id = XET.SubElement(fmt, n("FileFormatIdentifier")) + XET.SubElement(xml_id, n("IdentifierType")).text = id_type + XET.SubElement(xml_id, n("Identifier")).text = ident + + XET.SubElement(fmt, n("FormatName")).text = "Test Format" + XET.SubElement(fmt, n("FormatVersion")).text = "1.0" + XET.SubElement(fmt, n("FormatAliases")).text = "Alias" + XET.SubElement(fmt, n("FormatID")).text = "100" + XET.SubElement(fmt, n("FormatDescription")).text = "Description" + XET.SubElement(fmt, n("ReleaseDate")).text = "2020-01-01" + XET.SubElement(fmt, n("FormatTypes")).text = "Text" + XET.SubElement(fmt, n("ProvenanceName")).text = "Creator" + XET.SubElement(fmt, n("ProvenanceSourceDate")).text = "2020-01-01" + XET.SubElement(fmt, n("LastUpdatedDate")).text = "2021-01-01" + XET.SubElement(fmt, n("ProvenanceDescription")).text = "Meta" + + devs = XET.SubElement(fmt, n("Developers")) + XET.SubElement(devs, n("DeveloperCompoundName")).text = "Dev" + XET.SubElement(devs, n("OrganisationName")).text = "Org" + + ext_sig = XET.SubElement(fmt, n("ExternalSignature")) + XET.SubElement(ext_sig, n("Signature")).text = "tst" + + rel = XET.SubElement(fmt, n("RelatedFormat")) + XET.SubElement(rel, n("RelationshipType")).text = "Has priority over" + XET.SubElement(rel, n("RelatedFormatID")).text = "200" + + internal = XET.SubElement(fmt, n("InternalSignature")) + XET.SubElement(internal, n("SignatureName")).text = "sig" + XET.SubElement(internal, n("SignatureNote")).text = "note" + byte_seq = XET.SubElement(internal, n("ByteSequence")) + XET.SubElement(byte_seq, n("PositionType")).text = "Absolute from BOF" + XET.SubElement(byte_seq, n("ByteSequenceValue")).text = "ab" + XET.SubElement(byte_seq, n("Offset")).text = "0" + XET.SubElement(byte_seq, n("MaxOffset")).text = "" + + doc = XET.SubElement(fmt, n("Document")) + XET.SubElement(doc, n("TitleText")).text = "Doc" + author = XET.SubElement(doc, n("Author")) + XET.SubElement(author, n("AuthorCompoundName")).text = "A" + publisher = XET.SubElement(doc, n("Publisher")) + XET.SubElement(publisher, n("PublisherCompoundName")).text = "P" + XET.SubElement(doc, n("PublicationDate")).text = "2020" + doc_id = XET.SubElement(doc, n("DocumentIdentifier")) + XET.SubElement(doc_id, n("IdentifierType")).text = "URL" + XET.SubElement(doc_id, n("Identifier")).text = "example.test/spec" + XET.SubElement(doc, n("DocumentNote")).text = "note" + XET.SubElement(doc, n("DocumentType")).text = "type" + XET.SubElement(doc, n("AvailabilityDescription")).text = "avail" + XET.SubElement(doc, n("AvailabilityNote")).text = "note" + XET.SubElement(doc, n("DocumentIPR")).text = "ipr" + + ref_file = XET.SubElement(fmt, n("ReferenceFile")) + XET.SubElement(ref_file, n("ReferenceFileName")).text = "sample.bin" + XET.SubElement(ref_file, n("ReferenceFileDescription")).text = "desc" + XET.SubElement(ref_file, n("ReferenceFileIPR")).text = "ipr" + ref_id = XET.SubElement(ref_file, n("ReferenceFileIdentifier")) + XET.SubElement(ref_id, n("IdentifierType")).text = "URL" if include_reference_url else "DOI" + XET.SubElement(ref_id, n("Identifier")).text = "example.test/file" if include_reference_url else "10.1/xyz" + + return XET.tostring(root, encoding="utf-8") + + +def _tna(tag): + return "{http://pronom.nationalarchives.gov.uk}" + tag + + +def test_parse_pronom_xml_extracts_core_fields(monkeypatch): + payload = b"checksum-data" + + class DummySock: + def read(self): + return payload + + def close(self): + return None + + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + monkeypatch.setattr(prepare, "urlopen", lambda _url: DummySock()) + + result = info.parse_pronom_xml(io.BytesIO(_minimal_pronom_xml())) + + assert result.find("puid").text == "x-fmt/263" + assert result.find("container").text == "zip" + assert result.find("mime").text == "application/test" + assert result.find("apple_uti").text == "public.test" + assert result.find("has_priority_over").text == "200" + assert result.find("signature/pattern/regex").text + ref_identifiers = [node.text for node in result.findall("details/reference/*") if node.tag == "dc:identifier"] + assert "http://example.test/spec" in ref_identifiers + checksum = result.find("details/example_file/checksum") + assert checksum.attrib["type"] == "md5" + assert checksum.text == hashlib.md5(payload).hexdigest() + + +def test_parse_pronom_xml_respects_puid_filter(monkeypatch): + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + + assert info.parse_pronom_xml(io.BytesIO(_minimal_pronom_xml(puid="fmt/1")), puid_filter="fmt/2") is None + + +def test_parse_pronom_xml_skips_incompatible_signature(monkeypatch, capsys): + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + monkeypatch.setattr(prepare, "convert_to_regex", lambda *_a, **_k: prepare.FLG_INCOMPATIBLE) + + result = info.parse_pronom_xml(io.BytesIO(_minimal_pronom_xml(include_reference_url=False))) + + assert result.findall("signature") == [] + assert "incompatible PRONOM signature found" in capsys.readouterr().err + + +def test_prettify_outputs_xml_string(): + elem = XET.Element("root") + XET.SubElement(elem, "child").text = "value" + + pretty = prepare.prettify(elem) + + assert "value" in pretty + + +def test_formatinfo_save_writes_formats_xml(monkeypatch, tmp_path): + info = prepare.FormatInfo("dummy.zip") + fmt = XET.Element("format") + XET.SubElement(fmt, "puid").text = "fmt/1" + info.formats = [fmt] + monkeypatch.setattr(prepare, "ET", XET) + + out = tmp_path / "formats.xml" + info.save(str(out)) + + content = out.read_text(encoding="utf-8") + assert "fmt/1" in content + + +def test_sort_formats_orders_by_priority_relationship(monkeypatch): + info = prepare.FormatInfo("dummy.zip") + f1 = _make_format_element("fmt/1", "100", priority_over="fmt/2") + f2 = _make_format_element("fmt/2", "200") + + sorted_formats = info._sort_formats([f2, f1]) + + assert [f.find("puid").text for f in sorted_formats] == ["fmt/1", "fmt/2"] + + +@pytest.mark.parametrize( + "position, expected", + [ + ("Absolute from BOF", "BOF"), + ("Absolute from EOF", "EOF"), + ("Variable", "VAR"), + ("Indirect From BOF", "IFB"), + ], +) +def test_fido_position_known_values(position, expected): + assert prepare.fido_position(position) == expected + + +def test_do_byte_rejects_invalid_hex_pair(): + with pytest.raises(Exception) as exc: + prepare.do_byte("ZZ", 0, True) + + assert "bad byte sequence" in str(exc.value) + + +def test_convert_to_regex_rejects_invalid_start_char(): + with pytest.raises(ValueError): + prepare.convert_to_regex("^") + + +def test_convert_to_regex_question_mark_requires_double_question_mark(): + with pytest.raises(Exception) as exc: + prepare.convert_to_regex("?a") + + assert "Illegal character after ?" in str(exc.value) + + +def test_convert_to_regex_bracket_returns_incompatible_for_bad_separator(): + assert prepare.convert_to_regex("[0102]") == prepare.FLG_INCOMPATIBLE + + +def test_convert_to_regex_paren_raises_on_illegal_char(): + with pytest.raises(Exception): + prepare.convert_to_regex("(01!)") + + +def test_load_pronom_xml_reports_unmapped_priority(monkeypatch, capsys): + class DummyStream: + def close(self): + return None + + class DummyZip: + def __init__(self, *_args, **_kwargs): + self.items = ["one.xml"] + + def infolist(self): + return list(self.items) + + def open(self, _item): + return DummyStream() + + def close(self): + return None + + info = prepare.FormatInfo("dummy.zip") + fmt = _make_format_element("fmt/1", "100", priority_over="999") + monkeypatch.setattr(prepare.zipfile, "ZipFile", DummyZip) + monkeypatch.setattr(info, "parse_pronom_xml", lambda _stream, _puid: fmt) + monkeypatch.setattr(info, "_sort_formats", lambda formats: formats) + + info.load_pronom_xml() + + assert "Error looking up priority over PRONOM ID 999" in capsys.readouterr().err + + +def test_load_pronom_xml_exits_when_zip_close_fails(monkeypatch): + class DummyStream: + def close(self): + return None + + class DummyZip: + def __init__(self, *_args, **_kwargs): + self.items = ["one.xml"] + + def infolist(self): + return list(self.items) + + def open(self, _item): + return DummyStream() + + def close(self): + raise RuntimeError("close failed") + + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare.zipfile, "ZipFile", DummyZip) + monkeypatch.setattr(info, "parse_pronom_xml", lambda _stream, _puid: None) + + with pytest.raises(SystemExit): + info.load_pronom_xml() + + +def test_parse_pronom_xml_extracts_super_and_subtype_relationships(monkeypatch): + root = XET.fromstring(_minimal_pronom_xml()) + fmt = root.find(".//" + _tna("FileFormat")) + + rel_super = XET.SubElement(fmt, _tna("RelatedFormat")) + XET.SubElement(rel_super, _tna("RelationshipType")).text = "Is supertype of" + XET.SubElement(rel_super, _tna("RelatedFormatID")).text = "300" + + rel_sub = XET.SubElement(fmt, _tna("RelatedFormat")) + XET.SubElement(rel_sub, _tna("RelationshipType")).text = "Is subtype of" + XET.SubElement(rel_sub, _tna("RelatedFormatID")).text = "301" + + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + monkeypatch.setattr(prepare, "urlopen", lambda _url: io.BytesIO(b"x")) + + result = info.parse_pronom_xml(io.BytesIO(XET.tostring(root, encoding="utf-8"))) + + assert result.find("details/is_supertype_of").text == "300" + assert result.find("details/is_subtype_of").text == "301" + + +def test_parse_pronom_xml_reference_file_http_404_continues(monkeypatch, capsys): + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + monkeypatch.setattr( + prepare, + "urlopen", + lambda _url: (_ for _ in ()).throw( + urllib.error.HTTPError(url="http://example.test/file", code=404, msg="missing", hdrs=None, fp=None) + ), + ) + + result = info.parse_pronom_xml(io.BytesIO(_minimal_pronom_xml())) + + checksum = result.find("details/example_file/checksum") + assert checksum is not None + assert checksum.text in (None, "") + assert "HTTP 404 error loading resource" in capsys.readouterr().err + + +def test_parse_pronom_xml_reference_file_http_non_404_uses_empty_hash(monkeypatch, capsys): + info = prepare.FormatInfo("dummy.zip") + monkeypatch.setattr(prepare, "ET", XET) + monkeypatch.setattr( + prepare, + "urlopen", + lambda _url: (_ for _ in ()).throw( + urllib.error.HTTPError(url="http://example.test/file", code=500, msg="error", hdrs=None, fp=None) + ), + ) + + result = info.parse_pronom_xml(io.BytesIO(_minimal_pronom_xml())) + + checksum = result.find("details/example_file/checksum") + assert checksum.text == hashlib.md5(b"").hexdigest() + assert "HTTP 500 error loading resource" in capsys.readouterr().err diff --git a/tests/test_toxml.py b/tests/test_toxml.py new file mode 100644 index 00000000..4c386189 --- /dev/null +++ b/tests/test_toxml.py @@ -0,0 +1,43 @@ +import io +from types import SimpleNamespace + +from fido import toxml + + +def test_toxml_main_renders_single_row(monkeypatch, capsys): + monkeypatch.setattr(toxml, "get_local_versions", lambda: SimpleNamespace(pronom_version="999")) + monkeypatch.setattr( + toxml.sys, + "stdin", + io.StringIO('OK,0,fmt/1000,"Sample Format","Sample Sig",9,sample.bin,application/test,byte\n'), + ) + + toxml.main() + + output = capsys.readouterr().out + assert "" in output + assert "999" in output + assert "sample.bin" in output + assert "fmt/1000" in output + assert "9" in output + assert output.strip().endswith("") + + +def test_toxml_main_renders_multiple_rows(monkeypatch, capsys): + monkeypatch.setattr(toxml, "get_local_versions", lambda: SimpleNamespace(pronom_version="116")) + monkeypatch.setattr( + toxml.sys, + "stdin", + io.StringIO( + "OK,1,fmt/1,Format A,Sig A,4,a.bin,text/plain,byte\n" + "KO,2,,Unknown,,8,b.bin,,extension\n" + ), + ) + + toxml.main() + + output = capsys.readouterr().out + assert output.count("") == 2 + assert "a.bin" in output + assert "b.bin" in output + assert "KO" in output From ff7253f3f4f391c2883f0d5985ebee9d7df81b07 Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Mon, 3 Aug 2026 21:51:24 +0200 Subject: [PATCH 31/33] turn on github actions --- .github/workflows/pytest.yml | 41 ++++++++++++++++ README.md | 3 +- coverage.svg | 21 ++++++++ tests/pronom/test_soap.py | 94 +++++++++++++++++++----------------- 4 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/pytest.yml create mode 100644 coverage.svg diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000..ff62d75b --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,41 @@ +name: Pytest + +on: + push: + +jobs: + test: + name: Run Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov coverage-badge + pip install -e . + - name: Lint with flake8 + run: flake8 tests --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=120 --statistics + - name: Test with pytest + run: pytest --cov=fido --cov-report=xml + - name: Generate coverage badge + if: matrix.python-version == '3.12' && github.ref == 'refs/heads/master' + run: coverage-badge -f -o coverage.svg + - name: Commit coverage badge + if: matrix.python-version == '3.12' && github.ref == 'refs/heads/master' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add coverage.svg + git diff --cached --quiet || git commit -m "chore: update coverage badge [skip ci]" + git push diff --git a/README.md b/README.md index b3c82086..0d39f495 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ Format Identification for Digital Objects (fido) By [Open Preservation Foundation](http://www.openpreservation.org) -[![Code Coverage](https://codecov.io/gh/openpreserve/fido/branch/master/graph/badge.svg)](https://codecov.io/gh/openpreserve/fido) +[![Pytest](https://github.com/openpreserve/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) +[![Coverage](./coverage.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) FIDO is a command-line tool to identify the file formats of digital objects. It is designed for simple integration into automated work-flows. diff --git a/coverage.svg b/coverage.svg new file mode 100644 index 00000000..6963b3e1 --- /dev/null +++ b/coverage.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + coverage + coverage + 87% + 87% + + diff --git a/tests/pronom/test_soap.py b/tests/pronom/test_soap.py index f28a6ecd..7a3b32e3 100644 --- a/tests/pronom/test_soap.py +++ b/tests/pronom/test_soap.py @@ -9,37 +9,37 @@ class FakeResponse: - """Small response shim used to mock urllib responses.""" + """Small response shim used to mock urllib responses.""" - def __init__(self, payload): - self.payload = payload + def __init__(self, payload): + self.payload = payload - def read(self): - return self.payload + def read(self): + return self.payload - def __enter__(self): - return self + def __enter__(self): + return self - def __exit__(self, exc_type, exc, tb): - return False + def __exit__(self, exc_type, exc, tb): + return False class FakeRequest: - """Simple urllib Request shim with header support.""" + """Simple urllib Request shim with header support.""" - def __init__(self, url, data=None): - self.url = url - self.data = data - self.headers = {} + def __init__(self, url, data=None): + self.url = url + self.data = data + self.headers = {} - def add_header(self, key, value): - self.headers[key] = value + def add_header(self, key, value): + self.headers[key] = value @unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) def test_pronom_version_from_mocked_soap_response(mock_request): - """Parse signature version from a mocked SOAP payload.""" - xml = b""" + """Parse signature version from a mocked SOAP payload.""" + xml = b""" @@ -51,16 +51,18 @@ def test_pronom_version_from_mocked_soap_response(mock_request): """ - mock_request.Request.side_effect = lambda url, data=None: FakeRequest(url, data=data) - mock_request.urlopen.return_value = FakeResponse(xml) + mock_request.Request.side_effect = lambda url, data=None: FakeRequest( + url, data=data + ) + mock_request.urlopen.return_value = FakeResponse(xml) - assert fido.pronom.soap.get_pronom_sig_version() == 116 + assert fido.pronom.soap.get_pronom_sig_version() == 116 @unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) def test_get_droid_signatures_counts_file_formats(mock_request): - """Count FileFormat elements from a mocked DROID signature XML file.""" - sig_xml = b""" + """Count FileFormat elements from a mocked DROID signature XML file.""" + sig_xml = b""" @@ -69,44 +71,46 @@ def test_get_droid_signatures_counts_file_formats(mock_request): """ - mock_request.urlopen.return_value = FakeResponse(sig_xml) + mock_request.urlopen.return_value = FakeResponse(sig_xml) - xml, count = fido.pronom.soap.get_droid_signatures(116) - assert "SignatureFile" in xml - assert count == 2 + xml, count = fido.pronom.soap.get_droid_signatures(116) + assert "SignatureFile" in xml + assert count == 2 @unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) def test_get_droid_signatures_handles_http_error(mock_request, capsys): - """Return fallback values and log an error when download fails.""" + """Return fallback values and log an error when download fails.""" - mock_request.urlopen.side_effect = urllib.error.HTTPError(url="http://example.test", code=500, msg="boom", hdrs=None, fp=None) + mock_request.urlopen.side_effect = urllib.error.HTTPError( + url="http://example.test", code=500, msg="boom", hdrs=None, fp=None + ) - xml, count = fido.pronom.soap.get_droid_signatures(116) - captured = capsys.readouterr() - assert xml == [] - assert count is False - assert "could not download signature file v116" in captured.err + xml, count = fido.pronom.soap.get_droid_signatures(116) + captured = capsys.readouterr() + assert xml == [] + assert count is False + assert "could not download signature file v116" in captured.err @unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) def test_get_sig_xml_for_puid_returns_raw_xml(mock_request): - """Return unmodified PRONOM XML bytes for a PUID.""" - payload = b"bar" - mock_request.urlopen.return_value = FakeResponse(payload) + """Return unmodified PRONOM XML bytes for a PUID.""" + payload = b"bar" + mock_request.urlopen.return_value = FakeResponse(payload) - assert fido.pronom.soap.get_sig_xml_for_puid("fmt/18") == payload + assert fido.pronom.soap.get_sig_xml_for_puid("fmt/18") == payload @unittest.mock.patch("fido.pronom.soap.urllib.request", create=True) def test_get_soap_response_exits_on_request_error(mock_request, capsys): - """Exit with code 1 when creating the request fails.""" + """Exit with code 1 when creating the request fails.""" - mock_request.Request.side_effect = urllib.error.URLError("offline") + mock_request.Request.side_effect = urllib.error.URLError("offline") - with pytest.raises(SystemExit) as exc: - fido.pronom.soap._get_soap_response('"action"', b"") + with pytest.raises(SystemExit) as exc: + fido.pronom.soap._get_soap_response('"action"', b"") - captured = capsys.readouterr() - assert exc.value.code == 1 - assert "There was a problem contacting the PRONOM service" in captured.out + captured = capsys.readouterr() + assert exc.value.code == 1 + assert "There was a problem contacting the PRONOM service" in captured.out From 9424b5c6d7ea3f2e031b762970c5409a5de4bd71 Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Tue, 4 Aug 2026 20:38:41 +0200 Subject: [PATCH 32/33] switch repository url for demonstration --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d39f495..014a81db 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Format Identification for Digital Objects (fido) By [Open Preservation Foundation](http://www.openpreservation.org) -[![Pytest](https://github.com/openpreserve/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) -[![Coverage](./coverage.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) +[![Pytest](https://github.com/m3ssman/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/m3ssman/fido/actions/workflows/pytest.yml) +[![Coverage](./coverage.svg)](https://github.com/m3ssman/fido/actions/workflows/pytest.yml) FIDO is a command-line tool to identify the file formats of digital objects. It is designed for simple integration into automated work-flows. From c04891119fac7b3b5bede7190798dfbfe7e7ee2c Mon Sep 17 00:00:00 2001 From: Uwe Hartwig Date: Wed, 16 Sep 2026 09:37:51 +0200 Subject: [PATCH 33/33] exchange actions url --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 014a81db..0d39f495 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Format Identification for Digital Objects (fido) By [Open Preservation Foundation](http://www.openpreservation.org) -[![Pytest](https://github.com/m3ssman/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/m3ssman/fido/actions/workflows/pytest.yml) -[![Coverage](./coverage.svg)](https://github.com/m3ssman/fido/actions/workflows/pytest.yml) +[![Pytest](https://github.com/openpreserve/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) +[![Coverage](./coverage.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml) FIDO is a command-line tool to identify the file formats of digital objects. It is designed for simple integration into automated work-flows.