Dev/2.0 upgrade

fido/pronom/http.py → .attic/http.py

@@ -21,6 +21,8 @@
 """
 from six.moves import urllib
 
+# NOTE: from fido/pronom/
+
 
 def get_sig_xml_for_puid(puid):
     """Return the full PRONOM signature XML for the passed PUID."""

.github/workflows/pytest.yml

@@ -0,0 +1,41 @@
+name: Pytest
+
+on:
+  push:
+
+jobs:
+  test:
+    name: Run Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest pytest-cov coverage-badge
+          pip install -e .
+      - name: Lint with flake8
+        run: flake8 tests --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=120 --statistics
+      - name: Test with pytest
+        run: pytest --cov=fido --cov-report=xml
+      - name: Generate coverage badge
+        if: matrix.python-version == '3.12' && github.ref == 'refs/heads/master'
+        run: coverage-badge -f -o coverage.svg
+      - name: Commit coverage badge
+        if: matrix.python-version == '3.12' && github.ref == 'refs/heads/master'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add coverage.svg
+          git diff --cached --quiet || git commit -m "chore: update coverage badge [skip ci]"
+          git push

.github/workflows/test-pr.yml

@@ -11,7 +11,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: [3.8, 3.9, 3.10]
 
     steps:
       - uses: actions/checkout@v3
@@ -25,10 +25,7 @@ jobs:
           pip install -U flake8 pep257 pytest-cov codecov codacy-coverage pluggy
           pip install -e .
       - name: Lint code with flake8
-        run: flake8 . --count --show-source --max-line-length=127 --statistics
-      - name: Lint code with pep257
-        if: matrix.python-version == 2.7
-        run: pep257 --match="(?!fido).*\.py" ./fido
+        run: flake8 . --count --show-source --ignore=E231,E241,E501,W503,E203 --max-line-length=120 --statistics
       - name: Test using pytest
         run: pytest --cov=fido
       - name: Generate LCOV coverage report

.gitignore

@@ -1,5 +1,6 @@
 *.egg-info
 *.py[co]
+.venv/
 
 /MANIFEST
 /dist
@@ -8,6 +9,9 @@
 /.eggs
 .coverage
 .env/
+.pytest_cache/
+.ruff_cache
+__pycache__/
 
 /fmtinfo.csv
 /exp

CONTRIBUTING.md

@@ -44,7 +44,6 @@ The checklist below contains some of our expectations, and will help you create
 - New code should contributions should adhere to the [PEP 8 -- Style Guide for Python Code](https://www.python.org/dev/peps/pep-0008/) and [PEP 257 -- Docstring Conventions](https://www.python.org/dev/peps/pep-0257/).
 - Linebreaks should be used to limit line length within reason; we do not strictly enforce the 80-character line limit of PEP 8.
 - Non-trivial changes should be accompanied by corresponding unit tests.
-- FIDO runs on Python 2 and 3 (specifically versions 2.7, 3.4 and 3.5); changes must preserve this 2/3 compatibility.
 - A pull request should resolve an existing GitHub issue and the name of its git branch should reference that issue by using the following naming convention: `dev/issue-<ISSUE_NO>-short-description`, e.g., `dev/issue-126-add-contributing-doc`.
 - Git commits should be of a manageable size and should introduce one logical change; git commit messages should adhere to the [seven rules of a great Git commit message](https://chris.beams.io/posts/git-commit/):
   - Separate subject from body with a blank line
@@ -55,12 +54,10 @@ The checklist below contains some of our expectations, and will help you create
   - Wrap the body at 72 characters
   - Use the body to explain what and why vs. how
 
-FIDO's Travis Continuous Integration configuration runs `pytest` to execute the tests, `flake8` to check PEP 8 conformance, and `pep257` to check PEP 257 (docstring) conformance.
 You should run these tools locally before pushing a commit by running the following commands:
 
-    $ python setup.py test
+    $ pytest
     $ flake8 --ignore=E501 ./fido
-    $ pep257 --match='(?!fido).*\.py' ./fido
 
 
 ## Code Review & Approval

README.md

@@ -3,7 +3,8 @@ Format Identification for Digital Objects (fido)
 
 By [Open Preservation Foundation](http://www.openpreservation.org)
 
-[![Build Status](https://travis-ci.org/openpreserve/fido.svg?branch=master)](https://travis-ci.org/openpreserve/fido) [![Code Coverage](https://codecov.io/gh/openpreserve/fido/branch/master/graph/badge.svg)](https://codecov.io/gh/openpreserve/fido)
+[![Pytest](https://github.com/openpreserve/fido/actions/workflows/pytest.yml/badge.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml)
+[![Coverage](./coverage.svg)](https://github.com/openpreserve/fido/actions/workflows/pytest.yml)
 
 FIDO is a command-line tool to identify the file formats of digital objects.
 It is designed for simple integration into automated work-flows.
@@ -69,8 +70,9 @@ Any platform
 1. Download the latest zip release from <https://github.com/openpreserve/fido/releases>
 2. Unzip into some directory
 3. Open a command shell, cd to the directory that you placed the zip contents into
-4. Run `python setup.py install` to install FIDO and dependencies.  This may require sudo on Linux/OSX or admin privileges on Windows.
-5. You should now be able to see the help text:
+4. Create and activate local Python Environment in this directory.
+5. Run `python -m pip install .` to install FIDO and dependencies.  This may require sudo on Linux/OSX or admin privileges on Windows.
+6. You should now be able to see the help text:
    `fido -h`
 
 Using pip
@@ -137,6 +139,8 @@ FIDO 1.3.3 and later have experimental Python 3 support.
 
 FIDO 1.4 and later have Python 3 support.
 
+FIDO 2.0 and later use pyproject Configuration for Installation and Development.
+
 Format Definitions
 ------------------
 
@@ -185,40 +189,40 @@ Examples running FIDO
 
 Identify all files in the current directory and below, sending output
 into file-info.csv:
-   `python fido.py -recurse . > file-info.csv`
+   `fido -recurse . > file-info.csv`
 
 Do the same as above, but also look inside of zip or tar files:
-   `python fido.py -recurse -zip . > file-info.csv`
+   `fido -recurse -zip . > file-info.csv`
 
 Take input from a list of files:
 
 Linux:
 
 ```shell
 ls > files.txt
-python fido.py -input files.txt
+fido -input files.txt
 ```
 
 Windows:
 
 ```shell
 dir /b > files.txt
-python fido.py -input files.txt
+fido -input files.txt
 ```
 
 Take input from a pipe:
 
 Linux:
-   `find . -type f | python fido.py -input -`
+   `find . -type f | fido -input -`
 
 Windows:
-   `dir /b | python fido.py -input -`
+   `dir /b | fido -input -`
 
 Only show files that could not be identified:
-   `python fido.py -matchprintf "" .`
+   `fido -matchprintf "" .`
 
 Only show files that could be identified:
-   `python fido.py -nomatchprintf "" .`
+   `fido -nomatchprintf "" .`
 
 Deep scan of container objects
 ------------------------------

fido/__init__.py

@@ -7,43 +7,9 @@
 It is designed for simple integration into automated work-flows.
 """
 
-from __future__ import print_function
+__version__ = "2.0.0-dev"
 
+# todo: move this to a conf/conf.py or something rather than init.py. Would require some cascading updates, though
 from os.path import abspath, dirname, join
 
-from six.moves import input as rinput
-
-
-__version__ = '1.6.1'
-
-
-CONFIG_DIR = join(abspath(dirname(__file__)), 'conf')
-
-
-def query_yes_no(question, default='yes'):
-    """
-    Ask a yes/no question via input() and return their answer.
-
-    `question` is a string that is presented to the user. `default` is the
-    presumed answer if the user just hits <Enter>. It must be "yes" (the
-    default), "no" or None (meaning an answer is required of the user).
-
-    The "answer" return value is True for "yes" or False for "no".
-    """
-    valid = {'yes': True, 'y': True, 'no': False, 'n': False}
-    if default is None:
-        prompt = ' [y/n] '
-    elif default == 'yes':
-        prompt = ' [Y/n] '
-    elif default == 'no':
-        prompt = ' [y/N] '
-    else:
-        raise ValueError('Invalid default answer: "%s"' % default)
-    while True:
-        print(question + prompt, end='')
-        choice = rinput().lower()
-        if default is not None and choice == '':
-            return valid[default]
-        if choice in valid:
-            return valid[choice]
-        print('Please respond with "yes" or "no" (or "y" or "n").')
+CONFIG_DIR = join(abspath(dirname(__file__)), "conf")

fido/cli_args.py

@@ -0,0 +1,130 @@
+import argparse
+from argparse import ArgumentParser, RawTextHelpFormatter
+from typing import Any, Dict, List
+
+
+def parse_cli_args(argv: List[str], defaults: Dict[str, Any]) -> argparse.Namespace:
+    """
+    Parse command-line arguments.
+    Args:
+        argv (list[str]): List of command-line arguments. Could be sys.argv
+        defaults (dict): Dictionary of default values. Expects to find configdir, bufsize and container_bufsize.
+    Returns:
+        argparse.Namespace: Parsed command-line arguments. Reference via name as in args.v or args.recurse.
+    """
+
+    parser = ArgumentParser(
+        description=defaults["description"],
+        epilog=defaults["epilog"],
+        fromfile_prefix_chars="@",
+        formatter_class=RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "-v", default=False, action="store_true", help="show version information"
+    )
+    parser.add_argument(
+        "-q", default=False, action="store_true", help="run (more) quietly"
+    )
+    parser.add_argument(
+        "-recurse",
+        default=False,
+        action="store_true",
+        help="recurse into subdirectories",
+    )
+    parser.add_argument(
+        "-zip",
+        default=False,
+        action="store_true",
+        help="recurse into zip and tar files",
+    )
+    parser.add_argument(
+        "-noextension",
+        default=False,
+        action="store_true",
+        help="disable extension matching, reduces number of matches but may reduce false positives",
+    )
+    parser.add_argument(
+        "-nocontainer",
+        default=False,
+        action="store_true",
+        help="disable deep scan of container documents, increases speed but may reduce accuracy with big files",
+    )
+    parser.add_argument(
+        "-pronom_only",
+        default=False,
+        action="store_true",
+        help="disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results",
+    )
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "-input",
+        default=False,
+        help="file containing a list of files to check, one per line. - means stdin",
+    )
+    group.add_argument(
+        "files",
+        nargs="*",
+        default=[],
+        metavar="FILE",
+        help="files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.",
+    )
+
+    parser.add_argument(
+        "-filename", default=None, help="filename if file contents passed through STDIN"
+    )
+    parser.add_argument(
+        "-useformats",
+        metavar="INCLUDEPUIDS",
+        default=None,
+        help="comma separated string of formats to use in identification",
+    )
+    parser.add_argument(
+        "-nouseformats",
+        metavar="EXCLUDEPUIDS",
+        default=None,
+        help="comma separated string of formats not to use in identification",
+    )
+    parser.add_argument(
+        "-matchprintf",
+        metavar="FORMATSTRING",
+        default=None,
+        help="format string (Python style) to use on match. See nomatchprintf, README.txt.",
+    )
+    parser.add_argument(
+        "-nomatchprintf",
+        metavar="FORMATSTRING",
+        default=None,
+        help="format string (Python style) to use if no match. See README.txt",
+    )
+    parser.add_argument(
+        "-bufsize",
+        type=int,
+        default=None,
+        help=f"size (in bytes) of the buffer to match against (default={defaults['bufsize']})",
+    )
+    parser.add_argument(
+        "-sigs",
+        default=None,
+        metavar="SIG_ACT",
+        help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.',
+    )
+    parser.add_argument(
+        "-container_bufsize",
+        type=int,
+        default=None,
+        help=f"size (in bytes) of the buffer to match against (default={defaults['container_bufsize']}).",
+    )
+    parser.add_argument(
+        "-loadformats",
+        default=None,
+        metavar="XML1,...,XMLn",
+        help="comma separated string of XML format files to add.",
+    )
+    parser.add_argument(
+        "-confdir",
+        default=defaults["config_dir"],
+        help="configuration directory to load_fido_xml, for example, the format specifications from.",
+    )
+
+    return parser.parse_args(argv)