Add data_manager_repeatmasker

data_managers/data_manager_repeatmasker/.shed.yml

@@ -0,0 +1,14 @@
+categories:
+- Data Managers
+description: Download Dfam FamDB libraries for RepeatMasker
+homepage_url: https://www.dfam.org/
+long_description: |
+  Downloads a Dfam repeat library in the partitioned FamDB (v3) format and
+  registers it in the repeatmasker_famdb data table, so RepeatMasker (>= 4.2.4)
+  can screen sequences against a real library instead of the empty placeholder
+  shipped in the RepeatMasker Conda package. The root partition is always
+  downloaded; curated/uncurated consensus and HMM components are selectable.
+owner: iuc
+name: data_manager_repeatmasker
+remote_repository_url: https://github.com/galaxyproject/tools-iuc/tree/main/data_managers/data_manager_repeatmasker
+type: unrestricted

data_managers/data_manager_repeatmasker/README.md

@@ -0,0 +1,35 @@
+# RepeatMasker Dfam library data manager
+
+This data manager downloads a [Dfam](https://www.dfam.org/) repeat library in
+the partitioned FamDB (v3) format and registers it in the `repeatmasker_famdb`
+tool data table for use by the RepeatMasker Galaxy tool (RepeatMasker >= 4.2.4).
+
+## Why this is needed
+
+The RepeatMasker Conda package no longer ships a populated repeat library: the
+`Dfam.h5` bundled in the package is a placeholder, and running RepeatMasker
+against it fails with `Species "..." is not known to RepeatMasker` because the
+library contains no families. The library must be downloaded separately; this
+data manager automates that and makes the result available through a data table.
+
+## What it downloads
+
+Dfam distributes its library as a required **root** partition plus optional
+**component** partitions, split by curation status (curated/uncurated) and model
+type (consensus sequences for the rmblast engine, profile HMMs for nhmmer):
+
+| Component | Engine | Notes |
+|-----------|--------|-------|
+| Curated consensus | rmblast | Recommended default |
+| Uncurated consensus | rmblast | Very large |
+| Curated HMM | nhmmer | |
+| Uncurated HMM | nhmmer | Very large (100+ partitions) |
+
+The root partition is always downloaded. Each file is verified against its Dfam
+md5 checksum, decompressed, and placed in a single directory that RepeatMasker
+consumes via `-libdir`.
+
+## Data table
+
+`repeatmasker_famdb` with columns `value, name, version, path`, where `path`
+points at the directory holding the FamDB `.h5` partition files.

data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+"""Download a Dfam FamDB library for RepeatMasker and register it in a data table.
+
+Dfam distributes its libraries in the partitioned FamDB (v3) format: a single
+required *root* partition plus one file per *component* partition.  The
+components are split by curation status (curated/uncurated) and model type
+(consensus sequences, used by the rmblast search engine, or profile HMMs, used
+by nhmmer).  This data manager downloads the root partition (always) together
+with whichever components the admin selects, laying the ``*.h5`` files out in a
+single directory that RepeatMasker (>= 4.2.4) can consume via ``-libdir``.
+
+See https://www.dfam.org/releases/current/families/FamDB/README.txt
+"""
+
+import argparse
+import gzip
+import hashlib
+import json
+import os
+import shutil
+import sys
+from datetime import date
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+# Map a Dfam release to its release directory and FamDB file-name prefix.
+# Adding a new release is a one-line change here (the file layout is stable).
+RELEASES = {
+    "4.0": {"dir": "Dfam_4.0", "prefix": "dfam40"},
+    "3.9": {"dir": "Dfam_3.9", "prefix": "dfam39"},
+    "3.8": {"dir": "Dfam_3.8", "prefix": "dfam38"},
+}
+
+BASE_URL = "https://www.dfam.org/releases/{dir}/families/FamDB/"
+
+# component key -> FamDB file-name infix
+COMPONENTS = {
+    "curated_consensus": "curated.consensus",
+    "uncurated_consensus": "uncurated.consensus",
+    "curated_hmm": "curated.hmm",
+    "uncurated_hmm": "uncurated.hmm",
+}
+
+# A tiny (~0.3 MB) real partition used to exercise the full download / gunzip /
+# checksum / registration path quickly during automated testing.
+TEST_COMPONENT = "uncurated_consensus"
+CHUNK = 2 ** 16
+
+
+def url_exists(url):
+    """Return True if a HEAD-like GET on url succeeds with status < 400."""
+    try:
+        return urlopen(Request(url)).getcode() < 400
+    except HTTPError:
+        return False
+
+
+def fetch_md5(url):
+    """Return the expected md5 hex digest from Dfam's ``<file>.md5`` sidecar."""
+    with urlopen(Request(url + ".md5")) as response:
+        # sidecar format: "<md5>  <filename>"
+        return response.read().decode().split()[0]
+
+
+def download_and_extract(url, target_directory):
+    """Download a gzipped ``.h5.gz`` partition, verify its md5, gunzip it.
+
+    The uncompressed ``.h5`` file is written into ``target_directory`` and the
+    compressed download is removed.  Raises on a checksum mismatch so a
+    truncated or corrupted download never registers as a usable library.
+    """
+    gz_name = url.rsplit("/", 1)[1]
+    gz_path = os.path.join(target_directory, gz_name)
+    h5_path = os.path.join(target_directory, gz_name[: -len(".gz")])
+
+    expected_md5 = fetch_md5(url)
+    md5 = hashlib.md5()
+    with urlopen(Request(url)) as src, open(gz_path, "wb") as dst:
+        while True:
+            chunk = src.read(CHUNK)
+            if not chunk:
+                break
+            md5.update(chunk)
+            dst.write(chunk)
+    if md5.hexdigest() != expected_md5:
+        sys.exit(
+            "Checksum mismatch for {}: expected {}, got {}".format(
+                gz_name, expected_md5, md5.hexdigest()
+            )
+        )
+
+    with gzip.open(gz_path, "rb") as f_in, open(h5_path, "wb") as f_out:
+        shutil.copyfileobj(f_in, f_out, CHUNK)
+    os.remove(gz_path)
+    return os.path.basename(h5_path)
+
+
+def iter_partition_urls(base_url, prefix, infix):
+    """Yield partition URLs for a component, probing 0,1,2,... until one is absent.
+
+    Dfam numbers component partitions contiguously from 0, so the first missing
+    index marks the end.  This adapts automatically to how many partitions a
+    component has in a given release instead of hard-coding per-release counts.
+    """
+    part = 0
+    while True:
+        url = "{}{}.{}.{}.h5.gz".format(base_url, prefix, infix, part)
+        if not url_exists(url):
+            break
+        yield url
+        part += 1
+
+
+def download(release, components, test, out_file):
+    if release not in RELEASES:
+        sys.exit(
+            "Unknown Dfam release '{}'. Known releases: {}".format(
+                release, ", ".join(sorted(RELEASES))
+            )
+        )
+    rel = RELEASES[release]
+    base_url = BASE_URL.format(dir=rel["dir"])
+    prefix = rel["prefix"]
+
+    with open(out_file) as fh:
+        params = json.load(fh)
+    target_directory = params["output_data"][0]["extra_files_path"]
+    os.makedirs(target_directory)
+
+    if test:
+        # Exercise the real code path cheaply: fetch only the smallest partition
+        # of a single component, and skip the large root/curated downloads.
+        components = [TEST_COMPONENT]
+        infix = COMPONENTS[TEST_COMPONENT]
+        installed = [
+            download_and_extract(
+                "{}{}.{}.0.h5.gz".format(base_url, prefix, infix), target_directory
+            )
+        ]
+    else:
+        # The root partition is always required.
+        installed = [
+            download_and_extract(
+                "{}{}.0.h5.gz".format(base_url, prefix), target_directory
+            )
+        ]
+        for component in components:
+            infix = COMPONENTS[component]
+            for url in iter_partition_urls(base_url, prefix, infix):
+                installed.append(download_and_extract(url, target_directory))
+
+    today = date.today().strftime("%Y-%m-%d")
+    selected = "+".join(components) if components else "root"
+    entry = {
+        "value": "dfam_{}_{}_{}".format(release, selected, today).replace(".", "_"),
+        "name": "Dfam {} ({}) [{}]".format(release, ", ".join(components) or "root only", today),
+        "version": release,
+        "path": target_directory,
+    }
+    data_manager_json = {"data_tables": {"repeatmasker_famdb": [entry]}}
+
+    sys.stderr.write(
+        "Installed {} FamDB partition file(s) into {}\n".format(
+            len(installed), target_directory
+        )
+    )
+    with open(out_file, "w") as fh:
+        json.dump(data_manager_json, fh, sort_keys=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--release", required=True, help="Dfam release, e.g. 4.0")
+    parser.add_argument(
+        "--components",
+        default="",
+        help="comma-separated component keys to install in addition to the root partition",
+    )
+    parser.add_argument("--out_file", required=True, help="JSON output file")
+    parser.add_argument(
+        "--test", action="store_true", help="download only a tiny partition for testing"
+    )
+    args = parser.parse_args()
+
+    components = [c for c in args.components.split(",") if c]
+    unknown = [c for c in components if c not in COMPONENTS]
+    if unknown:
+        sys.exit("Unknown component(s): {}".format(", ".join(unknown)))
+
+    download(args.release, components, args.test, args.out_file)

data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.xml

@@ -0,0 +1,77 @@
+<tool id="repeatmasker_download" name="RepeatMasker Dfam library downloader" tool_type="manage_data" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="@PROFILE@">
+    <description>download a Dfam FamDB library for RepeatMasker</description>
+    <macros>
+        <token name="@TOOL_VERSION@">4.0</token>
+        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@PROFILE@">22.01</token>
+    </macros>
+    <requirements>
+        <requirement type="package" version="3.12">python</requirement>
+    </requirements>
+    <command detect_errors="exit_code"><![CDATA[
+python '$__tool_directory__/repeatmasker_download.py'
+    --release '$release'
+    --components '$components'
+    --out_file '$out_file'
+    $test_data_manager
+    ]]></command>
+    <inputs>
+        <param name="release" type="select" label="Dfam release">
+            <option value="4.0" selected="true">Dfam 4.0</option>
+            <option value="3.9">Dfam 3.9</option>
+            <option value="3.8">Dfam 3.8</option>
+        </param>
+        <param name="components" type="select" multiple="true" optional="true" display="checkboxes"
+               label="Components to install in addition to the (always downloaded) root partition"
+               help="The curated consensus component is recommended for the default rmblast search engine. HMM components are only needed for the nhmmer engine. Uncurated components are very large.">
+            <option value="curated_consensus" selected="true">Curated consensus sequences (rmblast; recommended)</option>
+            <option value="uncurated_consensus">Uncurated consensus sequences (rmblast; very large)</option>
+            <option value="curated_hmm">Curated profile HMMs (nhmmer)</option>
+            <option value="uncurated_hmm">Uncurated profile HMMs (nhmmer; very large)</option>
+        </param>
+        <param name="test_data_manager" type="hidden" value="" />
+    </inputs>
+    <outputs>
+        <data name="out_file" format="data_manager_json" />
+    </outputs>
+    <tests>
+        <test>
+            <!-- Downloads only the smallest (~0.3 MB) partition to keep CI fast -->
+            <param name="release" value="4.0" />
+            <param name="test_data_manager" value="--test" />
+            <output name="out_file">
+                <assert_contents>
+                    <has_text text="repeatmasker_famdb" />
+                    <has_text text="dfam_4_0" />
+                    <has_text text="&quot;version&quot;: &quot;4.0&quot;" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+    <help><![CDATA[
+This data manager downloads a Dfam_ repeat library in the partitioned FamDB (v3)
+format and registers it in the ``repeatmasker_famdb`` data table, so that the
+RepeatMasker tool can screen sequences against it without relying on the empty
+placeholder library shipped in the RepeatMasker Conda package.
+
+**Components**
+
+Dfam is distributed as a required *root* partition (taxonomy and index) plus
+optional *component* partitions:
+
+- **Curated consensus** — curated families as consensus sequences; used by the
+  default rmblast search engine. Recommended for most installations.
+- **Uncurated consensus** — uncurated families as consensus sequences. Much
+  larger; only needed for broad, sensitive searches.
+- **Curated / Uncurated HMMs** — profile HMMs, used only by the nhmmer search
+  engine.
+
+The root partition is always downloaded; select the components you need. Each
+file's md5 checksum is verified after download.
+
+.. _Dfam: https://www.dfam.org/
+    ]]></help>
+    <citations>
+        <citation type="doi">10.1186/s13100-020-00230-y</citation>
+    </citations>
+</tool>

data_managers/data_manager_repeatmasker/data_manager_conf.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<data_managers>
+    <data_manager tool_file="data_manager/repeatmasker_download.xml" id="repeatmasker_download">
+        <data_table name="repeatmasker_famdb">
+            <output>
+                <column name="value" />
+                <column name="name" />
+                <column name="version" />
+                <column name="path" output_ref="out_file">
+                    <move type="directory" relativize_symlinks="True">
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">repeatmasker_famdb/${value}</target>
+                    </move>
+                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/repeatmasker_famdb/${value}</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+            </output>
+        </data_table>
+    </data_manager>
+</data_managers>

data_managers/data_manager_repeatmasker/test-data/repeatmasker_famdb.loc

@@ -0,0 +1 @@
+## Populated by the data manager during testing.

data_managers/data_manager_repeatmasker/tool-data/repeatmasker_famdb.loc.sample

@@ -0,0 +1,14 @@
+## Dfam FamDB libraries for RepeatMasker, installed by data_manager_repeatmasker
+##
+## This file lists directories holding a Dfam library in the partitioned FamDB
+## (v3) format (a root partition plus selected component partitions). Each such
+## directory can be passed to RepeatMasker (>= 4.2.4) via -libdir.
+##
+## The tab-separated columns are:
+##
+##   <value>	<name>	<version>	<path>
+##
+## For example:
+##
+##   dfam_4_0_curated_consensus_2026-07-22	Dfam 4.0 (curated_consensus) [2026-07-22]	4.0	/depot/repeatmasker_famdb/dfam_4_0_curated_consensus_2026-07-22
+##

data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.sample

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<tables>
+    <!-- Locations of Dfam FamDB libraries for RepeatMasker -->
+    <table name="repeatmasker_famdb" comment_char="#">
+        <columns>value, name, version, path</columns>
+        <file path="tool-data/repeatmasker_famdb.loc" />
+    </table>
+</tables>

data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.test

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<tables>
+    <!-- Locations of Dfam FamDB libraries for RepeatMasker -->
+    <table name="repeatmasker_famdb" comment_char="#">
+        <columns>value, name, version, path</columns>
+        <file path="${__HERE__}/test-data/repeatmasker_famdb.loc" />
+    </table>
+</tables>