diff --git a/data_managers/data_manager_repeatmasker/.shed.yml b/data_managers/data_manager_repeatmasker/.shed.yml new file mode 100644 index 00000000000..e01fa85511d --- /dev/null +++ b/data_managers/data_manager_repeatmasker/.shed.yml @@ -0,0 +1,14 @@ +categories: +- Data Managers +description: Download Dfam FamDB libraries for RepeatMasker +homepage_url: https://www.dfam.org/ +long_description: | + Downloads a Dfam repeat library in the partitioned FamDB (v3) format and + registers it in the repeatmasker_famdb data table, so RepeatMasker (>= 4.2.4) + can screen sequences against a real library instead of the empty placeholder + shipped in the RepeatMasker Conda package. The root partition is always + downloaded; curated/uncurated consensus and HMM components are selectable. +owner: iuc +name: data_manager_repeatmasker +remote_repository_url: https://github.com/galaxyproject/tools-iuc/tree/main/data_managers/data_manager_repeatmasker +type: unrestricted diff --git a/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.py b/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.py new file mode 100644 index 00000000000..2d353f88032 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +"""Download a Dfam FamDB library for RepeatMasker and register it in a data table. + +Dfam distributes its libraries in the partitioned FamDB (v3) format: a single +required *root* partition plus one file per *component* partition. The +components are split by curation status (curated/uncurated) and model type +(consensus sequences, used by the rmblast search engine, or profile HMMs, used +by nhmmer). This data manager downloads the root partition (always) together +with whichever components the admin selects, laying the ``*.h5`` files out in a +single directory that RepeatMasker (>= 4.2.4) can consume via ``-libdir``. The +selected components, and the search engines they support, are recorded in the +data table so consuming tools can filter on them. + +See https://www.dfam.org/releases/current/families/FamDB/README.txt +""" + +import argparse +import gzip +import hashlib +import json +import os +import shutil +import sys +from datetime import date +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +# Map a Dfam release to its release directory and FamDB file-name prefix. +# Adding a new release is a one-line change here (the file layout is stable). +RELEASES = { + "4.0": {"dir": "Dfam_4.0", "prefix": "dfam40"}, + "3.9": {"dir": "Dfam_3.9", "prefix": "dfam39"}, + "3.8": {"dir": "Dfam_3.8", "prefix": "dfam38"}, +} + +BASE_URL = "https://www.dfam.org/releases/{dir}/families/FamDB/" + +# component key -> FamDB file-name infix and the search engine that can use it. +# Consensus sequences are searched with rmblast, profile HMMs with nhmmer, so the +# selected components determine which engines the installed library supports. +COMPONENTS = { + "curated_consensus": {"infix": "curated.consensus", "engine": "rmblast"}, + "uncurated_consensus": {"infix": "uncurated.consensus", "engine": "rmblast"}, + "curated_hmm": {"infix": "curated.hmm", "engine": "nhmmer"}, + "uncurated_hmm": {"infix": "uncurated.hmm", "engine": "nhmmer"}, +} + +# Search engines, in the order they are reported in the data table. +ENGINES = ("rmblast", "nhmmer") + +# A tiny (~0.3 MB) real partition used to exercise the full download / gunzip / +# checksum / registration path quickly during automated testing. +TEST_COMPONENT = "uncurated_consensus" +CHUNK = 2 ** 16 + + +def url_exists(url): + """Return True if a HEAD-like GET on url succeeds with status < 400.""" + try: + return urlopen(Request(url)).getcode() < 400 + except HTTPError: + return False + + +def fetch_md5(url): + """Return the expected md5 hex digest from Dfam's ``.md5`` sidecar.""" + with urlopen(Request(url + ".md5")) as response: + # sidecar format: " " + return response.read().decode().split()[0] + + +def download_and_extract(url, target_directory): + """Download a gzipped ``.h5.gz`` partition, verify its md5, gunzip it. + + The uncompressed ``.h5`` file is written into ``target_directory`` and the + compressed download is removed. Raises on a checksum mismatch so a + truncated or corrupted download never registers as a usable library. + """ + gz_name = url.rsplit("/", 1)[1] + gz_path = os.path.join(target_directory, gz_name) + h5_path = os.path.join(target_directory, gz_name[: -len(".gz")]) + + expected_md5 = fetch_md5(url) + md5 = hashlib.md5() + with urlopen(Request(url)) as src, open(gz_path, "wb") as dst: + while True: + chunk = src.read(CHUNK) + if not chunk: + break + md5.update(chunk) + dst.write(chunk) + if md5.hexdigest() != expected_md5: + sys.exit( + "Checksum mismatch for {}: expected {}, got {}".format( + gz_name, expected_md5, md5.hexdigest() + ) + ) + + with gzip.open(gz_path, "rb") as f_in, open(h5_path, "wb") as f_out: + shutil.copyfileobj(f_in, f_out, CHUNK) + os.remove(gz_path) + return os.path.basename(h5_path) + + +def iter_partition_urls(base_url, prefix, infix): + """Yield partition URLs for a component, probing 0,1,2,... until one is absent. + + Dfam numbers component partitions contiguously from 0, so the first missing + index marks the end. This adapts automatically to how many partitions a + component has in a given release instead of hard-coding per-release counts. + """ + part = 0 + while True: + url = "{}{}.{}.{}.h5.gz".format(base_url, prefix, infix, part) + if not url_exists(url): + break + yield url + part += 1 + + +def download(release, components, test, out_file): + if release not in RELEASES: + sys.exit( + "Unknown Dfam release '{}'. Known releases: {}".format( + release, ", ".join(sorted(RELEASES)) + ) + ) + rel = RELEASES[release] + base_url = BASE_URL.format(dir=rel["dir"]) + prefix = rel["prefix"] + + with open(out_file) as fh: + params = json.load(fh) + target_directory = params["output_data"][0]["extra_files_path"] + os.makedirs(target_directory) + + if test: + # Exercise the real code path cheaply: fetch only the smallest partition + # of a single component, and skip the large root/curated downloads. + components = [TEST_COMPONENT] + infix = COMPONENTS[TEST_COMPONENT]["infix"] + installed = [ + download_and_extract( + "{}{}.{}.0.h5.gz".format(base_url, prefix, infix), target_directory + ) + ] + else: + # The root partition is always required. + installed = [ + download_and_extract( + "{}{}.0.h5.gz".format(base_url, prefix), target_directory + ) + ] + for component in components: + infix = COMPONENTS[component]["infix"] + for url in iter_partition_urls(base_url, prefix, infix): + installed.append(download_and_extract(url, target_directory)) + + today = date.today().strftime("%Y-%m-%d") + selected = "+".join(components) + # Record what was installed so consuming tools can filter on it, e.g. offer a + # library only for a search engine its components actually support. + engines = [ + engine + for engine in ENGINES + if any(COMPONENTS[c]["engine"] == engine for c in components) + ] + entry = { + "value": "dfam_{}_{}_{}".format(release, selected, today).replace(".", "_"), + "name": "Dfam {} ({}) [{}]".format(release, ", ".join(components), today), + "version": release, + "components": ",".join(components), + "engines": ",".join(engines), + "path": target_directory, + } + data_manager_json = {"data_tables": {"repeatmasker_famdb": [entry]}} + + sys.stderr.write( + "Installed {} FamDB partition file(s) into {}\n".format( + len(installed), target_directory + ) + ) + with open(out_file, "w") as fh: + json.dump(data_manager_json, fh, sort_keys=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--release", required=True, help="Dfam release, e.g. 4.0") + parser.add_argument( + "--components", + default="", + help="comma-separated component keys to install in addition to the root partition", + ) + parser.add_argument("--out_file", required=True, help="JSON output file") + parser.add_argument( + "--test", action="store_true", help="download only a tiny partition for testing" + ) + args = parser.parse_args() + + # Cheetah renders an empty multi-select as the string "None". + components = [c for c in args.components.split(",") if c and c != "None"] + if not components: + sys.exit( + "No components selected. Select at least one of: {}".format( + ", ".join(sorted(COMPONENTS)) + ) + ) + unknown = [c for c in components if c not in COMPONENTS] + if unknown: + sys.exit("Unknown component(s): {}".format(", ".join(unknown))) + + download(args.release, components, args.test, args.out_file) diff --git a/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.xml b/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.xml new file mode 100644 index 00000000000..7a5350f86c4 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/data_manager/repeatmasker_download.xml @@ -0,0 +1,98 @@ + + download a Dfam FamDB library for RepeatMasker + + 4.0 + 0 + 22.01 + + + python + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + = 4.2.4). + +**Why this is needed** + +The RepeatMasker Conda package no longer ships a populated repeat library: the +``Dfam.h5`` bundled in the package is a placeholder, and running RepeatMasker +against it fails with ``Species "..." is not known to RepeatMasker`` because the +library contains no families. The library has to be downloaded separately; this +data manager automates that and makes the result available through a data table. + +**Components** + +Dfam is distributed as a required *root* partition (taxonomy and index) plus +optional *component* partitions, split by curation status (curated/uncurated) +and model type (consensus sequences for the rmblast search engine, profile HMMs +for nhmmer): + +- **Curated consensus** — curated families as consensus sequences; used by the + default rmblast search engine. Recommended for most installations. +- **Uncurated consensus** — uncurated families as consensus sequences. Much + larger; only needed for broad, sensitive searches. +- **Curated / Uncurated HMMs** — profile HMMs, used only by the nhmmer search + engine. The uncurated HMM component has 100+ partitions. + +The root partition is always downloaded, and at least one component must be +selected. Each file is verified against its Dfam md5 checksum, decompressed, and +placed in a single directory that RepeatMasker consumes via ``-libdir``. + +**Data table** + +``repeatmasker_famdb``, with the columns +``value, name, version, components, engines, path``, where ``path`` points at +the directory holding the FamDB ``.h5`` partition files. ``components`` records +which components were selected and ``engines`` the search engines they can be +used with (``rmblast`` and/or ``nhmmer``); both are comma-separated so a +consuming tool can filter on them with a ``multiple_splitter`` filter. + +.. _Dfam: https://www.dfam.org/ + ]]> + + 10.1186/s13100-020-00230-y + + diff --git a/data_managers/data_manager_repeatmasker/data_manager_conf.xml b/data_managers/data_manager_repeatmasker/data_manager_conf.xml new file mode 100644 index 00000000000..c3c2d1e1bbb --- /dev/null +++ b/data_managers/data_manager_repeatmasker/data_manager_conf.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + repeatmasker_famdb/${value} + + ${GALAXY_DATA_MANAGER_DATA_PATH}/repeatmasker_famdb/${value} + abspath + + + + + diff --git a/data_managers/data_manager_repeatmasker/test-data/repeatmasker_famdb.loc b/data_managers/data_manager_repeatmasker/test-data/repeatmasker_famdb.loc new file mode 100644 index 00000000000..dc309ef95a3 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/test-data/repeatmasker_famdb.loc @@ -0,0 +1 @@ +## Populated by the data manager during testing. diff --git a/data_managers/data_manager_repeatmasker/tool-data/repeatmasker_famdb.loc.sample b/data_managers/data_manager_repeatmasker/tool-data/repeatmasker_famdb.loc.sample new file mode 100644 index 00000000000..3050930f3e6 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/tool-data/repeatmasker_famdb.loc.sample @@ -0,0 +1,20 @@ +## Dfam FamDB libraries for RepeatMasker, installed by data_manager_repeatmasker +## +## This file lists directories holding a Dfam library in the partitioned FamDB +## (v3) format (a root partition plus selected component partitions). Each such +## directory can be passed to RepeatMasker (>= 4.2.4) via -libdir. +## +## The tab-separated columns are: +## +## +## +## lists the component partitions installed alongside the root +## partition, and the search engines those components can be used with +## (consensus sequences -> rmblast, profile HMMs -> nhmmer). Both are +## comma-separated, so a consuming tool can filter on them with a +## multiple_splitter filter. +## +## For example: +## +## dfam_4_0_curated_consensus_2026-07-22 Dfam 4.0 (curated_consensus) [2026-07-22] 4.0 curated_consensus rmblast /depot/repeatmasker_famdb/dfam_4_0_curated_consensus_2026-07-22 +## diff --git a/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.sample b/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.sample new file mode 100644 index 00000000000..87355cc5e48 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.sample @@ -0,0 +1,8 @@ + + + + + value, name, version, components, engines, path + +
+ diff --git a/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.test b/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.test new file mode 100644 index 00000000000..79374b04533 --- /dev/null +++ b/data_managers/data_manager_repeatmasker/tool_data_table_conf.xml.test @@ -0,0 +1,8 @@ + + + + + value, name, version, components, engines, path + +
+