diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index c024d3a..96e748c 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -38,11 +38,16 @@ runs: - name: Setup apptainer if: contains(inputs.profile, 'singularity') uses: eWaterCycle/setup-apptainer@4bb22c52d4f63406c49e94c804632975787312b3 # v2.0.0 + with: + apptainer-version: 1.5.3 - name: Set up Singularity if: contains(inputs.profile, 'singularity') shell: bash run: | + # The runner image restricts unprivileged user namespaces, which non-suid + # apptainer needs to start a container at all. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 91f761d..4f60919 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -24,7 +24,6 @@ jobs: - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@51565b514bff1827cf34620de25d0055759f1fc9 # v2 - # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters with: diff --git a/.nf-core.yml b/.nf-core.yml index d33098c..de98ff1 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -3,7 +3,9 @@ lint: - assets/nf-core-funcprofiler_logo_light.png - .github/PULL_REQUEST_TEMPLATE.md - .github/workflows/linting.yml - - .github/workflows/linting_comment.yml + # The 4.0.3 template renders `HEAD_REPO:` with two spaces, which the prettier + # pin used by prek collapses to one, so this file can never match the template. + - .github/workflows/branch.yml - .gitignore nf_core_version: 4.0.3 repository_type: pipeline diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml index d1de92d..7900625 100644 --- a/assets/methods_description_template.yml +++ b/assets/methods_description_template.yml @@ -3,7 +3,6 @@ description: "Suggested text and references to use when describing pipeline usag section_name: "nf-core/funcprofiler Methods Description" section_href: "https://github.com/nf-core/funcprofiler" plot_type: "html" -## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline ## You inject any metadata in the Nextflow '${workflow}' object data: |

Methods

diff --git a/conf/base.config b/conf/base.config index f61c036..49af73f 100644 --- a/conf/base.config +++ b/conf/base.config @@ -10,7 +10,6 @@ process { - // TODO nf-core: Check the defaults for all processes cpus = { 1 * task.attempt } memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } @@ -24,7 +23,6 @@ process { // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. // If possible, it would be nice to keep the same label naming convention when // adding in your local modules too. - // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { cpus = { 1 } diff --git a/conf/test.config b/conf/test.config index 347dec0..7d055c2 100644 --- a/conf/test.config +++ b/conf/test.config @@ -21,11 +21,8 @@ process { params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - //input = params.pipelines_testdata_base_path + 'samplesheets/test.csv' - input = 'https://github.com/nickp60/test-datasets/raw/refs/heads/funcprofiler/samplesheets/test.csv' + input = params.pipelines_testdata_base_path + 'samplesheets/test.csv' databases = params.pipelines_testdata_base_path + 'database.csv' - outdir = "tmp" - cleanup = false run_fmhfunprofiler = true run_humann_v3 = true run_mifaser = true diff --git a/conf/test_full.config b/conf/test_full.config index 340f238..148780b 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -13,12 +13,8 @@ params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' -// input = params.pipelines_testdata_base_path + 'samplesheets/test.csv' -// databases = params.pipelines_testdata_base_path + 'database_full.csv' input = 'https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/taxprofiler/samplesheet_shortreadsonly.csv' - databases = 'https://raw.githubusercontent.com/nickp60/test-datasets/refs/heads/funcprofiler/database_full.csv' - outdir = "fulltest_results" - cleanup = false + databases = params.pipelines_testdata_base_path + 'database_full.csv' run_fmhfunprofiler = true run_humann_v3 = true run_mifaser = true diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 30309b6..95b80c6 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -182,4 +182,9 @@ If you update images or graphics, follow the nf-core [style guidelines](https:// ## Pipeline specific contribution guidelines - +When adding a new profiler: + +- Add it to [nf-core/modules](https://github.com/nf-core/modules) rather than to `modules/local`. A local module is only a stopgap for tools that are not yet packaged on Bioconda. +- Gate it behind a `run_` parameter, and take its database from the database sheet so it is grouped by `dbprep` like every other tool. +- Add it to the `profile` subworkflow, to `docs/output.md`, and to `CITATIONS.md`. +- Cover it with an nf-test, and regenerate snapshots with `nf-test test --update-snapshot`. diff --git a/main.nf b/main.nf index cfb7114..d45ad6e 100644 --- a/main.nf +++ b/main.nf @@ -38,7 +38,7 @@ workflow NFCORE_FUNCPROFILER { // // WORKFLOW: Run pipeline // - FUNCPROFILER ( + FUNCPROFILER( samplesheet, databases, params.multiqc_config, diff --git a/modules.json b/modules.json index 1189520..68fa76c 100644 --- a/modules.json +++ b/modules.json @@ -12,7 +12,7 @@ }, "diamond/blastx": { "branch": "master", - "git_sha": "1b534ad2336876dbe82e9fea2773334cb624b426", + "git_sha": "83ce9b314157a24d9a92f17a07ed5d55f30b78f9", "installed_by": ["modules"] }, "eggnogmapper": { @@ -32,7 +32,7 @@ }, "gunzip": { "branch": "master", - "git_sha": "96c57dfd98a0641886a67bd449fe33ee2ec0e374", + "git_sha": "0902eac3012baaf4f9ab6513c8c55acc9353c96c", "installed_by": ["modules"] }, "humann3/humann": { @@ -57,7 +57,7 @@ }, "mifaser": { "branch": "master", - "git_sha": "6b43745037429413ca11e0e4a1029d90493c3135", + "git_sha": "522d0a5709833aaa28c58d4464b4348d03316a62", "installed_by": ["modules"] }, "multiqc": { @@ -77,12 +77,12 @@ }, "seqkit/fq2fa": { "branch": "master", - "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120", + "git_sha": "d07a945f421a4a91ce4e5a22db63e0e7f8741607", "installed_by": ["modules"] }, "untar": { "branch": "master", - "git_sha": "447f7bc0fa41dfc2400c8cad4c0291880dc060cf", + "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120", "installed_by": ["modules"] } } @@ -101,7 +101,7 @@ }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "ce3424257de288c0ca25f097a68d3a289da23f12", + "git_sha": "a7b27fd25bfa8dcc07d299e88bd790585901a436", "installed_by": ["subworkflows"] } } diff --git a/modules/local/humann4/humann/main.nf b/modules/local/humann4/humann/main.nf index ad5b87d..861ef5e 100644 --- a/modules/local/humann4/humann/main.nf +++ b/modules/local/humann4/humann/main.nf @@ -1,6 +1,6 @@ // Taken 98% from https://github.com/nf-core/modules/pull/1089/files process HUMANN4 { - tag "$meta.id" + tag "${meta.id}" label 'process_high' conda "${moduleDir}/environment.yml" @@ -14,10 +14,10 @@ process HUMANN4 { path utility_db output: - tuple val(meta), path("*_genefamilies.tsv.gz") , emit: genefamilies + tuple val(meta), path("*_genefamilies.tsv.gz"), emit: genefamilies tuple val(meta), path("*_pathabundance.tsv.gz"), emit: pathabundance - tuple val(meta), path("*_reactions.tsv.gz") , emit: reactions - tuple val(meta), path("*.log") , emit: log + tuple val(meta), path("*_reactions.tsv.gz"), emit: reactions + tuple val(meta), path("*.log"), emit: log tuple val("${task.process}"), val('HUMAnN'), eval("humann --version 2>&1 | sed 's/humann v//'"), emit: versions_humann, topic: versions tuple val("${task.process}"), val('MetaPHLan'), eval("metaphlan --version 2>&1 | sed 's/metaphlan v//'"), emit: versions_metaphlan, topic: versions @@ -39,13 +39,13 @@ process HUMANN4 { find \${NUCS_DB} humann \\ - $args \\ + ${args} \\ --threads ${task.cpus} \\ - --input $input \\ + --input ${input} \\ --protein-database \${PROTS_DB} \\ --nucleotide-database \${NUCS_DB} \\ - --output-basename $prefix \\ - $pangenome_string \\ + --output-basename ${prefix} \\ + ${pangenome_string} \\ ${args} \\ --o-log ${prefix}.log \\ --output . @@ -54,11 +54,12 @@ process HUMANN4 { gzip -n *.tsv """ + stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - echo $args + echo ${args} for suf in genefamilies.tsv.gz pathabundance.tsv.gz reactions.tsv.gz do diff --git a/modules/nf-core/diamond/blastx/environment.yml b/modules/nf-core/diamond/blastx/environment.yml index cca5cbb..5dc13ad 100644 --- a/modules/nf-core/diamond/blastx/environment.yml +++ b/modules/nf-core/diamond/blastx/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::diamond=2.1.24 + - bioconda::diamond=2.2.1 diff --git a/modules/nf-core/diamond/blastx/main.nf b/modules/nf-core/diamond/blastx/main.nf index 2b03de0..453170c 100644 --- a/modules/nf-core/diamond/blastx/main.nf +++ b/modules/nf-core/diamond/blastx/main.nf @@ -4,8 +4,8 @@ process DIAMOND_BLASTX { conda "${moduleDir}/environment.yml" container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container - ? 'https://depot.galaxyproject.org/singularity/diamond:2.1.24--hf93d47f_0' - : 'quay.io/biocontainers/diamond:2.1.24--hf93d47f_0'}" + ? 'https://depot.galaxyproject.org/singularity/diamond:2.2.1--he361c42_0' + : 'quay.io/biocontainers/diamond:2.2.1--he361c42_0'}" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/diamond/blastx/meta.yml b/modules/nf-core/diamond/blastx/meta.yml index e00b8fa..019a3e9 100644 --- a/modules/nf-core/diamond/blastx/meta.yml +++ b/modules/nf-core/diamond/blastx/meta.yml @@ -176,3 +176,4 @@ maintainers: - "@jfy133" - "@mjamy" - "@vagkaratzas" + - "@sofstam" diff --git a/modules/nf-core/diamond/blastx/tests/main.nf.test.snap b/modules/nf-core/diamond/blastx/tests/main.nf.test.snap index 80340d2..b3cf211 100644 --- a/modules/nf-core/diamond/blastx/tests/main.nf.test.snap +++ b/modules/nf-core/diamond/blastx/tests/main.nf.test.snap @@ -6,16 +6,16 @@ [ "DIAMOND_BLASTX", "diamond", - "2.1.24" + "2.2.1" ] ] } ], + "timestamp": "2026-06-09T15:06:53.562101168", "meta": { - "nf-test": "0.9.3", - "nextflow": "25.10.4" - }, - "timestamp": "2026-04-22T15:47:51.928317786" + "nf-test": "0.9.5", + "nextflow": "26.04.3" + } }, "Should search for transcriptome hits against a DIAMOND db and return the default tab separated output file of hits": { "content": [ @@ -32,15 +32,15 @@ [ "DIAMOND_BLASTX", "diamond", - "2.1.24" + "2.2.1" ] ] } ], + "timestamp": "2026-06-09T15:06:47.844654623", "meta": { - "nf-test": "0.9.3", - "nextflow": "25.10.4" - }, - "timestamp": "2026-04-22T15:47:45.383426326" + "nf-test": "0.9.5", + "nextflow": "26.04.3" + } } } \ No newline at end of file diff --git a/modules/nf-core/gunzip/main.nf b/modules/nf-core/gunzip/main.nf index a8533e7..6edffc5 100644 --- a/modules/nf-core/gunzip/main.nf +++ b/modules/nf-core/gunzip/main.nf @@ -3,7 +3,7 @@ process GUNZIP { label 'process_single' conda "${moduleDir}/environment.yml" - container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/52/52ccce28d2ab928ab862e25aae26314d69c8e38bd41ca9431c67ef05221348aa/data' : 'community.wave.seqera.io/library/coreutils_grep_gzip_lbzip2_pruned:838ba80435a629f8'}" @@ -19,8 +19,9 @@ process GUNZIP { script: def args = task.ext.args ?: '' - def extension = (archive.toString() - '.gz').tokenize('.')[-1] - def name = archive.toString() - '.gz' - ".${extension}" + def nameWithoutGz = archive.extension == 'gz' ? archive.baseName : archive.name + def extension = file(nameWithoutGz).extension + def name = file(nameWithoutGz).baseName def prefix = task.ext.prefix ?: name gunzip = prefix + ".${extension}" """ @@ -35,8 +36,9 @@ process GUNZIP { """ stub: - def extension = (archive.toString() - '.gz').tokenize('.')[-1] - def name = archive.toString() - '.gz' - ".${extension}" + def nameWithoutGz = archive.extension == 'gz' ? archive.baseName : archive.name + def extension = file(nameWithoutGz).extension + def name = file(nameWithoutGz).baseName def prefix = task.ext.prefix ?: name gunzip = prefix + ".${extension}" """ diff --git a/modules/nf-core/gunzip/tests/main.nf.test b/modules/nf-core/gunzip/tests/main.nf.test index 776211a..33cb75a 100644 --- a/modules/nf-core/gunzip/tests/main.nf.test +++ b/modules/nf-core/gunzip/tests/main.nf.test @@ -89,6 +89,60 @@ nextflow_process { } + test("Should decompress file with extension appearing multiple times in filename") { + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([ + [], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true).copyTo('test.fa.v1.fa.gz') + ] + ) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert file(process.out.gunzip[0][1]).name == 'test.fa.v1.fa' } + ) + } + + } + + test("Should decompress file with extension appearing multiple times in filename - prefix") { + + config './nextflow.config' + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([ + [ id: 'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true).copyTo('test.fa.v1.fa.gz') + ] + ) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + test("Should run without failures - prefix - stub") { options '-stub' diff --git a/modules/nf-core/gunzip/tests/main.nf.test.snap b/modules/nf-core/gunzip/tests/main.nf.test.snap index 111ba1b..980e95a 100644 --- a/modules/nf-core/gunzip/tests/main.nf.test.snap +++ b/modules/nf-core/gunzip/tests/main.nf.test.snap @@ -1,4 +1,45 @@ { + "Should decompress file with extension appearing multiple times in filename - prefix": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.xyz.fa:md5,4161df271f9bfcd25d5845a1e220dbec" + ] + ], + "1": [ + [ + "GUNZIP", + "gunzip", + "1.13" + ] + ], + "gunzip": [ + [ + { + "id": "test" + }, + "test.xyz.fa:md5,4161df271f9bfcd25d5845a1e220dbec" + ] + ], + "versions_gunzip": [ + [ + "GUNZIP", + "gunzip", + "1.13" + ] + ] + } + ], + "timestamp": "2026-04-15T15:20:52.59447", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.4" + } + }, "Should run without failures - prefix - stub": { "content": [ { @@ -34,11 +75,11 @@ ] } ], + "timestamp": "2026-01-19T17:21:56.633550769", "meta": { "nf-test": "0.9.3", "nextflow": "25.10.2" - }, - "timestamp": "2026-01-19T17:21:56.633550769" + } }, "Should run without failures - stub": { "content": [ @@ -75,11 +116,11 @@ ] } ], + "timestamp": "2026-01-19T17:21:51.435621199", "meta": { "nf-test": "0.9.3", "nextflow": "25.10.2" - }, - "timestamp": "2026-01-19T17:21:51.435621199" + } }, "Should run without failures": { "content": [ @@ -116,11 +157,11 @@ ] } ], + "timestamp": "2026-01-19T17:21:40.613975821", "meta": { "nf-test": "0.9.3", "nextflow": "25.10.2" - }, - "timestamp": "2026-01-19T17:21:40.613975821" + } }, "Should run without failures - prefix": { "content": [ @@ -157,10 +198,10 @@ ] } ], + "timestamp": "2026-01-19T17:21:46.086880414", "meta": { "nf-test": "0.9.3", "nextflow": "25.10.2" - }, - "timestamp": "2026-01-19T17:21:46.086880414" + } } } \ No newline at end of file diff --git a/modules/nf-core/mifaser/tests/main.nf.test b/modules/nf-core/mifaser/tests/main.nf.test index fa7a239..954ba2b 100644 --- a/modules/nf-core/mifaser/tests/main.nf.test +++ b/modules/nf-core/mifaser/tests/main.nf.test @@ -33,7 +33,7 @@ nextflow_process { [ id:'test', single_end:true ], [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] ] - input[1] = UNTAR.out.untar.map{ meta, dir -> dir.resolve("database/mifaser/GS-24-all") } + input[1] = UNTAR.out.untar.map { it[1] } """ } } diff --git a/modules/nf-core/seqkit/fq2fa/environment.yml b/modules/nf-core/seqkit/fq2fa/environment.yml index b26fb1e..09abb79 100644 --- a/modules/nf-core/seqkit/fq2fa/environment.yml +++ b/modules/nf-core/seqkit/fq2fa/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::seqkit=2.9.0 + - bioconda::seqkit=2.13.0 diff --git a/modules/nf-core/seqkit/fq2fa/main.nf b/modules/nf-core/seqkit/fq2fa/main.nf index 313af89..0722455 100644 --- a/modules/nf-core/seqkit/fq2fa/main.nf +++ b/modules/nf-core/seqkit/fq2fa/main.nf @@ -1,11 +1,11 @@ process SEQKIT_FQ2FA { - tag "$meta.id" + tag "${meta.id}" label 'process_single' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/seqkit:2.9.0--h9ee0642_0' : - 'quay.io/biocontainers/seqkit:2.9.0--h9ee0642_0' }" + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/4f/4fe272ab9a519cf418160471a485b5ef50ea3f571a8e4555a826f70a4d8243ae/data' + : 'community.wave.seqera.io/library/seqkit:2.13.0--05c0a96bf9fb2751'}" input: tuple val(meta), path(fastq) @@ -20,19 +20,21 @@ process SEQKIT_FQ2FA { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - """ seqkit \\ fq2fa \\ - $args \\ - -j $task.cpus \\ + ${args} \\ + -j ${task.cpus} \\ -o ${prefix}.fa.gz \\ - $fastq + ${fastq} """ stub: + def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ + echo ${args} + echo "" | gzip > ${prefix}.fa.gz """ } diff --git a/modules/nf-core/seqkit/fq2fa/tests/main.nf.test b/modules/nf-core/seqkit/fq2fa/tests/main.nf.test index 08f399e..b20ac23 100644 --- a/modules/nf-core/seqkit/fq2fa/tests/main.nf.test +++ b/modules/nf-core/seqkit/fq2fa/tests/main.nf.test @@ -10,47 +10,44 @@ nextflow_process { tag "seqkit/fq2fa" test("sarscov2 - bam") { - when { process { """ - input[0] = [[ id:'test', single_end:false ], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] """ } } then { + assert process.success assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(sanitizeOutput(process.out)).match() } ) } - } test("sarscov2 - bam - stub") { - - options "-stub" + options '-stub' when { process { """ - input[0] = [[ id:'test', single_end:false ], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] """ } } then { + assert process.success assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(sanitizeOutput(process.out)).match() } ) } - } - } diff --git a/modules/nf-core/seqkit/fq2fa/tests/main.nf.test.snap b/modules/nf-core/seqkit/fq2fa/tests/main.nf.test.snap index b96e153..ea4763d 100644 --- a/modules/nf-core/seqkit/fq2fa/tests/main.nf.test.snap +++ b/modules/nf-core/seqkit/fq2fa/tests/main.nf.test.snap @@ -2,22 +2,6 @@ "sarscov2 - bam - stub": { "content": [ { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fa.gz:md5,68b329da9893e34099c7d8ad5cb9c940" - ] - ], - "1": [ - [ - "SEQKIT_FQ2FA", - "seqkit", - "2.9.0" - ] - ], "fasta": [ [ { @@ -31,36 +15,20 @@ [ "SEQKIT_FQ2FA", "seqkit", - "2.9.0" + "2.13.0" ] ] } ], - "timestamp": "2026-03-12T13:05:56.983767141", + "timestamp": "2026-06-15T10:19:21.836507096", "meta": { - "nf-test": "0.9.4", - "nextflow": "25.10.4" + "nf-test": "0.9.5", + "nextflow": "26.04.3" } }, "sarscov2 - bam": { "content": [ { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.fa.gz:md5,f0c5c9110ce19e9ebbc9a6b6baf9e105" - ] - ], - "1": [ - [ - "SEQKIT_FQ2FA", - "seqkit", - "2.9.0" - ] - ], "fasta": [ [ { @@ -74,15 +42,15 @@ [ "SEQKIT_FQ2FA", "seqkit", - "2.9.0" + "2.13.0" ] ] } ], - "timestamp": "2026-03-12T13:05:49.874408711", + "timestamp": "2026-06-12T17:24:20.019577498", "meta": { - "nf-test": "0.9.4", - "nextflow": "25.10.4" + "nf-test": "0.9.5", + "nextflow": "26.04.3" } } } \ No newline at end of file diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index b9c324d..bf2c056 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -3,7 +3,7 @@ process UNTAR { label 'process_single' conda "${moduleDir}/environment.yml" - container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/52/52ccce28d2ab928ab862e25aae26314d69c8e38bd41ca9431c67ef05221348aa/data' : 'community.wave.seqera.io/library/coreutils_grep_gzip_lbzip2_pruned:838ba80435a629f8'}" diff --git a/nextflow_schema.json b/nextflow_schema.json index f63b96f..e855d7e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -129,19 +129,16 @@ }, "run_rgi": { "type": "boolean", - "description": "Turn on profiling with RGI. Requires database to be present CSV file passed to --databases", - "default": false + "description": "Turn on profiling with RGI. Requires database to be present CSV file passed to --databases" }, "run_mifaser": { "type": "boolean", - "description": "Turn on profiling with mi-faser. Requires database to be present CSV file passed to --databases", - "default": false + "description": "Turn on profiling with mi-faser. Requires database to be present CSV file passed to --databases" }, "run_eggnogmapper": { "type": "boolean", "fa_icon": "fas fa-toggle-on", - "description": "Turn on functional annotation with EggNOG-mapper. Requires `eggnogmapper`-tagged database entries in the CSV file passed to --databases", - "default": false + "description": "Turn on functional annotation with EggNOG-mapper. Requires `eggnogmapper`-tagged database entries in the CSV file passed to --databases" } } }, diff --git a/subworkflows/local/dataprep/meta.yml b/subworkflows/local/dataprep/meta.yml index 40b2a5f..915faa4 100644 --- a/subworkflows/local/dataprep/meta.yml +++ b/subworkflows/local/dataprep/meta.yml @@ -1,50 +1,42 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "dataprep" -## TODO nf-core: Add a description of the subworkflow and list keywords -description: Sort SAM/BAM/CRAM file +description: | + Validate the input samplesheet and prepare read channels for profiling. Sets + `single_end`, `run_accession` and `instrument_platform` on the meta map, merges + multiple runs belonging to the same sample, and produces an additional channel + in which read pairs are concatenated into a single file for tools that cannot + take paired input. keywords: - - sort - - bam - - sam - - cram -## TODO nf-core: Add a list of the modules and/or subworkflows used in the subworkflow + - samplesheet + - fastq + - merging + - concatenation + - metagenome components: - - samtools/sort - - samtools/index -## TODO nf-core: List all of the channels used as input with a description and their structure + - cat/fastq input: - - ch_bam: + - samplesheet: type: file description: | - The input channel containing the BAM/CRAM/SAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.{bam/cram/sam}" -## TODO nf-core: List all of the channels used as output with a descriptions and their structure + Channel of validated samplesheet rows. Long-read (`OXFORD_NANOPORE`) + platforms and FASTA input are rejected, and `fastq_1` is required. + Structure: [ val(meta), val(run_accession), val(instrument_platform), path(fastq_1), path(fastq_2), path(fasta) ] + pattern: "*.{fq,fastq}{,.gz}" output: - - bam: + - reads: type: file description: | - Channel containing BAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.bam" - - bai: + Channel of reads per sample, with runs merged where a sample has more + than one. Paired-end samples carry two files, single-end samples one. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fq,fastq}{,.gz}" + - reads_concat: type: file description: | - Channel containing indexed BAM (BAI) files - Structure: [ val(meta), path(bai) ] - pattern: "*.bai" - - csi: - type: file - description: | - Channel containing CSI files - Structure: [ val(meta), path(csi) ] - pattern: "*.csi" - - versions: - type: file - description: | - File containing software versions - Structure: [ path(versions.yml) ] - pattern: "versions.yml" + Channel of the same reads concatenated into a single file per sample, + with `single_end` forced to `true`, for tools that accept only one FASTQ. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fq,fastq}{,.gz}" authors: - "@nickp60" maintainers: diff --git a/subworkflows/local/dbprep/meta.yml b/subworkflows/local/dbprep/meta.yml index c915730..89e987e 100644 --- a/subworkflows/local/dbprep/meta.yml +++ b/subworkflows/local/dbprep/meta.yml @@ -1,50 +1,35 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "dbprep" -## TODO nf-core: Add a description of the subworkflow and list keywords -description: Sort SAM/BAM/CRAM file +description: | + Prepare the databases declared in the database sheet. Archives are decompressed + once per file and only for tools the user has enabled, then all entities + belonging to the same database are grouped so that each profiler receives a + single entry per database name and parameter combination. keywords: - - sort - - bam - - sam - - cram -## TODO nf-core: Add a list of the modules and/or subworkflows used in the subworkflow + - database + - untar + - decompression + - metagenome components: - - samtools/sort - - samtools/index -## TODO nf-core: List all of the channels used as input with a description and their structure + - untar input: - - ch_bam: + - databases: type: file description: | - The input channel containing the BAM/CRAM/SAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.{bam/cram/sam}" -## TODO nf-core: List all of the channels used as output with a descriptions and their structure + Channel of database entries from the database sheet. `meta.tool` selects + which profiler the database belongs to, `meta.db_entity` names the part + of a multi-file database, and `meta.db_params` carries tool-specific + arguments. Entries ending in `.tar`, `.tar.gz`, `.tar.bz2` or `.tar.xz` + are decompressed, the rest are passed through untouched. + Structure: [ val(meta), path(database) ] + pattern: "*.{tar,tar.gz,tar.bz2,tar.xz}" output: - - bam: + - dbs: type: file description: | - Channel containing BAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.bam" - - bai: - type: file - description: | - Channel containing indexed BAM (BAI) files - Structure: [ val(meta), path(bai) ] - pattern: "*.bai" - - csi: - type: file - description: | - Channel containing CSI files - Structure: [ val(meta), path(csi) ] - pattern: "*.csi" - - versions: - type: file - description: | - File containing software versions - Structure: [ path(versions.yml) ] - pattern: "versions.yml" + Channel of databases grouped by tool, name and parameters. The second + element is a list of maps, each holding the entity name and its path. + Structure: [ val(meta), [ [ db_entity: val(entity), db_path: path(db) ] ] ] authors: - "@nickp60" maintainers: diff --git a/subworkflows/local/profile/main.nf b/subworkflows/local/profile/main.nf index 931005b..21ddf47 100644 --- a/subworkflows/local/profile/main.nf +++ b/subworkflows/local/profile/main.nf @@ -2,21 +2,23 @@ // Run profiling // -include { MIFASER } from '../../../modules/nf-core/mifaser/main' -include { HUMANN3_HUMANN } from '../../../modules/nf-core/humann3/humann/main' -include { HUMANN4 } from '../../../modules/local/humann4/humann/main' -include { HUMANN3_REGROUP } from '../../../modules/nf-core/humann3/regroup/main' -include { HUMANN4_REGROUP } from '../../../modules/local/humann4/regroup/main' +include { MIFASER } from '../../../modules/nf-core/mifaser/main' +include { HUMANN3_HUMANN } from '../../../modules/nf-core/humann3/humann/main' +include { HUMANN4 } from '../../../modules/local/humann4/humann/main' +include { HUMANN3_REGROUP } from '../../../modules/nf-core/humann3/regroup/main' +include { HUMANN4_REGROUP } from '../../../modules/local/humann4/regroup/main' //include { FMHFUNPROFILER } from '../../../modules/local/fmhfunprofiler/main' -include { FMHFUNPROFILER } from '../../../modules/nf-core/fmhfunprofiler/main' -include { METAPHLAN_METAPHLAN as MPAHUMANN3; - METAPHLAN_METAPHLAN as MPAHUMANN4 } from '../../../modules/nf-core/metaphlan/metaphlan/main' -include { DIAMOND_BLASTX } from '../../../modules/nf-core/diamond/blastx/main' -include { RGI_BWT } from '../../../modules/nf-core/rgi/bwt/main' -include { RGI_CARDANNOTATION } from '../../../modules/nf-core/rgi/cardannotation/main' -include { EGGNOGMAPPER } from '../../../modules/nf-core/eggnogmapper/main' -include { SEQKIT_FQ2FA } from '../../../modules/nf-core/seqkit/fq2fa/main' -include { GUNZIP } from '../../../modules/nf-core/gunzip/main' +include { FMHFUNPROFILER } from '../../../modules/nf-core/fmhfunprofiler/main' +include { + METAPHLAN_METAPHLAN as MPAHUMANN3 ; + METAPHLAN_METAPHLAN as MPAHUMANN4 +} from '../../../modules/nf-core/metaphlan/metaphlan/main' +include { DIAMOND_BLASTX } from '../../../modules/nf-core/diamond/blastx/main' +include { RGI_BWT } from '../../../modules/nf-core/rgi/bwt/main' +include { RGI_CARDANNOTATION } from '../../../modules/nf-core/rgi/cardannotation/main' +include { EGGNOGMAPPER } from '../../../modules/nf-core/eggnogmapper/main' +include { SEQKIT_FQ2FA } from '../../../modules/nf-core/seqkit/fq2fa/main' +include { GUNZIP } from '../../../modules/nf-core/gunzip/main' // Custom Functions @@ -31,11 +33,13 @@ include { GUNZIP } from '../../../modules * @return A multiMap'ed output channel with two sub channels, one with the profile and the other with the db */ def sanitizeId(str) { - return str.toString() - .replaceAll(/_/, '-') // underscore to hyphens - .replaceAll(/\s+/, '-') // spaces to hyphens - .replaceAll(/[^\w\-.]/, '') // remove special chars - .replaceAll(/-+/, '-') // collapse multiple hyphens + return str + .toString() + // underscores to hyphens, spaces to hyphens, drop special chars, collapse hyphens + .replaceAll(/_/, '-') + .replaceAll(/\s+/, '-') + .replaceAll(/[^\w\-.]/, '') + .replaceAll(/-+/, '-') } def prepareInputs(pairedreads, databases, tool_name, singleFqTool = false) { /* @@ -53,52 +57,45 @@ def prepareInputs(pairedreads, databases, tool_name, singleFqTool = false) { */ // Step 1: Filter databases to only the requested tool, then group by db_name and db_params def ch_dbs_grouped = databases - .flatMap { meta_db, file_list -> + .flatMap { meta_db, file_list -> // Flatten: emit one tuple per file object file_list.collect { file_obj -> - // Merge the file object's db_entity into the metadata - def meta_with_entity = meta_db + [db_entity: file_obj.db_entity] - [meta_with_entity, file_obj] + // Merge the file object's db_entity into the metadata + def meta_with_entity = meta_db + [db_entity: file_obj.db_entity] + [meta_with_entity, file_obj] } - } - .filter { meta_db, file -> + } + .filter { meta_db, file -> meta_db.tool == tool_name - } - .map { meta_db, file -> + } + .map { meta_db, file -> // Create grouping key: [tool, db_name, db_params] - def group_key = [ - meta_db.tool, - meta_db.db_name ?: '', - meta_db.db_params ?: '' - ] + def group_key = [meta_db.tool, meta_db.db_name ?: '', meta_db.db_params ?: ''] [group_key, meta_db, file] - } - .groupTuple() // Group all files for same tool+db_name+db_params - .map { group_key, meta_db_list, files -> - def tool = group_key[0] - def db_name = group_key[1] - def db_params = group_key[2] - - // Convert files list to Map keyed by db_entity for deterministic snapshots - // Map structure: entity_name -> db_path (entity is already in the key) - def files_map = [:] - [meta_db_list, files].transpose().each { meta_db, file -> - files_map[meta_db.db_entity] = file.db_path // Store only the path } - - // Create consolidated metadata with db_entities as a Set - def meta_db_grouped = [ - id: sanitizeId("${tool}--${db_name}--${db_params}"), - tool: tool, - db_name: db_name, - db_params: db_params, - db_entities: files_map.keySet() as Set, // Set of entity names - num_files: files_map.size() - ] - - // Return files as map + // Group all files for same tool+db_name+db_params + .groupTuple() + .map { group_key, meta_db_list, files -> + def tool = group_key[0] + def db_name = group_key[1] + def db_params = group_key[2] + + // Convert files list to Map keyed by db_entity for deterministic snapshots + // Map structure: entity_name -> db_path (entity is already in the key) + def files_map = [:] + [meta_db_list, files] + .transpose() + .each { meta_db, file -> + // Store only the path + files_map[meta_db.db_entity] = file.db_path + } + + // Create consolidated metadata with db_entities as a Set + def meta_db_grouped = [id: sanitizeId("${tool}--${db_name}--${db_params}"), tool: tool, db_name: db_name, db_params: db_params, db_entities: files_map.keySet() as Set, num_files: files_map.size()] + + // Return files as map [meta_db_grouped, files_map.toSorted()] - } + } // Step 2: Combine reads with ALL grouped databases (cartesian product) // Each sample will get one entry per unique db_name+db_params combination for this tool def reads_with_dbs = pairedreads @@ -120,44 +117,43 @@ def prepareInputs(pairedreads, databases, tool_name, singleFqTool = false) { } [meta, reads, db_meta, db_files] } - .multiMap { it -> - reads: [it[0] , it[1]] - db: [ it[2], it[3]] + .multiMap { it -> + reads: [it[0], it[1]] + db: [it[2], it[3]] } return result - } -def getDbPath(groupeddb, entity='main', asTuple=false){ +def getDbPath(groupeddb, entity = 'main', asTuple = false) { // Extract the relevant database file path by entity key from the files map - def dbpath = groupeddb - .map { meta_db, files_map -> - // files_map is now a Map[entity -> db_path], so direct lookup - if (!files_map.containsKey(entity)) { - error("No entity '${entity}' file found in database ${meta_db.id}") - } + def dbpath = groupeddb.map { meta_db, files_map -> + // files_map is now a Map[entity -> db_path], so direct lookup + if (!files_map.containsKey(entity)) { + error("No entity '${entity}' file found in database ${meta_db.id}") + } - def db_path = files_map[entity] // Direct access to path + def db_path = files_map[entity] + // Direct access to path - if (asTuple){ - return [meta_db, db_path] - } else { - return db_path - } + if (asTuple) { + return [meta_db, db_path] + } + else { + return db_path } + } return dbpath } workflow PROFILING { take: - reads // [ [ meta ], [ reads ] ] - reads_concat // [ [ meta ], [ reads ] ] - databases // [ [ meta ], path ] + reads // [ [ meta ], [ reads ] ] + reads_concat // [ [ meta ], [ reads ] ] + databases // [ [ meta ], path ] main: - ch_versions = Channel.empty() - ch_multiqc_files = Channel.empty() - ch_raw_profiles = Channel.empty() // These are count tables + // These are count tables + ch_raw_profiles = channel.empty() /* COMBINE READS WITH POSSIBLE DATABASES @@ -178,109 +174,109 @@ workflow PROFILING { ch_input_for_humann_v4 = prepareInputs(reads_concat, databases, 'humann_v4', true) ch_input_for_mifaser = prepareInputs(reads_concat, databases, 'mifaser', true) - if ( params.run_fmhfunprofiler ) { - // this tool needs the db_params at runtime, so it takes a [[meta], path] tuple instead of just a path - getDbPath(ch_input_for_fmhfunprofiler.db, "main", true) - .multiMap { db_meta, db_path -> - def args = db_meta.db_params.split(" ") - - if (args.size() != 2) { - throw new IllegalArgumentException("fmh-funcprofiler's db_params must be configured with 2 ints (kmer and sketch db args) , but got ${args.size()}: ${db_meta.db_params}") - } - db_path: db_path - kmer: args[0] - sketch: args[1] - } - .set {fmh_db} - FMHFUNPROFILER ( + if (params.run_fmhfunprofiler) { + // this tool needs the db_params at runtime, so it takes a [[meta], path] tuple instead of just a path + getDbPath(ch_input_for_fmhfunprofiler.db, "main", true) + .multiMap { db_meta, db_path -> + def args = db_meta.db_params.split(" ") + + if (args.size() != 2) { + throw new IllegalArgumentException("fmh-funcprofiler's db_params must be configured with 2 ints (kmer and sketch db args) , but got ${args.size()}: ${db_meta.db_params}") + } + db_path: db_path + kmer: args[0] + sketch: args[1] + } + .set { fmh_db } + FMHFUNPROFILER( ch_input_for_fmhfunprofiler.reads, fmh_db.db_path, fmh_db.kmer, - fmh_db.sketch - - ) - ch_raw_profiles = ch_raw_profiles.mix( FMHFUNPROFILER.out.csv ) + fmh_db.sketch, + ) + ch_raw_profiles = ch_raw_profiles.mix(FMHFUNPROFILER.out.csv) } - if ( params.run_mifaser ) { - ch_input_for_mifaser = prepareInputs(reads_concat, databases, 'mifaser', true) - MIFASER ( ch_input_for_mifaser.reads, getDbPath(ch_input_for_mifaser.db, 'main')) - ch_raw_profiles = ch_raw_profiles.mix( MIFASER.out.ec_counts ) + if (params.run_mifaser) { + ch_input_for_mifaser = prepareInputs(reads_concat, databases, 'mifaser', true) + MIFASER(ch_input_for_mifaser.reads, getDbPath(ch_input_for_mifaser.db, 'main')) + ch_raw_profiles = ch_raw_profiles.mix(MIFASER.out.ec_counts) } - if ( params.run_humann_v3 ) { - MPAHUMANN3 ( - ch_input_for_humann_v3.reads, - getDbPath(ch_input_for_humann_v3.db, 'humann_metaphlan'),false - ) - // JOIN the original reads with the profile output - ch_humann3_input = ch_input_for_humann_v3.reads - .join(MPAHUMANN3.out.profile, by: 0) // Join on meta map + if (params.run_humann_v3) { + MPAHUMANN3( + ch_input_for_humann_v3.reads, + getDbPath(ch_input_for_humann_v3.db, 'humann_metaphlan'), + false, + ) + // JOIN the original reads with the profile output + ch_humann3_input = ch_input_for_humann_v3.reads.join(MPAHUMANN3.out.profile, by: 0) + // Join on meta map HUMANN3_HUMANN( - ch_humann3_input.map { it -> [it[0], it[1]] }, // Extract reads - ch_humann3_input.map { it -> [it[0], it[2]] }, // Extract profile + ch_humann3_input.map { it -> [it[0], it[1]] }, + ch_humann3_input.map { it -> [it[0], it[2]] }, getDbPath(ch_input_for_humann_v3.db, 'humann_nucleotide'), getDbPath(ch_input_for_humann_v3.db, 'humann_protein'), getDbPath(ch_input_for_humann_v3.db, 'humann_utility'), ) - HUMANN3_REGROUP(HUMANN3_HUMANN.out.genefamilies, "uniref90_level4ec", getDbPath(ch_input_for_humann_v3.db, 'humann_utility')) - ch_raw_profiles = ch_raw_profiles.mix( MPAHUMANN3.out.profile ) - ch_raw_profiles = ch_raw_profiles.mix( HUMANN3_HUMANN.out.pathabundance ) - .mix( HUMANN3_HUMANN.out.genefamilies ) - .mix( HUMANN3_HUMANN.out.pathcoverage ) + HUMANN3_REGROUP(HUMANN3_HUMANN.out.genefamilies, "uniref90_level4ec", getDbPath(ch_input_for_humann_v3.db, 'humann_utility')) + ch_raw_profiles = ch_raw_profiles.mix(MPAHUMANN3.out.profile) + ch_raw_profiles = ch_raw_profiles + .mix(HUMANN3_HUMANN.out.pathabundance) + .mix(HUMANN3_HUMANN.out.genefamilies) + .mix(HUMANN3_HUMANN.out.pathcoverage) } - if ( params.run_humann_v4 ) { - MPAHUMANN4 ( - ch_input_for_humann_v4.reads, - getDbPath(ch_input_for_humann_v4.db, 'humann_metaphlan'),false - ) - ch_humann4_input = ch_input_for_humann_v4.reads - .join(MPAHUMANN4.out.profile, by: 0) // Join on meta map + if (params.run_humann_v4) { + MPAHUMANN4( + ch_input_for_humann_v4.reads, + getDbPath(ch_input_for_humann_v4.db, 'humann_metaphlan'), + false, + ) + ch_humann4_input = ch_input_for_humann_v4.reads.join(MPAHUMANN4.out.profile, by: 0) + // Join on meta map HUMANN4( - ch_humann4_input.map { it -> [it[0], it[1]] }, // Extract reads - ch_humann4_input.map { it -> [it[0], it[2]] }, // Extract profile + ch_humann4_input.map { it -> [it[0], it[1]] }, + ch_humann4_input.map { it -> [it[0], it[2]] }, getDbPath(ch_input_for_humann_v4.db, 'humann_nucleotide'), getDbPath(ch_input_for_humann_v4.db, 'humann_protein'), getDbPath(ch_input_for_humann_v4.db, 'humann_utility'), ) - HUMANN4_REGROUP(HUMANN4.out.genefamilies, "uniclust90_level4ec", getDbPath(ch_input_for_humann_v4.db, 'humann_utility')) - ch_raw_profiles = ch_raw_profiles.mix( HUMANN4.out.pathabundance ) - .mix( HUMANN4.out.genefamilies ) - .mix( HUMANN4.out.reactions ) + HUMANN4_REGROUP(HUMANN4.out.genefamilies, "uniclust90_level4ec", getDbPath(ch_input_for_humann_v4.db, 'humann_utility')) + ch_raw_profiles = ch_raw_profiles + .mix(HUMANN4.out.pathabundance) + .mix(HUMANN4.out.genefamilies) + .mix(HUMANN4.out.reactions) } - if ( params.run_diamond ) { - DIAMOND_BLASTX ( ch_input_for_diamond.reads, getDbPath(ch_input_for_diamond.db, "main"), 'tsv', '' ) - ch_raw_profiles = ch_raw_profiles.mix( DIAMOND_BLASTX.out.tsv ) + if (params.run_diamond) { + DIAMOND_BLASTX(ch_input_for_diamond.reads, getDbPath(ch_input_for_diamond.db, "main"), 'tsv', '') + ch_raw_profiles = ch_raw_profiles.mix(DIAMOND_BLASTX.out.tsv) } - if ( params.run_rgi ) { + if (params.run_rgi) { RGI_CARDANNOTATION(getDbPath(ch_input_for_rgi.db, "main")) - RGI_BWT( ch_input_for_rgi.reads, RGI_CARDANNOTATION.out.db, [] ) - ch_raw_profiles = ch_raw_profiles.mix( RGI_BWT.out.tsv ) + RGI_BWT(ch_input_for_rgi.reads, RGI_CARDANNOTATION.out.db, []) + ch_raw_profiles = ch_raw_profiles.mix(RGI_BWT.out.tsv) } - if ( params.run_eggnogmapper ) { - SEQKIT_FQ2FA(ch_input_for_eggnogmapper.reads) - GUNZIP(SEQKIT_FQ2FA.out.fasta) - EGGNOGMAPPER ( + if (params.run_eggnogmapper) { + SEQKIT_FQ2FA(ch_input_for_eggnogmapper.reads) + GUNZIP(SEQKIT_FQ2FA.out.fasta) + EGGNOGMAPPER( GUNZIP.out.gunzip, - getDbPath(ch_input_for_eggnogmapper.db, "eggnogmapper_db"), - getDbPath(ch_input_for_eggnogmapper.db, "") + getDbPath(ch_input_for_eggnogmapper.db, "eggnogmapper_db"), + getDbPath(ch_input_for_eggnogmapper.db, ""), ) - ch_raw_profiles = ch_raw_profiles.mix( EGGNOGMAPPER.out.annotations ) - + ch_raw_profiles = ch_raw_profiles.mix(EGGNOGMAPPER.out.annotations) } - emit: - profiles = ch_raw_profiles // channel: [ val(meta), [ reads ] ] - should be text files or biom - versions = ch_versions // channel: [ versions.yml ] - mqc = ch_multiqc_files + profiles = ch_raw_profiles // channel: [ val(meta), path(profile) ] + // Software versions are published to the `versions` topic by each module } +// due to https://github.com/askimed/nf-test/issues/309 workflow TEST_PREPAREINPUTS_WRAPPER { - // due to https://github.com/askimed/nf-test/issues/309) take: reads databases @@ -289,7 +285,8 @@ workflow TEST_PREPAREINPUTS_WRAPPER { main: testresult = prepareInputs(reads, databases, tool_name, singleFqTool) + emit: reads = testresult.reads - db = testresult.db + db = testresult.db } diff --git a/subworkflows/local/profile/meta.yml b/subworkflows/local/profile/meta.yml index 78bc923..d69de4d 100644 --- a/subworkflows/local/profile/meta.yml +++ b/subworkflows/local/profile/meta.yml @@ -1,57 +1,60 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "profile" -## -description: Execute functional profiling tools +description: | + Run the enabled read-based functional profilers. Reads are combined with every + database matching the tool, so a sample is profiled once per database name and + parameter combination. Tools that take paired input read from `reads`, tools + that take a single FASTQ read from `reads_concat`, and each profiler runs only + when its `run_` parameter is set. keywords: - microbiome - metagenome - fastq - - function + - functional profiling components: - - humann/humann - fmhfunprofiler - mifaser + - humann3/humann + - humann3/regroup + - humann4 + - humann4/regroup + - metaphlan/metaphlan + - diamond/blastx - rgi/bwt - -## TODO nf-core: List all of the channels used as input with a description and their structure + - rgi/cardannotation + - eggnogmapper + - seqkit/fq2fa + - gunzip input: - reads: type: file description: | - The input channel containing paired fastq files - Structure: [ val(meta), path(fastq) ] - pattern: "*.{.fastq}*" + Channel of paired-end (or single-end) reads, used by the profilers that + accept a read pair. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fq,fastq}{,.gz}" - reads_concat: type: file description: | - The input channel containing concatenated fastq pairs as a single fastq - Structure: [ val(meta), path(fastq) ] - pattern: "*.{.fastq}*" + Channel of reads concatenated into a single file per sample, used by the + profilers that accept only one FASTQ. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fq,fastq}{,.gz}" - databases: type: file description: | - The input channel containing the database files - Structure: [ val(meta), path(file) ] -## TODO nf-core: List all of the channels used as output with a descriptions and their structure + Channel of grouped databases as emitted by the `dbprep` subworkflow. + Structure: [ val(meta), [ [ db_entity: val(entity), db_path: path(db) ] ] ] output: - profiles: type: file description: | - Channel containing tabular profiling results - Structure: [ val(meta), path(paths) ] - pattern: "*.bam" - - versions: - type: file - description: | - File containing software versions - Structure: [ path(versions.yml) ] - pattern: "versions.yml" - - mqc: - type: file - description: | - File containing MultiQC files - Structure: [ path(file) ] - pattern: "*" + Channel of the tabular profiles produced by each enabled tool. The meta + map identifies the sample, the tool and the database used. + Structure: [ val(meta), path(profile) ] + pattern: "*.{tsv,csv,txt}" +# Software versions are published to the `versions` topic by each module rather +# than emitted as a channel. authors: - "@nickp60" maintainers: diff --git a/subworkflows/local/profile/tests/main.nf.test.snap b/subworkflows/local/profile/tests/main.nf.test.snap index d34209d..071f27e 100644 --- a/subworkflows/local/profile/tests/main.nf.test.snap +++ b/subworkflows/local/profile/tests/main.nf.test.snap @@ -48,26 +48,26 @@ "content": [ [ "minigut.merged.fastq.gz:md5,1fbae8f5fe6fdfa6552cfca32d9c3129", - "minigut_KOs_sketched_scaled_1000_demo.sig.fmhfunprofiler.csv:md5,16b7196a90c1d2fd3e3109c8d252cd08" + "minigut_KOs_sketched_scaled_1000_demo.sig.fmhfunprofiler.csv:md5,1081039ab9e9b378d2dfd3206e96e917" ] ], - "timestamp": "2026-07-23T16:38:21.384784", + "timestamp": "2026-07-29T16:20:47.621463", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.4" + "nextflow": "26.04.6" } }, "run multisampleinputs - grouped databases (humann_v3)": { "content": [ [ "minigut.merged.fastq.gz:md5,1fbae8f5fe6fdfa6552cfca32d9c3129", - "minigut_KOs_sketched_scaled_1000_demo.sig.fmhfunprofiler.csv:md5,16b7196a90c1d2fd3e3109c8d252cd08" + "minigut_KOs_sketched_scaled_1000_demo.sig.fmhfunprofiler.csv:md5,1081039ab9e9b378d2dfd3206e96e917" ] ], - "timestamp": "2026-07-24T09:47:36.521206", + "timestamp": "2026-07-29T16:20:29.367839", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.4" + "nextflow": "26.04.6" } } } \ No newline at end of file diff --git a/subworkflows/local/utils_nfcore_funcprofiler_pipeline/main.nf b/subworkflows/local/utils_nfcore_funcprofiler_pipeline/main.nf index 9d65ed8..a261053 100644 --- a/subworkflows/local/utils_nfcore_funcprofiler_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_funcprofiler_pipeline/main.nf @@ -99,13 +99,15 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // - Channel.fromList(samplesheetToList(params.input, "assets/schema_input.json")) + Channel + .fromList(samplesheetToList(params.input, "assets/schema_input.json")) .set { ch_samplesheet } // // Create channel from databases file provided through params.databases // - Channel.fromList(samplesheetToList(params.databases, "assets/schema_database.json")) + Channel + .fromList(samplesheetToList(params.databases, "assets/schema_database.json")) .set { ch_databases } emit: @@ -182,9 +184,7 @@ def validateInputSamplesheet(input) { // def toolCitationText() { - def text_qc = [ - "Sequencing quality control was performed with FastQC (Andrews 2010)." - ].join(' ').trim() + def text_qc = ["Sequencing quality control was performed with FastQC (Andrews 2010)."].join(' ').trim() def text_humann = [ "Functional profiling was performed with", @@ -196,25 +196,15 @@ def toolCitationText() { "using MetaPhlAn (Blanco-Míguez et al. 2023) for taxonomic marker-based profiling.", ].join(' ').trim() - def text_diamond = [ - "Protein-level sequence alignment was performed with DIAMOND (Buchfink et al. 2021)." - ].join(' ').trim() + def text_diamond = ["Protein-level sequence alignment was performed with DIAMOND (Buchfink et al. 2021)."].join(' ').trim() - def text_fmhfunprofiler = [ - "Functional profiling was additionally performed with fmhfunprofiler (Hera et al. 2024)." - ].join(' ').trim() + def text_fmhfunprofiler = ["Functional profiling was additionally performed with fmhfunprofiler (Hera et al. 2024)."].join(' ').trim() - def text_mifaser = [ - "Enzyme function annotation was performed with mi-faser (Zhu et al. 2017)." - ].join(' ').trim() + def text_mifaser = ["Enzyme function annotation was performed with mi-faser (Zhu et al. 2017)."].join(' ').trim() - def text_eggnogmapper = [ - "Functional Annotation, Orthology Assignments, and Domain Prediction was performed with eggNOG-mapper v2 (Cantalapiedra et. al 2021)" - ].join(' ').trim() + def text_eggnogmapper = ["Functional Annotation, Orthology Assignments, and Domain Prediction was performed with eggNOG-mapper v2 (Cantalapiedra et. al 2021)"].join(' ').trim() - def text_rgi = [ - "Resistome prediction was performed using RGI (Alcock et. al 2023)" - ].join(' ').trim() + def text_rgi = ["Resistome prediction was performed using RGI (Alcock et. al 2023)"].join(' ').trim() def citation_text = [ "Tools used in the workflow included:", @@ -233,22 +223,16 @@ def toolCitationText() { def toolBibliographyText() { - def text_qc = [ - !params.skip_preprocessing_qc ? "
  • Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. Available at: http://www.bioinformatics.babraham.ac.uk/projects/fastqc/
    • " : "" - ].join(' ').trim() + def text_qc = [!params.skip_preprocessing_qc ? "
  • Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. Available at: http://www.bioinformatics.babraham.ac.uk/projects/fastqc/
    • " : ""].join(' ').trim() def text_humann = [ params.run_humann_v3 || params.run_humann_v4 ? "
  • Beghini, F., McIver, L. J., Blanco-M\u00edguez, A., Dubois, L., Asnicar, F., Maharjan, S., Mailyan, A., Thomas, A. M., Manghi, P., Valles-Colomer, M., Weingart, G., Zhang, Y., Zolfo, M., Huttenhower, C., Franzosa, E. A., & Segata, N. (2021). Integrating taxonomic, functional, and strain-level profiling of diverse microbial communities with bioBakery 3. eLife, 10, e65088. 10.7554/eLife.65088
    • " : "", params.run_humann_v3 || params.run_humann_v4 ? "
  • Blanco-M\u00edguez, A., Beghini, F., Cumbo, F., McIver, L. J., Thompson, K. N., Zolfo, M., Manghi, P., Dubois, L., Huang, K. D., Thomas, A. M., Nickols, W. A., Piccinno, G., Piperni, E., Pun\u010doch\u00e1\u0159, M., Valles-Colomer, M., Tett, A., Giordano, F., Davies, R., Wolf, J., \u2026 Segata, N. (2023). Extending and improving metagenomic taxonomic profiling with uncharacterized species using MetaPhlAn 4. Nature Biotechnology, 41, 1633\u20131645. 10.1038/s41587-023-01688-w
    • " : "", ].join(' ').trim() - def text_diamond = [ - params.run_diamond ? "
  • Buchfink, B., Reuter, K., & Drost, H.-G. (2021). Sensitive protein alignments at tree-of-life scale using DIAMOND. Nature Methods, 18(4), 366–368. 10.1038/s41592-021-01101-x
    • " : "" - ].join(' ').trim() + def text_diamond = [params.run_diamond ? "
  • Buchfink, B., Reuter, K., & Drost, H.-G. (2021). Sensitive protein alignments at tree-of-life scale using DIAMOND. Nature Methods, 18(4), 366–368. 10.1038/s41592-021-01101-x
    • " : ""].join(' ').trim() - def text_fmhfunprofiler = [ - params.run_fmhfunprofiler ? "
  • Hera, M. R., Liu, S., Wei, W., Rodriguez, J. S., Ma, C., & Koslicki, D. (2024). Metagenomic functional profiling: to sketch or not to sketch? Bioinformatics, 40(Suppl 2), ii165–ii173. 10.1093/bioinformatics/btae397
    • " : "" - ].join(' ').trim() + def text_fmhfunprofiler = [params.run_fmhfunprofiler ? "
  • Hera, M. R., Liu, S., Wei, W., Rodriguez, J. S., Ma, C., & Koslicki, D. (2024). Metagenomic functional profiling: to sketch or not to sketch? Bioinformatics, 40(Suppl 2), ii165–ii173. 10.1093/bioinformatics/btae397
    • " : ""].join(' ').trim() def text_mifaser = [ params.run_mifaser ? "
  • Zhu, C., Miller, M., Marpaka, S., Vaysberg, P., R\u00fchlemann, M. C., Wu, G., Heinsen, F.-A., Tempel, M., Woodhouse, L., Burkhardt, L., Tams, R., Knecht, C., Heinig, M., Franke, A., Huser, T., & Bromberg, Y. (2017). Functional sequencing read annotation for high precision microbiome analysis. Nucleic Acids Research, 46(4), e23. 10.1093/nar/gkx1209
    • " : "", @@ -256,13 +240,9 @@ def toolBibliographyText() { params.run_mifaser ? "
  • Zhu, C., Delmont, T. O., Vogel, T. M., & Bromberg, Y. (2015). Functional basis of microorganism classification. PLoS Computational Biology, 11(8), e1004472. 10.1371/journal.pcbi.1004472
    • " : "", ].join(' ').trim() - def text_eggnggmapper = [ - params.run_eggnogmapper ? "
  • Carlos P Cantalapiedra, Ana Hernández-Plaza, Ivica Letunic, Peer Bork, Jaime Huerta-Cepas, eggNOG-mapper v2: Functional Annotation, Orthology Assignments, and Domain Prediction at the Metagenomic Scale, Molecular Biology and Evolution, Volume 38, Issue 12, December 2021, 0.1093/molbev/msab293
    • " : "" - ].join(' ').trim() + def text_eggnggmapper = [params.run_eggnogmapper ? "
  • Carlos P Cantalapiedra, Ana Hernández-Plaza, Ivica Letunic, Peer Bork, Jaime Huerta-Cepas, eggNOG-mapper v2: Functional Annotation, Orthology Assignments, and Domain Prediction at the Metagenomic Scale, Molecular Biology and Evolution, Volume 38, Issue 12, December 2021, 0.1093/molbev/msab293
    • " : ""].join(' ').trim() - def text_rgi = [ - params.run_rgi ? "
  • Alcock et al. 2023. CARD 2023: expanded curation, support for machine learning, and resistome prediction at the Comprehensive Antibiotic Resistance Database. Nucleic Acids Researchpubmed.ncbi.nlm.nih.gov/36263822
    • " : "" - ].join(' ').trim() + def text_rgi = [params.run_rgi ? "
  • Alcock et al. 2023. CARD 2023: expanded curation, support for machine learning, and resistome prediction at the Comprehensive Antibiotic Resistance Database. Nucleic Acids Researchpubmed.ncbi.nlm.nih.gov/36263822
    • " : ""].join(' ').trim() def reference_text = [ text_qc, diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 0d9d4e0..9ff0681 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -35,7 +35,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { fullHelp: help_full, ] if(parameters_schema) { - help_options << [parametersSchema: parameters_schema] + help_options << [parameters_schema: parameters_schema] } log.info paramsHelp( help_options, @@ -51,7 +51,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { summary_options = [:] if(parameters_schema) { - summary_options << [parametersSchema: parameters_schema] + summary_options << [parameters_schema: parameters_schema] } log.info before_text log.info paramsSummaryLog(summary_options, input_workflow) @@ -64,10 +64,10 @@ workflow UTILS_NFSCHEMA_PLUGIN { if(validate_params) { validateOptions = [:] if(parameters_schema) { - validateOptions << [parametersSchema: parameters_schema] + validateOptions << [parameters_schema: parameters_schema] } if(cli_typecast != null) { - validateOptions << [cliTypecast: cli_typecast] + validateOptions << [cast_cli_params: cli_typecast] } validateParameters(validateOptions) } diff --git a/tests/nextflow.config b/tests/nextflow.config index 3dc70e4..25639c6 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -4,8 +4,6 @@ ======================================================================================== */ -// TODO nf-core: Specify any additional parameters here -// Or any resources requirements params { modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' diff --git a/workflows/funcprofiler.nf b/workflows/funcprofiler.nf index cebaf4b..a68d6f6 100644 --- a/workflows/funcprofiler.nf +++ b/workflows/funcprofiler.nf @@ -3,9 +3,9 @@ IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_funcprofiler_pipeline' @@ -35,16 +35,15 @@ include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_func // // MODULE: Installed directly from nf-core/modules // -include { UNTAR } from '../modules/nf-core/untar/main' -include { PROFILING } from '../subworkflows/local/profile/main' -include { DATAPREP } from '../subworkflows/local/dataprep/main' -include { DBPREP } from '../subworkflows/local/dbprep/main' +include { UNTAR } from '../modules/nf-core/untar/main' +include { PROFILING } from '../subworkflows/local/profile/main' +include { DATAPREP } from '../subworkflows/local/dataprep/main' +include { DBPREP } from '../subworkflows/local/dbprep/main' workflow FUNCPROFILER { - take: samplesheet // channel: samplesheet read in from --input databases // channel: databases from --databases @@ -58,25 +57,47 @@ workflow FUNCPROFILER { ch_versions = Channel.empty() ch_multiqc_files = Channel.empty() - DATAPREP ( - samplesheet + DATAPREP( + samplesheet ) - DBPREP ( - databases + DBPREP( + databases ) - PROFILING ( - DATAPREP.out.reads, - DATAPREP.out.reads_concat, - DBPREP.out.dbs + PROFILING( + DATAPREP.out.reads, + DATAPREP.out.reads_concat, + DBPREP.out.dbs, ) - def ch_collated_versions = softwareVersionsToYAML(ch_versions) + // + // Collate and save software versions + // + def topic_versions = channel + .topic("versions") + .distinct() + .branch { entry -> + versions_file: entry instanceof Path + versions_tuple: true + } + + def topic_versions_string = topic_versions.versions_tuple + .map { process, tool, version -> + [process[process.lastIndexOf(':') + 1..-1], " ${tool}: ${version}"] + } + .groupTuple(by: 0) + .map { process, tool_versions -> + tool_versions.unique().sort() + "${process}:\n${tool_versions.join('\n')}" + } + + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + .mix(topic_versions_string) .collectFile( storeDir: "${outdir}/pipeline_info", - name: 'nf_core_' + 'funcprofiler_software_' + 'mqc_' + 'versions.yml', + name: 'nf_core_' + 'funcprofiler_software_' + 'mqc_' + 'versions.yml', sort: true, - newLine: true + newLine: true, ) // @@ -108,11 +129,5 @@ workflow FUNCPROFILER { emit: multiqc_report = MULTIQC.out.report.map { _meta, report -> [report] }.toList() // channel: /path/to/multiqc_report.html - versions = ch_versions // channel: [ path(versions.yml) ] + versions = ch_versions // channel: [ path(versions.yml) ] } - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - THE END -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/