diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69794531..49807357 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ exclude: '\.(tsv|fasta|gb)$|^shared/vendored/|^phylogenetic/rules/config.smk' repos: - repo: https://github.com/snakemake/snakefmt - rev: v0.11.1 + rev: v2.0.3 hooks: - id: snakefmt - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 + rev: v1.7.12 hooks: - id: actionlint entry: env SHELLCHECK_OPTS='--exclude=SC2027' actionlint @@ -16,11 +16,11 @@ repos: # additional_dependencies: # - tomli - repo: https://github.com/google/yamlfmt - rev: v0.17.2 + rev: v0.21.0 hooks: - id: yamlfmt - repo: https://github.com/pappasam/toml-sort - rev: v0.24.2 + rev: v0.24.4 hooks: - id: toml-sort-fix - repo: https://github.com/pre-commit/pre-commit-hooks @@ -42,7 +42,7 @@ repos: - id: end-of-file-fixer - id: fix-byte-order-marker - repo: https://github.com/pre-commit/sync-pre-commit-deps - rev: v0.0.3 + rev: v0.0.4 hooks: - id: sync-pre-commit-deps - repo: https://github.com/shellcheck-py/shellcheck-py diff --git a/ingest/Snakefile b/ingest/Snakefile index ac5e56fb..558cc875 100644 --- a/ingest/Snakefile +++ b/ingest/Snakefile @@ -27,7 +27,11 @@ rule all: include: "rules/curate.smk" + + include: "rules/nextclade.smk" + + include: "rules/fetch_from_ppx.smk" @@ -44,18 +48,18 @@ if "custom_rules" in config: rule extract_open_data: - input: - metadata="results/metadata.tsv", - sequences="results/sequences.fasta", - output: - metadata="results/metadata_open.tsv", - sequences="results/sequences_open.fasta", - shell: - """ - augur filter --metadata {input.metadata} \ - --sequences {input.sequences} \ - --metadata-id-columns PPX_accession \ - --exclude-where "dataUseTerms=RESTRICTED" \ - --output-metadata {output.metadata} \ - --output-sequences {output.sequences} - """ + input: + metadata="results/metadata.tsv", + sequences="results/sequences.fasta", + output: + metadata="results/metadata_open.tsv", + sequences="results/sequences_open.fasta", + shell: + """ + augur filter --metadata {input.metadata} \ + --sequences {input.sequences} \ + --metadata-id-columns PPX_accession \ + --exclude-where "dataUseTerms=RESTRICTED" \ + --output-metadata {output.metadata} \ + --output-sequences {output.sequences} + """ diff --git a/ingest/build-configs/nextstrain-automation/nextstrain_automation.smk b/ingest/build-configs/nextstrain-automation/nextstrain_automation.smk index bf47d559..e57c377a 100644 --- a/ingest/build-configs/nextstrain-automation/nextstrain_automation.smk +++ b/ingest/build-configs/nextstrain-automation/nextstrain_automation.smk @@ -63,12 +63,12 @@ rule custom_subset_metadata: metadata="data/all_metadata_added.tsv", output: subset_metadata="data/subset_metadata.tsv", - params: - metadata_fields=",".join(config["curate"]["metadata_columns"]), - benchmark: - "benchmarks/subset_metadata.txt" log: "logs/subset_metadata.txt", + benchmark: + "benchmarks/subset_metadata.txt" + params: + metadata_fields=",".join(config["curate"]["metadata_columns"]), shell: r""" exec &> >(tee {log:q}) @@ -76,7 +76,7 @@ rule custom_subset_metadata: csvtk cut -t -f {params.metadata_fields:q} \ {input.metadata:q} \ | csvtk mutate -t -f date_released -n date_submitted \ - > {output.subset_metadata:q} + >{output.subset_metadata:q} """ diff --git a/phylogenetic/rules/annotate_phylogeny.smk b/phylogenetic/rules/annotate_phylogeny.smk index c70d2333..94376bf7 100644 --- a/phylogenetic/rules/annotate_phylogeny.smk +++ b/phylogenetic/rules/annotate_phylogeny.smk @@ -29,6 +29,10 @@ rule ancestral: alignment=build_dir + "/{build_name}/masked.fasta", output: node_data=build_dir + "/{build_name}/nt_muts.json", + log: + "logs/{build_name}/ancestral.txt", + benchmark: + "benchmarks/{build_name}/ancestral.txt" params: inference="joint", root_sequence=lambda w: ( @@ -36,10 +40,6 @@ rule ancestral: if config.get("ancestral_root_seq") else "" ), - log: - "logs/{build_name}/ancestral.txt", - benchmark: - "benchmarks/{build_name}/ancestral.txt" shell: r""" exec &> >(tee {log:q}) @@ -83,20 +83,21 @@ rule traits: """ Inferring ancestral traits for {params.columns!s} - increase uncertainty of reconstruction by {params.sampling_bias_correction} to partially account for sampling bias + """ input: tree=build_dir + "/{build_name}/tree.nwk", metadata=build_dir + "/{build_name}/metadata.tsv", output: node_data=build_dir + "/{build_name}/traits.json", - params: - columns=config["traits"]["columns"], - sampling_bias_correction=config["traits"]["sampling_bias_correction"], - strain_id=config["strain_id_field"], log: "logs/{build_name}/traits.txt", benchmark: "benchmarks/{build_name}/traits.txt" + params: + columns=config["traits"]["columns"], + sampling_bias_correction=config["traits"]["sampling_bias_correction"], + strain_id=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) @@ -188,12 +189,12 @@ rule recency: metadata=build_dir + "/{build_name}/metadata.tsv", output: node_data=build_dir + "/{build_name}/recency.json", - params: - strain_id=config["strain_id_field"], log: "logs/{build_name}/recency.txt", benchmark: "benchmarks/{build_name}/recency.txt" + params: + strain_id=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) diff --git a/phylogenetic/rules/export.smk b/phylogenetic/rules/export.smk index 01ed2699..bbd7e4ef 100644 --- a/phylogenetic/rules/export.smk +++ b/phylogenetic/rules/export.smk @@ -56,12 +56,12 @@ rule colors: metadata=build_dir + "/{build_name}/metadata.tsv", output: colors=build_dir + "/{build_name}/colors.tsv", - params: - ignore_categories=config.get("colors", {}).get("ignore_categories", []), log: "logs/{build_name}/colors.txt", benchmark: "benchmarks/{build_name}/colors.txt" + params: + ignore_categories=config.get("colors", {}).get("ignore_categories", []), shell: r""" exec &> >(tee {log:q}) @@ -108,12 +108,12 @@ rule export: output: auspice_json=build_dir + "/{build_name}/tree.json", root_sequence=build_dir + "/{build_name}/tree_root-sequence.json", - params: - strain_id=config["strain_id_field"], log: "logs/{build_name}/export.txt", benchmark: "benchmarks/{build_name}/export.txt" + params: + strain_id=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) diff --git a/phylogenetic/rules/merge_inputs.smk b/phylogenetic/rules/merge_inputs.smk index 0c576101..f1ead375 100644 --- a/phylogenetic/rules/merge_inputs.smk +++ b/phylogenetic/rules/merge_inputs.smk @@ -98,7 +98,7 @@ if len(_input_metadata) == 1: r""" exec &> >(tee {log:q}) - augur read-file {input.metadata:q} > {output.metadata:q} + augur read-file {input.metadata:q} >{output.metadata:q} """ else: @@ -114,15 +114,15 @@ else: for name, info in input_sources.items() if info.get("metadata", None) }, - params: - metadata=lambda w, input: list(map("=".join, input.items())), - id_field=config["strain_id_field"], output: metadata="results/metadata.tsv", log: "logs/merge_metadata.txt", benchmark: "benchmarks/merge_metadata.txt" + params: + metadata=lambda w, input: list(map("=".join, input.items())), + id_field=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) @@ -154,7 +154,7 @@ if len(_input_sequences) == 1: r""" exec &> >(tee {log:q}) - augur read-file {input.sequences:q} > {output.sequences:q} + augur read-file {input.sequences:q} >{output.sequences:q} """ else: diff --git a/phylogenetic/rules/prepare_sequences.smk b/phylogenetic/rules/prepare_sequences.smk index 8345030b..589b8ac0 100644 --- a/phylogenetic/rules/prepare_sequences.smk +++ b/phylogenetic/rules/prepare_sequences.smk @@ -31,12 +31,12 @@ rule map_accessions: script="scripts/map_accessions.py", output: accession_list=build_dir + "/{build_name}/{in_ex_clude}_ppx.txt", - wildcard_constraints: - in_ex_clude="(include|exclude)", log: "logs/{build_name}/map_accessions_{in_ex_clude}.txt", benchmark: "benchmarks/{build_name}/map_accessions_{in_ex_clude}.txt" + wildcard_constraints: + in_ex_clude="(include|exclude)", shell: r""" exec &> >(tee {log:q}) @@ -60,6 +60,10 @@ rule filter: sequences=build_dir + "/{build_name}/good_sequences.fasta", metadata=build_dir + "/{build_name}/good_metadata.tsv", log=build_dir + "/{build_name}/good_filter.log", + log: + "logs/{build_name}/filter.txt", + benchmark: + "benchmarks/{build_name}/download.txt" params: min_date=config["filter"]["min_date"], min_length=config["filter"]["min_length"], @@ -70,10 +74,6 @@ rule filter: if "exclude_where" in config["filter"] else "" ), - log: - "logs/{build_name}/filter.txt", - benchmark: - "benchmarks/{build_name}/download.txt" shell: r""" exec &> >(tee {log:q}) @@ -135,13 +135,13 @@ rule subsample: output: strains=build_dir + "/{build_name}/{sample}_strains.txt", log=build_dir + "/{build_name}/{sample}_filter.log", - params: - augur_filter_args=lambda w: config["subsample"][w.sample], - strain_id=config["strain_id_field"], log: "logs/{build_name}/{sample}_subsample.txt", benchmark: "benchmarks/{build_name}/{sample}_subsample.txt" + params: + augur_filter_args=lambda w: config["subsample"][w.sample], + strain_id=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) @@ -175,12 +175,12 @@ rule combine_samples: output: sequences=build_dir + "/{build_name}/filtered.fasta", metadata=build_dir + "/{build_name}/metadata.tsv", - params: - strain_id=config["strain_id_field"], log: "logs/{build_name}/combine_samples.txt", benchmark: "benchmarks/{build_name}/combine_samples.txt" + params: + strain_id=config["strain_id_field"], shell: r""" exec &> >(tee {log:q}) @@ -190,8 +190,7 @@ rule combine_samples: --sequences {input.sequences:q} \ --metadata {input.metadata:q} \ --exclude-all \ - --include {input.strains:q} {input.include:q}\ - --output-sequences {output.sequences:q} \ + --include {input.strains:q} {input.include:q} --output-sequences {output.sequences:q} \ --output-metadata {output.metadata:q} """ @@ -206,6 +205,11 @@ rule align: genome_annotation=config["genome_annotation"], output: alignment=build_dir + "/{build_name}/aligned.fasta", + log: + "logs/{build_name}/align.txt", + benchmark: + "benchmarks/{build_name}/align.txt" + threads: workflow.cores params: # Alignment params from all-clades nextclade dataset excess_bandwidth=100, @@ -214,11 +218,6 @@ rule align: min_seed_cover=0.1, allowed_mismatches=8, gap_alignment_side="left", - threads: workflow.cores - log: - "logs/{build_name}/align.txt", - benchmark: - "benchmarks/{build_name}/align.txt" shell: r""" exec &> >(tee {log:q}) @@ -235,7 +234,7 @@ rule align: --gap-alignment-side {params.gap_alignment_side:q} \ --output-fasta - \ {input.sequences:q} \ - | seqkit seq -i > {output.alignment:q} + | seqkit seq -i >{output.alignment:q} """ @@ -250,13 +249,13 @@ rule mask: mask=config["mask"]["maskfile"], output: build_dir + "/{build_name}/masked.fasta", - params: - from_start=config["mask"]["from_beginning"], - from_end=config["mask"]["from_end"], log: "logs/{build_name}/mask.txt", benchmark: "benchmarks/{build_name}/mask.txt" + params: + from_start=config["mask"]["from_beginning"], + from_end=config["mask"]["from_end"], shell: r""" exec &> >(tee {log:q})