Skip to content

feat: Automate configuration YAML generation (Resolves #24) #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dyrpsf wants to merge 1 commit into
base: dev
Choose a base branch
from
+169 −65
Open

feat: Automate configuration YAML generation (Resolves #24) #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions scripts/sync_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import os
import re
Comment on lines +1 to +2

# Define paths relative to the root directory
CONFIG_DIR = os.path.join("src", "specimen", "data", "config")
DEFAULT_CONFIG = os.path.join(CONFIG_DIR, "hqtb_config_default.yaml")
ADVANCED_CONFIG = os.path.join(CONFIG_DIR, "hqtb_advanced_config_expl.yaml")
BASIC_CONFIG = os.path.join(CONFIG_DIR, "hqtb_basic_config_expl.yaml")

# Developer tags to strip out for non-default configs
DEV_TAGS = ["# @IDEA", "# @TODO", "# @DEV"]

# Blocks to entirely skip for the basic "quick-and-dirty" configuration
ADVANCED_BLOCKS = [
"refinement_cleanup:",
"refinement_smoothing:",
"GeneGapFiller:",
"media_gap:",
"mcc:",
"egc:",
]
Comment on lines +13 to +21


def is_dev_note(line: str) -> bool:
"""Checks if a line contains developer-specific tags."""
return any(tag in line for tag in DEV_TAGS)


def generate_advanced(lines: list[str]) -> list[str]:
"""Generates advanced config by stripping dev notes."""
# Advanced contains all params and user comments, just no dev notes
return [line for line in lines if not is_dev_note(line)]


def generate_basic(lines: list[str]) -> list[str]:
"""Generates basic config by stripping dev notes and advanced blocks."""
basic_lines = []
skip_block = False
current_indent = 0

for line in lines:
if is_dev_note(line):
continue

stripped = line.lstrip()
indent = len(line) - len(stripped)

# Handle skipping nested blocks
if skip_block:
# If we find text that is at the same or lower indentation level, the block is over
if stripped and indent <= current_indent:
skip_block = False
else:
continue

# Check if we should start skipping a new block
if any(stripped.startswith(key) for key in ADVANCED_BLOCKS):
skip_block = True
current_indent = indent
continue

basic_lines.append(line)

return basic_lines


def main():
print("Syncing SPECIMEN YAML configurations...")
if not os.path.exists(DEFAULT_CONFIG):
print(
f"Error: Could not find {DEFAULT_CONFIG}. Run this script from the project root."
)
return

with open(DEFAULT_CONFIG, "r", encoding="utf-8") as f:
lines = f.readlines()

# Generate and write Advanced Config
advanced_lines = generate_advanced(lines)
with open(ADVANCED_CONFIG, "w", encoding="utf-8") as f:
f.writelines(advanced_lines)
print(f" [+] Successfully generated {ADVANCED_CONFIG}")

# Generate and write Basic Config
basic_lines = generate_basic(lines)
with open(BASIC_CONFIG, "w", encoding="utf-8") as f:
f.writelines(basic_lines)
print(f" [+] Successfully generated {BASIC_CONFIG}")
print("Done!")


if __name__ == "__main__":
main()
41 changes: 22 additions & 19 deletions src/specimen/data/config/hqtb_advanced_config_expl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
# Configuration file for the SPECIMEN HQTB pipeline

# This is the advaned config file. For a shorter version,
# download the hqtb-basic configuration file.
# Configuration file for the SPECIMEN HTQB pipeline

# Meaning of the default parameters:
# The value __USER__ indicates parameters required to be specified by the user
Expand Down Expand Up @@ -30,6 +27,7 @@ subject:
template:
annotated_genome: __USER__
model: __USER__
namespace: BiGG

# information about the output
# ----------------------------
Expand Down Expand Up @@ -71,7 +69,6 @@ data:
# -----------------------------------------------

parameters:

########## general ##########
general:
idprefix: 'HQTB' # prefix to use for fantasy IDs, if IDs for
Expand All @@ -81,37 +78,34 @@ parameters:

########## step 1: bidirectional blast ##########
bidirectional_blast:
# options for giving customised names and setting a header
# -> default uses information from the models/files directly
# default should suffice except special cases
template_name: null
input_name: null
temp_header: null
in_header: null
# change the sensitivity of DIAMOND
# can be set by user if wanted, but not necessary
sensitivity: more-sensitive

########## step 2: generaft draft model ##########
generate_draft_model:
# if the gene ids of the template model do not match the
# identifiers of the annotated genome file, use this option to edit them
edit_names: "no"
# filter for genes based on percentage identity value
pid: 80.0
# Which medium to save the model with.
medium: default
medium: default

########## step 3: refinement ###########
########## step 3.1: extension ##########
refinement_extension:
# params for DIAMOND
sensitivity: more-sensitive # change the sensitivity of DIAMOND
coverage: 95.0 # Set the min. coverage
pid: 90.0 # set the minimal percentage identity value for valid hits
# params for adding entities during extension
pid: 90.0 # set the minimal percentage identify value for valid hits
# params for adding entities during
Comment on lines 101 to +104
formula-check: 'existence' # When checking, if a metabolite can be added to the model
# also check the formula. For more information about
# available options, please refer to the docs of
# the function isreaction_complete().
# the function isreaction_comlete().
exclude-dna: True # Exclude reactions containing 'DNA' in their name
# from being added to the model.
exclude-rna: True # Exclude reactions containing 'RNA' in their name
Expand All @@ -123,6 +117,9 @@ parameters:
########## step 3.2: cleanup ##########
refinement_cleanup:

# gap-filling
# """""""""""

# general options _________________________________________
# parameters, that apply to all the gap filling algorithms
idprefix: 'HQTB' # prefix to use for fantasy IDs, if IDs for
Expand All @@ -140,7 +137,10 @@ parameters:
# the high probability of false positives.

# enable algorithms ________________________________________

# via KEGG ..................
# ...
# via BioCyc .................
# ...
# via GFF ....................
GeneGapFiller: False # Activate gap filling via GFF
GeneGapFiller parameters:
Expand All @@ -160,6 +160,7 @@ parameters:
media_gap: null # path to a medium config file. Set it to null to skip this gapfiller
growth_threshold: 0.05 # Threshold value for minimal growth, that is required for the media.


# duplicate handling
# """"""""""""""""""

Expand All @@ -169,6 +170,7 @@ parameters:
remove_dupl_reac: True
remove_dupl_meta: True


########## step 3.3: annotation ##########
refinement_annotation:
# for KEGG pathway annotation
Expand All @@ -190,15 +192,15 @@ parameters:
# if set to True, overwrites the following option
tests: # list of tests to run
# currently available: modelpolisher, cobra
- "modelpolisher"
- "cobra"
# - "modelpolisher"
# - "cobra"

########## step 5: analysis ##########
analysis:
# How to compare to pan-core model (current option only id)
pc_based_on: id
# simulate growth on different media (media config file (refineGEMs) required)
media_analysis: __USER__
media_analysis: __USER__
# test for amino acid auxotrophies
test_aa_auxotrophies: True
# perform pathway analysis with KEGG
Expand All @@ -215,5 +217,6 @@ performance:
iterations: 3
chunk_size: 2000
egcs:
chunk_size: 10 # chunk soze for parallel processing during solving EGCs
chunk_size: 1 # chunk size for parallel processing during solving EGCs
limit: 2 # max numbers of cores to use while trying to solve EGCs

100 changes: 54 additions & 46 deletions src/specimen/data/config/hqtb_basic_config_expl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
# Configuration file for the SPECIMEN HTQB pipeline

# This is the shortend version. More more options,
# download the hqtb-advaned configuration file.

# Meaning of the default parameters:
# The value __USER__ indicates parameters required to be specified by the user
# The value USER indicates parameters required only in specific cases
Expand Down Expand Up @@ -51,10 +48,10 @@ data:
# is used and uses this path to a directory as the parent folder for the
# following paths (assumes all data paths are relative ones)
data_direc: null
# and the relative paths to the data file below:
# required
diamond: __USER__
# optional, but speed up the program by a large margin

# optional, but good and manual
ncbi_map: null

# optional for directionality control
Expand All @@ -81,7 +78,12 @@ parameters:

########## step 1: bidirectional blast ##########
bidirectional_blast:
# set the sensitivity for DIAMOND
# default should suffice except special cases
template_name: null
input_name: null
temp_header: null
in_header: null
# can be set by user if wanted, but not necessary
sensitivity: more-sensitive

########## step 2: generaft draft model ##########
Expand All @@ -90,6 +92,7 @@ parameters:
# identifiers of the annotated genome file, use this option to edit them
edit_names: "no"
pid: 80.0
medium: default

########## step 3: refinement ###########
########## step 3.1: extension ##########
Expand All @@ -103,48 +106,53 @@ parameters:
# also check the formula. For more information about
# available options, please refer to the docs of
# the function isreaction_comlete().
exclude-dna: True # Exclude reactions containing 'DNA' in their name
Comment on lines 106 to +109
# from being added to the model.
exclude-rna: True # Exclude reactions containing 'RNA' in their name
# from being added to the model.
threshold_add_reacs: 5 # Threshold values. If the mapping of an EC against reactions
# exceeds this value, the reactions will not be added, due to
# the high probability of false positives.

########## step 3.2: cleanup ##########
refinement_cleanup:

# gap-filling
# """""""""""

# general options _________________________________________
# parameters, that apply to all the gap filling algorithms
# current default means no gapfilling
formula-check: 'existence' # When checking, if a metabolite can be added to the model
# also check the formula. For more information about
# available options, please refer to the docs of
# the function isreaction_comlete().

# enable algorithms ________________________________________
GeneGapFiller: False # Activate gap filling via GFF
GeneGapFiller parameters:
fasta: USER # FASTA (containing CDS) for the GeneGapFiller.
# Depending on the case, might be the same as annotated genome.
gff: USER # Path to a gff file (does not have to be the RefSeq).
# Needs to be from the same genome the model was build on.
type: 'swissprot' # Type of database. Can either be 'swissprot' or 'user'.
dmnd-database: USER # Path to the SwissProt/User DIAMOND database file.
database-mapping: USER # Path to the SwissProt/User mapping file (against EC / BRENDA in case of SwissProt)
check-NCBI: False # Enable checking NCBI accession numbers for EC numbers - time costly.
sensitivity: 'more-sensitive' # Sensitivity option for the DIAMOND run.
coverage: 90.0 # Coverage (parameter for DIAMOND).
percentage identity: 90.0 # Percentage identity threshold value for accepting
# matches found by DIAMOND as homologous.
# via COBRApy - medium ........
media_gap: null # path to a medium config file. Set it to null to skip this gapfiller
########## step 3.3: annotation ##########
Comment on lines 117 to +118
refinement_annotation:
# for KEGG pathway annotation
viaEC: False
viaRC: False

########## step 3.4: smoothing ##########
refinement_smoothing:
mcc: skip # skip, apply or extra for MassChargeCuration
# ECG correction
egc: null
# BOFdat weight fractions: depend on organism (current: Klebsiella )
dna_weight_frac: 0.023
ion_weight_frac: 0.05

########## step 5: analysis ##########
########## step 4: validation ##########
validation:
run_all: True # run all validation tests
# if set to True, overwrites the following option
tests: # list of tests to run
# currently available: modelpolisher, cobra
# - "modelpolisher"
# - "cobra"

########## step 5: analysis ##########
analysis:
media_analysis: __USER__
# How to compare to pan-core model (current option only id)
pc_based_on: id
# simulate growth on different media (media config file (refineGEMs) required)
media_analysis: __USER__
# test for amino acid auxotrophies
test_aa_auxotrophies: True
# perform pathway analysis with KEGG
pathway: True

########## technical params ##########
performance:
threads: 2
# for the gapfilling, if iterations and chunk_size are set (not null)
# use a heuristic for faster performance:
# instead of using all reactions that can be added at once,
# run x interations of gapfilling with n-size randomised chunks of reactions
gapfilling:
iterations: 3
chunk_size: 2000
egcs:
chunk_size: 1 # chunk size for parallel processing during solving EGCs
limit: 2 # max numbers of cores to use while trying to solve EGCs