From 801930366b24231afaffa4f799424cc3c0dc2548 Mon Sep 17 00:00:00 2001 From: dyrpsf Date: Tue, 14 Jul 2026 22:27:24 +0530 Subject: [PATCH] feat(config): add developer script to automate YAML syncing Resolves #24. Introduces scripts/sync_configs.py to automatically parse hqtb_config_default.yaml and generate basic and advanced variations, stripping dev tags (@TODO, @IDEA) and omitting advanced parameter blocks. --- scripts/sync_configs.py | 93 ++++++++++++++++ .../config/hqtb_advanced_config_expl.yaml | 41 +++---- .../data/config/hqtb_basic_config_expl.yaml | 100 ++++++++++-------- 3 files changed, 169 insertions(+), 65 deletions(-) create mode 100644 scripts/sync_configs.py diff --git a/scripts/sync_configs.py b/scripts/sync_configs.py new file mode 100644 index 0000000..54b4659 --- /dev/null +++ b/scripts/sync_configs.py @@ -0,0 +1,93 @@ +import os +import re + +# Define paths relative to the root directory +CONFIG_DIR = os.path.join("src", "specimen", "data", "config") +DEFAULT_CONFIG = os.path.join(CONFIG_DIR, "hqtb_config_default.yaml") +ADVANCED_CONFIG = os.path.join(CONFIG_DIR, "hqtb_advanced_config_expl.yaml") +BASIC_CONFIG = os.path.join(CONFIG_DIR, "hqtb_basic_config_expl.yaml") + +# Developer tags to strip out for non-default configs +DEV_TAGS = ["# @IDEA", "# @TODO", "# @DEV"] + +# Blocks to entirely skip for the basic "quick-and-dirty" configuration +ADVANCED_BLOCKS = [ + "refinement_cleanup:", + "refinement_smoothing:", + "GeneGapFiller:", + "media_gap:", + "mcc:", + "egc:", +] + + +def is_dev_note(line: str) -> bool: + """Checks if a line contains developer-specific tags.""" + return any(tag in line for tag in DEV_TAGS) + + +def generate_advanced(lines: list[str]) -> list[str]: + """Generates advanced config by stripping dev notes.""" + # Advanced contains all params and user comments, just no dev notes + return [line for line in lines if not is_dev_note(line)] + + +def generate_basic(lines: list[str]) -> list[str]: + """Generates basic config by stripping dev notes and advanced blocks.""" + basic_lines = [] + skip_block = False + current_indent = 0 + + for line in lines: + if is_dev_note(line): + continue + + stripped = line.lstrip() + indent = len(line) - len(stripped) + + # Handle skipping nested blocks + if skip_block: + # If we find text that is at the same or lower indentation level, the block is over + if stripped and indent <= current_indent: + skip_block = False + else: + continue + + # Check if we should start skipping a new block + if any(stripped.startswith(key) for key in ADVANCED_BLOCKS): + skip_block = True + current_indent = indent + continue + + basic_lines.append(line) + + return basic_lines + + +def main(): + print("Syncing SPECIMEN YAML configurations...") + if not os.path.exists(DEFAULT_CONFIG): + print( + f"Error: Could not find {DEFAULT_CONFIG}. Run this script from the project root." + ) + return + + with open(DEFAULT_CONFIG, "r", encoding="utf-8") as f: + lines = f.readlines() + + # Generate and write Advanced Config + advanced_lines = generate_advanced(lines) + with open(ADVANCED_CONFIG, "w", encoding="utf-8") as f: + f.writelines(advanced_lines) + print(f" [+] Successfully generated {ADVANCED_CONFIG}") + + # Generate and write Basic Config + basic_lines = generate_basic(lines) + with open(BASIC_CONFIG, "w", encoding="utf-8") as f: + f.writelines(basic_lines) + print(f" [+] Successfully generated {BASIC_CONFIG}") + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/src/specimen/data/config/hqtb_advanced_config_expl.yaml b/src/specimen/data/config/hqtb_advanced_config_expl.yaml index 2b519de..d8cb379 100644 --- a/src/specimen/data/config/hqtb_advanced_config_expl.yaml +++ b/src/specimen/data/config/hqtb_advanced_config_expl.yaml @@ -1,7 +1,4 @@ -# Configuration file for the SPECIMEN HQTB pipeline - -# This is the advaned config file. For a shorter version, -# download the hqtb-basic configuration file. +# Configuration file for the SPECIMEN HTQB pipeline # Meaning of the default parameters: # The value __USER__ indicates parameters required to be specified by the user @@ -30,6 +27,7 @@ subject: template: annotated_genome: __USER__ model: __USER__ + namespace: BiGG # information about the output # ---------------------------- @@ -71,7 +69,6 @@ data: # ----------------------------------------------- parameters: - ########## general ########## general: idprefix: 'HQTB' # prefix to use for fantasy IDs, if IDs for @@ -81,13 +78,12 @@ parameters: ########## step 1: bidirectional blast ########## bidirectional_blast: - # options for giving customised names and setting a header - # -> default uses information from the models/files directly + # default should suffice except special cases template_name: null input_name: null temp_header: null in_header: null - # change the sensitivity of DIAMOND + # can be set by user if wanted, but not necessary sensitivity: more-sensitive ########## step 2: generaft draft model ########## @@ -95,10 +91,8 @@ parameters: # if the gene ids of the template model do not match the # identifiers of the annotated genome file, use this option to edit them edit_names: "no" - # filter for genes based on percentage identity value pid: 80.0 - # Which medium to save the model with. - medium: default + medium: default ########## step 3: refinement ########### ########## step 3.1: extension ########## @@ -106,12 +100,12 @@ parameters: # params for DIAMOND sensitivity: more-sensitive # change the sensitivity of DIAMOND coverage: 95.0 # Set the min. coverage - pid: 90.0 # set the minimal percentage identity value for valid hits - # params for adding entities during extension + pid: 90.0 # set the minimal percentage identify value for valid hits + # params for adding entities during formula-check: 'existence' # When checking, if a metabolite can be added to the model # also check the formula. For more information about # available options, please refer to the docs of - # the function isreaction_complete(). + # the function isreaction_comlete(). exclude-dna: True # Exclude reactions containing 'DNA' in their name # from being added to the model. exclude-rna: True # Exclude reactions containing 'RNA' in their name @@ -123,6 +117,9 @@ parameters: ########## step 3.2: cleanup ########## refinement_cleanup: + # gap-filling + # """"""""""" + # general options _________________________________________ # parameters, that apply to all the gap filling algorithms idprefix: 'HQTB' # prefix to use for fantasy IDs, if IDs for @@ -140,7 +137,10 @@ parameters: # the high probability of false positives. # enable algorithms ________________________________________ - + # via KEGG .................. + # ... + # via BioCyc ................. + # ... # via GFF .................... GeneGapFiller: False # Activate gap filling via GFF GeneGapFiller parameters: @@ -160,6 +160,7 @@ parameters: media_gap: null # path to a medium config file. Set it to null to skip this gapfiller growth_threshold: 0.05 # Threshold value for minimal growth, that is required for the media. + # duplicate handling # """""""""""""""""" @@ -169,6 +170,7 @@ parameters: remove_dupl_reac: True remove_dupl_meta: True + ########## step 3.3: annotation ########## refinement_annotation: # for KEGG pathway annotation @@ -190,15 +192,15 @@ parameters: # if set to True, overwrites the following option tests: # list of tests to run # currently available: modelpolisher, cobra - - "modelpolisher" - - "cobra" + # - "modelpolisher" + # - "cobra" ########## step 5: analysis ########## analysis: # How to compare to pan-core model (current option only id) pc_based_on: id # simulate growth on different media (media config file (refineGEMs) required) - media_analysis: __USER__ + media_analysis: __USER__ # test for amino acid auxotrophies test_aa_auxotrophies: True # perform pathway analysis with KEGG @@ -215,5 +217,6 @@ performance: iterations: 3 chunk_size: 2000 egcs: - chunk_size: 10 # chunk soze for parallel processing during solving EGCs + chunk_size: 1 # chunk size for parallel processing during solving EGCs limit: 2 # max numbers of cores to use while trying to solve EGCs + diff --git a/src/specimen/data/config/hqtb_basic_config_expl.yaml b/src/specimen/data/config/hqtb_basic_config_expl.yaml index 29d4ec2..a2660b4 100644 --- a/src/specimen/data/config/hqtb_basic_config_expl.yaml +++ b/src/specimen/data/config/hqtb_basic_config_expl.yaml @@ -1,8 +1,5 @@ # Configuration file for the SPECIMEN HTQB pipeline -# This is the shortend version. More more options, -# download the hqtb-advaned configuration file. - # Meaning of the default parameters: # The value __USER__ indicates parameters required to be specified by the user # The value USER indicates parameters required only in specific cases @@ -51,10 +48,10 @@ data: # is used and uses this path to a directory as the parent folder for the # following paths (assumes all data paths are relative ones) data_direc: null - # and the relative paths to the data file below: # required diamond: __USER__ - # optional, but speed up the program by a large margin + + # optional, but good and manual ncbi_map: null # optional for directionality control @@ -81,7 +78,12 @@ parameters: ########## step 1: bidirectional blast ########## bidirectional_blast: - # set the sensitivity for DIAMOND + # default should suffice except special cases + template_name: null + input_name: null + temp_header: null + in_header: null + # can be set by user if wanted, but not necessary sensitivity: more-sensitive ########## step 2: generaft draft model ########## @@ -90,6 +92,7 @@ parameters: # identifiers of the annotated genome file, use this option to edit them edit_names: "no" pid: 80.0 + medium: default ########## step 3: refinement ########### ########## step 3.1: extension ########## @@ -103,48 +106,53 @@ parameters: # also check the formula. For more information about # available options, please refer to the docs of # the function isreaction_comlete(). + exclude-dna: True # Exclude reactions containing 'DNA' in their name + # from being added to the model. + exclude-rna: True # Exclude reactions containing 'RNA' in their name + # from being added to the model. + threshold_add_reacs: 5 # Threshold values. If the mapping of an EC against reactions + # exceeds this value, the reactions will not be added, due to + # the high probability of false positives. ########## step 3.2: cleanup ########## - refinement_cleanup: - - # gap-filling - # """"""""""" - - # general options _________________________________________ - # parameters, that apply to all the gap filling algorithms - # current default means no gapfilling - formula-check: 'existence' # When checking, if a metabolite can be added to the model - # also check the formula. For more information about - # available options, please refer to the docs of - # the function isreaction_comlete(). - - # enable algorithms ________________________________________ - GeneGapFiller: False # Activate gap filling via GFF - GeneGapFiller parameters: - fasta: USER # FASTA (containing CDS) for the GeneGapFiller. - # Depending on the case, might be the same as annotated genome. - gff: USER # Path to a gff file (does not have to be the RefSeq). - # Needs to be from the same genome the model was build on. - type: 'swissprot' # Type of database. Can either be 'swissprot' or 'user'. - dmnd-database: USER # Path to the SwissProt/User DIAMOND database file. - database-mapping: USER # Path to the SwissProt/User mapping file (against EC / BRENDA in case of SwissProt) - check-NCBI: False # Enable checking NCBI accession numbers for EC numbers - time costly. - sensitivity: 'more-sensitive' # Sensitivity option for the DIAMOND run. - coverage: 90.0 # Coverage (parameter for DIAMOND). - percentage identity: 90.0 # Percentage identity threshold value for accepting - # matches found by DIAMOND as homologous. - # via COBRApy - medium ........ - media_gap: null # path to a medium config file. Set it to null to skip this gapfiller + ########## step 3.3: annotation ########## + refinement_annotation: + # for KEGG pathway annotation + viaEC: False + viaRC: False ########## step 3.4: smoothing ########## - refinement_smoothing: - mcc: skip # skip, apply or extra for MassChargeCuration - # ECG correction - egc: null - # BOFdat weight fractions: depend on organism (current: Klebsiella ) - dna_weight_frac: 0.023 - ion_weight_frac: 0.05 - - ########## step 5: analysis ########## + ########## step 4: validation ########## + validation: + run_all: True # run all validation tests + # if set to True, overwrites the following option + tests: # list of tests to run + # currently available: modelpolisher, cobra + # - "modelpolisher" + # - "cobra" + + ########## step 5: analysis ########## analysis: - media_analysis: __USER__ + # How to compare to pan-core model (current option only id) + pc_based_on: id + # simulate growth on different media (media config file (refineGEMs) required) + media_analysis: __USER__ + # test for amino acid auxotrophies + test_aa_auxotrophies: True + # perform pathway analysis with KEGG + pathway: True + +########## technical params ########## +performance: + threads: 2 + # for the gapfilling, if iterations and chunk_size are set (not null) + # use a heuristic for faster performance: + # instead of using all reactions that can be added at once, + # run x interations of gapfilling with n-size randomised chunks of reactions + gapfilling: + iterations: 3 + chunk_size: 2000 + egcs: + chunk_size: 1 # chunk size for parallel processing during solving EGCs + limit: 2 # max numbers of cores to use while trying to solve EGCs +