diff --git a/src/integrationtest/data_classes.py b/src/integrationtest/data_classes.py index 8a3bf5c..33431bf 100644 --- a/src/integrationtest/data_classes.py +++ b/src/integrationtest/data_classes.py @@ -73,6 +73,12 @@ class integtest_param_base_class: # command-line arguments to be passed to run control dunerc_cmd_args: list[str] = field(default_factory=list) + # TRACE debug levels that should be enabled + # example: {"fast": {"ModuleX": 5}, "slow": {"ModuleY": 7}} + # Additional info is available in the comments for the + # trace_debug_settings fixture in integration_drunc.py. + trace_debug_levels: dict = field(default_factory=dict) + @dataclass class integtest_params_for_generated_dunedaq_config(integtest_param_base_class): # *** Parameters that are needed for both generated and predefined configs, diff --git a/src/integrationtest/integrationtest_drunc.py b/src/integrationtest/integrationtest_drunc.py index 9da8e19..5dbf2bd 100644 --- a/src/integrationtest/integrationtest_drunc.py +++ b/src/integrationtest/integrationtest_drunc.py @@ -44,6 +44,9 @@ from daqconf.set_session_env_var import ( set_session_env_var, ) +from daqconf.get_session_env_var import ( + get_session_env_var, +) from daqconf.get_session_apps import get_segment_apps import time import random @@ -300,14 +303,6 @@ def create_config_files(request, tmp_path_factory, check_system_resources): # 05-Nov-2025, KAB, MiR: added the setting of a random RC port set_rc_controller_port(oksfile=str(config_db), session_name=integtest_params.config_session_name, rc_port=0) - # 03-Jul-2025, KAB: added the setting of the TRACE_FILE env var in the OKS Session, - # if it is set in the user's environment, and if it is not already set in the configuration. - try: - trace_file_env_var = os.environ["TRACE_FILE"] - set_session_env_var(str(config_db), integtest_params.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False) - except KeyError: - pass - dal = conffwk.dal.module("generated", "schema/appmodel/fdmodules.schema.xml") db = conffwk.Configuration("oksconflibs:" + str(config_db)) @@ -398,7 +393,8 @@ def apply_update(obj, substitution): @pytest.fixture(scope="module") -def run_dunerc(request, create_config_files, process_manager_type, cleanup_hdf5_files, tmp_path_factory): +def run_dunerc(request, create_config_files, process_manager_type, trace_debug_settings, + cleanup_hdf5_files, tmp_path_factory): """Run drunc with the OKS DB files created by `create_config_files`. The commands specified by the `dunerc_command_list` variable in the test module are executed. If `dunerc_command_list`'s items are @@ -796,6 +792,115 @@ def check_system_resources(request): print(f"\n*** Note: {resval_report_string}") +@pytest.fixture(scope="module") +def trace_debug_settings(request, create_config_files): + """Set the appropriate env vars and trace levels for debugging, if requested. + """ + + # There are a number of things that we want this fixture to do for us. + # + # 1) If the user has the TRACE_FILE env var set in their shell environment, + # we want to copy that into the execution environment of the DAQ processes. + # That is done by setting the appropriate parameters in the OKS configuration. + # * We do this even if the OKS configuration already has a setting for the + # TRACE_FILE env var in it. This gives users the most control - if they + # set up trace locally before running integtests, those are the settings + # that get used. + # + # 2) If the user has specified one or more TRACE debug levels that should be + # enabled, we do that. + # * The format of a TRACE level request is: + # (conf_dict.)trace_debug_levels = {"": + # {"": 0: + + # check if TRACE is already enabled in the OKS configuration + # (we trust the logic above to copy a user-environment TRACE_FILE into the OKS config) + trace_file_value = get_session_env_var(str(create_config_files.dunedaq_config_file), + create_config_files.integtest_params.config_session_name, + "TRACE_FILE", quiet=True) + + # if not, then enable it by creating a temporary TRACE_FILE + if trace_file_value is None: + trace_file_value = str(create_config_files.dunedaq_config_dir) + "/integtest_dunedaq.trace" + set_session_env_var(str(create_config_files.dunedaq_config_file), + create_config_files.integtest_params.config_session_name, + "TRACE_FILE", trace_file_value, overwrite=True) + + # set the env var in the environment of this process + os.environ["TRACE_FILE"] = trace_file_value + + # fetch information from TRACE that we'll need in the next step + tlvls_result = subprocess.run(["trace_cntl", "tids"], capture_output=True, text=True, check=True) + tlvls_output = tlvls_result.stdout # the full listing of the current level settings + + # set the requested debug levels, and + # build up the list of commands that we'll use to restore the original TRACE settings + for trace_type in create_config_files.integtest_params.trace_debug_levels.keys(): # fast or slow + requested_levels = create_config_files.integtest_params.trace_debug_levels[trace_type] + if type(requested_levels) == dict: + for key, value in requested_levels.items(): + mask_result = subprocess.run(["bitN_to_mask", f"DEBUG+{value}"], capture_output=True, + text=True, check=True) + enable_mask = mask_result.stdout + + fast_mask = "0x1ff" + slow_mask = "0xff" + for text_line in tlvls_output.splitlines(): + tokens = text_line.split() + if key == tokens[1]: + fast_mask = tokens[2] + slow_mask = tokens[3] + break + + lc_trace_type = trace_type.lower() + if "fast" in lc_trace_type: + subprocess.run(["trace_cntl", "-n", key, "lvlset", str(enable_mask), "0", "0"], check=True) + subprocess.run(["trace_cntl", "modeM", "1"], check=True) + restore_trace_settings.append(["trace_cntl", "-n", key, "lvlmskM", fast_mask]) + if "slow" in lc_trace_type: + subprocess.run(["trace_cntl", "-n", key, "lvlset", "0", str(enable_mask), "0"], check=True) + subprocess.run(["trace_cntl", "modeS", "1"], check=True) + restore_trace_settings.append(["trace_cntl", "-n", key, "lvlmskS", slow_mask]) + + # pause here to let the DAQ system and pytest tests run + yield + + # restore the original TRACE level settings, if needed + for restore_cmd in restore_trace_settings: + subprocess.run(restore_cmd, check=True) + + @pytest.fixture(scope="module") def cleanup_hdf5_files(request, create_config_files): """Delete the HDF5 files that are produced by the test, if requested diff --git a/src/integrationtest/opmon_metric_checks.py b/src/integrationtest/opmon_metric_checks.py index 47351f9..cfdf119 100644 --- a/src/integrationtest/opmon_metric_checks.py +++ b/src/integrationtest/opmon_metric_checks.py @@ -1,3 +1,4 @@ +import re from opmonlib.info_file_collator import collate_info_files from integrationtest.verbosity_helper import ( IntegtestVerbosityLevels, @@ -118,8 +119,8 @@ def check_metric_value_sum(collated_opmon_data: dict, dict_key_list: list, min_v return False value_sum = 0; - for timestamp_string in working_dict.keys(): - value_sum += int(working_dict[timestamp_string]) + for timestamp_string, value_string in working_dict.items(): + value_sum += int(value_string) if max_value_sum == -1: if value_sum < min_value_sum: print(f"\N{POLICE CARS REVOLVING LIGHT} The sum of metric values for key \"{full_key_path}\" ({value_sum}) is outside the expected range ({min_value_sum}..unbounded). \N{POLICE CARS REVOLVING LIGHT}") @@ -136,3 +137,52 @@ def check_metric_value_sum(collated_opmon_data: dict, dict_key_list: list, min_v verbosity_helper.lvl_print(IntegtestVerbosityLevels.drunc_transitions, f"\N{WHITE HEAVY CHECK MARK} The sum of metric values for key \"{full_key_path}\" ({value_sum}) is within the expected range ({min_value_sum}..{max_value_sum}).") return True + + +# Function to check whether the strings reported in the specified collection of opmon data +# for the specified metric name match the specified regular expression. +# The metric name is specified as a list of dictionary keys. For example: +# [daq_session_name, "df-01", "appfwk.AppInfo", "state"] +# Returns False if a problem is encountered (e.g. parsing the collated JSON metric data) +# or the metric (string) values do not match the specified regex. True otherwise. +def check_metric_value_string(collated_opmon_data: dict, dict_key_list: list, pattern_string: str, + verbosity_helper: VerbosityHelper = VerbosityHelper(99)): + full_key_path = dict_key_list[0] + for key_name in dict_key_list[1:]: + full_key_path += "/" + str(key_name) + "Checking that the sum of {full_key_path} metric values is within its allowed range" + + # sanity check - make sure that we really have a dictionary + working_dict = collated_opmon_data + if not isinstance(working_dict, dict): + print(f"\N{POLICE CARS REVOLVING LIGHT} The data type of the collated opmon data ({type(working_dict)}) is not 'dictionary', as it needs to be. \N{POLICE CARS REVOLVING LIGHT}") + return False + + # work our way down the nested dictionaries until we get to the requested metric values + for key_name in dict_key_list: + # if the user specified a key name of "*", we just use the first available key + if key_name == "*": + key_list = list(working_dict.keys()) + first_key = key_list[0] + working_dict = working_dict[first_key] + # otherwise, we fetch the requested key, if it's available + elif key_name in working_dict.keys(): + working_dict = working_dict[key_name] + # otherwise, we bail out + else: + print(f"\N{POLICE CARS REVOLVING LIGHT} Unable to find the data for key \"{key_name}\" when looking up metric \"{full_key_path}\" in collated opmon data. \N{POLICE CARS REVOLVING LIGHT}") + return False + + # sanity check - make sure that we really have a dictionary at the next level in the tree + if not isinstance(working_dict, dict): + print(f"\N{POLICE CARS REVOLVING LIGHT} The data type of the opmon data associated with the '{key_name}' key ({type(working_dict)}) is not 'dictionary', as it needs to be. \N{POLICE CARS REVOLVING LIGHT}") + return False + + for timestamp_string, value_string in working_dict.items(): + if not re.search(pattern_string, value_string): + print(f"\N{POLICE CARS REVOLVING LIGHT} One of the metric values for key \"{full_key_path}\" ({repr(value_string)}) does not match the expected pattern ({pattern_string}). \N{POLICE CARS REVOLVING LIGHT}") + return False + + verbosity_helper.lvl_print(IntegtestVerbosityLevels.drunc_transitions, + f"\N{WHITE HEAVY CHECK MARK} All of the metric values for key \"{full_key_path}\" match the expected pattern ({pattern_string}).") + return True