New fixture for enabling TRACE levels and new function for checking opmon value strings

src/integrationtest/data_classes.py

@@ -73,6 +73,12 @@ class integtest_param_base_class:
     # command-line arguments to be passed to run control
     dunerc_cmd_args: list[str] = field(default_factory=list)
 
+    # TRACE debug levels that should be enabled
+    # example: {"fast": {"ModuleX": 5}, "slow": {"ModuleY": 7}}
+    # Additional info is available in the comments for the
+    # trace_debug_settings fixture in integration_drunc.py.
+    trace_debug_levels: dict = field(default_factory=dict)
+
 @dataclass
 class integtest_params_for_generated_dunedaq_config(integtest_param_base_class):
     # *** Parameters that are needed for both generated and predefined configs,

src/integrationtest/integrationtest_drunc.py

@@ -44,6 +44,9 @@
 from daqconf.set_session_env_var import (
     set_session_env_var,
 )
+from daqconf.get_session_env_var import (
+    get_session_env_var,
+)
 from daqconf.get_session_apps import get_segment_apps
 import time
 import random
@@ -300,14 +303,6 @@ def create_config_files(request, tmp_path_factory, check_system_resources):
     # 05-Nov-2025, KAB, MiR: added the setting of a random RC port
     set_rc_controller_port(oksfile=str(config_db), session_name=integtest_params.config_session_name, rc_port=0)
 
-    # 03-Jul-2025, KAB: added the setting of the TRACE_FILE env var in the OKS Session,
-    # if it is set in the user's environment, and if it is not already set in the configuration.
-    try:
-        trace_file_env_var = os.environ["TRACE_FILE"]
-        set_session_env_var(str(config_db), integtest_params.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False)
-    except KeyError:
-        pass
-
     dal = conffwk.dal.module("generated", "schema/appmodel/fdmodules.schema.xml")
     db = conffwk.Configuration("oksconflibs:" + str(config_db))
 
@@ -398,7 +393,8 @@ def apply_update(obj, substitution):
 
 
 @pytest.fixture(scope="module")
-def run_dunerc(request, create_config_files, process_manager_type, cleanup_hdf5_files, tmp_path_factory):
+def run_dunerc(request, create_config_files, process_manager_type, trace_debug_settings,
+               cleanup_hdf5_files, tmp_path_factory):
     """Run drunc with the OKS DB files created by `create_config_files`. The
     commands specified by the `dunerc_command_list` variable in the
     test module are executed. If `dunerc_command_list`'s items are
@@ -796,6 +792,115 @@ def check_system_resources(request):
             print(f"\n*** Note: {resval_report_string}")
 
 
+@pytest.fixture(scope="module")
+def trace_debug_settings(request, create_config_files):
+    """Set the appropriate env vars and trace levels for debugging, if requested.
+    """
+
+    # There are a number of things that we want this fixture to do for us.
+    #
+    # 1) If the user has the TRACE_FILE env var set in their shell environment,
+    #    we want to copy that into the execution environment of the DAQ processes.
+    #    That is done by setting the appropriate parameters in the OKS configuration.
+    #    * We do this even if the OKS configuration already has a setting for the
+    #      TRACE_FILE env var in it. This gives users the most control - if they
+    #      set up trace locally before running integtests, those are the settings
+    #      that get used.
+    #
+    # 2) If the user has specified one or more TRACE debug levels that should be
+    #    enabled, we do that.
+    #    * The format of a TRACE level request is:
+    #      (conf_dict.)trace_debug_levels = {"<path type, fast or slow>":
+    #            {"<trace name>": <trace level}}
+    #      For example: {"fast": {"ModuleX": 5}, "slow": {"ModuleY": 7}}
+    #    * We use the TRACE_FILE that is defined in the OKS configuration to set
+    #      the requested levels. If none is defined, then we make up a temporary
+    #      one in the users pytest directory, *and* we write the information about
+    #      the temporary file into the OKS configuration.
+    #    * We set the TRACE_FILE in the Linux environment in which the integtest
+    #      is running.  We do this so that subsequent 'trace_cntl', etc.  commands
+    #      know which TRACE_FILE to use.
+    #    * We set the requested TRACE levels using the specified path ('fast' or
+    #      'slow', the requested TRACE name, and the requested level.
+    #    * As the test is being torn down (after the "yield" command), we restore
+    #      the original levels for any TRACE names that we touched.
+    #    * For all of the TRACE shell commands that we run, we set the
+    #      subprocess.run() "check" option to True so that an exception will get
+    #      thrown if there is a problem.
+
+    # Set the TRACE_FILE env var in the OKS Session, if it is set in the user's environment.
+    try:
+        trace_file_env_var = os.environ["TRACE_FILE"]
+        set_session_env_var(str(create_config_files.dunedaq_config_file),
+                            create_config_files.integtest_params.config_session_name,
+                            "TRACE_FILE", trace_file_env_var, overwrite=True)
+    except KeyError:
+        # if the env var is not set in the user's environment, we simply continue
+        pass
+
+    # If the integtest specifies one or more TRACE levels to be set, we do that here.
+    # We build up a list of commands that will be used to restore the TRACE levels
+    # to their original values once the test is done.
+    restore_trace_settings = []
+    if len(create_config_files.integtest_params.trace_debug_levels) > 0:
+
+        # check if TRACE is already enabled in the OKS configuration
+        # (we trust the logic above to copy a user-environment TRACE_FILE into the OKS config)
+        trace_file_value = get_session_env_var(str(create_config_files.dunedaq_config_file),
+                                               create_config_files.integtest_params.config_session_name,
+                                               "TRACE_FILE", quiet=True)
+
+        # if not, then enable it by creating a temporary TRACE_FILE
+        if trace_file_value is None:
+            trace_file_value = str(create_config_files.dunedaq_config_dir) + "/integtest_dunedaq.trace"
+            set_session_env_var(str(create_config_files.dunedaq_config_file),
+                                create_config_files.integtest_params.config_session_name,
+                                "TRACE_FILE", trace_file_value, overwrite=True)
+
+        # set the env var in the environment of this process
+        os.environ["TRACE_FILE"] = trace_file_value
+
+        # fetch information from TRACE that we'll need in the next step
+        tlvls_result = subprocess.run(["trace_cntl", "tids"], capture_output=True, text=True, check=True)
+        tlvls_output = tlvls_result.stdout  # the full listing of the current level settings
+
+        # set the requested debug levels, and
+        # build up the list of commands that we'll use to restore the original TRACE settings
+        for trace_type in create_config_files.integtest_params.trace_debug_levels.keys(): # fast or slow
+            requested_levels = create_config_files.integtest_params.trace_debug_levels[trace_type]
+            if type(requested_levels) == dict:
+                for key, value in requested_levels.items():
+                    mask_result = subprocess.run(["bitN_to_mask", f"DEBUG+{value}"], capture_output=True,
+                                                 text=True, check=True)
+                    enable_mask = mask_result.stdout
+
+                    fast_mask = "0x1ff"
+                    slow_mask = "0xff"
+                    for text_line in tlvls_output.splitlines():
+                        tokens = text_line.split()
+                        if key == tokens[1]:
+                            fast_mask = tokens[2]
+                            slow_mask = tokens[3]
+                            break
+
+                    lc_trace_type = trace_type.lower()
+                    if "fast" in lc_trace_type:
+                        subprocess.run(["trace_cntl", "-n", key, "lvlset", str(enable_mask), "0", "0"], check=True)
+                        subprocess.run(["trace_cntl", "modeM", "1"], check=True)
+                        restore_trace_settings.append(["trace_cntl", "-n", key, "lvlmskM", fast_mask])
+                    if "slow" in lc_trace_type:
+                        subprocess.run(["trace_cntl", "-n", key, "lvlset", "0", str(enable_mask), "0"], check=True)
+                        subprocess.run(["trace_cntl", "modeS", "1"], check=True)
+                        restore_trace_settings.append(["trace_cntl", "-n", key, "lvlmskS", slow_mask])
+
+    # pause here to let the DAQ system and pytest tests run
+    yield
+
+    # restore the original TRACE level settings, if needed
+    for restore_cmd in restore_trace_settings:
+        subprocess.run(restore_cmd, check=True)
+
+
 @pytest.fixture(scope="module")
 def cleanup_hdf5_files(request, create_config_files):
     """Delete the HDF5 files that are produced by the test, if requested

src/integrationtest/opmon_metric_checks.py

@@ -1,3 +1,4 @@
+import re
 from opmonlib.info_file_collator import collate_info_files
 from integrationtest.verbosity_helper import (
     IntegtestVerbosityLevels,
@@ -118,8 +119,8 @@ def check_metric_value_sum(collated_opmon_data: dict, dict_key_list: list, min_v
             return False
 
     value_sum = 0;
-    for timestamp_string in working_dict.keys():
-        value_sum += int(working_dict[timestamp_string])
+    for timestamp_string, value_string in working_dict.items():
+        value_sum += int(value_string)
     if max_value_sum == -1:
         if value_sum < min_value_sum:
             print(f"\N{POLICE CARS REVOLVING LIGHT} The sum of metric values for key \"{full_key_path}\" ({value_sum}) is outside the expected range ({min_value_sum}..unbounded). \N{POLICE CARS REVOLVING LIGHT}")
@@ -136,3 +137,52 @@ def check_metric_value_sum(collated_opmon_data: dict, dict_key_list: list, min_v
             verbosity_helper.lvl_print(IntegtestVerbosityLevels.drunc_transitions,
                                        f"\N{WHITE HEAVY CHECK MARK} The sum of metric values for key \"{full_key_path}\" ({value_sum}) is within the expected range ({min_value_sum}..{max_value_sum}).")
             return True
+
+
+# Function to check whether the strings reported in the specified collection of opmon data
+# for the specified metric name match the specified regular expression.
+# The metric name is specified as a list of dictionary keys.  For example:
+#   [daq_session_name, "df-01", "appfwk.AppInfo", "state"]
+# Returns False if a problem is encountered (e.g. parsing the collated JSON metric data)
+# or the metric (string) values do not match the specified regex.  True otherwise.
+def check_metric_value_string(collated_opmon_data: dict, dict_key_list: list, pattern_string: str,
+                              verbosity_helper: VerbosityHelper = VerbosityHelper(99)):
+    full_key_path = dict_key_list[0]
+    for key_name in dict_key_list[1:]:
+        full_key_path += "/" + str(key_name)
+    "Checking that the sum of {full_key_path} metric values is within its allowed range"
+
+    # sanity check - make sure that we really have a dictionary
+    working_dict = collated_opmon_data
+    if not isinstance(working_dict, dict):
+        print(f"\N{POLICE CARS REVOLVING LIGHT} The data type of the collated opmon data ({type(working_dict)}) is not 'dictionary', as it needs to be. \N{POLICE CARS REVOLVING LIGHT}")
+        return False
+
+    # work our way down the nested dictionaries until we get to the requested metric values
+    for key_name in dict_key_list:
+        # if the user specified a key name of "*", we just use the first available key
+        if key_name == "*":
+            key_list = list(working_dict.keys())
+            first_key = key_list[0]
+            working_dict = working_dict[first_key]
+        # otherwise, we fetch the requested key, if it's available
+        elif key_name in working_dict.keys():
+            working_dict = working_dict[key_name]
+        # otherwise, we bail out
+        else:
+            print(f"\N{POLICE CARS REVOLVING LIGHT} Unable to find the data for key \"{key_name}\" when looking up metric \"{full_key_path}\" in collated opmon data. \N{POLICE CARS REVOLVING LIGHT}")
+            return False
+
+        # sanity check - make sure that we really have a dictionary at the next level in the tree
+        if not isinstance(working_dict, dict):
+            print(f"\N{POLICE CARS REVOLVING LIGHT} The data type of the opmon data associated with the '{key_name}' key ({type(working_dict)}) is not 'dictionary', as it needs to be. \N{POLICE CARS REVOLVING LIGHT}")
+            return False
+
+    for timestamp_string, value_string in working_dict.items():
+        if not re.search(pattern_string, value_string):
+            print(f"\N{POLICE CARS REVOLVING LIGHT} One of the metric values for key \"{full_key_path}\" ({repr(value_string)}) does not match the expected pattern ({pattern_string}). \N{POLICE CARS REVOLVING LIGHT}")
+            return False
+
+    verbosity_helper.lvl_print(IntegtestVerbosityLevels.drunc_transitions,
+                               f"\N{WHITE HEAVY CHECK MARK} All of the metric values for key \"{full_key_path}\" match the expected pattern ({pattern_string}).")
+    return True