From decc956a9c9e96948f3f415c9cd7930638034f68 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 23 Jun 2026 05:23:00 -0500 Subject: [PATCH 1/5] ENH: Declare pkl files in ninja graph via per-module stamp file igenerator.py writes N pkl files per module as side effects that ninja cannot track because their names (ClassName.SubmoduleName.pkl) are not enumerable at CMake configure time. When pkl files are deleted while the .index.txt byproducts survive, ninja considers igenerator up-to-date and pyi_generator.py fails with "No pickle files were found." Add a --pkl_stamp argument to igenerator.py. The stamp is written after all pkl files for the module are complete and is declared as a CMake OUTPUT of the igenerator add_custom_command. Ninja now re-runs igenerator whenever the stamp is absent, which guarantees the pkl files are regenerated before pyi_generator.py reads the .index.txt manifests. --- Wrapping/Generators/SwigInterface/igenerator.py | 12 ++++++++++++ Wrapping/macro_files/itk_end_wrap_module.cmake | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index bcda8695eab..c5a0f157c8e 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -207,6 +207,15 @@ def argument_parser(): type=str, help="The directory for .pyi files to be generated", ) + cmdln_arg_parser.add_argument( + "--pkl_stamp", + action="store", + dest="pkl_stamp", + default="", + type=str, + help="Stamp file written after all pkl files are complete; declared as a " + "CMake OUTPUT so ninja re-runs igenerator when pkl files are missing.", + ) cmdln_arg_parser.add_argument( "-d", action="store_true", @@ -2054,6 +2063,9 @@ def main(): ff.write("'" + function + "', ") ff.write(")\n") + if options.pkl_stamp: + Path(options.pkl_stamp).touch() + if __name__ == "__main__": main() diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 04bde56dc4b..45f825ab57e 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,11 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.pkl.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.pkl.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -188,7 +190,7 @@ macro(itk_end_wrap_module) --library-output-dir "${WRAPPER_LIBRARY_OUTPUT_DIR}" --submodule-order "${THIS_MODULE_SUBMODULE_ORDER}" --pyi_index_list "${ITK_PYI_INDEX_FILES}" --pyi_dir "${ITK_STUB_DIR}" --pkl_dir - "${ITK_PKL_DIR}" + "${ITK_PKL_DIR}" --pkl_stamp "${pkl_stamp_file}" DEPENDS ${IGENERATOR} ${ITK_WRAP_DOC_DOCSTRING_FILES} @@ -202,6 +204,7 @@ macro(itk_end_wrap_module) VERBATIM ) + unset(pkl_stamp_file) unset(snake_case_config_file) else() #message(FATAL_ERROR "Number of interface files is 0 :${WRAPPER_LIBRARY_NAME}:") From 4878f705d9c3a4b1a09b7a2e35750b63cffe5722 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 23 Jun 2026 06:40:34 -0500 Subject: [PATCH 2/5] ENH: Replace per-class .pkl files with a per-build SQLite pkl database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit igenerator.py now writes all pickle data for a module into a single WAL-mode SQLite database (itk-pkl-v3.db) kept in the build tree's ITK_PKL_DIR. The DB is keyed by bare class.submodule strings and is intermediate handoff state between igenerator (writer) and pyi_generator (reader), so it stays local to one build tree and is never shared via ITK_WRAP_CACHE. The .index.txt manifests now hold DB keys instead of file paths. pyi_generator.py reads keys from the manifests, queries the DB, and prunes any row whose key is absent from the current build's manifests (exact keyset cleanup, replacing the previous per-tree *.pkl glob). The schema version is embedded in the filename so a stale database from an older schema is ignored rather than migrated. itk_end_wrap_module.cmake passes --module_name to igenerator to identify the manifest partition; the per-module stamp file (.stamp) remains the CMake OUTPUT that ninja tracks. Assisted-by: Claude Code — design, implementation, and pre-commit validation --- Wrapping/CMakeLists.txt | 10 +- .../Generators/Python/itk/pyi_generator.py | 139 ++++++++++++------ .../Generators/SwigInterface/CMakeLists.txt | 7 + .../Generators/SwigInterface/igenerator.py | 61 ++++---- Wrapping/Generators/pkl_db.py | 31 ++++ .../macro_files/itk_end_wrap_module.cmake | 5 +- 6 files changed, 170 insertions(+), 83 deletions(-) create mode 100644 Wrapping/Generators/pkl_db.py diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 1f4d8c95cd3..13cd681e01b 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -233,6 +233,13 @@ if(ITK_WRAP_PYTHON) # ${GLOBAL_IdxFilesList}: is described within SwigInterface/CMakeLists.txt # The variable is passed to pyi_generator to make sure that the function only uses current index files # All index files found that are not in the given list are assumed to be outdated and are removed. + # Prune only for ITK's own build: external wrap projects share ITK_PKL_DIR + # but their manifests cover only their own modules, so pruning there would + # delete every ITK-core row. + set(_pyi_prune_arg) + if(NOT EXTERNAL_WRAP_ITK_PROJECT) + set(_pyi_prune_arg --prune) + endif() add_custom_target( itk-stub-files ALL @@ -242,13 +249,14 @@ if(ITK_WRAP_PYTHON) COMMAND ${Python3_EXECUTABLE} ${PYI_GENERATOR} --pyi_dir "${ITK_STUB_DIR}" --pkl_dir "${ITK_PKL_DIR}" --index_list_file - "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" + "${CMAKE_CURRENT_BINARY_DIR}/GlobalIdxFilesList.txt" ${_pyi_prune_arg} DEPENDS ${PYI_GENERATOR} ${GLOBAL_IdxFilesList} COMMENT "Generating .pyi files for Python wrapper interface" VERBATIM ) + unset(_pyi_prune_arg) # Add module dependencies to ensure .index.txt files have been generated unique(WRAPPED_MODULE_LIST "${WRAP_ITK_MODULES}") diff --git a/Wrapping/Generators/Python/itk/pyi_generator.py b/Wrapping/Generators/Python/itk/pyi_generator.py index d8007fb6aed..ce2aa4787e5 100644 --- a/Wrapping/Generators/Python/itk/pyi_generator.py +++ b/Wrapping/Generators/Python/itk/pyi_generator.py @@ -5,17 +5,20 @@ """ import os +import sys +import sqlite3 from os import remove from argparse import ArgumentParser from io import StringIO from pathlib import Path, PurePath import pickle import glob -import re from collections import defaultdict +sys.path.insert(0, str(Path(__file__).parents[2])) +from pkl_db import open_pkl_db # noqa: E402 + -# The ITKClass is duplicated in igenerator.py class ITKClass: def __init__(self, l_class_name): # Structure @@ -379,13 +382,21 @@ def write_class_proxy_pyi( pyi_file.write(interfaces_code) -def unpack(file_names: [str], save_dir: str) -> str | None: - class_definitions = [] +def unpack(keys: list[str], db_conn: sqlite3.Connection, save_dir: str) -> str | None: + placeholders = ",".join("?" * len(keys)) + rows = { + k: data + for k, data in db_conn.execute( + f"SELECT key, data FROM pkl_data WHERE key IN ({placeholders})", keys + ) + } - for file_name in file_names: - with open(file_name, "rb") as pickled_file: - itk_class = pickle.load(pickled_file) - class_definitions.append(itk_class) + class_definitions = [] + for key in keys: + if key not in rows: + print(f"WARNING: pkl key {key!r} not found in database, skipping.") + continue + class_definitions.append(pickle.loads(rows[key])) base = merge(class_definitions) @@ -419,9 +430,9 @@ def unpack(file_names: [str], save_dir: str) -> str | None: return base.class_name else: - files = "\n\t".join(file_names) + files = "\n\t".join(keys) print( - f"WARNING: The following files do not contain a class definition for Python stub wrapping:\n" + f"WARNING: The following keys do not contain a class definition for Python stub wrapping:\n" f"\t{files}" ) @@ -522,6 +533,13 @@ def merge(class_definitions: []) -> ITKClass | None: type=str, help="Configured file listing the index files containing pickle file references", ) + cmdln_arg_parser.add_argument( + "--prune", + action="store_true", + dest="prune", + help="Delete pkl DB rows whose keys are absent from the index manifests. " + "Only safe when the manifests cover every module sharing the DB.", + ) cmdln_arg_parser.add_argument( "-d", action="store_true", @@ -530,7 +548,7 @@ def merge(class_definitions: []) -> ITKClass | None: ) options = cmdln_arg_parser.parse_args() - if not Path(options.pkl_dir).exists: + if not Path(options.pkl_dir).exists(): except_comment = f"Invalid directory provided '{options.pkl_dir}'" raise Exception(except_comment) @@ -556,7 +574,6 @@ def merge(class_definitions: []) -> ITKClass | None: remove(invalid_index_file) for missing_index_file in missing_index_files: - # continue on without missing file, display warning print( f"WARNING: index file {missing_index_file} is missing, " f"Python stub hints will not be generated for this file. " @@ -568,52 +585,84 @@ def merge(class_definitions: []) -> ITKClass | None: print(f"Exception: {except_comment}") raise Exception(except_comment) - indexed_pickled_files = set() + # Collect DB keys from all .index.txt manifests (keys, not file paths). + indexed_pkl_keys: set[str] = set() for index_file in sorted(index_files): with open(index_file) as file: for line in file: - indexed_pickled_files.add(line.strip()) + key = line.strip() + if key: + indexed_pkl_keys.add(key) - existing_pickled_files = { - filepath.replace(os.sep, "/") - for filepath in glob.glob(f"{options.pkl_dir}/*.pkl") - } - - invalid_pickled_files = existing_pickled_files - indexed_pickled_files - missing_pickled_files = indexed_pickled_files - existing_pickled_files - - if options.debug_code: - for invalid_pickle_file in invalid_pickled_files: - print( - f"WARNING: Outdated pickle file {invalid_pickle_file} has been removed" - ) - remove(invalid_pickle_file) + if len(indexed_pkl_keys) == 0: + raise Exception(f"No pkl keys were found in index files in '{options.pkl_dir}'") - for missing_file in missing_pickled_files: - # continue on without missing file, display warning - print( - f"WARNING: pickle file {missing_file} is missing, Python stub hints will not be generated for this file." - ) - indexed_pickled_files.remove(missing_file) - - if len(indexed_pickled_files) == 0: - raise Exception(f"No pickle files were found in directory {options.pkl_dir}") - - indexed_pickled_files = sorted(list(indexed_pickled_files)) + indexed_pkl_keys_sorted = sorted(indexed_pkl_keys) output_template_import_list: list[str] = [] output_proxy_import_list: list[str] = [] - class_name_dict = defaultdict(list) - for file in indexed_pickled_files: - current_class_name = re.split(r"\.", PurePath(file).parts[-1])[0] - class_name_dict[current_class_name].append(file) + class_name_dict: defaultdict[str, list[str]] = defaultdict(list) + for key in indexed_pkl_keys_sorted: + current_class_name = key.split(".")[0] + class_name_dict[current_class_name].append(key) + + db_conn = open_pkl_db(options.pkl_dir) + + # Indexed keys absent from the DB mean it was deleted or damaged while the + # stamp files stayed fresh, so ninja would never re-run igenerator; drop + # the stamps and fail so the next build regenerates the database. + present_keys: set[str] = set() + for chunk_start in range(0, len(indexed_pkl_keys_sorted), 500): + chunk = indexed_pkl_keys_sorted[chunk_start : chunk_start + 500] + placeholders = ",".join("?" * len(chunk)) + present_keys.update( + row[0] + for row in db_conn.execute( + f"SELECT key FROM pkl_data WHERE key IN ({placeholders})", chunk + ) + ) + absent_keys = set(indexed_pkl_keys_sorted) - present_keys + if absent_keys: + db_conn.close() + for stamp_file in glob.glob(f"{options.pkl_dir}/*.stamp"): + remove(stamp_file) + raise Exception( + f"{len(absent_keys)} indexed pkl keys are missing from the pkl " + f"database in '{options.pkl_dir}'. The igenerator stamp files " + "were removed; re-run the build to regenerate the database." + ) - for current_class_name, class_files in class_name_dict.items(): - class_name = unpack(class_files, options.pyi_dir) + for current_class_name, class_keys in class_name_dict.items(): + class_name = unpack(class_keys, db_conn, options.pyi_dir) if class_name is not None: output_template_import_list.append(f"from .{class_name}Template import *\n") output_proxy_import_list.append(f"from .{class_name}Proxy import *\n") + if not output_template_import_list: + raise Exception( + f"No Python stub classes were generated from pkl keys in '{options.pkl_dir}'" + ) + + if options.prune: + if missing_index_files: + print( + "WARNING: skipping pkl DB pruning: the index manifest set is " + "incomplete, so live keys cannot be determined safely." + ) + else: + with db_conn: + db_conn.execute("CREATE TEMP TABLE _live_keys (key TEXT PRIMARY KEY)") + db_conn.executemany( + "INSERT OR IGNORE INTO _live_keys VALUES(?)", + [(k,) for k in indexed_pkl_keys_sorted], + ) + db_conn.execute( + "DELETE FROM pkl_data WHERE key NOT IN (SELECT key FROM _live_keys)" + ) + db_conn.execute("DROP TABLE _live_keys") + + db_conn.close() + output_init_import_file = init_init_import_file() output_proxy_import_file = init_proxy_import_file() diff --git a/Wrapping/Generators/SwigInterface/CMakeLists.txt b/Wrapping/Generators/SwigInterface/CMakeLists.txt index af8526463b6..0f554984117 100644 --- a/Wrapping/Generators/SwigInterface/CMakeLists.txt +++ b/Wrapping/Generators/SwigInterface/CMakeLists.txt @@ -397,6 +397,13 @@ set( "igenerator.py path" FORCE ) +set( + IGENERATOR_PKL_DB + "${CMAKE_CURRENT_SOURCE_DIR}/../pkl_db.py" + CACHE INTERNAL + "pkl_db.py path (shared module imported by igenerator.py)" + FORCE +) macro(itk_wrap_simple_type_swig_interface wrap_class swig_name) string(APPEND SWIG_INTERFACE_TYPEDEFS "using ${swig_name} = ${wrap_class};\n") diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index c5a0f157c8e..f0f5e761b08 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -62,12 +62,14 @@ import re from argparse import ArgumentParser from io import StringIO -from os.path import exists from pathlib import Path from keyword import iskeyword from typing import Any import logging +sys.path.insert(0, str(Path(__file__).parents[1])) +from pkl_db import open_pkl_db # noqa: E402 + def argument_parser(): cmdln_arg_parser = ArgumentParser() @@ -205,7 +207,7 @@ def argument_parser(): dest="pkl_dir", default="", type=str, - help="The directory for .pyi files to be generated", + help="Directory for per-submodule .index.txt manifests and per-module stamp files.", ) cmdln_arg_parser.add_argument( "--pkl_stamp", @@ -213,8 +215,8 @@ def argument_parser(): dest="pkl_stamp", default="", type=str, - help="Stamp file written after all pkl files are complete; declared as a " - "CMake OUTPUT so ninja re-runs igenerator when pkl files are missing.", + help="Stamp file touched after the pkl DB transaction commits; " + "declared as a CMake OUTPUT so ninja re-runs igenerator when stale.", ) cmdln_arg_parser.add_argument( "-d", @@ -1910,7 +1912,7 @@ def get_submodule_namespace( def generate_swig_input( submodule_name, - pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, @@ -1933,45 +1935,23 @@ def generate_swig_input( swig_input_generator.snakeCaseProcessObjectFunctions ) - # Write index list of generated .pkl files index_file_contents: StringIO = StringIO() all_keys = swig_input_generator.classes.keys() if len(all_keys): for itk_class in all_keys: - # Future problem will be that a few files will be empty - # Can either somehow detect this or accept it - # pickle class here class_name: str = swig_input_generator.classes[itk_class].class_name submodule_name: str = swig_input_generator.classes[itk_class].submodule_name - pickled_filename: str = f"{pkl_dir}/{class_name}.{submodule_name}.pkl" - - # Only write to the pickle file if it does not match what is already saved. - overwrite: bool = False - pickle_exists: bool = exists(pickled_filename) - if pickle_exists: - with open(pickled_filename, "rb") as pickled_file: - existing_itk_class = pickle.load(pickled_file) - overwrite = not ( - existing_itk_class == swig_input_generator.classes[itk_class] - ) - if overwrite or not pickle_exists: - with open(pickled_filename, "wb") as pickled_file: - pickle.dump(swig_input_generator.classes[itk_class], pickled_file) - - index_file_contents.write(pickled_filename + "\n") + key: str = f"{class_name}.{submodule_name}" + pkl_db_rows.append( + (key, pickle.dumps(swig_input_generator.classes[itk_class])) + ) + index_file_contents.write(key + "\n") else: - # The following warning is useful for debugging, and eventually we - # may wish to find a way to remove modules that are not currently part - # of the build. For example, currently all *.wrap files are processed and listed - # as module dependencies. If FFTW is not enabled, that causes empty submodules - # to be created as dependencies unnecessarily. - # Changing that behavior will require structural code changes, or alternate - # mechanisms to be implemented. if glb_options.debug_code: print( f"WARNING: {submodule_name} has no classes identified, but was listed as a dependent submodule." ) - generate_pyi_index_files(submodule_name, index_file_contents, pkl_dir) + generate_pyi_index_files(submodule_name, index_file_contents, options.pkl_dir) def main(): @@ -1980,7 +1960,7 @@ def main(): raise ValueError(f"Required directory missing '{options.pyi_dir}'") if options.pkl_dir == "": - raise ValueError(f"Required directory missing '{options.pkl_dir}'") + raise ValueError("--pkl_dir is required for .index.txt manifests") # Ensure that the requested stub file directory exists if options.pyi_dir != "": @@ -2041,15 +2021,26 @@ def main(): ordered_submodule_list.append(submodule_name) del submodule_names_list + pkl_db_rows: list[tuple[str, bytes]] = [] for submodule_name in ordered_submodule_list: generate_swig_input( submodule_name, - options.pkl_dir, + pkl_db_rows, pygccxml_config, options, snake_case_process_object_functions, ) + if pkl_db_rows: + conn = open_pkl_db(options.pkl_dir) + with conn: + conn.executemany( + "INSERT INTO pkl_data(key, data) VALUES(?,?)" + " ON CONFLICT(key) DO UPDATE SET data=excluded.data", + pkl_db_rows, + ) + conn.close() + snake_case_file = options.snake_case_file if len(snake_case_file) > 1: with open(snake_case_file, "w") as ff: diff --git a/Wrapping/Generators/pkl_db.py b/Wrapping/Generators/pkl_db.py new file mode 100644 index 00000000000..bdc58801ce9 --- /dev/null +++ b/Wrapping/Generators/pkl_db.py @@ -0,0 +1,31 @@ +import sqlite3 +from pathlib import Path + +PKL_DB_SCHEMA_VERSION = 3 + + +def _pkl_db_path(pkl_dir: str) -> Path: + """Return the build-tree-local pkl SQLite DB path. + + The DB is intermediate handoff state keyed by bare class name, so it must + stay local to one build tree; it is never shared via ITK_WRAP_CACHE. + """ + return Path(pkl_dir) / f"itk-pkl-v{PKL_DB_SCHEMA_VERSION}.db" + + +def open_pkl_db(pkl_dir: str) -> sqlite3.Connection: + """Open (creating if absent) the build-tree-local pkl DB.""" + db_path = _pkl_db_path(pkl_dir) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path, timeout=30) + try: + # WAL needs shared memory; on filesystems without it (e.g. NFS) fall + # back to the default rollback journal, serialized by the busy timeout. + conn.execute("PRAGMA journal_mode=WAL") + except sqlite3.OperationalError: + pass + conn.execute( + "CREATE TABLE IF NOT EXISTS pkl_data (key TEXT PRIMARY KEY, data BLOB NOT NULL)" + ) + conn.commit() + return conn diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 45f825ab57e..fd44a660e18 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,13 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") - set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.pkl.stamp") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py - list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.pkl.stamp + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -193,6 +193,7 @@ macro(itk_end_wrap_module) "${ITK_PKL_DIR}" --pkl_stamp "${pkl_stamp_file}" DEPENDS ${IGENERATOR} + ${IGENERATOR_PKL_DB} ${ITK_WRAP_DOC_DOCSTRING_FILES} ${CastXML_OUTPUT_FILES} ${typedef_in_files} From 4adb6db5d1f5e6dc6324774dd66b578a2bbf083d Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 08:17:43 -0500 Subject: [PATCH 3/5] ENH: Defer pygccxml initialization until after submodule list is built pygccxml is now imported lazily via _load_pygccxml() called in main() after the submodule_names_list is populated from the mdx files. The pygccxml_config is constructed immediately after, so the processing loop is unchanged. This separates the submodule-discovery phase (pure file I/O) from the pygccxml/CastXML-dependent phase, enabling a future cache-hit early return before pygccxml is ever loaded. --- .../Generators/SwigInterface/igenerator.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index f0f5e761b08..f05465ed710 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -230,8 +230,16 @@ def argument_parser(): glb_options = argument_parser() -sys.path.insert(1, glb_options.pygccxml_path) -import pygccxml # noqa: E402 +pygccxml = None # populated lazily by _load_pygccxml() + + +def _load_pygccxml(): + """Import pygccxml into the global namespace; loaded lazily, not at import time.""" + global pygccxml + import importlib + + sys.path.insert(1, glb_options.pygccxml_path) + pygccxml = importlib.import_module("pygccxml") # Global debugging variables @@ -1968,18 +1976,6 @@ def main(): if options.pkl_dir != "": Path(options.pkl_dir).mkdir(exist_ok=True, parents=True) - # init the pygccxml stuff - pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) - pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False - pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True - - pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( - xml_generator_path=options.castxml_path, - xml_generator="castxml", - # Use castxml-output=1 to take advantage of the newer XML format - flags=["--castxml-output=1"], - ) - submodule_names_list: list[str] = [] # The first mdx file is the master index file for this module. master_mdx_filename: Path = Path(options.mdx[0]) @@ -2000,6 +1996,18 @@ def main(): if submodule_name not in submodule_names_list: submodule_names_list.append(submodule_name) + _load_pygccxml() + + pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) + pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False + pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True + + pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( + xml_generator_path=options.castxml_path, + xml_generator="castxml", + flags=["--castxml-output=1"], + ) + for submodule_name in submodule_names_list: wrappers_namespace: Any = global_submodule_cache.get_submodule_namespace( submodule_name, options.library_output_dir, pygccxml_config From 920e629e5c776d09920cb141f9d543e55836f50f Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 07:21:26 -0500 Subject: [PATCH 4/5] ENH: Add two-level content-addressed cache for CastXML wrapping On 2-core CI runners, 816 CastXML invocations (~32 min) account for ~10% of a cold Python wrapping build. A warm cache reduces that to ~23 sec, cutting total build time by 32 min on Azure DevOps (19%) and 3h 12m on GHA when combined with a warm ccache (64%). itk-castxml-cache.py wraps the CastXML binary transparently: - L1 key: sha256 of cxx + castxml.inc + flags (no subprocess, ~0.2s) - L2 key: sha256 of castxml -E preprocessed output with line markers stripped, making keys path-independent across build directories Enable via ITK_WRAP_CASTXML_CACHE (default ON). Set ITK_WRAP_CACHE to override the cache root (default ~/.cache/itk-wrap). CI wiring: Cache@2 tasks added to Linux, macOS, and Windows Azure DevOps pipelines; arm.yml updated for GHA. pixi tasks added for configure-python-ci / build-python-ci / test-python-ci. --- .github/workflows/arm.yml | 18 + CMake/itkWrapCastXMLCacheSupport.cmake | 66 ++ Documentation/docs/contributing/index.md | 1 + .../contributing/wrapping_architecture.md | 220 +++++ .../AzurePipelinesLinuxPython.yml | 10 + .../AzurePipelinesMacOSPython.yml | 9 + .../AzurePipelinesWindowsPython.yml | 9 + Wrapping/CMakeLists.txt | 19 + .../Generators/CastXML/itk-castxml-cache.py | 833 ++++++++++++++++++ .../itk_auto_load_submodules.cmake | 26 +- pixi.lock | 50 ++ pyproject.toml | 30 +- 12 files changed, 1288 insertions(+), 3 deletions(-) create mode 100644 CMake/itkWrapCastXMLCacheSupport.cmake create mode 100644 Documentation/docs/contributing/wrapping_architecture.md create mode 100755 Wrapping/Generators/CastXML/itk-castxml-cache.py diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 5fa97d3b5e3..6c0d918d418 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -102,6 +102,7 @@ jobs: echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV" echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV" echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV" + echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV" if [ "$RUNNER_OS" == "Linux" ]; then sudo apt-get update -qq && sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 @@ -118,6 +119,16 @@ jobs: restore-keys: | ccache-v4-${{ runner.os }}-${{ matrix.name }}- + - name: Restore CastXML cache + if: matrix.python-version != '' + id: restore-castxml-cache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + restore-keys: | + castxml-v1-${{ runner.os }}-arm-python- + - name: Restore ExternalData object store id: restore-externaldata uses: actions/cache/restore@v5 @@ -202,6 +213,13 @@ jobs: path: ${{ runner.temp }}/ccache key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }} + - name: Save CastXML cache + if: ${{ !cancelled() && matrix.python-version != '' }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + # ExternalData object store is populated by # .github/workflows/populate-externaldata-cache.yml — a dedicated # workflow whose only job is to prefetch every CID and write the diff --git a/CMake/itkWrapCastXMLCacheSupport.cmake b/CMake/itkWrapCastXMLCacheSupport.cmake new file mode 100644 index 00000000000..57a650acc22 --- /dev/null +++ b/CMake/itkWrapCastXMLCacheSupport.cmake @@ -0,0 +1,66 @@ +# Locate the wrapper script relative to this file so the default also +# resolves when ITK_SOURCE_DIR is not defined in the including scope. +get_filename_component( + _itk_wrap_castxml_cache_src_root + "${CMAKE_CURRENT_LIST_DIR}/.." + ABSOLUTE +) +set( + _ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT + "${_itk_wrap_castxml_cache_src_root}/Wrapping/Generators/CastXML/itk-castxml-cache.py" +) +unset(_itk_wrap_castxml_cache_src_root) + +option( + ITK_WRAP_CASTXML_CACHE + "Use a content-addressed two-level cache for CastXML wrapping steps." + ON +) +mark_as_advanced(ITK_WRAP_CASTXML_CACHE) + +if(ITK_WRAP_CASTXML_CACHE) + set( + ITK_WRAP_CASTXML_CACHE_SCRIPT + "${_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT}" + CACHE FILEPATH + "Path to the CastXML content-addressed cache wrapper script" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_SCRIPT) + + if(NOT EXISTS "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE is ON but the wrapper script was not found:\n" + " ${ITK_WRAP_CASTXML_CACHE_SCRIPT}\n" + "Set ITK_WRAP_CASTXML_CACHE_SCRIPT to the correct path or turn off ITK_WRAP_CASTXML_CACHE." + ) + endif() + + if(NOT Python3_EXECUTABLE) + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE requires Python3_EXECUTABLE to be set." + ) + endif() + + set( + ITK_WRAP_CASTXML_CACHE_MAX_DAYS + "13.9" + CACHE STRING + "Days before a CastXML cache entry is evicted by --evict after each build (default 13.9)" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_MAX_DAYS) + + message(STATUS "CastXML content-addressed cache enabled") + message(STATUS " Script: ${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + STATUS + " Cache root: set ITK_WRAP_CACHE env var at build time (default: ~/.cache/itk-wrap)" + ) + message( + STATUS + " Eviction (after each build): entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then oldest trimmed to ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0)" + ) +endif() + +unset(_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT) diff --git a/Documentation/docs/contributing/index.md b/Documentation/docs/contributing/index.md index 69651949f96..de86176ec99 100644 --- a/Documentation/docs/contributing/index.md +++ b/Documentation/docs/contributing/index.md @@ -461,6 +461,7 @@ CDash Dashboard dashboard.md updating_third_party.md python_packaging.md +wrapping_architecture.md ../README.md ``` diff --git a/Documentation/docs/contributing/wrapping_architecture.md b/Documentation/docs/contributing/wrapping_architecture.md new file mode 100644 index 00000000000..51a426383b5 --- /dev/null +++ b/Documentation/docs/contributing/wrapping_architecture.md @@ -0,0 +1,220 @@ +Python Wrapping Architecture +============================ + +ITK's Python wrapping pipeline converts C++ template declarations into +importable Python extension modules (`.abi3.so`) and type-stub files +(`.pyi`). The pipeline runs in two distinct phases: a **configure phase** +driven by CMake and a **build phase** driven by Ninja. + +## Configure phase: `.wrap` → `castxml_inputs/` + +Each ITK module that supports wrapping contains a `wrapping/` subdirectory +with one `.wrap` file per submodule. A `.wrap` file is a CMake script that +calls macros such as `itk_wrap_class()` and `itk_wrap_template()` to declare +which C++ template instantiations should be exposed to Python. + +CMake processes every `.wrap` file at configure time and writes three +files per submodule into `/Wrapping/castxml_inputs/`: + +| Generated file | How | Contents | +|---|---|---| +| `.cxx` | `configure_file` | `#include` directives + `_wrapping_` namespace with `using` aliases for every requested template instantiation | +| `.castxml.inc` | `file(GENERATE …)` | Compiler `-I` and `-D` flags needed to parse the `.cxx` file | +| `SwigInterface.h.in` | `configure_file` | `#include` list used by SWIG | + +CMake also registers one `add_custom_command` per submodule (for CastXML) +and one per ITK module (for `igenerator.py`). No compilation happens at +configure time; only the input files and build rules are written. + +## Build phase: CastXML → igenerator → SWIG → compile → link + +### Step 1 — CastXML (816 independent jobs) + +Each submodule produces exactly one XML file: + +``` +castxml_inputs/.cxx +castxml_inputs/.castxml.inc ──▶ itk-castxml-cache.py ──▶ castxml_inputs/.xml +Modules/.../include/.h (many) +``` + +`itk-castxml-cache.py` wraps the CastXML binary with a two-level +content-addressed cache (`~/.cache/itk-wrap` or `$ITK_WRAP_CACHE`): + +- **L1** — hash of the `.cxx` file content → L2 key +- **L2** — `castxml -E` (preprocessor only) output hash → cached `output.xml.gz` + +A cache hit avoids running CastXML entirely. All 816 CastXML jobs are +independent and run fully in parallel. No CastXML job reads or modifies +another submodule's `.xml` output. + +### Step 2 — `igenerator.py` (96 per-module jobs) + +Each ITK module (e.g. `ITKImageIntensity`) batches all of its submodules +into a single `igenerator.py` invocation: + +``` +castxml_inputs/itkAbsImageFilter.xml ──┐ +castxml_inputs/itkImage.xml ──┤ +castxml_inputs/itkOffset.xml ──┤ igenerator.py [ITKImageIntensity] +... (all N submodule XMLs) ──┘ --submodule-order "sub1;sub2;...;subN" + │ + ┌──────────────────────────┼─────────────────────────────┐ + │ per submodule (×N) │ │ + ▼ ▼ ▼ + Typedefs/.i itk-pkl/.index.txt itk-pkl-v3.db (SQLite) + Typedefs/.idx (lists DB keys; byproduct) (class metadata; WAL mode) + Typedefs/SwigInterface.h + + per module (×1): + itk-pkl/.stamp + itk/Configuration/_snake_case.py +``` + +`igenerator.py` uses `pygccxml` to parse each `.xml` file and emit SWIG +interface (`.i`) and index (`.idx`) files, class-metadata rows in a +build-local SQLite database consumed later by `pyi_generator.py`, and a +`.index.txt` manifest listing the DB keys for each submodule. + +The SQLite database (`itk-pkl-v3.db`) is written to the `itk-pkl/` directory +inside the build tree. It is intermediate handoff state keyed by bare class +name, so it is always build-tree-local and is never shared via +`ITK_WRAP_CACHE`. + +**Ninja scheduling**: an `igenerator.py` job for module A starts as soon +as all of A's CastXML jobs are complete, even while CastXML is still +running for module B. There is no global barrier between the CastXML and +`igenerator.py` layers. + +### Step 3 — SWIG, compile, link (per submodule) + +``` +Typedefs/.i +Typedefs/SwigInterface.h ──▶ swig ──▶ Modules/.../Python.cpp + Generators/Python/itk/Python.py + +Modules/.../Python.cpp ──▶ ccache + g++ ──▶ .o ──▶ link ──▶ _Python.abi3.so +``` + +### Step 4 — `pyi_generator.py` (one global job) + +After **all** 816 `.index.txt` files exist, `pyi_generator.py` reads every +`.index.txt`, queries the SQLite database for each key, and writes the +type-stub files used by IDEs: + +``` +itk-pkl/.index.txt (×816) ──▶ pyi_generator.py ──▶ _proxies.pyi +itk-pkl-v3.db (SQLite) (queries DB via keys in .index.txt) __init__.pyi +``` + +## Key file reference + +| Path pattern | Written by | Read by | +|---|---|---| +| `Wrapping/castxml_inputs/.cxx` | CMake `configure_file` | CastXML | +| `Wrapping/castxml_inputs/.castxml.inc` | CMake `file(GENERATE)` | `itk-castxml-cache.py` | +| `Wrapping/castxml_inputs/.xml` | `itk-castxml-cache.py` / CastXML | `igenerator.py` | +| `Wrapping/Typedefs/.i` | `igenerator.py` | SWIG | +| `Wrapping/Typedefs/.idx` | `igenerator.py` | SWIG | +| `Wrapping/Generators/Python/itk-pkl/.index.txt` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/itk-pkl-v3.db` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/.stamp` | `igenerator.py` | ninja (tracks DB write completeness) | +| `Wrapping/Generators/Python/itk/_Python.abi3.so` | linker | Python `import itk` | +| `Wrapping/Generators/Python/itk/_proxies.pyi` | `pyi_generator.py` | IDEs | + +## Caches + +### CastXML cache (`itk-castxml-cache.py`) + +| Variable | Default | Purpose | +|---|---|---| +| `ITK_WRAP_CACHE` | `~/.cache/itk-wrap` | Cache root for CastXML `.xml.gz` files | +| `ITK_WRAP_CACHE_VERBOSE` | unset | Set to `1` to log HIT/MISS per file | + +The CastXML cache is content-addressed and generator-version-stamped +(`_KEY_VERSION` in `itk-castxml-cache.py`). It is shared across build +directories; a fresh configure reuses XML from a previous build on the same +machine. + +### pkl SQLite database (`igenerator.py` / `pyi_generator.py`) + +The pkl database (`itk-pkl-v3.db`) lives in the build tree's `itk-pkl/` +directory and is a build artifact, not a user-level cache. Unlike the +content-addressed CastXML cache, it is keyed by bare `class.submodule` +strings, so it is **always build-tree-local and never honors +`ITK_WRAP_CACHE`** — sharing it across build trees would collide same-named +classes from different ITK sources. + +`igenerator.py` writes the rows; `pyi_generator.py` reads them and, when +invoked with `--prune` (ITK's own build only), prunes any row whose key is +absent from the current build's `.index.txt` manifests (exact keyset +cleanup). External wrap projects share the ITK build tree's database but +their manifests cover only their own modules, so they never prune. The +schema version in the filename means a database from an older schema is +ignored rather than migrated. + +### ccache + +CastXML re-runs produce identical `.xml` files (the content is deterministic) +but are slow. The CastXML cache eliminates that cost. For the C++ compilation +steps (Step 3), ccache caches compiled `.o` files keyed on source content. +Both caches are independent and complement each other. + +### Build-phase timing on 2-core CI runners + +On a 2-core CI runner (cold caches throughout): + +| Phase | Approx. time | Notes | +|---|---|---| +| CastXML (816 jobs, 2 cores) | ~32 min | Eliminated on warm-cache runs | +| igenerator + SWIG + C++ compile | ~225 min | Reduced by ccache on subsequent runs | +| Tests | ~44 min | | + +CastXML is ~10 % of the cold-cache total. The C++ compilation phase +dominates; `ccache` is the primary lever there. + +## Ninja dependency graph summary + +``` +.wrap files (configure time, not in graph) + │ cmake configure_file / file(GENERATE) + ▼ +castxml_inputs/.cxx + .castxml.inc + .h headers + │ itk-castxml-cache.py [816 parallel jobs] + ▼ +castxml_inputs/.xml (write-once; never mutated after creation) + │ igenerator.py [96 per-module jobs; starts per-module, not globally gated] + ▼ +Typedefs/.i + .idx + SwigInterface.h +itk-pkl/.index.txt (DB keys) + itk-pkl-v3.db (SQLite) + .stamp + │ swig + ccache compile + link [parallel per submodule] + ▼ +_Python.abi3.so + │ pyi_generator.py [1 global job; needs all .index.txt] + ▼ +_proxies.pyi + __init__.pyi +``` + +## Troubleshooting + +**CastXML cache not being used** +: Set `ITK_WRAP_CACHE_VERBOSE=1` and rebuild one module to confirm HIT or + MISS log lines. Ensure `ITK_WRAP_CASTXML_CACHE=ON` is set in CMake. + A version bump to `_KEY_VERSION` in `itk-castxml-cache.py` forces a cold + cache for all entries. + +**`indexed pkl keys are missing from the pkl database`** +: The `.index.txt` manifests exist (so ninja considers `igenerator.py` + up-to-date) but the pkl database is absent or damaged. `pyi_generator.py` + detects this, deletes the stamp files itself, and fails the build once; + the next build re-runs `igenerator.py` and repopulates the DB. Manual + recovery is only needed if the automatic pass keeps failing: + ```bash + find /Wrapping/Generators/Python/itk-pkl -name "*.index.txt" -o -name "*.stamp" | xargs rm -f + ninja -C + ``` + +**Adding a new wrapped class** +: Add `itk_wrap_class()` / `itk_wrap_template()` calls to the relevant + `.wrap` file. Re-run CMake to regenerate the `.cxx` and `.castxml.inc` + files, then build normally. CMake will automatically include the new + submodule in the `--submodule-order` passed to `igenerator.py`. diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index 8cc9be6c2dc..b4d162752f8 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Linux timeoutInMinutes: 0 @@ -55,6 +56,7 @@ jobs: df -h / displayName: 'Free preinstalled software' + - checkout: self clean: true fetchDepth: 5 @@ -102,6 +104,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "LinuxPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index 898ba9743cf..ee84c981241 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: macOS timeoutInMinutes: 0 @@ -106,6 +107,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "macOSPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index 3e834a7dfda..c021e87db2e 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Windows timeoutInMinutes: 0 @@ -84,6 +85,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "WindowsPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "WindowsPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 13cd681e01b..a7a9defb416 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -152,6 +152,12 @@ endif() include(ConfigureWrapping.cmake) +# The cache support module is not installed and its script lives in the ITK +# source tree, so external wrap projects build without the CastXML cache. +if(ITK_WRAP_PYTHON AND NOT EXTERNAL_WRAP_ITK_PROJECT) + include(itkWrapCastXMLCacheSupport) +endif() + ############################################################################### # Configure specific wrapper modules ############################################################################### @@ -272,6 +278,19 @@ if(ITK_WRAP_PYTHON) endif() endforeach() + if(ITK_WRAP_CASTXML_CACHE) + add_custom_command( + TARGET itk-stub-files + POST_BUILD + COMMAND + ${Python3_EXECUTABLE} "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" --evict + "${ITK_WRAP_CASTXML_CACHE_MAX_DAYS}" + COMMENT + "CastXML cache: evicting entries older than ${ITK_WRAP_CASTXML_CACHE_MAX_DAYS} days, then trimming to the size cap" + VERBATIM + ) + endif() + unset(ITK_STUB_DIR CACHE) unset(ITK_PKL_DIR CACHE) endif() diff --git a/Wrapping/Generators/CastXML/itk-castxml-cache.py b/Wrapping/Generators/CastXML/itk-castxml-cache.py new file mode 100755 index 00000000000..d9aa8961a6c --- /dev/null +++ b/Wrapping/Generators/CastXML/itk-castxml-cache.py @@ -0,0 +1,833 @@ +#!/usr/bin/env python3 +""" +Two-level content-addressed cache wrapper for ITK's CastXML wrapping step. + +Invocation (replaces ccache + castxml in the cmake COMMAND): + python3 itk-castxml-cache.py /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --no-cache /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --evict DAYS [--max-size GB] # e.g. --evict 13.9 --max-size 2.0 + +Cache key hierarchy: + L1 (fast, no subprocess): sha256 of key version + castxml content-hash + + inc file + cxx file + flags. + The castxml binary is fingerprinted by SHA-256 of its content (not mtime), so + `ninja -t clean` re-links the binary without changing the L1 key. Each L1 + entry also records the `-E` header set with content hashes plus a listing + fingerprint of every -I include dir; an L1 hit is honored only when both + still match, so a changed header, or a new header appearing in any include + dir (shadowing), correctly misses instead of restoring stale XML. A new + header inside an existing subdirectory of an include dir is not detected + (same residual gap as compiler depfiles). + L2 (content-addressed): sha256 of key version + castxml content-hash + + normalised `castxml -E` preprocessed output. Preprocessor line markers + (# N "path") are stripped before hashing, making L2 keys path-independent + across build directories; the castxml content-hash ensures a castxml + upgrade never restores XML produced by the previous binary. + +Lookup: + L1 HIT (recorded header hashes still match) → restore L2 → DONE (no subprocess) + L1 miss / header changed → run castxml -E → compute L2 key + L2 HIT → restore; refresh L1 map + L2 miss → run full castxml; store; write L1 map + +Storage formats (ITK_WRAP_CACHE_FORMAT): + gzip (default): output.xml.gz, ~8x smaller than raw XML. Decompressed on + restore. 253 MB for a full 807-module ITK build; each build directory + gets its own decompressed copy (copy is nearly as fast as a hardlink). + uncompressed: output.xml, plain copy on restore. ~2.2 G per full build. + Set ITK_WRAP_CACHE_FORMAT=uncompressed to opt in. + +Multi-path cascade (ITK_WRAP_CACHE, colon-separated): + Reads search all paths in order, returning the first hit. Writes go to the + first path that accepts them (atomic rename succeeds). A read-only shared + NFS cache can be listed after the user's writable local SSD cache, e.g.: + export ITK_WRAP_CACHE=/local/ssd/cache:/nfs/lab/shared-cache + Students benefit from the lab cache for L2 hits (saving the full castxml + run) while writing L1 maps only to their own writable local cache. + +Eviction: run once after a full wrapping build via --evict DAYS. + Skipped immediately when nothing was stored since the previous eviction, so + null builds pay two stat calls instead of a cache-tree walk. Time pass + removes entries older than DAYS (the CMake ITK_WRAP_CASTXML_CACHE_MAX_DAYS + variable supplies DAYS in ITK builds); size pass then removes oldest entries + until total is under ITK_WRAP_CACHE_MAX_SIZE GB (default 2.0). Staging + leftovers and entries lacking their _ok marker are removed after a one-hour + grace period. --evict touches only the first writable cache in the list. + +Environment: + ITK_WRAP_CACHE colon-separated cache roots (default: ~/.cache/itk-wrap) + ITK_WRAP_CACHE_FORMAT storage format: gzip (default) or uncompressed + ITK_WRAP_CACHE_VERBOSE set to 1 for hit/miss logging to stderr + ITK_WRAP_CACHE_BYPASS set to 1 to bypass all caching (same as --no-cache) + ITK_WRAP_CACHE_MAX_SIZE max cache size in GB enforced by --evict (default: 2.0) +""" + +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import zlib + +# Bump when the key algorithm changes; old entries orphan and age out. +# Mixed into both the L1 and L2 keys so a bump invalidates every lookup path. +_KEY_VERSION = b"v2\x00" + +# Strip "# N " preprocessor line markers so the L2 hash is path-independent. +_LINE_MARKER_RE = re.compile(rb"^# \d+ ", re.MULTILINE) + + +def _strip_line_markers(data: bytes) -> bytes: + return b"\n".join( + line for line in data.splitlines() if not _LINE_MARKER_RE.match(line) + ) + + +# Captures the file path from a preprocessor line marker: # N "path" flags +_DEP_MARKER_RE = re.compile(rb'^# \d+ "([^"]*)"', re.MULTILINE) + + +def _extract_dep_paths(preprocessed: bytes) -> list: + """Return sorted real file paths named in `castxml -E` line markers.""" + paths = set() + for m in _DEP_MARKER_RE.finditer(preprocessed): + p = m.group(1).decode("utf-8", "surrogateescape") + # Line markers escape backslashes on Windows (C:\\path\\file.h) + p = p.replace("\\\\", "\\") + if p and not p.startswith("<"): # skip , + paths.add(p) + return sorted(paths) + + +def _hash_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + return h.hexdigest() + + +def _deps_manifest(paths: list) -> list: + """Return [[path, sha256], ...] for the preprocessed dependency set.""" + manifest = [] + for p in paths: + try: + manifest.append([p, _hash_file(p)]) + except OSError: + manifest.append([p, ""]) + return manifest + + +def _deps_unchanged(deps: list) -> bool: + """True when every recorded dependency still hashes to its stored value.""" + for dep in deps: + if not (isinstance(dep, list) and len(dep) == 2): + return False # malformed manifest entry: force a miss + path, expected = dep + try: + actual = _hash_file(path) + except OSError: + actual = "" # matches _deps_manifest's record for unreadable paths + if actual != expected: + return False + return True + + +def _incdirs_from_inc_file(inc_file: str) -> list: + """Return the ordered, deduplicated -I directories from an @inc response file.""" + dirs = [] + seen = set() + try: + with open(inc_file, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + token = line.strip().strip('"') + if token.startswith("-I"): + d = token[2:].strip() + if d and d not in seen: + seen.add(d) + dirs.append(d) + except OSError: + pass + return dirs + + +def _incdirs_fingerprint(inc_file: str) -> str: + """Hash of the top-level entry names of every -I dir. + + A header appearing in (or vanishing from) any include dir changes the + fingerprint, so a newly-added shadowing header forces an L1 miss and the + `castxml -E` pass re-resolves includes against the real search path. + """ + h = hashlib.sha256(_KEY_VERSION) + for d in _incdirs_from_inc_file(inc_file): + h.update(d.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + try: + names = sorted(os.listdir(d)) + except OSError: + names = [] + for name in names: + h.update(name.encode("utf-8", "surrogateescape")) + h.update(b"\x00") + h.update(b"\x01") + return h.hexdigest() + + +def _cache_roots(): + """Return ordered list of cache root directories from ITK_WRAP_CACHE. + + The env var is a colon-separated list (like PATH). Reads search all + roots in order; writes go to the first root that accepts them. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if not raw: + return [os.path.join(os.path.expanduser("~"), ".cache", "itk-wrap")] + sep = ";" if sys.platform == "win32" else ":" + return [p for p in raw.split(sep) if p] + + +def _verbose(): + return os.environ.get("ITK_WRAP_CACHE_VERBOSE", "") == "1" + + +def _log(msg): + if _verbose(): + print(f"itk-castxml-cache: {msg}", file=sys.stderr) + + +def _use_uncompressed(): + """Return True when uncompressed storage is explicitly requested via ITK_WRAP_CACHE_FORMAT.""" + return os.environ.get("ITK_WRAP_CACHE_FORMAT", "").lower() in ( + "uncompressed", + "raw", + "plain", + ) + + +def _max_cache_gb(): + """Parse ITK_WRAP_CACHE_MAX_SIZE (default 2.0) into GB as a float.""" + try: + return float(os.environ.get("ITK_WRAP_CACHE_MAX_SIZE", "2.0")) + except ValueError: + return 2.0 + + +def _remove_if_stale(path, grace_cutoff): + """Remove a dead dir (crashed staging / missing _ok) past the grace period.""" + try: + if os.stat(path).st_mtime < grace_cutoff: + shutil.rmtree(path, ignore_errors=True) + return 1 + except OSError: + pass + return 0 + + +def _evict(cache_root, max_size=2.0, days=13.9): + """Evict cache entries; run once after a full wrapping build via --evict. + + Exits immediately when nothing was stored since the previous eviction. + L2: remove entries whose _ok mtime is older than `days`, then remove the + oldest by mtime until the total is under `max_size` GB. + L1: remove mapping entries whose mtime is older than `days`. + Staging leftovers and entries without _ok are removed after a one-hour + grace period so crashed stores cannot grow the cache unbounded. + """ + l2_root = os.path.join(cache_root, "l2") + if not os.path.isdir(l2_root): + return + + last_store = os.path.join(cache_root, "_last_store") + last_evict = os.path.join(cache_root, "_last_evict") + try: + evict_mtime = os.stat(last_evict).st_mtime + except OSError: + evict_mtime = None + if evict_mtime is not None: + try: + store_mtime = os.stat(last_store).st_mtime + except OSError: + store_mtime = 0.0 + if evict_mtime >= store_mtime: + _log("evict: no stores since last eviction — skipping") + return + + max_bytes = int(max_size * (1 << 30)) + cutoff = time.time() - days * 86400 + grace_cutoff = time.time() - 3600 + kept = [] + total = 0 + removed = 0 + + for shard in os.listdir(l2_root): + shard_dir = os.path.join(l2_root, shard) + if not os.path.isdir(shard_dir): + continue + for key in os.listdir(shard_dir): + entry = os.path.join(shard_dir, key) + if ".tmp" in key or key.endswith(".old"): + removed += _remove_if_stale(entry, grace_cutoff) + continue + ok = os.path.join(entry, "_ok") + try: + mtime = os.stat(ok).st_mtime + except OSError: + removed += _remove_if_stale(entry, grace_cutoff) + continue + try: + entry_bytes = sum( + os.path.getsize(os.path.join(entry, fn)) for fn in os.listdir(entry) + ) + if mtime < cutoff: + shutil.rmtree(entry) + removed += 1 + else: + kept.append((mtime, entry_bytes, entry)) + total += entry_bytes + except OSError: + pass + + if total > max_bytes: + kept.sort(key=lambda x: x[0]) # oldest first + for _mtime, entry_bytes, entry in kept: + if total <= max_bytes: + break + try: + shutil.rmtree(entry) + total -= entry_bytes + removed += 1 + except OSError: + pass + + l1_root = os.path.join(cache_root, "l1") + if os.path.isdir(l1_root): + for shard in os.listdir(l1_root): + shard_dir = os.path.join(l1_root, shard) + if not os.path.isdir(shard_dir): + continue + for key_name in os.listdir(shard_dir): + key_dir = os.path.join(shard_dir, key_name) + l1f = os.path.join(key_dir, "l2_key") + try: + if os.stat(l1f).st_mtime < cutoff: + shutil.rmtree(key_dir) + removed += 1 + except OSError: + removed += _remove_if_stale(key_dir, grace_cutoff) + + try: + with open(last_evict, "w"): + pass + except OSError: + pass + + _log( + f"evict: removed {removed} entries" + f" (age>{days}d or size>{max_size:.1f}GB);" + f" {total / (1 << 30):.2f} GB remaining" + ) + + +def _bypass_mode(): + """Return True when caching should be skipped entirely. + + Controlled by --no-cache as the first positional arg (before castxml binary) + or by ITK_WRAP_CACHE_BYPASS=1 in the environment. Use for single-use builds + where the castxml -E overhead would cost more than the cache saves. + """ + return os.environ.get("ITK_WRAP_CACHE_BYPASS", "") == "1" + + +def _parse_args(argv): + """ + Parse a castxml command line into structured components. + + Strips a leading --no-cache flag (before the castxml binary) when present. + Returns (castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache) + where passthrough_flags preserves the original ordering for subprocess use. + """ + no_cache = False + if argv and argv[0] == "--no-cache": + no_cache = True + argv = argv[1:] + + if not argv: + return None, None, None, None, [], no_cache + + castxml_bin = argv[0] + output_xml = None + inc_file = None + cxx_file = None + passthrough_flags = [] + + i = 1 + while i < len(argv): + arg = argv[i] + if arg == "-o" and i + 1 < len(argv): + output_xml = argv[i + 1] + # Include in passthrough so castxml writes its output normally + passthrough_flags.extend([arg, argv[i + 1]]) + i += 2 + elif arg.startswith("@"): + inc_file = arg[1:] + passthrough_flags.append(arg) + i += 1 + elif arg.endswith(".cxx"): + cxx_file = arg + passthrough_flags.append(arg) + i += 1 + else: + passthrough_flags.append(arg) + i += 1 + + return castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache + + +def _castxml_content_hash(castxml_bin, primary_root): + """Return a stable SHA-256 of the castxml binary, cached on disk. + + Sidecar file stores "size mtime_ns sha256" so re-hashing only happens when + size or mtime changes. After `ninja -t clean`, castxml is re-linked with + the same content → same hash → stable L1 key → L1 hits on warm rebuilds. + Sidecar lives in the first (writable) cache root. + """ + try: + st = os.stat(castxml_bin) + except OSError: + return "missing" + + # One sidecar per binary path (path key avoids slashes in filename) + path_key = hashlib.sha256(castxml_bin.encode()).hexdigest()[:16] + sidecar = os.path.join(primary_root, "_binhash", path_key) + + try: + with open(sidecar) as f: + parts = f.read().split() + if ( + len(parts) == 3 + and int(parts[0]) == st.st_size + and int(parts[1]) == st.st_mtime_ns + ): + return parts[2] + except (OSError, ValueError): + pass + + # Sidecar miss or stale — hash the binary (one-time cost per unique binary) + h = hashlib.sha256() + try: + with open(castxml_bin, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + except OSError: + return "unreadable" + + content_hash = h.hexdigest() + try: + os.makedirs(os.path.dirname(sidecar), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(sidecar), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(f"{st.st_size} {st.st_mtime_ns} {content_hash}\n") + os.replace(tmp, sidecar) + except OSError: + os.unlink(tmp) + except OSError: + pass + return content_hash + + +def _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags): + """Compute L1 cache key from direct inputs only (~0.2s, no subprocess).""" + h = hashlib.sha256(_KEY_VERSION) + + # Stable content fingerprint — survives re-link with unchanged binary. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + + # Response file: include dirs + defines passed via @file + if inc_file: + try: + with open(inc_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00inc_miss\x00") + h.update(b"\x00") + + if cxx_file: + try: + with open(cxx_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00cxx_miss\x00") + h.update(b"\x00") + + # Remaining flags (--castxml-cc-gnu, compiler path, std flags, etc.) + # Skip -o and the output xml path — irrelevant to content. + skip_next = False + for flag in passthrough_flags: + if skip_next: + skip_next = False + continue + if flag == "-o": + skip_next = True + continue + if flag.endswith(".xml"): + continue + h.update(flag.encode()) + h.update(b"\x00") + + return h.hexdigest() + + +def _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_output): + """ + Build a `castxml -E` command that preprocesses the same inputs. + + Strips --castxml-output, --castxml-start (XML-generation flags), + replaces -o with the temp preprocess output path, appends -E. + """ + cmd = [castxml_bin] + skip_next = False + for arg in passthrough_flags: + if skip_next: + skip_next = False + continue + if arg.startswith("--castxml-output"): + continue + if arg.startswith("--castxml-start"): + if "=" not in arg: + skip_next = True + continue + if arg == "-o": + skip_next = True # drop -o and its value (xml output path) + continue + if arg.endswith(".xml"): + continue + cmd.append(arg) + cmd.extend(["-E", "-o", pre_output]) + return cmd + + +def _compute_l2_key(castxml_bin, passthrough_flags, bin_hash): + """Run castxml -E; return (l2_key, deps_manifest) or (None, None) on failure.""" + pre_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".i", delete=False) as tmp: + pre_path = tmp.name + cmd = _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_path) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + _log(f"castxml -E failed (exit {result.returncode})") + return None, None + with open(pre_path, "rb") as f: + pre = f.read() + h = hashlib.sha256(_KEY_VERSION) + # Same preprocessed text from a different castxml must never collide. + h.update(f"castxml\x00{bin_hash}\x00".encode()) + h.update(_strip_line_markers(pre)) + return h.hexdigest(), _deps_manifest(_extract_dep_paths(pre)) + except OSError: + return None, None + finally: + if pre_path is not None: + try: + os.unlink(pre_path) + except OSError: + pass + + +def _l1_file(cache_root, l1_key): + return os.path.join(cache_root, "l1", l1_key[:2], l1_key, "l2_key") + + +def _l2_dir(cache_root, l2_key): + return os.path.join(cache_root, "l2", l2_key[:2], l2_key) + + +def _restore_xml(cache_root, l2_key, output_xml): + """Restore cached XML to output_xml from one cache root. Returns True on success.""" + entry = _l2_dir(cache_root, l2_key) + ok_file = os.path.join(entry, "_ok") + xml_plain = os.path.join(entry, "output.xml") + xml_gz = os.path.join(entry, "output.xml.gz") + + if not os.path.isfile(ok_file): + return False + + try: + os.makedirs(os.path.dirname(os.path.abspath(output_xml)), exist_ok=True) + if os.path.isfile(xml_plain): + # copyfile (not copy2): a fresh mtime keeps ninja's edge up to date + shutil.copyfile(xml_plain, output_xml) + elif os.path.isfile(xml_gz): + with gzip.open(xml_gz, "rb") as src, open(output_xml, "wb") as dst: + shutil.copyfileobj(src, dst) + else: + return False + except (OSError, zlib.error, EOFError): + return False # truncated/corrupt cache entry: miss, fall through to -E + + try: + if os.path.getsize(output_xml) == 0: + return False # empty decompression: treat as corrupt + except OSError: + return False + + try: + os.utime(ok_file, None) + except OSError: + pass + + return True + + +def _restore_from_caches(roots, l2_key, output_xml): + """Search all cache roots for l2_key, restore on first hit.""" + for root in roots: + if _restore_xml(root, l2_key, output_xml): + return root # return the root that had the hit + return None + + +def _write_l1_mapping(l1f, l2_key, deps, incfp): + """Atomically write the L1 manifest (L2 key, dependency hashes, incdir fingerprint).""" + os.makedirs(os.path.dirname(l1f), exist_ok=True) + # Unique staging name + os.replace: concurrency-safe and refreshable on Windows + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(l1f), suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + json.dump({"l2_key": l2_key, "deps": deps, "incfp": incfp}, f) + os.replace(tmp, l1f) + except OSError: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _store(roots, l1_key, l2_key, output_xml, deps, incfp): + """Store output_xml to L2 cache and write L1→L2 mapping atomically. + + Tries each root in order and writes to the first that accepts an atomic + rename. Read-only roots (e.g. a shared NFS lab cache) are silently + skipped. + """ + uncompressed = _use_uncompressed() + + # L2 entry — write to first writable root + write_root = None + for root in roots: + entry = _l2_dir(root, l2_key) + tmp = None + try: + if not os.path.isfile(output_xml): + continue + os.makedirs(os.path.dirname(entry), exist_ok=True) + # Unique staging dir: concurrent stores of the same key cannot + # delete or interleave into each other's staging area. + tmp = tempfile.mkdtemp(dir=os.path.dirname(entry), prefix=l2_key + ".tmp") + # mkdtemp creates 0700; restore umask-derived perms for shared caches + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp, 0o777 & ~umask) + + if uncompressed: + shutil.copy2(output_xml, os.path.join(tmp, "output.xml")) + else: + with ( + open(output_xml, "rb") as src, + gzip.open( + os.path.join(tmp, "output.xml.gz"), "wb", compresslevel=6 + ) as dst, + ): + shutil.copyfileobj(src, dst) + + with open(os.path.join(tmp, "_meta.json"), "w") as f: + json.dump( + { + "l1_key": l1_key, + "l2_key": l2_key, + "format": "uncompressed" if uncompressed else "gzip", + }, + f, + ) + open(os.path.join(tmp, "_ok"), "w").close() # noqa: WPS515 + + old = entry + ".old" + shutil.rmtree( + old, ignore_errors=True + ) # clear stale .old from a prior crash + if os.path.exists(entry): + os.rename(entry, old) + try: + os.rename(tmp, entry) + except OSError: + if os.path.exists(old): + os.rename(old, entry) + raise + shutil.rmtree(old, ignore_errors=True) + write_root = root + break + except OSError as exc: + _log(f"L2 store failed for {root}: {exc}") + if tmp is not None: + shutil.rmtree(tmp, ignore_errors=True) + + if write_root is None: + _log("L2 store failed in all cache roots — no entry written") + return + + _touch_store_marker(write_root) + + # L1→L2 mapping — write to same root that accepted the L2 entry + try: + _write_l1_mapping(_l1_file(write_root, l1_key), l2_key, deps, incfp) + except OSError as exc: + _log(f"L1 map store failed: {exc}") + + +def _touch_store_marker(root): + """Record that this root received a write, so --evict knows to run.""" + try: + with open(os.path.join(root, "_last_store"), "w"): + pass + except OSError: + pass + + +def _store_l1_mapping(roots, l1_key, l2_key, deps, incfp): + """Write an L1→L2 mapping to the first writable root (no L2 write).""" + for root in roots: + try: + _write_l1_mapping(_l1_file(root, l1_key), l2_key, deps, incfp) + _touch_store_marker(root) + return + except OSError: + continue + + +def _run_castxml(castxml_bin, passthrough_flags): + result = subprocess.run([castxml_bin] + passthrough_flags) + return result.returncode + + +def main(): + argv = sys.argv[1:] + if not argv: + print(__doc__, file=sys.stderr) + return 1 + + # Stand-alone eviction subcommand: python3 itk-castxml-cache.py --evict DAYS + if argv[0] == "--evict": + import argparse + + p = argparse.ArgumentParser(prog="itk-castxml-cache.py --evict") + p.add_argument( + "days", + type=float, + help="Remove entries not used in this many days (e.g. 13.9)", + ) + p.add_argument( + "--max-size", + type=float, + default=None, + help="Max cache size in GB after time eviction (default: ITK_WRAP_CACHE_MAX_SIZE or 2.0)", + ) + p.add_argument( + "--cache-dir", default=None, help="Cache root (overrides ITK_WRAP_CACHE)" + ) + args = p.parse_args(argv[1:]) + max_size = args.max_size if args.max_size is not None else _max_cache_gb() + roots = [args.cache_dir] if args.cache_dir else _cache_roots() + _evict(roots[0], max_size=max_size, days=args.days) + return 0 + + castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache = ( + _parse_args(argv) + ) + + if castxml_bin is None: + print("itk-castxml-cache: no castxml binary specified", file=sys.stderr) + return 1 + if no_cache or _bypass_mode() or not output_xml or not cxx_file: + return _run_castxml(castxml_bin, passthrough_flags) + + roots = _cache_roots() + # primary_root is used for binary sidecar storage; writes go to first writable + primary_root = roots[0] + + bin_hash = _castxml_content_hash(castxml_bin, primary_root) + incfp = _incdirs_fingerprint(inc_file) if inc_file else "" + + # ── L1 check (fast, no subprocess) ────────────────────────────────────── + l1_key = _l1_key(bin_hash, inc_file, cxx_file, passthrough_flags) + + stored = None + stored_l1f = None + for root in roots: + l1f = _l1_file(root, l1_key) + if os.path.isfile(l1f): + try: + with open(l1f) as f: + entry = json.load(f) + except (OSError, ValueError): + continue + if isinstance(entry, dict) and entry.get("l2_key"): + stored = entry + stored_l1f = l1f + break + + # ── L1 HIT: skip castxml -E only when recorded headers are unchanged and + # no header has appeared in (or left) any include dir since the store ── + if ( + stored is not None + and stored.get("deps") + and stored.get("incfp") == incfp + and _deps_unchanged(stored["deps"]) + ): + hit_root = _restore_from_caches(roots, stored["l2_key"], output_xml) + if hit_root is not None: + try: + os.utime(stored_l1f, None) # mark used so age eviction keeps it + except OSError: + pass + _log(f"HIT {os.path.basename(cxx_file)} (l1→l2={stored['l2_key'][:8]})") + return 0 + # L2 entry missing or corrupt despite L1 hit — fall through to -E check. + _log(f"L2 entry missing for {cxx_file}, re-running castxml -E") + + # ── castxml -E to compute actual L2 key (L1 miss or header changed) ───── + actual_l2_key, deps = _compute_l2_key(castxml_bin, passthrough_flags, bin_hash) + + if actual_l2_key is None: + _log(f"preprocess failed for {cxx_file} — passing through") + return _run_castxml(castxml_bin, passthrough_flags) + + # ── L2 store lookup (handles cross-dir hits: L1 miss, L2 populated) ───── + hit_root = _restore_from_caches(roots, actual_l2_key, output_xml) + if hit_root is not None: + _log(f"HIT {os.path.basename(cxx_file)} (l2={actual_l2_key[:8]})") + # Populate L1 map so the next rebuild skips castxml -E + _store_l1_mapping(roots, l1_key, actual_l2_key, deps, incfp) + return 0 + + # ── Full castxml run ───────────────────────────────────────────────────── + _log(f"MISS {os.path.basename(cxx_file)}") + try: + os.unlink(output_xml) + except OSError: + pass + rc = _run_castxml(castxml_bin, passthrough_flags) + if rc == 0: + _store(roots, l1_key, actual_l2_key, output_xml, deps, incfp) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/macro_files/itk_auto_load_submodules.cmake b/Wrapping/macro_files/itk_auto_load_submodules.cmake index 4ca246ebc1e..1a0edec86ba 100644 --- a/Wrapping/macro_files/itk_auto_load_submodules.cmake +++ b/Wrapping/macro_files/itk_auto_load_submodules.cmake @@ -199,12 +199,33 @@ function(generate_castxml_commandline_flags) endforeach() # ===== Run the castxml command + if( + ITK_WRAP_CASTXML_CACHE + AND + Python3_EXECUTABLE + AND + ITK_WRAP_CASTXML_CACHE_SCRIPT + ) + set( + _castxml_cmd + ${Python3_EXECUTABLE} + "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" + ${CASTXML_EXECUTABLE} + ) + list(APPEND _castxml_depends "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + else() + set( + _castxml_cmd + ${_ccache_cmd} + ${CASTXML_EXECUTABLE} + ) + endif() add_custom_command( OUTPUT ${xml_file} COMMAND - ${_build_env} ${_ccache_cmd} ${CASTXML_EXECUTABLE} -o ${xml_file} - --castxml-output=1 ${_target} --castxml-start _wrapping_ ${_castxml_cc} -w + ${_build_env} ${_castxml_cmd} -o ${xml_file} --castxml-output=1 ${_target} + --castxml-start _wrapping_ ${_castxml_cc} -w -c # needed for ccache to think we are not calling for link @${castxml_inc_file} ${cxx_file} VERBATIM @@ -214,6 +235,7 @@ function(generate_castxml_commandline_flags) ${castxml_inc_file} ${_hdrs} ) + unset(_castxml_cmd) unset(cxx_file) unset(castxml_inc_file) unset(_build_env) diff --git a/pixi.lock b/pixi.lock index d476f4e4cce..0bea1db9e08 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1592,6 +1592,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.11.0-h4d9bdce_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/castxml-0.7.0-hde8d07d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ccache-4.13.6-hedf47ba_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.1.2-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-14.3.0-hb991d5c_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.11.0-hfcd1e18_0.conda @@ -1618,6 +1619,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-hcd61629_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.3.0-h5888daf_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-38_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.8-hf7376ad_1.conda @@ -1645,6 +1647,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/rhash-1.4.6-hb9d3cd8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/swig-4.4.1-h7a96c5f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.3-hb47aa4a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1671,6 +1674,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-compiler-1.11.0-hdceaead_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/castxml-0.7.0-ha3e84ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.13.6-h185addb_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.1.2-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/conda-gcc-specs-14.3.0-h92dcf8a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cxx-compiler-1.11.0-h7b35c40_0.conda @@ -1697,6 +1701,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-15.2.0-he9431aa_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-15.2.0-h87db57e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.3.0-h5ad3122_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-38_h88aeb00_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libllvm20-20.1.8-hfd2ba90_1.conda @@ -1724,6 +1729,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rhash-1.4.6-h86ecc28_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/swig-4.4.1-h512d76c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xxhash-0.8.3-hd794028_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1759,6 +1765,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-compiler-1.11.0-h7a00415_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/castxml-0.7.0-hb171174_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ccache-4.13.6-h894318c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools_osx-64-1024.3-llvm19_1_h3b512aa_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/clang-19-19.1.7-default_hc369343_5.conda @@ -1787,6 +1794,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran-15.2.0-h306097a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran5-15.2.0-h336fb69_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libhiredis-1.3.0-h240833e_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/liblapack-3.9.0-38_h859234e_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libllvm19-19.1.7-h56e7563_2.conda @@ -1816,6 +1824,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/swig-4.4.1-hdac4ec2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tapi-1300.6.5-h390ca13_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xxhash-0.8.3-h13e91ac_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h8210216_2.conda osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda @@ -1835,6 +1844,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-compiler-1.11.0-h61f9b84_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/castxml-0.7.0-hfc9ce51_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ccache-4.13.6-h414bf82_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools_osx-arm64-1024.3-llvm19_1_h8c76c84_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clang-19-19.1.7-default_h73dfc95_5.conda @@ -1864,6 +1874,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-hfcf01ff_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-h742603c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhiredis-1.3.0-h286801f_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.9.0-38_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libllvm19-19.1.7-h8e0c9ce_2.conda @@ -1893,6 +1904,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/swig-4.4.1-h4366dc5_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tapi-1300.6.5-h03f4b80_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xxhash-0.8.3-haa4e116_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-h4c7d964_0.conda @@ -1911,6 +1923,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/win-64/castxml-0.7.0-ha22e26b_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ccache-4.13.6-h7fd822b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.1.2-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cxx-compiler-1.11.0-h1c1089f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda @@ -1922,6 +1935,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libgcc-15.2.0-h1383e82_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgfortran5-15.2.0-hf2bee02_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgomp-15.2.0-h1383e82_7.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libhiredis-1.3.0-he0c23c2_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.9.0-38_hf9ab0e9_mkl.conda @@ -1950,6 +1964,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vs2022_win-64-19.44.35207-ha74f236_32.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/xxhash-0.8.3-hbba6f48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-hbeecb71_2.conda packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2211,6 +2226,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 855698 timestamp: 1777926446042 - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py313hf46b229_1.conda @@ -3206,6 +3222,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140759 timestamp: 1748219397797 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda @@ -4283,6 +4302,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108219 timestamp: 1746457673761 - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda @@ -4560,6 +4582,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 843745 timestamp: 1777926452238 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py313h897158f_1.conda @@ -5499,6 +5522,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140290 timestamp: 1748220539026 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda @@ -6499,6 +6525,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105762 timestamp: 1746457675564 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda @@ -7376,6 +7405,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 657026 timestamp: 1777926755291 - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda @@ -8299,6 +8329,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 59830 timestamp: 1748219625377 - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda @@ -9048,6 +9081,9 @@ packages: - __osx >=10.13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108449 timestamp: 1746457796808 - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda @@ -9210,6 +9246,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 601285 timestamp: 1777926636412 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda @@ -10143,6 +10180,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 56746 timestamp: 1748219528586 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda @@ -10893,6 +10933,9 @@ packages: - __osx >=11.0 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 98913 timestamp: 1746457827085 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda @@ -11048,6 +11091,7 @@ packages: - xxhash >=0.8.3,<0.8.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 692199 timestamp: 1777926529520 - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda @@ -11530,6 +11574,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 64205 timestamp: 1748219812303 - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda @@ -12394,6 +12441,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105768 timestamp: 1746458183583 - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda diff --git a/pyproject.toml b/pyproject.toml index 2fe4d3b2644..70b13a55ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ libopenblas = ">=0.3.30,<0.4" libgfortran5 = ">=15.2.0,<16" swig = ">=4.4.1" castxml = ">=0.7.0,<0.8" +ccache = ">=4.13.6,<5" [tool.pixi.feature.cxx.tasks.configure] cmd = '''cmake \ @@ -236,10 +237,37 @@ cmd = '''cmake \ -DITK_USE_SYSTEM_SWIG:BOOL=ON \ -DITK_USE_SYSTEM_CASTXML:BOOL=ON \ -DCMAKE_BUILD_TYPE:STRING=Release \ - -DBUILD_TESTING:BOOL=ON''' + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' description = "Configure ITK Python" outputs = ["build-python/CMakeFiles/"] +[tool.pixi.feature.python.tasks.configure-python-ci] +cmd = '''cmake \ + -Bbuild-python \ + -S. \ + -GNinja \ + -DITK_WRAP_PYTHON:BOOL=ON \ + -DITK_WRAP_CASTXML_CACHE:BOOL=ON \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' +description = "Configure ITK Python for CI (castxml cache enabled; set ITK_WRAP_CACHE before building)" +outputs = ["build-python/CMakeFiles/"] + +[tool.pixi.feature.python.tasks.build-python-ci] +cmd = "cmake --build build-python" +description = "Build ITK Python for CI" +outputs = ["build-python/lib/**"] +depends-on = ["configure-python-ci"] + +[tool.pixi.feature.python.tasks.test-python-ci] +cmd = "ctest -j3 --test-dir build-python --output-on-failure" +description = "Test ITK Python for CI" +depends-on = ["build-python-ci"] + [tool.pixi.feature.python.tasks.build-python] cmd = "cmake --build build-python" description = "Build ITK Python" From 196f1396706c1c5d59cd8bcc64ad30aad60ac044 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 30 Jun 2026 07:41:37 -0500 Subject: [PATCH 5/5] STYLE: Explicitly set ITK_WRAP_CASTXML_CACHE=ON in all Python CI configs Relies on CMake default (ON) elsewhere; explicit here avoids silent no-op if the option is ever overridden upstream in the dashboard script. --- .github/workflows/arm.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml | 1 + Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 6c0d918d418..46a28b95647 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -80,6 +80,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_USE_CLANG_FORMAT:BOOL=OFF ITK_COMPUTER_MEMORY_SIZE:STRING=11 ctest-options: "-E itkPyBufferMemoryLeakTest" diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index b4d162752f8..ed2e182fe58 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -133,6 +133,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=OFF BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=4.5 diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index ee84c981241..5429d5f51bd 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -136,6 +136,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON CMAKE_C_COMPILER_LAUNCHER:STRING=ccache CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache ITK_COMPUTER_MEMORY_SIZE:STRING=11 diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index c021e87db2e..822ccb75d93 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -113,6 +113,7 @@ jobs: BUILD_SHARED_LIBS:BOOL=ON BUILD_EXAMPLES:BOOL=OFF ITK_WRAP_PYTHON:BOOL=ON + ITK_WRAP_CASTXML_CACHE:BOOL=ON ITK_BUILD_DEFAULT_MODULES:BOOL=OFF ITKGroup_Core:BOOL=ON Module_ITKThresholding:BOOL=ON