diff --git a/CHANGELOG.md b/CHANGELOG.md index 9781860d..6faed724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,15 @@ date) and use that section as the basis for the GitHub release notes. `F`) instead of relying on ruff's shipped default. (#594) ### Changed +- `TDL_refinement.fit()` (`ontolearn/learners/tree_learner_refinement_inherit.py`) + duplicated `TDL.fit()` (`ontolearn/learners/tree_learner.py`) almost + line-for-line for grid search, classifier training, classification + reporting, and plotting, with only the SHAP-based refinement loop spliced + in; likewise `create_training_data` was a byte-for-byte copy of `TDL`'s. + Extracted the shared steps into `_fit_decision_tree`/`_report_and_plot`/ + `_finalize_predictions`/`_maybe_plot_embeddings` hooks on the base `TDL` + class, and `TDL_refinement` now inherits `create_training_data` instead of + redefining it. (#616) - `quality_funcs.f1`/`acc` reimplemented the tp/tn/fp/fn and precision/recall arithmetic already implemented in `metrics.py`'s `F1`/`Accuracy` classes; now delegate to those classes instead of duplicating the math. @@ -35,6 +44,14 @@ date) and use that section as the basis for the GitHub release notes. by default, matching the other learners. (#585) ### Fixed +- `TDL_refinement.classification_report` always raised `AttributeError`: + `TDL_refinement.fit()` set `self.__classification_report` from within its + own class body, which Python name-mangles to a different attribute than + the one the `classification_report` property (defined on `TDL`) reads. + Fixed as part of deduplicating `fit()` into shared `TDL` hooks, above; the + report is now always set from the base class's `_report_and_plot`, so both + classes agree on the mangled name. `tests/test_tdl_refinement.py`'s + `test_classification_report` re-enabled. (#616) - `NCESTrainer.train()` created a fresh `DataLoader(num_workers=...)` worker pool per architecture in a loop; the existing `os.cpu_count()`-based cap on `num_workers` only engaged when `cpu_count <= num_workers`, so it silently diff --git a/Improvements.md b/Improvements.md index 807de5a6..68a42284 100644 --- a/Improvements.md +++ b/Improvements.md @@ -15,6 +15,10 @@ remain open as of this writing. ## 2. Code duplication +- **`ontolearn/quality_funcs.py:31-56`** duplicates the tp/tn/fp/fn and + precision/recall math already implemented in `ontolearn/metrics.py:67-92`'s + `F1`/`Accuracy` classes. Have `quality_funcs.py` delegate to those classes + instead of reimplementing the arithmetic. - **`ontolearn/learners/tree_learner_refinement_inherit.py` vs `tree_learner.py`** — `TDL_refinement(TDL)` is ~70% copy-pasted rather than extended. `fit()` (tree_learner_refinement_inherit.py:604-761) duplicates diff --git a/ontolearn/learners/tree_learner.py b/ontolearn/learners/tree_learner.py index 0835f3e0..cf22b002 100644 --- a/ontolearn/learners/tree_learner.py +++ b/ontolearn/learners/tree_learner.py @@ -661,38 +661,14 @@ def construct_owl_expression_from_tree(self, X: pd.DataFrame, y: pd.DataFrame) - prediction_per_example = {pred for pred, positive_example in prediction_per_example} return list(prediction_per_example) - def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None): - """Fit the learner to the given learning problem - - (1) Extract multi-hop information about E^+ and E^-. - (2) Create OWL Class Expressions from (1) - (3) Build a binary sparse training data X where - first |E+| rows denote the binary representations of positives - Remaining rows denote the binary representations of E⁻ - (4) Create binary labels. - (4) Construct a set of DL concept for each e \in E^+ - (5) Union (4) - - :param learning_problem: The learning problem - :param max_runtime:total runtime of the learning - - """ - assert learning_problem is not None, "Learning problem cannot be None." - assert isinstance( - learning_problem, PosNegLPStandard - ), f"Learning problem must be PosNegLPStandard. Currently:{learning_problem}." - - if max_runtime is not None: - self.max_runtime = max_runtime - - start_time = time.time() - X: pd.DataFrame - y: Union[pd.DataFrame, pd.Series] - X, y = self.create_training_data(learning_problem=learning_problem, start_time=start_time) - # CD: Remember so that if user wants to use them - self.X, self.y = X, y + def _maybe_plot_embeddings(self, X: pd.DataFrame, y: pd.DataFrame) -> None: + """Plot a UMAP-reduced view of the training features, if enabled. Shared by TDL and TDL_refinement.""" if self.plot_embeddings: plot_umap_reduced_embeddings(X, y.label.to_list(), "umap_visualization.pdf") + + def _fit_decision_tree(self, X: pd.DataFrame, y: pd.DataFrame) -> None: + """Optionally grid-search classifier hyperparameters, then train self.clf on (X, y). + Shared by TDL and TDL_refinement.""" if self.grid_search_over: grid_search = sklearn.model_selection.GridSearchCV( tree.DecisionTreeClassifier(**self.kwargs_classifier), @@ -700,10 +676,13 @@ def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None print(grid_search.best_params_) self.kwargs_classifier.update(grid_search.best_params_) # Training - if self.verbose>0: + if self.verbose > 0: print("Training starts!") self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=X.values, y=y.values) + def _report_and_plot(self, X: pd.DataFrame, y: pd.DataFrame) -> None: + """Report classification metrics and render the configured tree/feature-importance plots. + Shared by TDL and TDL_refinement.""" if self.report_classification: self.__classification_report = "Classification Report: Negatives: -1 and Positives 1 \n" self.__classification_report += sklearn.metrics.classification_report(y.values, @@ -719,23 +698,61 @@ def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None plot_topk_feature_importance(feature_names=[owl_expression_to_dl(f) for f in self.features], cart_tree=self.clf) + def _finalize_predictions(self, X: pd.DataFrame, y: pd.DataFrame) -> None: + """Build the conjunctive/disjunctive OWL predictions from the trained tree and optionally verbalize them. + Shared by TDL and TDL_refinement.""" self.owl_class_expressions.clear() # Each item can be considered is a path of OWL Class Expressions # starting from the root node in the decision tree and # ending in a leaf node. self.conjunctive_concepts: List[OWLObjectIntersectionOf] - if self.verbose >0: + if self.verbose > 0: print("Computing conjunctive_concepts...") self.conjunctive_concepts = self.construct_owl_expression_from_tree(X, y) for i in self.conjunctive_concepts: self.owl_class_expressions.add(i) - if self.verbose >0: + if self.verbose > 0: print("Computing disjunction_of_conjunctive_concepts...") self.disjunction_of_conjunctive_concepts = concepts_reducer(concepts=self.conjunctive_concepts, reduced_cls=OWLObjectUnionOf) if self.verbalize: verbalize_learner_prediction(self.disjunction_of_conjunctive_concepts) + def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None): + """Fit the learner to the given learning problem + + (1) Extract multi-hop information about E^+ and E^-. + (2) Create OWL Class Expressions from (1) + (3) Build a binary sparse training data X where + first |E+| rows denote the binary representations of positives + Remaining rows denote the binary representations of E⁻ + (4) Create binary labels. + (4) Construct a set of DL concept for each e \in E^+ + (5) Union (4) + + :param learning_problem: The learning problem + :param max_runtime:total runtime of the learning + + """ + assert learning_problem is not None, "Learning problem cannot be None." + assert isinstance( + learning_problem, PosNegLPStandard + ), f"Learning problem must be PosNegLPStandard. Currently:{learning_problem}." + + if max_runtime is not None: + self.max_runtime = max_runtime + + start_time = time.time() + X: pd.DataFrame + y: Union[pd.DataFrame, pd.Series] + X, y = self.create_training_data(learning_problem=learning_problem, start_time=start_time) + # CD: Remember so that if user wants to use them + self.X, self.y = X, y + self._maybe_plot_embeddings(X, y) + self._fit_decision_tree(X, y) + self._report_and_plot(X, y) + self._finalize_predictions(X, y) + return self @property diff --git a/ontolearn/learners/tree_learner_refinement_inherit.py b/ontolearn/learners/tree_learner_refinement_inherit.py index 4b55451c..eb14d4b3 100644 --- a/ontolearn/learners/tree_learner_refinement_inherit.py +++ b/ontolearn/learners/tree_learner_refinement_inherit.py @@ -6,13 +6,7 @@ from sklearn.inspection import permutation_importance from .tree_learner import concepts_reducer from ontolearn.learning_problem import PosNegLPStandard -from ..utils.static_funcs import ( - plot_umap_reduced_embeddings, - plot_decision_tree_of_expressions, - plot_topk_feature_importance, -) from owlapy.class_expression import ( - OWLObjectIntersectionOf, OWLClassExpression, OWLObjectUnionOf, OWLDataSomeValuesFrom, @@ -23,10 +17,8 @@ OWLDatatypeRestriction, ) from owlapy.owl_individual import OWLNamedIndividual -from ontolearn.verbalizer import verbalize_learner_prediction import numpy as np import pandas as pd -import sklearn from sklearn import tree import itertools from shap import TreeExplainer @@ -288,7 +280,10 @@ def _extract_data_property_features( if self.verbose > 0: print(f"Warning: Error extracting data property features: {e}") - def extract_expressions_from_owl_individuals(self, individuals: List[OWLNamedIndividual]) -> (Tuple)[np.ndarray, List[OWLClassExpression]]: + def extract_expressions_from_owl_individuals(self, individuals: List[OWLNamedIndividual], start_time: float = None) -> (Tuple)[np.ndarray, List[OWLClassExpression]]: + # start_time is accepted (but unused) so this override keeps the same call + # signature as TDL.extract_expressions_from_owl_individuals, letting + # create_training_data() be shared via inheritance instead of duplicated here. # () Store mappings from str dl concept to owl class expression objects. features_dict = dict() @@ -470,32 +465,8 @@ def refine_numerical_features(self, topk_expressions: list[OWLClassExpression], self.generated_dt_classexpressions_per_individual[new_class_expression].add(ind) return refined_features, individuals_to_refined_feature_mapping - def create_training_data(self, learning_problem: PosNegLPStandard) -> Tuple[pd.DataFrame, pd.DataFrame]: - - if self.verbose > 0: - print("Creating a Training Dataset") - positive_examples: List[OWLNamedIndividual] - negative_examples: List[OWLNamedIndividual] - positive_examples = [i for i in learning_problem.pos] - negative_examples = [i for i in learning_problem.neg] - # (2) Initialize labels for (1). - y = [1.0 for _ in positive_examples] + [0.0 for _ in negative_examples] - # (3) Iterate over examples to extract unique features. - examples = positive_examples + negative_examples - # For the sake of convenience. sort features in ascending order of string lengths of DL representations. - - X, features = self.extract_expressions_from_owl_individuals(examples) - - # (4) Creating a tabular data for the binary classification problem. - # X = self.sparse_binary_representations(features, examples, examples_to_features) - self.features = features - X = pd.DataFrame(data=X, index=examples, columns=self.features) - y = pd.DataFrame(data=y, index=examples, columns=["label"]) - # Remove redundant columns - same_value_columns = X.apply(lambda col: col.nunique() == 1) - X = X.loc[:, ~same_value_columns] - self.features = X.columns.values.tolist() - return X, y + # create_training_data() is inherited from TDL: identical logic aside from + # which extract_expressions_from_owl_individuals() override it dispatches to. def plot_importance_evolution(self, initial_dict, refined_dicts, top_k=10): # prepare data @@ -601,6 +572,67 @@ def _merge_binary_feature_matrices(self, lp: PosNegLPStandard, X: pd.DataFrame, X = X.loc[:, ~same_value_columns] return X + def _refine_features_with_shap(self, X: pd.DataFrame, y: pd.DataFrame, learning_problem: PosNegLPStandard) -> pd.DataFrame: + """Iteratively extend the feature set with SHAP-guided, distribution-based numeric data-property ranges. + + Retrains self.clf after every feature-set extension and returns the final binary feature matrix. + """ + tree_explainer = TreeExplainer(self.clf) + sVal = tree_explainer.shap_values(self.X.values) + if self.verbose >= 10: + self.initial_importance_dict = self._get_shap_importance_dict(X) + # map individuals to additional data property features + # refine base features here ( extract from data properties) + dp_features: dict = dict() + generated_dt_class_expressions = self._extract_ranges_from_data_properties(dp_features, self.individuals_to_feature_mapping, self.data_properties_dict, self.per_individual_data_properties) + self.generated_dt_classexpressions_per_individual = generated_dt_class_expressions + X = self._merge_binary_feature_matrices(learning_problem, X, dp_features, self.individuals_to_feature_mapping) + self.X = X + self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=self.X.values, y=self.y.values) + if self.verbose >= 10: + self.concept_per_iteration.append(concepts_reducer(concepts=self.construct_owl_expression_from_tree(X, y), reduced_cls=OWLObjectUnionOf)) + self.top_feature_dicts.append(self._get_shap_importance_dict(self.X)) + refined_expressions: Set[OWLClassExpression] = set() + for i in range(1, self.refine_iterations): + if self.verbose > 0: + print(f"Refinement iteration {i + 1}/{self.refine_iterations}") + # calculate SHAP global feature importance + tree_explainer = TreeExplainer(self.clf) + sVal = tree_explainer.shap_values(self.X.values) + if isinstance(sVal, np.ndarray): + if sVal.ndim == 3: + global_importance = np.abs(sVal[:, :, 1]).mean(axis=0) + else: + global_importance = np.abs(sVal).mean(axis=0) + topk_id = np.argsort(global_importance) + # top expressions sorted low to high + top_expressions: list[OWLClassExpression] + top_expressions = [self.features[i] for i in topk_id[-self.topk :].tolist()] + + # print(top_expressions[-5:]) + features_to_refine: set[OWLClassExpression] = (set(top_expressions) - refined_expressions) + refined_features, refined_individuals_to_feature_mapping = self.refine_numerical_features(features_to_refine, iteration=i + 1) + if self.verbose >0: + print("Expressions to be refined:" + str(len(features_to_refine))) + + if(len(features_to_refine) == 0): + if self.verbose > 0: + print("No new expressions to refine. Stopping refinement.") + break + refined_expressions = refined_expressions.union(set(top_expressions)) + X = self._merge_binary_feature_matrices(learning_problem, X, refined_features, refined_individuals_to_feature_mapping) + self.X = X + self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=self.X.values, y=self.y.values) + + if self.verbose >= 10: + self.top_feature_dicts.append(self._get_shap_importance_dict(self.X)) + self.concept_per_iteration.append(concepts_reducer(concepts=self.construct_owl_expression_from_tree(X, y), reduced_cls=OWLObjectUnionOf)) + + if self.plot_importance_evo and self.verbose >= 10: + self.plot_importance_evolution(self.initial_importance_dict, self.top_feature_dicts, top_k=10) + + return X + def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None): """Fit the learner to the given learning problem @@ -634,97 +666,15 @@ def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None X, y = self.create_training_data(learning_problem=learning_problem) # CD: Remember so that if user wants to use them self.X, self.y = X, y - if self.plot_embeddings: - plot_umap_reduced_embeddings(X, y.label.to_list(), "umap_visualization.pdf") - if self.grid_search_over: - grid_search = sklearn.model_selection.GridSearchCV( - tree.DecisionTreeClassifier(**self.kwargs_classifier), - param_grid=self.grid_search_over, - **self.kwargs_grid_search, - ).fit(X.values, y.values) - print(grid_search.best_params_) - self.kwargs_classifier.update(grid_search.best_params_) - # Training - if self.verbose > 0: - print("Training starts!") - self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=X.values, y=y.values) + self._maybe_plot_embeddings(X, y) + self._fit_decision_tree(X, y) if self.verbose >= 10: self.concept_per_iteration.append(concepts_reducer(concepts=self.construct_owl_expression_from_tree(X, y), reduced_cls=OWLObjectUnionOf)) if self.feature_refinement: - tree_explainer = TreeExplainer(self.clf) - sVal = tree_explainer.shap_values(self.X.values) - if self.verbose >= 10: - self.initial_importance_dict = self._get_shap_importance_dict(X) - # map individuals to additional data property features - # refine base features here ( extract from data properties) - dp_features: dict = dict() - generated_dt_class_expressions = self._extract_ranges_from_data_properties(dp_features, self.individuals_to_feature_mapping, self.data_properties_dict, self.per_individual_data_properties) - self.generated_dt_classexpressions_per_individual = generated_dt_class_expressions - X = self._merge_binary_feature_matrices(learning_problem, X, dp_features, self.individuals_to_feature_mapping) - self.X = X - self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=self.X.values, y=self.y.values) - if self.verbose >= 10: - self.concept_per_iteration.append(concepts_reducer(concepts=self.construct_owl_expression_from_tree(X, y), reduced_cls=OWLObjectUnionOf)) - self.top_feature_dicts.append(self._get_shap_importance_dict(self.X)) - refined_expressions: Set[OWLClassExpression] = set() - for i in range(1, self.refine_iterations): - if self.verbose > 0: - print(f"Refinement iteration {i + 1}/{self.refine_iterations}") - # calculate SHAP global feature importance - tree_explainer = TreeExplainer(self.clf) - sVal = tree_explainer.shap_values(self.X.values) - if isinstance(sVal, np.ndarray): - if sVal.ndim == 3: - global_importance = np.abs(sVal[:, :, 1]).mean(axis=0) - else: - global_importance = np.abs(sVal).mean(axis=0) - topk_id = np.argsort(global_importance) - # top expressions sorted low to high - top_expressions: list[OWLClassExpression] - top_expressions = [self.features[i] for i in topk_id[-self.topk :].tolist()] - - # print(top_expressions[-5:]) - features_to_refine: set[OWLClassExpression] = (set(top_expressions) - refined_expressions) - refined_features, refined_individuals_to_feature_mapping = self.refine_numerical_features(features_to_refine, iteration=i + 1) - if self.verbose >0: - print("Expressions to be refined:" + str(len(features_to_refine))) - - if(len(features_to_refine) == 0): - if self.verbose > 0: - print("No new expressions to refine. Stopping refinement.") - break - refined_expressions = refined_expressions.union(set(top_expressions)) - X = self._merge_binary_feature_matrices(learning_problem, X, refined_features, refined_individuals_to_feature_mapping) - self.X = X - self.clf = tree.DecisionTreeClassifier(**self.kwargs_classifier).fit(X=self.X.values, y=self.y.values) - - if self.verbose >= 10: - self.top_feature_dicts.append(self._get_shap_importance_dict(self.X)) - self.concept_per_iteration.append(concepts_reducer(concepts=self.construct_owl_expression_from_tree(X, y), reduced_cls=OWLObjectUnionOf)) - - if self.plot_importance_evo and self.verbose >= 10: - self.plot_importance_evolution(self.initial_importance_dict, self.top_feature_dicts, top_k=10) - - if self.report_classification: - self.__classification_report = "Classification Report: Negatives: -1 and Positives 1 \n" - self.__classification_report += sklearn.metrics.classification_report( - y.values, - self.clf.predict(X.values), - target_names=["Negative", "Positive"], - ) - if self.verbose > 0: - print(self.__classification_report) - if self.plot_tree: - plot_decision_tree_of_expressions( - feature_names=[owl_expression_to_dl(f) for f in self.features], - cart_tree=self.clf, - ) - if self.plot_feature_importance: - plot_topk_feature_importance( - feature_names=[owl_expression_to_dl(f) for f in self.features], - cart_tree=self.clf, - topk=10, - ) + X = self._refine_features_with_shap(X, y, learning_problem) + + self._report_and_plot(X, y) + perm_feature_topk = False if perm_feature_topk: # plot permutation feature importance @@ -742,20 +692,7 @@ def fit(self, learning_problem: PosNegLPStandard = None, max_runtime: int = None global_importance = np.abs(sVal).mean(axis=0) self.plot_shap_feature_importances(global_importance, [owl_expression_to_dl(f) for f in self.features]) - self.owl_class_expressions.clear() - # Each item can be considered is a path of OWL Class Expressions - # starting from the root node in the decision tree and - # ending in a leaf node. - self.conjunctive_concepts: List[OWLObjectIntersectionOf] - if self.verbose > 0: - print("Computing conjunctive_concepts...") - self.conjunctive_concepts = self.construct_owl_expression_from_tree(X, y) - for i in self.conjunctive_concepts: - self.owl_class_expressions.add(i) - if self.verbose > 0: - print("Computing disjunction_of_conjunctive_concepts...") - self.disjunction_of_conjunctive_concepts = concepts_reducer(concepts=self.conjunctive_concepts, reduced_cls=OWLObjectUnionOf) - if self.verbalize: - verbalize_learner_prediction(self.disjunction_of_conjunctive_concepts) + + self._finalize_predictions(X, y) return self diff --git a/tests/test_tdl_refinement.py b/tests/test_tdl_refinement.py index 70c585cd..3b998e08 100644 --- a/tests/test_tdl_refinement.py +++ b/tests/test_tdl_refinement.py @@ -528,25 +528,25 @@ def test_fit_with_different_configurations(self): model2.fit(lp) self.assertIsNotNone(model2.clf) - #def test_classification_report(self): - # """Test that classification report is generated.""" - # model = TDL_refinement(knowledge_base=self.kb, report_classification=True, verbose=0, - # kwargs_classifier={"random_state": 42}) - # - # examples = self.family_lps['problems']['Brother'] - # p = set(examples['positive_examples'][:5]) # Reduced for speed - # n = set(examples['negative_examples'][:5]) # Reduced for speed - # typed_pos = set(map(OWLNamedIndividual, map(IRI.create, p))) - # typed_neg = set(map(OWLNamedIndividual, map(IRI.create, n))) - # lp = PosNegLPStandard(pos=typed_pos, neg=typed_neg) - # - # model.fit(lp) - # - # # Check that report was generated - # report = model.classification_report - # self.assertIsNotNone(report) - # self.assertIsInstance(report, str) - # self.assertIn("Classification Report", report) + def test_classification_report(self): + """Test that classification report is generated.""" + model = TDL_refinement(knowledge_base=self.kb, report_classification=True, verbose=0, + kwargs_classifier={"random_state": 42}) + + examples = self.family_lps['problems']['Brother'] + p = set(examples['positive_examples'][:5]) # Reduced for speed + n = set(examples['negative_examples'][:5]) # Reduced for speed + typed_pos = set(map(OWLNamedIndividual, map(IRI.create, p))) + typed_neg = set(map(OWLNamedIndividual, map(IRI.create, n))) + lp = PosNegLPStandard(pos=typed_pos, neg=typed_neg) + + model.fit(lp) + + # Check that report was generated + report = model.classification_report + self.assertIsNotNone(report) + self.assertIsInstance(report, str) + self.assertIn("Classification Report", report) class TestTDLPerformance(unittest.TestCase):