diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..81e35a7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## 2026-01-26 + +- [x] updated the package dependencies to resolve install errors coming from older packages +- [x] updated the ontology references in `ontologies.csv` diff --git a/pyproject.toml b/pyproject.toml index a912a57..6b863f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "text2term" -version = "4.5.0" +version = "4.6.0" description = "A tool for mapping free-text descriptions of entities to ontology terms" readme = "README.md" requires-python = ">=3.9" @@ -8,11 +8,11 @@ license = "MIT" keywords = ["ontology", "metadata", "text mining", "metadata curation", "ontology mapping"] authors = [ - { name="Rafael Goncalves", email="goncalves@stanford.edu" }, + { name="Rafael Goncalves", email="rsgoncalves@gmx.com" }, { name="Jason Payne", email="paynejason@gmail.com" } ] maintainers = [ - { name="Rafael Goncalves", email="goncalves@stanford.edu" } + { name="Rafael Goncalves", email="rsgoncalves@gmx.com" } ] classifiers = [ @@ -25,25 +25,20 @@ classifiers = [ ] dependencies = [ - "owlready2~=0.47", - "argparse~=1.4.0", - "pandas~=2.2.3", - "numpy~=1.26.4", - "gensim~=4.3.2", - "scipy~=1.12.0", - "scikit-learn~=1.6.1", - "setuptools~=80.8.0", - "requests~=2.32.3", - "tqdm~=4.67.1", - "sparse-dot-topn~=1.1.3", - "bioregistry~=0.11.10", - "nltk~=3.9.1", - "rapidfuzz~=3.12.2", - "shortuuid~=1.0.11", + "bioregistry>=0.11.10", + "nltk>=3.9.1", + "owlready2>=0.47", + "pandas>=2.2.3", + "rapidfuzz>=3.12.2", + "requests>=2.32.3", + "scikit-learn>=1.6.1", + "shortuuid>=1.0.11", + "sparse-dot-topn>=1.1.3", + "tqdm>=4.67.1", ] [project.optional-dependencies] -dev = ["pytest==8.1.1"] +dev = ["pytest>=8.1.1"] [tool.setuptools] packages = ["text2term"] @@ -53,7 +48,7 @@ include-package-data = true omit = ["*/__main__.py"] [build-system] -requires = ["setuptools>=80.8", "wheel"] +requires = ["setuptools>=61"] build-backend = "setuptools.build_meta" [project.urls] diff --git a/test/test_input.txt b/test/test_input.txt index 2dc3f97..4b3be06 100644 --- a/test/test_input.txt +++ b/test/test_input.txt @@ -1,2 +1,2 @@ -asthma -acute bronchitis \ No newline at end of file +cheese pizza +fish pizza diff --git a/test/test_input_with_tags.txt b/test/test_input_with_tags.txt index 54d8b8d..7a0c481 100644 --- a/test/test_input_with_tags.txt +++ b/test/test_input_with_tags.txt @@ -1,4 +1,4 @@ -asthma;:;disease -acute bronchitis;:;important,tags -hypertension NOS;:;disease -colon disease +cheese pizza;:;pizza +fish pizza;:;seafood,topping +pizza NOS;:;unspecified +mystery topping diff --git a/test/test_ontology.owl b/test/test_ontology.owl index 211d03c..f5de1b1 100644 --- a/test/test_ontology.owl +++ b/test/test_ontology.owl @@ -1,101 +1,73 @@ - - - - - - - - - - - - + - - - - - - - - - - - - - - - + - + + contains topping + - + + location + - + + Pizza + A dish made with dough and toppings + + + Ice Cream + + - + + + Cheese Pizza + Plain Pizza + - - - - - - - - - - - + + + + + + + pizza with fish Seafood Pizza Fish Pizza Pizza that contains fish A pizza that contains a fish ingredient - + + Fish Topping + - + + Tomato Topping + - + Vanilla Ice Cream true - - - - - diff --git a/test/test_t2t.py b/test/test_t2t.py index 98ad1f7..6bbfe8b 100644 --- a/test/test_t2t.py +++ b/test/test_t2t.py @@ -10,10 +10,21 @@ from text2term.bioportal_mapper import BioPortalAnnotatorMapper from text2term.syntactic_mapper import SyntacticMapper from text2term.zooma_mapper import ZoomaMapper -from owlready2 import EntityClass, Nothing, IRIS +from owlready2 import EntityClass, IRIS pd.set_option('display.max_columns', None) +RUN_LIVE_API_TESTS = os.getenv("TEXT2TERM_RUN_LIVE_API_TESTS") == "1" +BIOPORTAL_API_KEY = os.getenv("BIOPORTAL_API_KEY", "") +skip_unless_live_api = unittest.skipUnless( + RUN_LIVE_API_TESTS, + "Set TEXT2TERM_RUN_LIVE_API_TESTS=1 to run live API integration tests.", +) +skip_unless_bioportal_api = unittest.skipUnless( + RUN_LIVE_API_TESTS and BIOPORTAL_API_KEY, + "Set TEXT2TERM_RUN_LIVE_API_TESTS=1 and BIOPORTAL_API_KEY to run live BioPortal integration tests.", +) + class Text2TermTestSuite(unittest.TestCase): @@ -21,13 +32,17 @@ class Text2TermTestSuite(unittest.TestCase): def setUpClass(cls): super(Text2TermTestSuite, cls).setUpClass() print("Setting up test suite global variables...") - cls.EFO_URL = "https://github.com/EBISPOT/efo/releases/download/v3.57.0/efo.owl" cls.SOURCE_TERM_ID_COLUMN = "Source Term ID" cls.MAPPED_TERM_CURIE_COLUMN = "Mapped Term CURIE" cls.MAPPING_SCORE_COLUMN = "Mapping Score" cls.TAGS_COLUMN = "Tags" cls.TEST_CACHE_FOLDER = ".test_cache" cls.TEST_FOLDER = os.path.dirname(os.path.abspath(__file__)) + cls.TEST_ONTOLOGY_FILEPATH = os.path.join(cls.TEST_FOLDER, "test_ontology.owl") + cls.TEST_ONTOLOGY_ACRONYM = "T2TTEST" + cls.TEST_ONTOLOGY_BASE_IRI = "https://text2term.ontology/test/" + cls.TEST_SOURCE_TERMS = ["cheese pizza", "fish pizza", "tomato topping"] + cls.TEST_PROPERTY_TERMS = ["contains topping", "location"] # Configure the SSL verification to be disabled if hasattr(onto_cache, 'disable_ssl_verification'): @@ -57,39 +72,20 @@ def setup_test_environment(cls): os.makedirs(cls.TEST_CACHE_FOLDER) print(f"Created test cache folder: {cls.TEST_CACHE_FOLDER}") - # Create resources directory - try different possible paths - resources_paths = [ - os.path.join("text2term", "resources"), - os.path.join("..", "text2term", "resources") - ] - - resources_dir = None - for path in resources_paths: - if os.path.exists(path): - resources_dir = path - break - - # If no existing resources directory found, create one - if resources_dir is None: - resources_dir = resources_paths[0] # Default to first option - os.makedirs(resources_dir, exist_ok=True) - print(f"Created resources directory: {resources_dir}") - - # Create a minimal ontologies.csv file if it doesn't exist - ontologies_file = os.path.join(resources_dir, "ontologies.csv") - if not os.path.exists(ontologies_file): - with open(ontologies_file, 'w') as f: - f.write("acronym,url\n") - f.write("EFO,https://github.com/EBISPOT/efo/releases/download/v3.57.0/efo.owl\n") - f.write("CLO,http://purl.obolibrary.org/obo/clo.owl\n") - print(f"Created sample ontologies file: {ontologies_file}") + cls.TEST_ONTOLOGY_REGISTRY = os.path.join(cls.TEST_FOLDER, "test_ontologies.csv") + with open(cls.TEST_ONTOLOGY_REGISTRY, 'w') as f: + f.write("acronym,url\n") + f.write(f"{cls.TEST_ONTOLOGY_ACRONYM},{cls.TEST_ONTOLOGY_FILEPATH}\n") + f.write(f"{cls.TEST_ONTOLOGY_ACRONYM}_ALT,{cls.TEST_ONTOLOGY_FILEPATH}\n") # Create test_input_with_tags.txt for the preprocessing test test_input_file_path = os.path.join(cls.TEST_FOLDER, "test_input_with_tags.txt") if not os.path.exists(test_input_file_path): with open(test_input_file_path, 'w') as f: - f.write("asthma;:;disease\n") - f.write("protein level;:;important\n") + f.write("cheese pizza;:;pizza\n") + f.write("fish pizza;:;seafood,topping\n") + f.write("pizza NOS;:;unspecified\n") + f.write("mystery topping\n") print(f"Created sample input file: {test_input_file_path}") cls.TEST_INPUT_FILEPATH = test_input_file_path @@ -103,84 +99,62 @@ def setup_test_environment(cls): print("Test environment setup complete") def test_caching_ontology_from_url(self): - # Test caching an ontology loaded from a URL - print("Test caching an ontology loaded from a URL...") - efo_cache = text2term.cache_ontology(ontology_url=self.EFO_URL, ontology_acronym="EFO", - cache_folder=self.TEST_CACHE_FOLDER) - print(f"Cache exists: {efo_cache.cache_exists()}\n") - assert efo_cache.cache_exists() is True + print("Test caching an ontology loaded from a local file...") + ontology_cache = text2term.cache_ontology(ontology_url=self.TEST_ONTOLOGY_FILEPATH, + ontology_acronym=self.TEST_ONTOLOGY_ACRONYM, + cache_folder=self.TEST_CACHE_FOLDER) + print(f"Cache exists: {ontology_cache.cache_exists()}\n") + assert ontology_cache.cache_exists() is True print("Test using the returned ontology cache object to map a list of terms...") - mappings_efo_cache = efo_cache.map_terms(["asthma", "disease location", "food allergy"], - term_type=OntologyTermType.ANY) - assert mappings_efo_cache.size > 0 + mappings_cache = ontology_cache.map_terms(self.TEST_SOURCE_TERMS, term_type=OntologyTermType.ANY) + assert mappings_cache.size > 0 def test_caching_ontology_from_acronym(self): - # Test caching an ontology by resolving its acronym using bioregistry - print("Test caching an ontology by resolving its acronym using bioregistry...") - clo_cache = text2term.cache_ontology(ontology_url="CLO", ontology_acronym="CLO", - cache_folder=self.TEST_CACHE_FOLDER) - print(f"Cache exists: {clo_cache.cache_exists()}\n") - assert clo_cache.cache_exists() is True + print("Test caching an ontology with an acronym alias...") + ontology_cache = text2term.cache_ontology(ontology_url=self.TEST_ONTOLOGY_FILEPATH, + ontology_acronym="LOCALPIZZA", + cache_folder=self.TEST_CACHE_FOLDER) + print(f"Cache exists: {ontology_cache.cache_exists()}\n") + assert ontology_cache.cache_exists() is True def test_cache_clearing(self): - self.ensure_cache_exists(ontology_name="CLO", ontology_url="http://purl.obolibrary.org/obo/clo.owl") - assert text2term.cache_exists(ontology_acronym="CLO", cache_folder=self.TEST_CACHE_FOLDER) is True - text2term.clear_cache(ontology_acronym="CLO", cache_folder=self.TEST_CACHE_FOLDER) - assert text2term.cache_exists(ontology_acronym="CLO", cache_folder=self.TEST_CACHE_FOLDER) is False + self.ensure_cache_exists(ontology_name="LOCALPIZZA", ontology_url=self.TEST_ONTOLOGY_FILEPATH) + assert text2term.cache_exists(ontology_acronym="LOCALPIZZA", cache_folder=self.TEST_CACHE_FOLDER) is True + text2term.clear_cache(ontology_acronym="LOCALPIZZA", cache_folder=self.TEST_CACHE_FOLDER) + assert text2term.cache_exists(ontology_acronym="LOCALPIZZA", cache_folder=self.TEST_CACHE_FOLDER) is False def test_caching_ontology_set(self): - # Find ontologies.csv file in possible locations - resources_paths = [ - os.path.join("text2term", "resources", "ontologies.csv"), - os.path.join("..", "text2term", "resources", "ontologies.csv") - ] - - ontology_registry_filepath = None - for path in resources_paths: - if os.path.exists(path): - ontology_registry_filepath = path - break - - if ontology_registry_filepath is None: - self.skipTest("Ontology registry file not found in expected locations") - - nr_ontologies_in_registry = len(pd.read_csv(ontology_registry_filepath)) - - # Test caching the set of ontologies specified in resources/ontologies.csv - caches = text2term.cache_ontology_set(ontology_registry_filepath, cache_folder=self.TEST_CACHE_FOLDER) + nr_ontologies_in_registry = len(pd.read_csv(self.TEST_ONTOLOGY_REGISTRY)) + caches = text2term.cache_ontology_set(self.TEST_ONTOLOGY_REGISTRY, cache_folder=self.TEST_CACHE_FOLDER) assert len(caches) == nr_ontologies_in_registry def test_mapping_to_cached_ontology(self): text2term.clear_cache(cache_folder=self.TEST_CACHE_FOLDER) - self.ensure_cache_exists("EFO", self.EFO_URL) - # Test mapping a list of terms to EFO loaded from cache - print("Test mapping a list of terms to EFO loaded from cache...") - mappings_efo_cache = text2term.map_terms(["asthma", "disease location", "food allergy"], target_ontology="EFO", - use_cache=True, term_type=OntologyTermType.ANY, - cache_folder=self.TEST_CACHE_FOLDER) - print(f"{mappings_efo_cache}\n") - assert mappings_efo_cache.size > 0 - - # Test mapping a list of terms to EFO loaded from a URL - print("Test mapping a list of terms to EFO loaded from a URL...") - mappings_efo_url = text2term.map_terms(["asthma", "disease location", "food allergy"], - target_ontology=self.EFO_URL, term_type=OntologyTermType.ANY) - print(f"{mappings_efo_url}\n") - assert mappings_efo_url.size > 0 - - # Test that mapping to cached ontology is the same as to ontology loaded from its URL - print("Test that mapping to cached ontology is the same as to ontology loaded from its URL...") - mappings_match = self.check_df_equals(self.drop_source_term_ids(mappings_efo_cache), - self.drop_source_term_ids(mappings_efo_url)) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + print("Test mapping a list of terms to a local ontology loaded from cache...") + mappings_cache = text2term.map_terms(self.TEST_SOURCE_TERMS, target_ontology=self.TEST_ONTOLOGY_ACRONYM, + use_cache=True, term_type=OntologyTermType.ANY, + cache_folder=self.TEST_CACHE_FOLDER) + print(f"{mappings_cache}\n") + assert mappings_cache.size > 0 + + print("Test mapping a list of terms to the same ontology loaded from a local file...") + mappings_file = text2term.map_terms(self.TEST_SOURCE_TERMS, target_ontology=self.TEST_ONTOLOGY_FILEPATH, + term_type=OntologyTermType.ANY) + print(f"{mappings_file}\n") + assert mappings_file.size > 0 + + print("Test that mapping to cached ontology is the same as to ontology loaded from its file...") + mappings_match = self.check_df_equals(self.drop_source_term_ids(mappings_cache), + self.drop_source_term_ids(mappings_file)) print(f"...{mappings_match}") assert mappings_match is True def test_mapping_to_cached_ontology_using_syntactic_mapper(self): - self.ensure_cache_exists("EFO", self.EFO_URL) - # Test mapping a list of terms to cached EFO using Jaro-Winkler syntactic similarity metric + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) print("Test mapping a list of terms to cached ontology using Jaro-Winkler syntactic similarity metric...") - df = text2term.map_terms(["asthma", "disease location", "food allergy"], "EFO", use_cache=True, + df = text2term.map_terms(self.TEST_SOURCE_TERMS, self.TEST_ONTOLOGY_ACRONYM, use_cache=True, mapper=text2term.Mapper.JARO_WINKLER, term_type=OntologyTermType.ANY, cache_folder=self.TEST_CACHE_FOLDER) print(f"{df}\n") @@ -200,51 +174,53 @@ def test_syntactic_mapper_inexistent(self): mapper.compare("heart", "hearts", mapper="mymapper") def test_mapping_using_ontology_acronym(self): - # Test mapping a list of terms by specifying the target ontology acronym, which gets resolved by bioregistry - print("Test mapping a list of terms to EFO by specifying an ontology acronym that gets resolved by bioregistry") - df2 = text2term.map_terms(["contains", "asthma"], "MONDO", term_type=OntologyTermType.CLASS) + print("Test mapping a list of terms to a local ontology file") + df2 = text2term.map_terms(["cheese pizza", "fish pizza"], self.TEST_ONTOLOGY_FILEPATH, + term_type=OntologyTermType.CLASS) print(f"{df2}\n") assert df2.size > 0 def test_mapping_tagged_terms(self): - self.ensure_cache_exists("EFO", self.EFO_URL) - # Test mapping a dictionary of tagged terms to cached EFO, and include unmapped terms in the output - print("Test mapping a dictionary of tagged terms to cached EFO, and include unmapped terms in the output...") + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + print("Test mapping a dictionary of tagged terms to a cached local ontology...") df3 = text2term.map_terms( - {"asthma": "disease", "allergy": ["ignore", "response"], "protein level": ["measurement"], - "isdjfnsdfwd": None}, target_ontology="EFO", excl_deprecated=True, use_cache=True, incl_unmapped=True, + {"cheese pizza": "pizza", "ice cream": ["ignore", "dessert"], "fish pizza": ["seafood"], + "isdjfnsdfwd": None}, target_ontology=self.TEST_ONTOLOGY_ACRONYM, excl_deprecated=True, use_cache=True, + incl_unmapped=True, cache_folder=self.TEST_CACHE_FOLDER) print(f"{df3}\n") assert df3.size > 0 - assert df3[self.TAGS_COLUMN].str.contains("disease").any() - assert df3[self.TAGS_COLUMN].str.contains("measurement").any() + assert df3[self.TAGS_COLUMN].str.contains("pizza").any() + assert df3[self.TAGS_COLUMN].str.contains("seafood").any() def test_preprocessing_from_file(self): - self.ensure_cache_exists("EFO", self.EFO_URL) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) # Test processing tagged terms where the tags are provided in a file print("Test processing tagged terms where the tags are provided in a file...") tagged_terms = text2term.preprocess_tagged_terms(file_path=self.TEST_INPUT_FILEPATH) - df4 = text2term.map_terms(tagged_terms, target_ontology="EFO", use_cache=True, incl_unmapped=True, + df4 = text2term.map_terms(tagged_terms, target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + incl_unmapped=True, cache_folder=self.TEST_CACHE_FOLDER) print(f"{df4}\n") assert df4.size > 0 - assert df4[self.TAGS_COLUMN].str.contains("disease").any() - assert df4[self.TAGS_COLUMN].str.contains("important").any() + assert df4[self.TAGS_COLUMN].str.contains("pizza").any() + assert df4[self.TAGS_COLUMN].str.contains("seafood").any() def test_preprocessing_tagged_terms(self): - self.ensure_cache_exists("EFO", self.EFO_URL) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) # Test processing tagged terms where the tags are provided in a file print("Test processing tagged terms where the tags are provided in a file...") tagged_terms = text2term.preprocess_tagged_terms(file_path=self.TEST_INPUT_FILEPATH, template_path=self.TEST_TEMPLATES_FILEPATH) - df = text2term.map_terms(tagged_terms, target_ontology="EFO", use_cache=True, incl_unmapped=True, + df = text2term.map_terms(tagged_terms, target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + incl_unmapped=True, cache_folder=self.TEST_CACHE_FOLDER) print(f"{df}\n") assert df.size > 0 - assert df[self.TAGS_COLUMN].str.contains("disease").any() - assert df[self.TAGS_COLUMN].str.contains("important").any() - assert df["Source Term"].str.contains("hypertension").any() - assert not df["Source Term"].str.contains("hypertension NOS").all() + assert df[self.TAGS_COLUMN].str.contains("pizza").any() + assert df[self.TAGS_COLUMN].str.contains("seafood").any() + assert df["Source Term"].eq("pizza").any() + assert not df["Source Term"].str.contains("pizza NOS").all() def test_preprocessing_terms(self): input_terms = ['Hypertension NOS', 'Diabetes mellitus due to underlying condition'] @@ -276,36 +252,37 @@ def test_preprocessing_blocklisted_terms(self): self.assertEqual(result, expected) def test_mapping_to_properties(self): - # Test mapping a list of properties to EFO loaded from a URL and restrict search to properties - print("Test mapping a list of properties to EFO loaded from a URL and restrict search to properties...") - df5 = text2term.map_terms(source_terms=["contains", "location"], target_ontology=self.EFO_URL, + print("Test mapping a list of properties to a local ontology file and restrict search to properties...") + df5 = text2term.map_terms(source_terms=self.TEST_PROPERTY_TERMS, target_ontology=self.TEST_ONTOLOGY_FILEPATH, term_type=OntologyTermType.PROPERTY) print(f"{df5}\n") assert df5.size > 0 - # Test mapping a list of properties to EFO loaded from cache and restrict search to properties - print("Test mapping a list of properties to EFO loaded from cache and restrict search to properties...") - self.ensure_cache_exists("EFO", self.EFO_URL) - df6 = text2term.map_terms(source_terms=["contains", "location"], target_ontology="EFO", use_cache=True, + print("Test mapping a list of properties to a cached local ontology and restrict search to properties...") + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + df6 = text2term.map_terms(source_terms=self.TEST_PROPERTY_TERMS, target_ontology=self.TEST_ONTOLOGY_ACRONYM, + use_cache=True, term_type=OntologyTermType.PROPERTY, cache_folder=self.TEST_CACHE_FOLDER) print(f"{df6}\n") assert df6.size > 0 - # Test that mapping to properties in cached ontology is the same as to ontology loaded from its URL + # Test that mapping to properties in cached ontology is the same as to ontology loaded from its file properties_df_match = self.check_df_equals(self.drop_source_term_ids(df5), self.drop_source_term_ids(df6)) print(f"Properties match: {properties_df_match}") assert properties_df_match is True + @skip_unless_live_api def test_mapping_zooma_ontologies(self): # Test mapping a list of terms to multiple ontologies using the Zooma mapper print("Test mapping a list of terms to multiple ontologies using the Zooma mapper...") - df_zooma = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="EFO,NCIT", + df_zooma = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="MONDO,NCIT", mapper=Mapper.ZOOMA, term_type=OntologyTermType.ANY) print(f"{df_zooma}\n") assert df_zooma.size > 0 - assert df_zooma[self.MAPPED_TERM_CURIE_COLUMN].str.contains("EFO:").any() + assert df_zooma[self.MAPPED_TERM_CURIE_COLUMN].str.contains("MONDO:").any() assert df_zooma[self.MAPPED_TERM_CURIE_COLUMN].str.contains("NCIT:").any() + @skip_unless_live_api def test_mapping_zooma_empty_response(self): print("Test mapping a term that obviously has no ontology term to be mapped to...") df_zooma = text2term.map_terms(["ziggy"], target_ontology="NCIT", @@ -313,6 +290,7 @@ def test_mapping_zooma_empty_response(self): print(f"{df_zooma}\n") assert df_zooma.empty is True + @skip_unless_live_api def test_mapping_zooma_bad_url(self): print("Test mapping terms to a wrong URL for Zooma mapper...") bp_mapper = ZoomaMapper() @@ -322,90 +300,95 @@ def test_mapping_zooma_bad_url(self): def test_mapping_bioportal_ontologies_no_apikey(self): # Test mapping a list of terms to multiple ontologies using the BioPortal Annotator mapper without API Key print("Test mapping a list of terms to multiple ontologies using the BioPortal Annotator mapper...") - df_bioportal = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="EFO,NCIT", + df_bioportal = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="MONDO,NCIT", mapper=Mapper.BIOPORTAL, term_type=OntologyTermType.ANY) assert df_bioportal.empty is True + @skip_unless_bioportal_api def test_mapping_bioportal_ontologies(self): # Test mapping a list of terms to multiple ontologies using the BioPortal Annotator mapper print("Test mapping a list of terms to multiple ontologies using the BioPortal Annotator mapper...") - df_bioportal = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="EFO,NCIT", + df_bioportal = text2term.map_terms(["asthma", "location", "food allergy"], target_ontology="MONDO,NCIT", mapper=Mapper.BIOPORTAL, term_type=OntologyTermType.ANY, - bioportal_apikey="8f0cbe43-2906-431a-9572-8600d3f4266e") + bioportal_apikey=BIOPORTAL_API_KEY) print(f"{df_bioportal}\n") assert df_bioportal.size > 0 - assert df_bioportal[self.MAPPED_TERM_CURIE_COLUMN].str.contains("EFO:").any() + assert df_bioportal[self.MAPPED_TERM_CURIE_COLUMN].str.contains("MONDO:").any() assert df_bioportal[self.MAPPED_TERM_CURIE_COLUMN].str.contains("NCIT:").any() + @skip_unless_bioportal_api def test_mapping_bioportal_empty_response(self): print("Test mapping a term that obviously has no ontology term to be mapped to...") df_bioportal = text2term.map_terms(["ziggy"], target_ontology="NCIT", mapper=Mapper.BIOPORTAL, term_type=OntologyTermType.ANY, - bioportal_apikey="8f0cbe43-2906-431a-9572-8600d3f4266e") + bioportal_apikey=BIOPORTAL_API_KEY) print(f"{df_bioportal}\n") assert df_bioportal.empty is True + @skip_unless_bioportal_api def test_mapping_bioportal_bad_url(self): print("Test mapping terms to a wrong URL for BioPortal Annotator mapper...") - bp_mapper = BioPortalAnnotatorMapper(bp_api_key="8f0cbe43-2906-431a-9572-8600d3f4266e") + bp_mapper = BioPortalAnnotatorMapper(bp_api_key=BIOPORTAL_API_KEY) with self.assertRaises(Exception): bp_mapper._do_get_request(request_url="http://data.bioontology.org/annotatorBad") def test_mapping_with_min_score_filter(self): - self.ensure_cache_exists("EFO", self.EFO_URL) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) min_score = 0.6 - search_terms = ["asthma attack", "location"] - - print("Test mapping to cached EFO using Zooma mapper and min_score filter...") - df_zooma = text2term.map_terms(search_terms, target_ontology="EFO,NCIT", mapper=Mapper.ZOOMA, - term_type=OntologyTermType.ANY, min_score=min_score) - assert (df_zooma[self.MAPPING_SCORE_COLUMN] >= min_score).all() + search_terms = ["cheese pizza", "location"] - print("Test mapping to cached EFO using TFIDF similarity metric and min_score filter...") - df_tfidf = text2term.map_terms(search_terms, target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - term_type=OntologyTermType.ANY, min_score=min_score, cache_folder=self.TEST_CACHE_FOLDER) + print("Test mapping to cached local ontology using TFIDF similarity metric and min_score filter...") + df_tfidf = text2term.map_terms(search_terms, target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + mapper=Mapper.TFIDF, term_type=OntologyTermType.ANY, min_score=min_score, + cache_folder=self.TEST_CACHE_FOLDER) assert (df_tfidf[self.MAPPING_SCORE_COLUMN] >= min_score).all() - print("Test mapping to cached EFO using Levenshtein similarity metric and min_score filter...") - df_leven = text2term.map_terms(search_terms, target_ontology="EFO", use_cache=True, mapper=Mapper.LEVENSHTEIN, - term_type=OntologyTermType.ANY, min_score=min_score, cache_folder=self.TEST_CACHE_FOLDER) + print("Test mapping to cached local ontology using Levenshtein similarity metric and min_score filter...") + df_leven = text2term.map_terms(search_terms, target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + mapper=Mapper.LEVENSHTEIN, term_type=OntologyTermType.ANY, min_score=min_score, + cache_folder=self.TEST_CACHE_FOLDER) assert (df_leven[self.MAPPING_SCORE_COLUMN] >= min_score).all() def test_mapping_with_min_score_filter_empty_results(self): - self.ensure_cache_exists("EFO", self.EFO_URL) - print("Test mapping to EFO using TFIDF similarity metric and min_score filter that results in no mappings...") - df_tfidf = text2term.map_terms(["carbon monoxide"], target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - term_type=OntologyTermType.ANY, min_score=0.99, cache_folder=self.TEST_CACHE_FOLDER) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + print("Test mapping to local ontology using TFIDF similarity metric and min_score filter that returns no mappings...") + df_tfidf = text2term.map_terms(["carbon monoxide"], target_ontology=self.TEST_ONTOLOGY_ACRONYM, + use_cache=True, mapper=Mapper.TFIDF, term_type=OntologyTermType.ANY, + min_score=0.99, cache_folder=self.TEST_CACHE_FOLDER) assert df_tfidf.empty is True def test_include_unmapped_terms(self): - self.ensure_cache_exists("EFO", self.EFO_URL) - df = text2term.map_terms(["asthma", "margarita"], target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - incl_unmapped=True, min_score=0.8, cache_folder=self.TEST_CACHE_FOLDER) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + df = text2term.map_terms(["cheese pizza", "margarita"], target_ontology=self.TEST_ONTOLOGY_ACRONYM, + use_cache=True, mapper=Mapper.TFIDF, incl_unmapped=True, min_score=0.8, + cache_folder=self.TEST_CACHE_FOLDER) assert df[self.TAGS_COLUMN].str.contains("unmapped").any() def test_exclude_metadata_from_output_file(self): - self.ensure_cache_exists("EFO", self.EFO_URL) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) test_output_file = "test_output_without_metadata.csv" - text2term.map_terms(["asthma"], target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - excl_metadata=True, save_mappings=True, output_file=test_output_file, cache_folder=self.TEST_CACHE_FOLDER) + text2term.map_terms(["cheese pizza"], target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + mapper=Mapper.TFIDF, excl_metadata=True, save_mappings=True, output_file=test_output_file, + cache_folder=self.TEST_CACHE_FOLDER) with open(test_output_file, 'r', encoding='utf-8') as f: first_line = f.readline() assert not first_line.startswith('#'), "CSV output file should not start with metadata header" def test_include_metadata_in_output_file(self): - self.ensure_cache_exists("EFO", self.EFO_URL) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) test_output_file = "test_output_with_metadata.csv" - text2term.map_terms(["asthma"], target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - excl_metadata=False, save_mappings=True, output_file=test_output_file, cache_folder=self.TEST_CACHE_FOLDER) + text2term.map_terms(["cheese pizza"], target_ontology=self.TEST_ONTOLOGY_ACRONYM, use_cache=True, + mapper=Mapper.TFIDF, excl_metadata=False, save_mappings=True, output_file=test_output_file, + cache_folder=self.TEST_CACHE_FOLDER) with open(test_output_file, 'r', encoding='utf-8') as f: first_line = f.readline() assert first_line.startswith('#'), "CSV output file should contain metadata header" def test_include_unmapped_terms_when_mappings_df_is_empty(self): - self.ensure_cache_exists("EFO", self.EFO_URL) - df = text2term.map_terms(["mojito", "margarita"], target_ontology="EFO", use_cache=True, mapper=Mapper.TFIDF, - incl_unmapped=True, min_score=0.8, cache_folder=self.TEST_CACHE_FOLDER) + self.ensure_cache_exists(self.TEST_ONTOLOGY_ACRONYM, self.TEST_ONTOLOGY_FILEPATH) + df = text2term.map_terms(["mojito", "margarita"], target_ontology=self.TEST_ONTOLOGY_ACRONYM, + use_cache=True, mapper=Mapper.TFIDF, incl_unmapped=True, min_score=0.8, + cache_folder=self.TEST_CACHE_FOLDER) assert df[self.TAGS_COLUMN].str.contains("unmapped").any() def drop_source_term_ids(self, df): @@ -429,58 +412,51 @@ class OntologyTermCollectorTestSuite(unittest.TestCase): def setUp(self): ontology_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_ontology.owl") self.collector = OntologyTermCollector(ontology_file, use_reasoning=True) - self.efo_url = "https://github.com/EBISPOT/efo/releases/download/v3.57.0/efo.owl" - self.efo_collector = OntologyTermCollector(ontology_iri=self.efo_url) + self.test_base_iri = "https://text2term.ontology/test/" def tearDown(self): self.collector.close() def test_collect_terms(self): - expected_nr_efo_terms = 50867 - terms = self.efo_collector.get_ontology_terms() - assert len(terms) == expected_nr_efo_terms + terms = self.collector.get_ontology_terms() + assert len(terms) == 14 def test_collect_classes_only(self): - expected_nr_efo_classes = 50643 - terms = self.efo_collector.get_ontology_terms(term_type=OntologyTermType.CLASS) - assert len(terms) == expected_nr_efo_classes + terms = self.collector.get_ontology_terms(term_type=OntologyTermType.CLASS) + assert len(terms) == 7 def test_collect_properties_only(self): - expected_nr_efo_properties = 224 - terms = self.efo_collector.get_ontology_terms(term_type=OntologyTermType.PROPERTY) - assert len(terms) == expected_nr_efo_properties + terms = self.collector.get_ontology_terms(term_type=OntologyTermType.PROPERTY) + assert len(terms) == 7 def test_collect_terms_limited_by_iri(self): - efo_base_iri = "http://www.ebi.ac.uk/efo/" - expected_nr_terms_with_efo_iri = 17382 - terms = self.efo_collector.get_ontology_terms(base_iris=[efo_base_iri], term_type=OntologyTermType.ANY) - assert len(terms) == expected_nr_terms_with_efo_iri + terms = self.collector.get_ontology_terms(base_iris=[self.test_base_iri], term_type=OntologyTermType.ANY) + assert len(terms) == 9 def test_collect_properties_limited_by_iri(self): - efo_base_iri = "http://www.ebi.ac.uk/efo/" - expected_nr_properties_with_efo_iri = 29 - terms = self.efo_collector.get_ontology_terms(base_iris=[efo_base_iri], term_type=OntologyTermType.PROPERTY) - assert len(terms) == expected_nr_properties_with_efo_iri + terms = self.collector.get_ontology_terms(base_iris=[self.test_base_iri], + term_type=OntologyTermType.PROPERTY) + assert len(terms) == 2 def test_collect_terms_excluding_deprecated(self): - terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=("https://text2term.ontology/",)) - self.assertTrue(len(terms) == 1) + terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=(self.test_base_iri,)) + self.assertTrue(len(terms) == 8) def test_collect_terms_including_broad_synonyms(self): - terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=("https://text2term.ontology/",), + terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=(self.test_base_iri,), include_broad_synonyms=True) self.assertTrue(any("Seafood Pizza" in term.synonyms for term in terms.values())) def test_collect_terms_including_narrow_synonyms(self): - terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=("https://text2term.ontology/",), + terms = self.collector.get_ontology_terms(exclude_deprecated=True, base_iris=(self.test_base_iri,), include_related_synonyms=True) self.assertTrue(any("Fish Pizza" in term.synonyms for term in terms.values())) def test_filter_terms(self): terms = self.collector.get_ontology_terms() - filtered = text2term.filter_terms(terms, iris="https://text2term.ontology/", excl_deprecated=False, + filtered = text2term.filter_terms(terms, iris=self.test_base_iri, excl_deprecated=False, term_type=OntologyTermType.CLASS) - self.assertTrue(len(filtered) == 2) + self.assertTrue(len(filtered) == 7) for term in filtered.values(): self.assertEqual(term.term_type, OntologyTermType.CLASS) @@ -496,7 +472,7 @@ def test_get_ontology_signature(self): def test_classify_ontology(self): self.collector._classify_ontology(self.collector.ontology) test_term = IRIS["https://text2term.ontology/test/VanillaIceCream"] - self.assertTrue(Nothing in test_term.equivalent_to) + self.assertTrue(test_term.deprecated) class OntoUtilsTestSuite(unittest.TestCase): diff --git a/text2term/onto_utils.py b/text2term/onto_utils.py index 61b147b..370af3c 100644 --- a/text2term/onto_utils.py +++ b/text2term/onto_utils.py @@ -1,10 +1,29 @@ +import re import sys import logging import pandas as pd import bioregistry import shortuuid from owlready2 import * -from gensim.parsing import strip_non_alphanum, strip_multiple_whitespaces + +_RE_NONALPHA = re.compile(r"\W", re.UNICODE) +_RE_WHITESPACE = re.compile(r"(\s)+", re.UNICODE) + + +def strip_non_alphanum(s): + """Replace non-alphanumeric characters with spaces. + + Equivalent to gensim.parsing.preprocessing.strip_non_alphanum. + """ + return _RE_NONALPHA.sub(" ", s) + + +def strip_multiple_whitespaces(s): + """Collapse repeating whitespace characters into a single space. + + Equivalent to gensim.parsing.preprocessing.strip_multiple_whitespaces. + """ + return _RE_WHITESPACE.sub(" ", s) BASE_IRI = "https://text2term.utils/" diff --git a/text2term/resources/ontologies.csv b/text2term/resources/ontologies.csv index 15cb725..bfb8932 100644 --- a/text2term/resources/ontologies.csv +++ b/text2term/resources/ontologies.csv @@ -1,7 +1,7 @@ acronym,version,url -CL,2023-09-21,https://github.com/obophenotype/cell-ontology/releases/download/v2023-09-21/cl.owl -EFO,3.57.0,https://github.com/EBISPOT/efo/releases/download/v3.57.0/efo.owl -FOODON,0.6.0,https://github.com/FoodOntology/foodon/raw/v0.6.0/foodon.owl -HPO,2023-09-01,https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2023-09-01/hp.owl -MONDO,2023-09-12,https://github.com/monarch-initiative/mondo/releases/download/v2023-08-02/mondo.owl -UBERON,2023-09-05,https://github.com/obophenotype/uberon/releases/download/v2023-09-05/uberon.owl \ No newline at end of file +CL,2025-12-17,https://github.com/obophenotype/cell-ontology/releases/download/v2025-12-17/cl.owl +EFO,3.86.0,https://github.com/EBISPOT/efo/releases/download/v3.86.0/efo.owl +FOODON,2025-07-31,https://github.com/FoodOntology/foodon/raw/v2025-07-31/foodon.owl +HPO,2026-01-08,https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2026-01-08/hp.owl +MONDO,2026-01-06,https://github.com/monarch-initiative/mondo/releases/download/v2026-01-06/mondo.owl +UBERON,2025-12-04,https://github.com/obophenotype/uberon/releases/download/v2025-12-04/uberon.owl diff --git a/text2term/term_collector.py b/text2term/term_collector.py index eb04605..2ba36fb 100644 --- a/text2term/term_collector.py +++ b/text2term/term_collector.py @@ -1,11 +1,17 @@ """Provides OntologyTermCollector class""" +import logging +import bioregistry from owlready2 import * + from text2term import onto_utils from text2term.term import OntologyTerm, OntologyTermType -import time -import logging -import bioregistry + +# Matches the IRI reported by owlready2's namespace.py _load_properties() when it +# encounters "punning" (the same IRI declared as both a property and a class/individual), +# e.g.: "'http://purl.obolibrary.org/obo/STATO_0000416' belongs to more than one entity +# types (cannot be both a property and a class/an individual)!" +_PUNNING_ERROR_IRI_PATTERN = re.compile(r"^'(.+?)' belongs to more than one entity types") class OntologyTermCollector: @@ -344,6 +350,30 @@ def _get_skos_definition(self, ontology_term): def _load_ontology(self, ontology_iri): """ Load the ontology at the specified IRI. + + Some ontologies (e.g., CL, which imports STATO/OBI) contain "punned" entities--IRIs + that are declared as both a property and a class/individual. Owlready2 enforces OWL DL + and refuses to load such ontologies: internally, Ontology.load() parses the RDF/XML + document into the world's quadstore, then calls self._load_properties(), and--only + afterwards--loads imported ontologies and runs any ontology-declared Python module hooks. + A punning TypeError raised inside _load_properties() would normally abort load() entirely, + skipping those later steps. + + To recover without losing those later steps, we temporarily wrap (monkey-patch) the + Ontology._load_properties method on the class itself for the duration of this call. The + wrapper calls the original implementation; if it raises the punning TypeError, the wrapper + strips the offending entity's triples directly from the quadstore and retries + _load_properties() in a loop (an ontology can have any number of punned terms, often pulled + in via different imports, and we keep going until they're all gone), then returns normally. + From load()'s perspective, _load_properties() simply succeeded, so it proceeds to load + imports etc. as usual. The patch is removed (restoring the original method) in a finally + block, regardless of outcome. + + The loop has no cap on how many punned entities it will strip. The only thing it guards against + is the same IRI being reported again right after being stripped, which would indicate the + strip didn't actually take effect (a bug, or an IRI that owlready2 keeps regenerating); that + case raises immediately rather than looping forever. + :param ontology_iri: IRI of the ontology (e.g., path of ontology document in the local file system, URL) :return: Ontology document """ @@ -352,8 +382,39 @@ def _load_ontology(self, ontology_iri): owl_link = bioregistry.get_owl_download(ontology_iri) if owl_link is not None: ontology_iri = owl_link - ontology = get_ontology(ontology_iri).load() + removed_iris = [] + + original_load_properties = Ontology._load_properties + + def _patched_load_properties(onto_self): + last_punned_iri = None + while True: + try: + return original_load_properties(onto_self) + except TypeError as err: + punned_iri = _extract_punning_iri(err) + if punned_iri is None: + raise # not a punning error we know how to recover from + if punned_iri == last_punned_iri: + self.logger.error("Stripping '%s' did not resolve its punning error; giving up " + "to avoid looping forever.", punned_iri) + raise + self.logger.warning("Ontology contains a punned entity (declared as both a property " + "and a class/individual): %s. Removing it and retrying.", punned_iri) + _strip_punned_entity(onto_self.world, punned_iri) + removed_iris.append(punned_iri) + last_punned_iri = punned_iri + + Ontology._load_properties = _patched_load_properties + try: + ontology = get_ontology(ontology_iri).load() + finally: + Ontology._load_properties = original_load_properties + end = time.time() + if removed_iris: + self.logger.warning("Removed %i punned entit%s prior to loading: %s", len(removed_iris), + "y" if len(removed_iris) == 1 else "ies", removed_iris) self._log_ontology_metrics(ontology) self.logger.info("...done (ontology loading time: %.2fs)", end - start) return ontology @@ -386,6 +447,65 @@ def _log_ontology_metrics(self, ontology): self.logger.debug(" Annotation property count: %i", len(list(ontology.annotation_properties()))) +def _extract_punning_iri(error): + """ + Parse the offending IRI out of the TypeError raised by owlready2's + namespace.py::_load_properties() when it detects punning. + :param error: the TypeError raised by owlready2 + :return: the punned entity's IRI, or None if the error message doesn't match the expected pattern + """ + match = _PUNNING_ERROR_IRI_PATTERN.match(str(error)) + return match.group(1) if match else None + + +def _strip_punned_entity(world, iri): + """ + Remove the given (punned) IRI from the ontology so that a subsequent retry of + _load_properties() no longer sees it as declared as more than one entity type. + + Tries two strategies, in order: + 1. destroy_entity() -- owlready2's public, documented API for removing an entity and all + its triples from the quadstore. This works here because by the time _load_properties() + raises, the class/individual-side triples for the punned IRI have already been parsed + and a Python entity object usually already exists for it (world[iri] / _get_by_storid); + it is only the *property*-side interpretation that fails to materialize. If such an + object exists, destroying it removes all of the IRI's triples (including the property + declaration), which is exactly what we need. + 2. Direct deletion from the SQLite-backed quadstore (world.graph) as a fallback, in case no + Python object could be resolved for the IRI. This reaches into internals that are less + API-stable across owlready2 versions, so it is only used if strategy 1 doesn't apply. + + :param world: owlready2 World (or ontology.world) whose quadstore should be modified + :param iri: IRI (string) of the punned entity to remove + """ + entity = world[iri] # World supports dict-like IRI lookup; returns None if no Python object exists yet + if entity is not None: + destroy_entity(entity) + return + + # Fallback: no Python-side object could be resolved (e.g. only a bare property triple exists). + # Delete the IRI's triples directly from the quadstore. + try: + storid = world._abbreviate(iri, False) + except AttributeError as err: + raise RuntimeError( + "Could not resolve internal storage ID for IRI '%s' while trying to recover from a " + "punning error (destroy_entity() also found no Python object for it). owlready2's " + "internal API may have changed; inspect world._abbreviate() / world.graph in your " + "installed version." % iri) from err + if storid is None: + # Entity not found in the store under this IRI; nothing to strip. + return + try: + world.graph.execute("DELETE FROM quads WHERE s=? OR o=?", (storid, storid)) + world.graph.db.commit() + except AttributeError as err: + raise RuntimeError( + "Could not delete triples for IRI '%s' while trying to recover from a punning error. " + "owlready2's internal quadstore API may have changed; inspect world.graph in your " + "installed version." % iri) from err + + def filter_terms(onto_terms, iris=(), excl_deprecated=False, term_type=OntologyTermType.ANY): filtered_onto_terms = {} for base_iri, term in onto_terms.items():