From 23525b7edc75d43ab53c86330e359360293533e6 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Tue, 19 May 2020 15:04:56 -0400 Subject: [PATCH 01/23] Importing number of folds constant from model --- medacy/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medacy/__main__.py b/medacy/__main__.py index 11c566d..dbe2e96 100644 --- a/medacy/__main__.py +++ b/medacy/__main__.py @@ -8,7 +8,7 @@ import logging from medacy.data.dataset import Dataset -from medacy.model.model import Model +from medacy.model.model import Model, DEFAULT_NUM_FOLDS from medacy.pipelines import bert_pipeline from medacy.tools.json_to_pipeline import json_to_pipeline @@ -145,7 +145,7 @@ def main(): # Cross Validation arguments parser_validate = subparsers.add_parser('validate', help='Cross validate a model on a given dataset.') - parser_validate.add_argument('-k', '--k_folds', default=5, type=int, help='Number of folds to use for cross-validation.') + parser_validate.add_argument('-k', '--k_folds', default=DEFAULT_NUM_FOLDS, type=int, help='Number of folds to use for cross-validation.') parser_validate.add_argument('-gt', '--groundtruth', type=str, default=None, help='Directory to write groundtruth files.') parser_validate.add_argument('-pd', '--predictions', type=str, default=None, help='Directory to write prediction files.') parser_validate.set_defaults(func=cross_validate) From bc1ffe7dbe277022d464dbd7b1163d902ae9b252 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Wed, 20 May 2020 12:41:39 -0400 Subject: [PATCH 02/23] Removed references to deprecated content --- guide/models/clinical_notes_model.md | 28 -------- guide/models/epa_systematic_review_model.md | 1 - guide/models/nanomedicine_drug_labels.md | 1 - guide/walkthrough/data_management.md | 52 ++------------ guide/walkthrough/model_training.md | 76 +-------------------- guide/walkthrough/model_utilization.md | 2 - 6 files changed, 7 insertions(+), 153 deletions(-) delete mode 100644 guide/models/clinical_notes_model.md delete mode 100644 guide/models/epa_systematic_review_model.md delete mode 100644 guide/models/nanomedicine_drug_labels.md diff --git a/guide/models/clinical_notes_model.md b/guide/models/clinical_notes_model.md deleted file mode 100644 index c4ca52d..0000000 --- a/guide/models/clinical_notes_model.md +++ /dev/null @@ -1,28 +0,0 @@ -# Clinical Notes Model -medaCy comes equipped with a powerful model for entity extraction from clinical notes. - -Two versions of this model are available for installation: - -| Type | Command | Description | -| :-------: | :----------------: |:-------------:| -|Light| `pip install git+https://github.com/NLPatVCU/medaCy_model_clinical_notes.git` | Does not contain a Metamap layer (drop of around 3% in micro F-measure)| -|Heavy| Not yet available for download | Contains full predictive pipeline| - - -# Generalization Ability -The model generalization results below are computed on the evaluation set provided by organizers of the N2C2 2018 Shared Tasks. This evaluation set contains 202 clinical records. - - ------- strict ------- ------ lenient ------- - Prec. Rec. F(b=1) Prec. Rec. F(b=1) - Drug 0.8576 0.7907 0.8228 0.9525 0.8680 0.9083 - Strength 0.8732 0.8418 0.8572 0.9772 0.9340 0.9552 - Duration 0.6555 0.3624 0.4668 0.9187 0.5079 0.6542 - Route 0.9259 0.8432 0.8826 0.9701 0.8782 0.9219 - Form 0.8473 0.7499 0.7957 0.9613 0.8435 0.8986 - Ade 0.1117 0.0352 0.0535 0.3990 0.1232 0.1883 - Dosage 0.7478 0.7057 0.7262 0.9152 0.8571 0.8852 - Reason 0.3428 0.2837 0.3105 0.5306 0.4263 0.4728 - Frequency 0.6767 0.6401 0.6579 0.9672 0.8886 0.9262 - ------------------------------------------------ - Overall (micro) 0.7905 0.7137 0.7502 0.9243 0.8235 0.8710 - Overall (macro) 0.7786 0.6981 0.7343 0.9152 0.8099 0.8569 diff --git a/guide/models/epa_systematic_review_model.md b/guide/models/epa_systematic_review_model.md deleted file mode 100644 index 8b13789..0000000 --- a/guide/models/epa_systematic_review_model.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/guide/models/nanomedicine_drug_labels.md b/guide/models/nanomedicine_drug_labels.md deleted file mode 100644 index 8b13789..0000000 --- a/guide/models/nanomedicine_drug_labels.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/guide/walkthrough/data_management.md b/guide/walkthrough/data_management.md index 82391e1..fcc3c5f 100644 --- a/guide/walkthrough/data_management.md +++ b/guide/walkthrough/data_management.md @@ -46,15 +46,13 @@ MedaCy **does not** alter the data you load in any way - it only reads from it. A common data work flow might look like this. -running: - -``` +```pythonstub >>> from medacy.data.datset import Dataset >>> from medacy.pipeline_components.feature_overlayers.metamap.metamap import MetaMap >>> dataset = Dataset('/home/medacy/data') >>> for data_file in dataset: -... data_file.file_name +... data_file.file_name 'file_one' 'file_two' >>> data @@ -63,7 +61,7 @@ running: False >>> metamap = Metamap('/home/path/to/metamap/binary') >>> with metamap: -... data.metamap(metamap) +... data.metamap(metamap) data.is_metamapped() True ``` @@ -81,47 +79,6 @@ home/medacy/data └── file_two.metamapped ``` - - -## Loading a medaCy compatible dataset -Using a *medaCy compatible dataset* package to manage your training data insures that data is easy and efficient to access, versioned for replicability, and distributable (selectively!). - -A *medaCy compatible dataset* is python package wrapping data that can be hooked into medaCy. We can install a *medaCy compatible dataset* just like any python package. For instance, - - -`pip install https://github.com/NanoNLP/medaCy_dataset_end/archive/v1.0.3.tar.gz#egg=medacy_dataset_end-1.0.3` - -will install `v1.0.03` of the [END](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5644562/) dataset. Alternatively, - -`pip install git+https://github.com/NanoNLP/medaCy_dataset_end.git` - -will install the latest version of the [END](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5644562/) dataset. - -After you have installed a *medaCy compatible dataset*, loading it returns a configured `Dataset` object alongside meta-data in a `tuple` as follows: - -```python -from medacy.data import Dataset - -training_dataset, evaluation_dataset, meta_data = Dataset.load_external('medacy_dataset_end') - -``` - -alternatively, import the datasets package and directly call the load method: - -```python -import medacy_dataset_end - -training_dataset, evaluation_dataset, meta_data = medacy_dataset_end.load() - -print(meta_data['entities']) #entities this dataset annotates -print(meta_data['relations']) #relations this dataset annotates (END has None) - -training_dataset = medacy_dataset_end.load_training_dataset() #access just training - -evaluation_dataset = medacy_dataset_end.load_evaluation_dataset() #access just evaluation - -``` - ## Using a Dataset A *Dataset* is utilized for two main tasks: @@ -139,7 +96,7 @@ from medacy.pipelines import FDANanoDrugLabelPipeline dataset = Dataset('/home/medacy/data') entities = ['Nanoparticle', 'Dose'] pipeline = FDANanoDrugLabelPipeline(entities=entities) -model = Model(pipeline, n_jobs=1) +model = Model(pipeline) model.fit(dataset) ``` @@ -172,4 +129,3 @@ By default, this creates a sub-directory in your prediction dataset named *predi ``` where all files under *predictions* are the trained models predictions over your test data. - diff --git a/guide/walkthrough/model_training.md b/guide/walkthrough/model_training.md index bfe512a..f6cf66f 100644 --- a/guide/walkthrough/model_training.md +++ b/guide/walkthrough/model_training.md @@ -64,15 +64,15 @@ The previously mentioned components make up a medaCy model. In summary training ```python import os -from medacy.data import Dataset +from medacy.data.dataset import Dataset from medacy.pipelines import ClinicalPipeline -from medacy.ner import Model +from medacy.model.model import Model entities = ['Drug', 'Strength'] training_dataset = Dataset('/home/medacy/clinical_training_data/') pipeline = ClinicalPipeline(metamap=None, entities=entities) -model = Model(pipeline, n_jobs=30) #distribute documents between 30 processes during training and prediction +model = Model(pipeline) output_file_path = '/home/medacy/clinical_model.pickle' # Protect against running fit() without having a valid place to save it @@ -81,74 +81,4 @@ assert os.path.isfile(output_file_path) model.fit(training_dataset) model.dump(output_file_path) - - -``` - -The `ClinicalPipeline` source looks like this: - -```python -import sklearn_crfsuite -import spacy - -from medacy.pipeline_components.feature_extractors.discrete_feature_extractor import FeatureExtractor -from medacy.pipeline_components.feature_overlayers.metamap.metamap import MetaMap -from medacy.pipeline_components.feature_overlayers.metamap.metamap_component import MetaMapOverlayer -from medacy.pipeline_components.tokenizers.clinical_tokenizer import ClinicalTokenizer -from medacy.pipelines.base.base_pipeline import BasePipeline - - -class ClinicalPipeline(BasePipeline): - """ - A pipeline for clinical named entity recognition. A special tokenizer that breaks down a clinical document - to character level tokens defines this pipeline. It was created for the extraction of ADE related entities - from the 2018 N2C2 Shared Task. - - Created by Andiy Mulyar (andriymulyar.com) of NLP@VCU - """ - - - def __init__(self, entities, metamap=None, **kwargs): - """ - Create a pipeline with the name 'clinical_pipeline' utilizing - by default spaCy's small english model. - - :param entities: a list of entities to use in this pipeline. - :param metamap: an instance of MetaMap if metamap should be used, defaults to None. - """ - - super().__init__(entities, spacy_pipeline=spacy.load("en_core_web_sm")) - - if isinstance(metamap, MetaMap): - self.add_component(MetaMapOverlayer, metamap) - - def get_learner(self): - return ("CRF_l2sgd", - sklearn_crfsuite.CRF( - algorithm='l2sgd', - c2=0.1, - max_iterations=100, - all_possible_transitions=True - ) - ) - - def get_tokenizer(self): - return ClinicalTokenizer(self.spacy_pipeline) - - def get_feature_extractor(self): - return FeatureExtractor(window_size=3, spacy_features=['pos_', 'shape_', 'prefix_', 'suffix_', 'text']) ``` - - -The `__init__` method defines pipeline meta-data along with initializing the sequence of components the pipeline will use to annotate custom token attributes over the document. Components are imported and initialized as part of the pipeline by calling the `add_component` method. The first paramater is a component and the subsequent parameters are any arguments that are passed to the component on initialization. Token attributes beginning with `feature_` are automically collected by the `FeatureExtractor` initialized in the `get_feature_extractor` method. Note the instantiation of the `FeatureExtractor` allows the definition of an array of `spacy_features` to utilize - these can be any attribute of a spaCy [Token](https://spacy.io/api/token#attributes). - -The `get_learner` method returns a configured instance of the machine learning algorithm to utilize for training a model. Currently only CRF models wrapped by the package [sklearn-crfsuite](https://sklearn-crfsuite.readthedocs.io/en/latest/) are allowed. - -The `get_tokenizer` method returns a configured medaCy tokenizer. An interface for building and maintaining a tokenizer is provided and the pattern from `ClinicalTokenizer` can be followed for engineering your own. - -The `get_feature_extractor` method returns a configured feature extractor. This defines how and what features from annotated documents are collected to be fed into the model during training or prediction. The example configuration means that all medaCy annotated features and the specified `spacy_features` are collected in a range of three tokens to the left and three tokens to the right of every token (ie. the `window_size`). - - - - - diff --git a/guide/walkthrough/model_utilization.md b/guide/walkthrough/model_utilization.md index ada88cb..9e7361a 100644 --- a/guide/walkthrough/model_utilization.md +++ b/guide/walkthrough/model_utilization.md @@ -31,11 +31,9 @@ One of medaCy's most powerful features is the ability to maintain, version and d Once a model has been [packaged](packaging_a_medacy_model.md) and installed it can be used as follows: ```python -import medacy_model_clinical_notes #import the python package wrapping the model from medacy.model.model import Model model = Model.load_external('medacy_model_clinical_notes') - annotations = model.predict("The patient took 5 mg of aspirin.") ``` From 7e3e5ba3f906d97efb14de761c87a785445cdebb Mon Sep 17 00:00:00 2001 From: swfansworth Date: Tue, 26 May 2020 21:41:25 -0400 Subject: [PATCH 03/23] Fixed error with paths inputed as strings not being parsed --- medacy/tools/json_to_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medacy/tools/json_to_pipeline.py b/medacy/tools/json_to_pipeline.py index aa635d7..3d93c4e 100644 --- a/medacy/tools/json_to_pipeline.py +++ b/medacy/tools/json_to_pipeline.py @@ -44,7 +44,7 @@ def json_to_pipeline(json_path): :return: a custom pipeline class """ - if isinstance(json_path, os.PathLike): + if isinstance(json_path, (str, os.PathLike)): with open(json_path, 'rb') as f: input_json = json.load(f) elif isinstance(json_path, dict): From d17625eac6e26b3974432d0b6eef98589859ece0 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Wed, 27 May 2020 13:28:32 -0400 Subject: [PATCH 04/23] CLI now prints command used and medaCy version number --- medacy/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/medacy/__main__.py b/medacy/__main__.py index dbe2e96..511278c 100644 --- a/medacy/__main__.py +++ b/medacy/__main__.py @@ -6,7 +6,9 @@ import importlib import json import logging +from sys import argv +from medacy import __version__ from medacy.data.dataset import Dataset from medacy.model.model import Model, DEFAULT_NUM_FOLDS from medacy.pipelines import bert_pipeline @@ -174,7 +176,8 @@ def main(): logger.addHandler(logging.StreamHandler()) if args.test_mode: logger.setLevel(logging.DEBUG) - logging.info("Test mode enabled: logging set to debug") + logging.info("Test mode enabled: logging set to debug") + logging.info(f"medaCy v{__version__}\nCommand: python -m medacy {' '.join(argv[1:])}") start_time = dt.datetime.now() start_timestamp = start_time.strftime('%Y-%m-%d %H:%M:%S') logging.info(f'\n\nSTART TIME: {start_timestamp}') From 75310cdc99802dc6699b009a9414788adc62d32e Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Wed, 27 May 2020 13:29:11 -0400 Subject: [PATCH 05/23] This will be part of a bugfix release --- medacy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medacy/__init__.py b/medacy/__init__.py index f2b236f..739de78 100644 --- a/medacy/__init__.py +++ b/medacy/__init__.py @@ -1,2 +1,2 @@ -__version__ = '1.0.0' +__version__ = '1.0.1' __authors__ = "Andriy Mulyar, Jorge Vargas, Corey Sutphin, Steele Farnsworth, Bobby Best, Bridget T. McInnes" From b1cd4e13b18b16ea8992853cfe8a3b8a4b4bb6c0 Mon Sep 17 00:00:00 2001 From: Daiki Katsuragawa <50144563+daikikatsuragawa@users.noreply.github.com> Date: Sun, 15 Nov 2020 21:10:18 +0900 Subject: [PATCH 06/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1919cff..9d926a9 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ To explore medaCy's other models or train your own, visit the [examples section] Reference ========= -``` +```bibtex @ARTICLE { author = "Andriy Mulyar, Natassja Lewinski and Bridget McInnes", title = "TAC SRIE 2018: Extracting Systematic Review Information with MedaCy", From a5643596ade5c45e2cacbd5325b34e99ad3879dd Mon Sep 17 00:00:00 2001 From: swfansworth Date: Fri, 11 Dec 2020 20:24:33 -0500 Subject: [PATCH 07/23] Restored Python3.6 support with a backport for dataclasses as a conditional dependency --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48b7bb4..3885e4e 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def run_tests(self): setup( name='medacy', version=__version__, - python_requires='>=3.7', + python_requires='>=3.6', license='GNU GENERAL PUBLIC LICENSE', description='Medical Natural Language Processing (NLP) with spaCy', long_description=readme(), @@ -72,6 +72,11 @@ def run_tests(self): 'gensim==3.8.0', 'en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz' ], + extras_require={ + ':python_version == "3.6"': [ + 'dataclasses', + ], + }, tests_require=[ "pytest", "pytest-cov", From 7693185ecf4c938260f6a1fdcbda2e2f2af035c3 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Mon, 12 Jul 2021 13:16:47 -0400 Subject: [PATCH 08/23] Emphasize CLI, a few other changes. --- README.md | 43 +++++++++++++------------------------------ 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9d926a9..3bf1d0d 100644 --- a/README.md +++ b/README.md @@ -14,47 +14,30 @@ workflow by providing utilities for model training, prediction and organization - Customizable pipelines with detailed development instructions and documentation. - Allows the designing of replicable NLP systems for reproducing results and encouraging the distribution of models whilst still allowing for privacy. - Active community development spearheaded and maintained by [NLP@VCU](https://nlp.cs.vcu.edu/). -- Detailed [API](https://medacy.readthedocs.io/en/latest/). - -## :thought_balloon: Where to ask questions - -MedaCy is actively maintained by a team of researchers at Virginia Commonwealth University. The best way to -receive immediate responses to any questions is to raise an issue. Make sure to first consult the -[API](https://medacy.readthedocs.io/en/latest/). See how to formulate a good issue or feature request in the [Contribution Guide](CONTRIBUTING.md). ## :computer: Installation Instructions -MedaCy can be installed for general use or for pipeline development / research purposes. +MedaCy supports Python >= 3.6 | Application | Run | | ----------- |:-------------:| | Prediction and Model Training (stable) | `pip install git+https://github.com/NLPatVCU/medaCy.git` | | Prediction and Model Training (latest) | `pip install git+https://github.com/NLPatVCU/medaCy.git@development` | -| Pipeline Development and Contribution | [See Contribution Instructions](/CONTRIBUTING.md) | - -# :books: Power of medaCy -After installing medaCy and [medaCy's clinical model](guide/models/clinical_notes_model.md), simply run: +# :books: How to use medaCy -```python -from medacy.model.model import Model +MedaCy's components can be imported into other Python programs, but is designed primarily to be used via its command line interface. +Once medaCy is installed, one can read the instructions at any time with this command. -model = Model.load_external('medacy_model_clinical_notes') -annotation = model.predict("The patient was prescribed 1 capsule of Advil for 5 days.") -print(annotation) -``` -and receive instant predictions: -```python -[ - ('Drug', 40, 45, 'Advil'), - ('Dosage', 27, 28, '1'), - ('Form', 29, 36, 'capsule'), - ('Duration', 46, 56, 'for 5 days') -] +```bash +python -m medacy --help ``` -MedaCy can also be used through its command line interface, documented [here](./guide/command_line_interface.md) +More thorough documentation is provided [here](./guide/command_line_interface.md). + +## :thought_balloon: Where to Ask Questions -To explore medaCy's other models or train your own, visit the [examples section](guide). +MedaCy is actively maintained by a team of researchers at Virginia Commonwealth University. The best way to +receive immediate responses to any questions is to open an issue in this repository. Reference ========= @@ -74,9 +57,9 @@ This package is licensed under the GNU General Public License. Authors ======= -Current contributors: Steele Farnsworth, Anna Conte, Gabby Gurdin, Aidan Kierans, Aidan Myers, and Bridget T. McInnes +Current contributors: Steele Farnsworth, Gabby Gurdin, Aidan Myers, and Bridget T. McInnes -Former contributors: Andriy Mulyar, Jorge Vargas, Corey Sutphin, and Bobby Best +Former contributors: Andriy Mulyar, Jorge Vargas, Corey Sutphin, Bobby Best, Anna Conte, and Aidan Kierans Acknowledgments =============== From aa223d7f21a93033dd16109d3dd538365ed74d9a Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:20:23 -0400 Subject: [PATCH 09/23] Optimization: use numpy functionality rather than a nested loop. --- .../nn/bert_crf_for_token_classification.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/medacy/pipeline_components/learners/nn/bert_crf_for_token_classification.py b/medacy/pipeline_components/learners/nn/bert_crf_for_token_classification.py index 67085ca..1d41ae4 100644 --- a/medacy/pipeline_components/learners/nn/bert_crf_for_token_classification.py +++ b/medacy/pipeline_components/learners/nn/bert_crf_for_token_classification.py @@ -4,6 +4,7 @@ from transformers import BertForTokenClassification from torchcrf import CRF + class BertCrfForTokenClassification(BertForTokenClassification): """Subclass of Transformers package BERT token classifier. @@ -42,19 +43,13 @@ def forward( labels=labels ) - # If labels were given, we'll want to use them to train the CRF layer as well - if labels is not None: - # Note that this mutates labels. This is only okay because we don't use reuse labels in - # the learner. If this ever changes you can fix this by using labels.clone() - for i in range(labels.shape[0]): - for j in range(labels.shape[1]): - if labels[i][j] == self.crf.num_tags: - # Change 'X' label to 'O' so crf doesn't try to access wrong index - # Using a mask does not fix this. - labels[i][j] = 0 + if labels is None: + return outputs - # After 'X' labels have been removed, pass the emission scores through the CRF layer - outputs = (-self.crf(emissions=outputs[1], tags=labels), outputs[1]) + # If labels were given, we'll want to use them to train the CRF layer as well + labels = labels.clone() + labels[labels == self.crf.num_tags] = 0 + # After 'X' labels have been removed, pass the emission scores through the CRF layer # Returns CRF output if there were labels or original emission scores otherwise - return outputs + return -self.crf(emissions=outputs[1], tags=labels), outputs[1] From 458e513799e296701db34959773c80338de319bb Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:20:52 -0400 Subject: [PATCH 10/23] Fix typo. --- medacy/pipeline_components/learners/bert_learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medacy/pipeline_components/learners/bert_learner.py b/medacy/pipeline_components/learners/bert_learner.py index 3f0a3ad..44f9df3 100644 --- a/medacy/pipeline_components/learners/bert_learner.py +++ b/medacy/pipeline_components/learners/bert_learner.py @@ -137,7 +137,7 @@ def decode_labels(self, sequence_labels, mappings): if map_value != null_label: remapped_labels.append(label) - # Decode list of label indices useing self.vectorizer + # Decode list of label indices using self.vectorizer decoded_labels.append(self.vectorizer.devectorize_tag(remapped_labels)) return decoded_labels From 439340abb8d2fcba15b3dbd81ac8867fb43a29f1 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:21:35 -0400 Subject: [PATCH 11/23] Use fstrings and `statistics.mean`. --- medacy/pipeline_components/learners/bilstm_crf_learner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medacy/pipeline_components/learners/bilstm_crf_learner.py b/medacy/pipeline_components/learners/bilstm_crf_learner.py index c55e61f..4a07251 100644 --- a/medacy/pipeline_components/learners/bilstm_crf_learner.py +++ b/medacy/pipeline_components/learners/bilstm_crf_learner.py @@ -3,6 +3,7 @@ """ import logging import random +from statistics import mean import torch import torch.nn as nn @@ -103,8 +104,7 @@ def fit(self, x_data, y_data): optimizer.step() epoch_losses.append(loss) - average_loss = sum(epoch_losses) / len(epoch_losses) - logging.info('Epoch %d average loss: %f', i, average_loss) + logging.info(f'Epoch {i} average loss: {mean(epoch_losses)}') self.model = model From 0ed1b6a926cf8fc983bc8ad6b4d79a80553afcb3 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:28:12 -0400 Subject: [PATCH 12/23] Refactoring. --- .../learners/nn/bilstm_crf.py | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/medacy/pipeline_components/learners/nn/bilstm_crf.py b/medacy/pipeline_components/learners/nn/bilstm_crf.py index f524e55..e281d24 100644 --- a/medacy/pipeline_components/learners/nn/bilstm_crf.py +++ b/medacy/pipeline_components/learners/nn/bilstm_crf.py @@ -12,6 +12,7 @@ CHARACTER_HIDDEN_DIM = 100 CHARACTER_EMBEDDING_SIZE = 100 + class BiLstmCrf(nn.Module): """ BiLSTM and CRF pytorch layers. @@ -46,13 +47,13 @@ def __init__(self, word_vectors, other_features, tagset_size, device): # Setup word embedding layer self.word_embeddings = nn.Embedding.from_pretrained(word_vectors) - # The LSTM takes word embeddings concatenated with character verctors as inputs and + # The LSTM takes word embeddings concatenated with character vectors as inputs and # outputs hidden states with dimensionality hidden_dim. - lstm_input_size = vector_size + CHARACTER_HIDDEN_DIM*2 + lstm_input_size = vector_size + (CHARACTER_HIDDEN_DIM * 2) self.lstm = nn.LSTM(lstm_input_size, HIDDEN_DIM, bidirectional=True) # The linear layer that maps from hidden state space to tag space - linear_input_size = HIDDEN_DIM*2 + other_features + linear_input_size = (HIDDEN_DIM * 2) + other_features self.hidden2tag = nn.Linear(linear_input_size, self.tagset_size) self.crf = CRF(self.tagset_size) @@ -65,19 +66,17 @@ def _get_character_features(self, sentence): """ # Separate and pad character indices into a batch - longest_token_length = max([len(token[1]) for token in sentence]) + longest_token_length = max(len(token[1]) for token in sentence) character_indices = [] for token in sentence: - indices = [character for character in token[1]] + indices = list(token[1]) if len(indices) < longest_token_length: padding = longest_token_length - len(indices) indices += [0] * padding character_indices.append(indices) character_indices = torch.tensor(character_indices, device=self.device) - features = self.character_lstm(character_indices) - - return features + return self.character_lstm(character_indices) def _get_lstm_features(self, sentence): """Get BiLSTM features from a list of tokens @@ -86,32 +85,25 @@ def _get_lstm_features(self, sentence): :return: Output from BiLSTM. """ # Create tensor of word embeddings - - embedding_indices = [token[0] for token in sentence] - embedding_indices = torch.tensor(embedding_indices, device=self.device) + embedding_indices = torch.tensor([token[0] for token in sentence], device=self.device) word_embeddings = self.word_embeddings(embedding_indices) character_vectors = self._get_character_features(sentence) # Turn rest of features into a tensor - other_features = [token[2:] for token in sentence] - other_features = torch.tensor(other_features, device=self.device) + other_features = torch.tensor([token[2:] for token in sentence], device=self.device) # Combine into one final input vector for LSTM token_vector = torch.cat((word_embeddings, character_vectors), 1) # Reshape because LSTM requires input of shape (seq_len, batch, input_size) token_vector = token_vector.view(len(sentence), 1, -1) - # token_vector = self.dropout(token_vector) lstm_out, _ = self.lstm(token_vector) - lstm_out = lstm_out.view(len(sentence), HIDDEN_DIM*2) + lstm_out = lstm_out.view(len(sentence), HIDDEN_DIM * 2) lstm_out = torch.cat((lstm_out, other_features), 1) - lstm_features = self.hidden2tag(lstm_out) - - return lstm_features + return self.hidden2tag(lstm_out) def forward(self, sentence): - lstm_features = self._get_lstm_features(sentence) - return lstm_features + return self._get_lstm_features(sentence) From 85189cc17f4c56874f9a43274fedff49ecb31a91 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:28:43 -0400 Subject: [PATCH 13/23] Make tests more granular (one for CRF, one without). --- .../pipeline_components/learners/test_bert.py | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/medacy/tests/pipeline_components/learners/test_bert.py b/medacy/tests/pipeline_components/learners/test_bert.py index febeda4..2774109 100644 --- a/medacy/tests/pipeline_components/learners/test_bert.py +++ b/medacy/tests/pipeline_components/learners/test_bert.py @@ -31,34 +31,41 @@ def tearDownClass(cls): pkg_resources.cleanup_resources() shutil.rmtree(cls.prediction_directory) + def _bert_test(self, pipe): + model = Model(pipe) + model.cross_validate(self.dataset, 2) + model.fit(self.dataset) + resulting_dataset = model.predict(self.dataset, prediction_directory=self.prediction_directory) + self.assertIsInstance(resulting_dataset, Dataset) + # Test that there is at least one prediction + if not any(resulting_dataset.generate_annotations()): + warn("The model did not generate any predictions") + @unittest.skipUnless(use_cuda, "This test only runs if a cuda device is set in the medaCy config file") - def test_cross_validate_fit_predict(self): + def test_cross_validate_fit_predict_no_crf(self): """Tests that a model created with BERT can be fitted and used to predict, with and without the CRF layer""" - pipeline = BertPipeline( - entities=self.entities, - pretrained_model='bert-base-cased', - batch_size=self.batch_size, - cuda_device=cuda_device - ) + self._bert_test( + BertPipeline( + entities=self.entities, + pretrained_model='bert-base-cased', + batch_size=self.batch_size, + cuda_device=cuda_device + ) + ) - pipeline_crf = BertPipeline( - entities=self.entities, - pretrained_model='bert-base-cased', - batch_size=self.batch_size, - cuda_device=cuda_device, - using_crf=True + @unittest.skipUnless(use_cuda, "This test only runs if a cuda device is set in the medaCy config file") + def test_cross_validate_fit_predict_with_crf(self): + """Tests that a model created with BERT can be fitted and used to predict, with and without the CRF layer""" + self._bert_test( + BertPipeline( + entities=self.entities, + pretrained_model='bert-base-cased', + batch_size=self.batch_size, + cuda_device=cuda_device, + using_crf=True + ) ) - for pipe in [pipeline, pipeline_crf]: - model = Model(pipe) - model.cross_validate(self.dataset, 2) - model.fit(self.dataset) - resulting_dataset = model.predict(self.dataset, prediction_directory=self.prediction_directory) - self.assertIsInstance(resulting_dataset, Dataset) - # Test that there is at least one prediction - if not any(resulting_dataset.generate_annotations()): - warn("The model did not generate any predictions") - if __name__ == '__main__': unittest.main() From f472831e6bd45a118442f7ee22b00c5946a0713b Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:29:29 -0400 Subject: [PATCH 14/23] Pull static methods out into nonpublic functions. --- .../learners/nn/vectorizer.py | 136 +++++++++--------- 1 file changed, 64 insertions(+), 72 deletions(-) diff --git a/medacy/pipeline_components/learners/nn/vectorizer.py b/medacy/pipeline_components/learners/nn/vectorizer.py index 7a95637..01ea176 100644 --- a/medacy/pipeline_components/learners/nn/vectorizer.py +++ b/medacy/pipeline_components/learners/nn/vectorizer.py @@ -9,6 +9,58 @@ from gensim.models import KeyedVectors +def _unicode_to_ascii(unicode_string): + """Convert unicode string to closest ASCII equivalent. Based on code found at: + https://stackoverflow.com/a/518232/2809427 + + :param unicode_string: String to convert to ASCII + :return: String with every character converted to most similar ASCII character. + """ + unicode_string = re.sub(u"\u2013", "-", unicode_string) # em dash + + return ''.join( + character for character in unicodedata.normalize('NFD', unicode_string) + if unicodedata.category(character) != 'Mn' + and character in string.printable + ) + + +def _create_feature_dictionary(feature_name, sentences): + """Get dictionary that maps all possible values of a specific feature to ids. + + :param feature_name: Name of feature. + :param sentences: Sentences to get feature for. + :return: Dictionary for given feature. + """ + feature_to_index = {} + feature_name = '0:' + feature_name + + for sentence in sentences: + for token in sentence: + feature = token[feature_name] + if feature not in feature_to_index: + feature_to_index[feature] = len(feature_to_index) + + return feature_to_index + + +def _one_hot(index_dictionary, value): + """ + Create a one-hot vector representation for discrete features that appear in the X_data + :param index_dictionary: A dictionary mapping discrete features to unique integers (ie the order + they appeared in the X_data; see self.create_feature_dictionary) + :param value: The discrete feature + :return: A one-hot vector for that discrete feature + """ + vector = [0.0] * len(index_dictionary) + + if value in index_dictionary: + index = index_dictionary[value] + vector[index] = 1.0 + + return vector + + class Vectorizer: """Vectorizer for medaCy PyTorch data. Contains encoding methods and tracking encoding values. @@ -41,9 +93,10 @@ def load_word_embeddings(self, embeddings_file): :param embeddings_file: Word embeddings file to use. Can be .bin or other common formats. """ - is_binary = embeddings_file.endswith('.bin') - word_vectors = KeyedVectors.load_word2vec_format(embeddings_file, binary=is_binary) - self.word_vectors = word_vectors + self.word_vectors = KeyedVectors.load_word2vec_format( + embeddings_file, + binary=embeddings_file.endswith('.bin') + ) def create_tag_dictionary(self, tags): """Setup self.tag_to_index @@ -66,24 +119,6 @@ def add_tag(self, tag): """ self.tag_to_index[tag] = len(self.tag_to_index) - def create_feature_dictionary(self, feature_name, sentences): - """Get dictionary that maps all possible values of a specific feature to ids. - - :param feature_name: Name of feature. - :param sentences: Sentences to get feature for. - :return: Dictionary for given feature. - """ - feature_to_index = {} - feature_name = '0:' + feature_name - - for sentence in sentences: - for token in sentence: - feature = token[feature_name] - if feature not in feature_to_index: - feature_to_index[feature] = len(feature_to_index) - - return feature_to_index - def find_other_features(self, example): """Get the names of the other word features being used. @@ -112,8 +147,7 @@ def find_window_size(self, x_data): longest_length = len(sentence) test_token = sentence[int(longest_length/2)] - lowest = 0 - highest = 0 + highest, lowest = 0, 0 # Loop through keys in test token to find highest and lowest window distances. for key in test_token: @@ -130,30 +164,14 @@ def find_window_size(self, x_data): self.window_size = highest - def unicode_to_ascii(self, unicode_string): - """Convert unicode string to closest ASCII equivalent. Based on code found at: - https://stackoverflow.com/a/518232/2809427 - - :param unicode_string: String to convert to ASCII - :return: String with every character converted to most similar ASCII character. - """ - unicode_string = re.sub(u"\u2013", "-", unicode_string) # em dash - - return ''.join( - character for character in unicodedata.normalize('NFD', unicode_string) - if unicodedata.category(character) != 'Mn' - and character in string.printable - ) - def devectorize_tag(self, tag_indices): """Devectorize a list of tag indices using self.tag_to_index :param tag_indices: List of tag indices. :return: List of tags. """ - to_tag = {y:x for x, y in self.tag_to_index.items()} - tags = [to_tag[index] for index in tag_indices] - return tags + to_tag = {y: x for x, y in self.tag_to_index.items()} + return [to_tag[index] for index in tag_indices] def find_window_indices(self, token): """Get relative indices of window words. Avoids trying to access keys that don't exist. @@ -161,32 +179,7 @@ def find_window_indices(self, token): :param token: Token the indexes are relative to. :return: List of indices """ - window = [] - window_range = range(-self.window_size, self.window_size + 1) - - for i in window_range: - test_key = 'text' - test_key = '%d:%s' % (i, test_key) - if test_key in token: - window.append(i) - - return window - - def one_hot(self, index_dictionary, value): - """ - Create a one-hot vector representation for discrete features that appear in the X_data - :param index_dictionary: A dictionary mapping discrete features to unique integers (ie the order - they appeared in the X_data; see self.create_feature_dictionary) - :param value: The discrete feature - :return: A one-hot vector for that discrete feature - """ - vector = [0.0] * len(index_dictionary) - - if value in index_dictionary: - index = index_dictionary[value] - vector[index] = 1.0 - - return vector + return [i for i in range(-self.window_size, self.window_size + 1) if f'{i}:text' in token] def vectorize_tokens(self, tokens): """Vectorize list of tokens. @@ -201,7 +194,7 @@ def vectorize_tokens(self, tokens): # Add text index for looking up word embedding token_text = token['0:text'] - token_text = self.unicode_to_ascii(token_text) + token_text = _unicode_to_ascii(token_text) # Look up word embedding index try: @@ -239,7 +232,7 @@ def vectorize_tokens(self, tokens): for feature_name in other_feature_names: key = '%d:%s' % (i, feature_name) feature = token[key] - vector = self.one_hot(self.other_features[feature_name], feature) + vector = _one_hot(self.other_features[feature_name], feature) token_vector.extend(vector) else: for feature in self.other_features: @@ -276,7 +269,7 @@ def vectorize_dataset(self, x_data, y_data): # Create feature dictionaries for feature in self.other_features: - self.other_features[feature] = self.create_feature_dictionary(feature, x_data) + self.other_features[feature] = _create_feature_dictionary(feature, x_data) # Vectorize data sentences = [] @@ -286,9 +279,8 @@ def vectorize_dataset(self, x_data, y_data): correct_tags_vector = self.vectorize_tags(sentence_tags) sentences.append(tokens_vector) correct_tags.append(correct_tags_vector) - data = list(zip(sentences, correct_tags)) - return data + return list(zip(sentences, correct_tags)) def get_values(self): """Get Vectorizer values so they can saved or migrated. From 55233a9829d674222847eda513a286e3c84bb705 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:54:50 -0400 Subject: [PATCH 15/23] Convert `loss` to a float (it was a Tensor with one element). --- medacy/pipeline_components/learners/bilstm_crf_learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medacy/pipeline_components/learners/bilstm_crf_learner.py b/medacy/pipeline_components/learners/bilstm_crf_learner.py index 4a07251..a762419 100644 --- a/medacy/pipeline_components/learners/bilstm_crf_learner.py +++ b/medacy/pipeline_components/learners/bilstm_crf_learner.py @@ -102,7 +102,7 @@ def fit(self, x_data, y_data): optimizer.zero_grad() loss.backward() optimizer.step() - epoch_losses.append(loss) + epoch_losses.append(float(loss)) logging.info(f'Epoch {i} average loss: {mean(epoch_losses)}') From c7773edadb438901979012e9d0f570b089c25b18 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 14:55:23 -0400 Subject: [PATCH 16/23] Migrate to pytest. --- .../pipeline_components/learners/test_bert.py | 83 +++++++------------ 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/medacy/tests/pipeline_components/learners/test_bert.py b/medacy/tests/pipeline_components/learners/test_bert.py index 2774109..045c349 100644 --- a/medacy/tests/pipeline_components/learners/test_bert.py +++ b/medacy/tests/pipeline_components/learners/test_bert.py @@ -1,71 +1,46 @@ import os -import shutil -import tempfile -import unittest from warnings import warn -import pkg_resources +import pytest from medacy.data.dataset import Dataset from medacy.model.model import Model from medacy.pipelines.bert_pipeline import BertPipeline +from medacy.tests.pipeline_components.learners import cuda_device from medacy.tests.sample_data import test_dir -from medacy.tests.pipeline_components.learners import use_cuda, cuda_device +BATCH_SIZE = 3 -class TestBert(unittest.TestCase): - """ - Tests for medacy.pipeline_components.learners.bert_learner.BertLearner - and, by extension, medacy.pipelines.bert_pipeline.BertPipeline - """ - @classmethod - def setUpClass(cls): - cls.dataset = Dataset(os.path.join(test_dir, 'sample_dataset_1'), data_limit=1) - cls.entities = cls.dataset.get_labels(as_list=True) - cls.prediction_directory = tempfile.mkdtemp() # Directory to store predictions - cls.batch_size = 3 +@pytest.fixture +def dataset(): + return Dataset(os.path.join(test_dir, 'sample_dataset_1'), data_limit=1) - @classmethod - def tearDownClass(cls): - pkg_resources.cleanup_resources() - shutil.rmtree(cls.prediction_directory) - def _bert_test(self, pipe): - model = Model(pipe) - model.cross_validate(self.dataset, 2) - model.fit(self.dataset) - resulting_dataset = model.predict(self.dataset, prediction_directory=self.prediction_directory) - self.assertIsInstance(resulting_dataset, Dataset) - # Test that there is at least one prediction - if not any(resulting_dataset.generate_annotations()): - warn("The model did not generate any predictions") +@pytest.fixture +def prediction_directory(tmp_path): + directory = tmp_path / 'preds' + directory.mkdir() + return directory - @unittest.skipUnless(use_cuda, "This test only runs if a cuda device is set in the medaCy config file") - def test_cross_validate_fit_predict_no_crf(self): - """Tests that a model created with BERT can be fitted and used to predict, with and without the CRF layer""" - self._bert_test( - BertPipeline( - entities=self.entities, - pretrained_model='bert-base-cased', - batch_size=self.batch_size, - cuda_device=cuda_device - ) - ) - @unittest.skipUnless(use_cuda, "This test only runs if a cuda device is set in the medaCy config file") - def test_cross_validate_fit_predict_with_crf(self): - """Tests that a model created with BERT can be fitted and used to predict, with and without the CRF layer""" - self._bert_test( - BertPipeline( - entities=self.entities, - pretrained_model='bert-base-cased', - batch_size=self.batch_size, - cuda_device=cuda_device, - using_crf=True - ) - ) +@pytest.mark.parametrize('use_crf', [True, False]) +@pytest.mark.slow +def test_bert(dataset, prediction_directory, use_crf): + pipe = BertPipeline( + entities=dataset.get_labels(as_list=True), + pretrained_model='bert-base-cased', + batch_size=BATCH_SIZE, + cuda_device=cuda_device, + using_crf=use_crf + ) + model = Model(pipe) + model.cross_validate(dataset, 2) + model.fit(dataset) + resulting_dataset = model.predict(dataset, prediction_directory=prediction_directory) + assert isinstance(resulting_dataset, Dataset) -if __name__ == '__main__': - unittest.main() + # Check that there is at least one prediction + if not any(resulting_dataset.generate_annotations()): + warn("The model did not generate any predictions") From cbde43c539cc36f4f6d2e64820a7ed5605f41ecc Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 15:30:55 -0400 Subject: [PATCH 17/23] Put fixtures in their own module. --- .../pipeline_components/learners/_fixtures.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 medacy/tests/pipeline_components/learners/_fixtures.py diff --git a/medacy/tests/pipeline_components/learners/_fixtures.py b/medacy/tests/pipeline_components/learners/_fixtures.py new file mode 100644 index 0000000..a4d8ef6 --- /dev/null +++ b/medacy/tests/pipeline_components/learners/_fixtures.py @@ -0,0 +1,18 @@ +import os + +import pytest + +from medacy.data.dataset import Dataset +from medacy.tests.sample_data import test_dir + + +@pytest.fixture +def dataset(): + return Dataset(os.path.join(test_dir, 'sample_dataset_1'), data_limit=1) + + +@pytest.fixture +def prediction_directory(tmp_path): + directory = tmp_path / 'preds' + directory.mkdir() + return directory From a3e5808a03b0e3c7e604195d3117b7b43ff2bb14 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 15:31:26 -0400 Subject: [PATCH 18/23] Create pytest.ini file for marking tests as slow. --- pytest.ini | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6df308d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + slow: marks tests as slow (deselect with '-m "not slow"') + serial From b4921500e5cc3972dea7adcea9fb526c0b58e9f5 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 15:32:27 -0400 Subject: [PATCH 19/23] Import fixtures from other module. Need to look into how to modularize tests with pytest, as I assume this system is not idiomatic. --- .../pipeline_components/learners/test_bert.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/medacy/tests/pipeline_components/learners/test_bert.py b/medacy/tests/pipeline_components/learners/test_bert.py index 045c349..70f2fda 100644 --- a/medacy/tests/pipeline_components/learners/test_bert.py +++ b/medacy/tests/pipeline_components/learners/test_bert.py @@ -1,4 +1,3 @@ -import os from warnings import warn import pytest @@ -6,22 +5,13 @@ from medacy.data.dataset import Dataset from medacy.model.model import Model from medacy.pipelines.bert_pipeline import BertPipeline +from medacy.tests.pipeline_components.learners import _fixtures from medacy.tests.pipeline_components.learners import cuda_device -from medacy.tests.sample_data import test_dir BATCH_SIZE = 3 - -@pytest.fixture -def dataset(): - return Dataset(os.path.join(test_dir, 'sample_dataset_1'), data_limit=1) - - -@pytest.fixture -def prediction_directory(tmp_path): - directory = tmp_path / 'preds' - directory.mkdir() - return directory +prediction_directory = _fixtures.prediction_directory +dataset = _fixtures.dataset @pytest.mark.parametrize('use_crf', [True, False]) From 1a59a28b1b7f91f3894c451e757293afcdf8174f Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 15:32:47 -0400 Subject: [PATCH 20/23] Migrate to pytest. --- .../learners/test_bilstm_crf.py | 60 +++++++------------ 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/medacy/tests/pipeline_components/learners/test_bilstm_crf.py b/medacy/tests/pipeline_components/learners/test_bilstm_crf.py index 648002e..ff66703 100644 --- a/medacy/tests/pipeline_components/learners/test_bilstm_crf.py +++ b/medacy/tests/pipeline_components/learners/test_bilstm_crf.py @@ -1,50 +1,34 @@ -import os -import shutil -import tempfile -import unittest from warnings import warn -import pkg_resources +import pytest from medacy.data.dataset import Dataset from medacy.model.model import Model from medacy.pipelines.lstm_systematic_review_pipeline import LstmSystematicReviewPipeline -from medacy.tests.sample_data import test_dir -from medacy.tests.pipeline_components.learners import use_cuda, cuda_device, word_embeddings +from medacy.tests.pipeline_components.learners import _fixtures +from medacy.tests.pipeline_components.learners import cuda_device +from medacy.tests.pipeline_components.learners import use_cuda, word_embeddings +BATCH_SIZE = 3 -class TestBiLstmCrf(unittest.TestCase): - """Tests for the BiLSTM+CRF""" +prediction_directory = _fixtures.prediction_directory +dataset = _fixtures.dataset - @classmethod - def setUpClass(cls): - cls.dataset = Dataset(os.path.join(test_dir, 'sample_dataset_1')) - cls.entities = cls.dataset.get_labels(as_list=True) - cls.prediction_directory = tempfile.mkdtemp() # Directory to store predictions - @classmethod - def tearDownClass(cls): - pkg_resources.cleanup_resources() - shutil.rmtree(cls.prediction_directory) +@pytest.mark.skipif(not (use_cuda and word_embeddings), reason='This test requires a cuda device and word embeddings to be set in the medaCy config file') +@pytest.mark.slow +def test_prediction(dataset, prediction_directory): + pipeline = LstmSystematicReviewPipeline( + entities=dataset.get_labels(as_list=True), + word_embeddings=word_embeddings, + cuda_device=cuda_device + ) - @unittest.skipUnless(use_cuda and word_embeddings, "This test only runs if a cuda device and word embeddings are set in the medaCy config file") - def test_prediction_with_testing_pipeline(self): - """Tests that a model created with the BiLSTM+CRF can be fitted and used to predict""" - pipeline = LstmSystematicReviewPipeline( - entities=self.entities, - word_embeddings=word_embeddings, - cuda_device=cuda_device - ) - - model = Model(pipeline) - model.fit(self.dataset) - resulting_dataset = model.predict(self.dataset, prediction_directory=self.prediction_directory) - self.assertIsInstance(resulting_dataset, Dataset) - # Test that there is at least one prediction - if not any(resulting_dataset.generate_annotations()): - warn("The model did not generate any predictions") - - -if __name__ == '__main__': - unittest.main() + model = Model(pipeline) + model.fit(dataset) + resulting_dataset = model.predict(dataset, prediction_directory=prediction_directory) + assert isinstance(resulting_dataset, Dataset) + # Check that there is at least one prediction + if not any(resulting_dataset.generate_annotations()): + warn("The model did not generate any predictions") From e2e422f6505aa962d07f6c23af38b4aaca5395fc Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 18 Jul 2021 15:36:42 -0400 Subject: [PATCH 21/23] Install pytest and skip slow tests. --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 162f04a..c007529 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,9 @@ dist: trusty group: edge python: - - "3.7" - - "3.8" + - "3.6" + - "3.7" + - "3.8" os: - linux @@ -16,7 +17,7 @@ os: install: - pip install --upgrade pip - pip install -e . + - pip install pytest script: - - "python setup.py test || exit 0" - - "python setup.py test" + - "python -m pytest ./medacy/tests/ -m 'not slow'" From 1a255bbba152219596e7f8c265aa384ce1dbcd63 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sat, 14 Aug 2021 19:38:16 -0400 Subject: [PATCH 22/23] General refactoring. Use list comprehensions where appropriate; uncouple a `zip` instance that could have been two list comps; flatten nested logic where possible. --- .../learners/nn/vectorizer.py | 57 +++++-------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/medacy/pipeline_components/learners/nn/vectorizer.py b/medacy/pipeline_components/learners/nn/vectorizer.py index 01ea176..873a244 100644 --- a/medacy/pipeline_components/learners/nn/vectorizer.py +++ b/medacy/pipeline_components/learners/nn/vectorizer.py @@ -3,9 +3,11 @@ """ import re import string +import unicodedata +from functools import reduce +from operator import or_ import torch -import unicodedata from gensim.models import KeyedVectors @@ -32,16 +34,9 @@ def _create_feature_dictionary(feature_name, sentences): :param sentences: Sentences to get feature for. :return: Dictionary for given feature. """ - feature_to_index = {} - feature_name = '0:' + feature_name - - for sentence in sentences: - for token in sentence: - feature = token[feature_name] - if feature not in feature_to_index: - feature_to_index[feature] = len(feature_to_index) - - return feature_to_index + feature_name = f'0:{feature_name}' + all_features = reduce(or_, ({token[feature_name] for token in sentence} for sentence in sentences)) + return {feature: index for index, feature in enumerate(all_features, 1)} def _one_hot(index_dictionary, value): @@ -103,14 +98,8 @@ def create_tag_dictionary(self, tags): :param tags: List of list of tag names. Usually all true labels for a dataset. """ - tag_to_index = {} - - for sequence in tags: - for tag in sequence: - if tag not in tag_to_index: - tag_to_index[tag] = len(tag_to_index) - - self.tag_to_index = tag_to_index + tag_set = reduce(or_, (set(tag_list) for tag_list in tags)) + self.tag_to_index = {tag: index for index, tag in enumerate(tag_set, 1)} def add_tag(self, tag): """Add tag to self.tag_to_index @@ -201,22 +190,14 @@ def vectorize_tokens(self, tokens): embedding_index = self.word_vectors.vocab[token_text].index except KeyError: embedding_index = len(self.word_vectors.vocab) - # Only for logging untrained tokens self.untrained_tokens.add(token_text) token_vector.append(embedding_index) # Add list of character indices as second item - character_indices = [] - for character in token_text: - index = self.character_to_index[character] - character_indices.append(index) - - # If there were no indices ex. special characters only - if not character_indices: - # Append the padding index - character_indices.append(0) + # If there were no indices (ex. special characters only), use only the padding index. + character_indices = [self.character_to_index[character] for character in token_text] or [0] token_vector.append(character_indices) # Find window indices @@ -224,14 +205,12 @@ def vectorize_tokens(self, tokens): # Add features to vector in order window_range = range(-self.window_size, self.window_size + 1) - other_feature_names = [key for key in self.other_features] - other_feature_names.sort() + other_feature_names = sorted(self.other_features) for i in window_range: if i in window: for feature_name in other_feature_names: - key = '%d:%s' % (i, feature_name) - feature = token[key] + feature = token[f'{i}:{feature_name}'] vector = _one_hot(self.other_features[feature_name], feature) token_vector.extend(vector) else: @@ -249,8 +228,7 @@ def vectorize_tags(self, tags): :param tags: List of tags to convert. :return: Torch tensor of indices. """ - indices = [self.tag_to_index[tag] for tag in tags] - return torch.tensor(indices, dtype=torch.long, device=self.device) + return torch.tensor([self.tag_to_index[tag] for tag in tags], dtype=torch.long, device=self.device) def vectorize_dataset(self, x_data, y_data): """Vectorize entire dataset. @@ -272,13 +250,8 @@ def vectorize_dataset(self, x_data, y_data): self.other_features[feature] = _create_feature_dictionary(feature, x_data) # Vectorize data - sentences = [] - correct_tags = [] - for sentence, sentence_tags in zip(x_data, y_data): - tokens_vector = self.vectorize_tokens(sentence) - correct_tags_vector = self.vectorize_tags(sentence_tags) - sentences.append(tokens_vector) - correct_tags.append(correct_tags_vector) + sentences = [self.vectorize_tokens(sentence) for sentence in x_data] + correct_tags = [self.vectorize_tags(sentence_tags) for sentence_tags in y_data] return list(zip(sentences, correct_tags)) From 2d2d72fe98ee93dc96bdc0cdd86fe7a098524586 Mon Sep 17 00:00:00 2001 From: swfarnsworth Date: Sun, 15 Aug 2021 15:19:20 -0400 Subject: [PATCH 23/23] Fix off-by-one errors. As it turns out, the numbering did start at 0 after all. --- medacy/pipeline_components/learners/nn/vectorizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/medacy/pipeline_components/learners/nn/vectorizer.py b/medacy/pipeline_components/learners/nn/vectorizer.py index 873a244..210a424 100644 --- a/medacy/pipeline_components/learners/nn/vectorizer.py +++ b/medacy/pipeline_components/learners/nn/vectorizer.py @@ -36,7 +36,7 @@ def _create_feature_dictionary(feature_name, sentences): """ feature_name = f'0:{feature_name}' all_features = reduce(or_, ({token[feature_name] for token in sentence} for sentence in sentences)) - return {feature: index for index, feature in enumerate(all_features, 1)} + return {feature: index for index, feature in enumerate(all_features)} def _one_hot(index_dictionary, value): @@ -80,7 +80,7 @@ def __init__(self, device): self.tag_to_index = {} self.character_to_index = { - character: index for index, character in enumerate(string.printable, 1) + character: index for index, character in enumerate(string.printable) } def load_word_embeddings(self, embeddings_file): @@ -99,7 +99,7 @@ def create_tag_dictionary(self, tags): :param tags: List of list of tag names. Usually all true labels for a dataset. """ tag_set = reduce(or_, (set(tag_list) for tag_list in tags)) - self.tag_to_index = {tag: index for index, tag in enumerate(tag_set, 1)} + self.tag_to_index = {tag: index for index, tag in enumerate(tag_set)} def add_tag(self, tag): """Add tag to self.tag_to_index