From 292edf4519a6f044da6a9cdc8d57a3cce2f4d0f5 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 27 Jul 2023 10:01:05 +0300 Subject: [PATCH 01/39] Added new classification models --- .../emotions_xlm_roberta_base.json | 105 +++++++++++++++++ .../classifiers/insults_xlm_roberta_base.json | 106 ++++++++++++++++++ .../intents_distilbert_base_multi.json | 105 +++++++++++++++++ .../sentiments_xlm_roberta_base.json | 104 +++++++++++++++++ .../classifiers/topics_xlm_roberta_base.json | 105 +++++++++++++++++ 5 files changed, 525 insertions(+) create mode 100644 deeppavlov/configs/classifiers/emotions_xlm_roberta_base.json create mode 100644 deeppavlov/configs/classifiers/insults_xlm_roberta_base.json create mode 100644 deeppavlov/configs/classifiers/intents_distilbert_base_multi.json create mode 100644 deeppavlov/configs/classifiers/sentiments_xlm_roberta_base.json create mode 100644 deeppavlov/configs/classifiers/topics_xlm_roberta_base.json diff --git a/deeppavlov/configs/classifiers/emotions_xlm_roberta_base.json b/deeppavlov/configs/classifiers/emotions_xlm_roberta_base.json new file mode 100644 index 0000000000..4422ff9600 --- /dev/null +++ b/deeppavlov/configs/classifiers/emotions_xlm_roberta_base.json @@ -0,0 +1,105 @@ +{ + "dataset_reader": { + "class_name": "huggingface_dataset_reader", + "path": "cedr", + "name": "main", + "train": "train", + "test": "test" + }, + "dataset_iterator": { + "class_name": "huggingface_dataset_iterator", + "features": "text", + "label": "labels", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 1e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 64, + "metrics": [ + "accuracy", + "f1_weighted", + "f1_macro" + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "test"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "xlm-roberta-base", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/emotions_classifier/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/classifiers/emotions/emotions_xlm_roberta_base.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/insults_xlm_roberta_base.json b/deeppavlov/configs/classifiers/insults_xlm_roberta_base.json new file mode 100644 index 0000000000..190b3e3f5a --- /dev/null +++ b/deeppavlov/configs/classifiers/insults_xlm_roberta_base.json @@ -0,0 +1,106 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "Comment", + "y": "Class", + "data_path": "{DOWNLOADS_PATH}/insults_data" + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 1e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 256, + "metrics": [ + "accuracy", + "f1_weighted", + "f1_macro" + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "xlm-roberta-base", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/insults_classifier/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/datasets/insults_data.tar.gz", + "subdir": "{DOWNLOADS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/v1/classifiers/insults/insults_xlm_roberta_base.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/intents_distilbert_base_multi.json b/deeppavlov/configs/classifiers/intents_distilbert_base_multi.json new file mode 100644 index 0000000000..18089bfb23 --- /dev/null +++ b/deeppavlov/configs/classifiers/intents_distilbert_base_multi.json @@ -0,0 +1,105 @@ +{ + "dataset_reader": { + "class_name": "huggingface_dataset_reader", + "path": "AmazonScience/massive", + "name": "all", + "train": "train", + "valid": "validation", + "test": "test" + }, + "dataset_iterator": { + "class_name": "huggingface_dataset_iterator", + "features": "utt", + "label": "intent", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 2e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 128, + "metrics": [ + "accuracy", + "f1_weighted" + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "distilbert-base-multilingual-cased", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/intents_classification/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/classifiers/intents/intents_classification_distilbert_multi.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/sentiments_xlm_roberta_base.json b/deeppavlov/configs/classifiers/sentiments_xlm_roberta_base.json new file mode 100644 index 0000000000..94f5dd0120 --- /dev/null +++ b/deeppavlov/configs/classifiers/sentiments_xlm_roberta_base.json @@ -0,0 +1,104 @@ +{ + "dataset_reader": { + "class_name": "huggingface_dataset_reader", + "path": "mteb/tweet_sentiment_extraction", + "train": "train", + "test": "test" + }, + "dataset_iterator": { + "class_name": "huggingface_dataset_iterator", + "features": "text", + "label": "label_text", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 1e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 64, + "metrics": [ + "accuracy", + "f1_weighted", + "f1_macro" + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "test"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "xlm-roberta-base", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/sentiments_classifier/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/classifiers/sentiments/sentiments_xlm_roberta_base.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} \ No newline at end of file diff --git a/deeppavlov/configs/classifiers/topics_xlm_roberta_base.json b/deeppavlov/configs/classifiers/topics_xlm_roberta_base.json new file mode 100644 index 0000000000..44ea6a3ac5 --- /dev/null +++ b/deeppavlov/configs/classifiers/topics_xlm_roberta_base.json @@ -0,0 +1,105 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "text", + "y": "topic", + "data_path": "{DOWNLOADS_PATH}/dp_topics_downsampled_data/", + "train" : "train.csv", + "valid" : "valid.csv", + "test": "test.csv" + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 1e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 256, + "metrics": [ + "accuracy", + "f1_weighted", + "f1_macro" + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid", "test"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "xlm-roberta-base", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/topics_classification/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/classifiers/topics/topics_xlm_roberta_base.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} From eff06dc30cd075adaf2ce452fd0c01f18111be3a Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 27 Jul 2023 10:01:18 +0300 Subject: [PATCH 02/39] Added new NER model --- .../configs/ner/ner_bert_base_mult.json | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 deeppavlov/configs/ner/ner_bert_base_mult.json diff --git a/deeppavlov/configs/ner/ner_bert_base_mult.json b/deeppavlov/configs/ner/ner_bert_base_mult.json new file mode 100644 index 0000000000..a8e9e4b626 --- /dev/null +++ b/deeppavlov/configs/ner/ner_bert_base_mult.json @@ -0,0 +1,103 @@ +{ + "dataset_reader": { + "class_name": "conll2003_reader", + "data_path": "{DOWNLOADS_PATH}/conll2003/", + "dataset_name": "conll2003", + "provide_pos": false + }, + "dataset_iterator": { + "class_name": "data_learning_iterator" + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_ner_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 256, + "in": ["x"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] + }, + { + "id": "tag_vocab", + "class_name": "simple_vocab", + "unk_token": ["O"], + "pad_with_zeros": true, + "save_path": "{MODEL_PATH}/tag.dict", + "load_path": "{MODEL_PATH}/tag.dict", + "fit_on": ["y"], + "in": ["y"], + "out": ["y_ind"] + }, + { + "class_name": "torch_transformers_sequence_tagger", + "n_tags": "#tag_vocab.len", + "pretrained_bert": "{BASE_MODEL}", + "attention_probs_keep_prob": 0.5, + "use_crf": true, + "encoder_layer_ids": [-1], + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 2e-05, + "weight_decay": 1e-06, + "betas": [0.9, 0.999], + "eps": 1e-06 + }, + "clip_norm": 1.0, + "min_learning_rate": 1e-07, + "learning_rate_drop_patience": 20, + "learning_rate_drop_div": 1.5, + "load_before_drop": true, + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "in": ["x_subword_tok_ids", "attention_mask", "startofword_markers"], + "in_y": ["y_ind"], + "out": ["y_pred_ind", "probas"] + }, + { + "ref": "tag_vocab", + "in": ["y_pred_ind"], + "out": ["y_pred"] + } + ], + "out": ["x_tokens", "y_pred"] + }, + "train": { + "batch_size": 128, + "metrics": [ + { + "name": "ner_f1", + "inputs": ["y", "y_pred"] + }, + { + "name": "ner_token_f1", + "inputs": ["y", "y_pred"] + } + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "bert-base-multilingual-cased", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/ner_amazon_massive/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/ner/ner_bert_base_mult.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} From d2d3ef7a42baa0379217ac24cbcee99ddde73cdf Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 27 Jul 2023 10:03:01 +0300 Subject: [PATCH 03/39] Added new models to tests --- tests/test_quick_start.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 6b5b2cc0cc..fba3353f81 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -121,6 +121,12 @@ ("classifiers/superglue/superglue_record_roberta.json", "classifiers", ('TI',)): [RECORD_ARGUMENTS_INFER_CHECK], ("classifiers/superglue/superglue_wic_bert.json", "classifiers", ('TI',)): [TWO_ARGUMENTS_INFER_CHECK], ("classifiers/topics_distilbert_base_uncased.json", "classifiers", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/topics_distilbert_base_uncased.json", "classifiers", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/emotions_xlm_roberta_base.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/insults_xlm_roberta_base.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/intents_distilbert_base_multi.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/sentiments_xlm_roberta_base.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/topics_xlm_roberta_base.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], ("classifiers/few_shot_roberta.json", "classifiers", ('IP',)): [ ('Dummy text', ['Dummy text Dummy text', 'Dummy class'], ('Dummy class',)) ] @@ -221,7 +227,8 @@ ("ner/ner_ontonotes_bert.json", "ner_ontonotes_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], ("ner/ner_ontonotes_bert_mult.json", "ner_ontonotes_bert_mult", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], ("ner/ner_rus_bert.json", "ner_rus_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], - ("ner/ner_collection3_bert.json", "ner_collection3_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK] + ("ner/ner_collection3_bert.json", "ner_collection3_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], + ("ner/ner_bert_base_mult.json", "ner_conll2003_bert", ('IP')): [ONE_ARGUMENT_INFER_CHECK] }, "sentence_segmentation": { ("sentence_segmentation/sentseg_dailydialog_bert.json", "sentseg_dailydialog_bert", ('IP', 'TI')): [ From 46ca69c683a3bc1233c76bb09fb46092ce16334a Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 27 Jul 2023 10:04:15 +0300 Subject: [PATCH 04/39] Fixed NER bug with multiple GPUs --- .../models/torch_bert/torch_transformers_sequence_tagger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py b/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py index a12fc9f82b..f08ca84457 100644 --- a/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py +++ b/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py @@ -205,6 +205,8 @@ def train_on_batch(self, labels=b_labels).loss if self.crf is not None: self.crf(y, y_masks) + if self.is_data_parallel: + loss = loss.mean() self._make_step(loss) return {'loss': loss.item()} From a30ee3733cc76dcf012048758a44c2fd91db5d58 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 27 Jul 2023 10:05:36 +0300 Subject: [PATCH 05/39] Added NER input clipping --- .../preprocessors/torch_transformers_preprocessor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py index effd4388d1..165c7909e3 100644 --- a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py +++ b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py @@ -552,6 +552,7 @@ def __call__(self, self._ner_bert_tokenize(toks, ys, self.tokenizer, + self.max_seq_length, self.max_subword_length, mode=self.mode, subword_mask_mode=self.subword_mask_mode, @@ -579,6 +580,7 @@ def __call__(self, attention_mask, startofword_markers, subword_tags else: nonmasked_tags = [[t for t in ts if t != 'X'] for ts in tags] + nonmasked_tags = nonmasked_tags[:self.max_seq_length] for swts, swids, swms, ts in zip(subword_tokens, subword_tok_ids, startofword_markers, @@ -612,6 +614,7 @@ def __call__(self, def _ner_bert_tokenize(tokens: List[str], tags: List[str], tokenizer: AutoTokenizer, + max_seq_length: int, max_subword_len: int = None, mode: str = None, subword_mask_mode: str = "first", @@ -639,6 +642,12 @@ def _ner_bert_tokenize(tokens: List[str], startofword_markers.extend([token_marker] + [0] * (len(subwords) - 1)) tags_subword.extend([tag] + ['X'] * (len(subwords) - 1)) + if len(tokens_subword) >= max_seq_length: + tokens_subword = tokens_subword[:max_seq_length - 1] + if len(startofword_markers) >= max_seq_length: + startofword_markers = startofword_markers[:max_seq_length - 1] + if len(tags_subword) >= max_seq_length: + tags_subword = tags_subword[:max_seq_length - 1] tokens_subword.append('[SEP]') startofword_markers.append(0) tags_subword.append('X') From 09fafcb8d053b66cb4f37cdc0d8ea13f090b74f1 Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 12:56:49 +0300 Subject: [PATCH 06/39] Add delimiter component --- deeppavlov/core/common/registry.json | 1 + 1 file changed, 1 insertion(+) diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index 9995d88370..5cb03dd9d8 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -36,6 +36,7 @@ "line_reader": "deeppavlov.dataset_readers.line_reader:LineReader", "logit_ranker": "deeppavlov.models.doc_retrieval.logit_ranker:LogitRanker", "mask": "deeppavlov.models.preprocessors.mask:Mask", + "sentence_delimiter": "deeppavlov.models.tokenizers.sentence_delimiter:SentenceDelimiter", "morphotagger_dataset_iterator": "deeppavlov.dataset_iterators.morphotagger_iterator:MorphoTaggerDatasetIterator", "morphotagger_dataset_reader": "deeppavlov.dataset_readers.morphotagging_dataset_reader:MorphotaggerDatasetReader", "multitask_reader":"deeppavlov.dataset_readers.multitask_reader:MultiTaskReader", From 8fd57eae4685809de4dc6850cb471324a246d830 Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 12:57:08 +0300 Subject: [PATCH 07/39] Add delimiter requirement --- deeppavlov/core/common/requirements_registry.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deeppavlov/core/common/requirements_registry.json b/deeppavlov/core/common/requirements_registry.json index 732803debb..d1c6255599 100644 --- a/deeppavlov/core/common/requirements_registry.json +++ b/deeppavlov/core/common/requirements_registry.json @@ -204,5 +204,8 @@ ], "wikitionary_100K_vocab": [ "{DEEPPAVLOV_PATH}/requirements/lxml.txt" + ], + "sentence_delimiter": [ + "{DEEPPAVLOV_PATH}/requirements/pysbd.txt" ] } From e93073b299a7f7ea1abd4fd5a5061551601f26a0 Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 12:58:04 +0300 Subject: [PATCH 08/39] Add support long sequences --- deeppavlov/configs/ner/ner_bert_base_mult.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_mult.json b/deeppavlov/configs/ner/ner_bert_base_mult.json index a8e9e4b626..e8d9389e4e 100644 --- a/deeppavlov/configs/ner/ner_bert_base_mult.json +++ b/deeppavlov/configs/ner/ner_bert_base_mult.json @@ -9,9 +9,14 @@ "class_name": "data_learning_iterator" }, "chainer": { - "in": ["x"], + "in": ["x_long"], "in_y": ["y"], "pipe": [ + { + "class_name": "sentence_delimiter", + "in": ["x_long"], + "out": ["x"] + }, { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", From 2ef420fc95e42fad0991f257e6b3be7f9c32d790 Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 12:58:31 +0300 Subject: [PATCH 09/39] Add delimiter --- deeppavlov/models/tokenizers/sentence_delimiter.py | 14 ++++++++++++++ deeppavlov/requirements/pysbd.txt | 1 + 2 files changed, 15 insertions(+) create mode 100644 deeppavlov/models/tokenizers/sentence_delimiter.py create mode 100644 deeppavlov/requirements/pysbd.txt diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py new file mode 100644 index 0000000000..f86a5e47aa --- /dev/null +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -0,0 +1,14 @@ +import pysbd + +from deeppavlov.core.models.component import Component +from deeppavlov.core.common.registry import register + + +@register('sentence_delimiter') +def SentenceDelimiter(x_long): + seg = pysbd.Segmenter(clean=False) + xs = seg.segment(x_long[0]) + return tuple(xs) + + + diff --git a/deeppavlov/requirements/pysbd.txt b/deeppavlov/requirements/pysbd.txt new file mode 100644 index 0000000000..c625687944 --- /dev/null +++ b/deeppavlov/requirements/pysbd.txt @@ -0,0 +1 @@ +pysbd==0.3.4 \ No newline at end of file From 658a85f0a3e81ffa0d767449eff61f6be7c7817d Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 16:12:00 +0300 Subject: [PATCH 10/39] Skip empty sequences --- deeppavlov/models/tokenizers/sentence_delimiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py index f86a5e47aa..06a7275acd 100644 --- a/deeppavlov/models/tokenizers/sentence_delimiter.py +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -7,7 +7,7 @@ @register('sentence_delimiter') def SentenceDelimiter(x_long): seg = pysbd.Segmenter(clean=False) - xs = seg.segment(x_long[0]) + xs = [a for a in seg.segment(x_long[0]) if len(a)>0] return tuple(xs) From df3aee0411a313162a5a87cf3116007f3e2b79fb Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 16:12:53 +0300 Subject: [PATCH 11/39] Restore the short variant --- deeppavlov/configs/ner/ner_bert_base_mult.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_mult.json b/deeppavlov/configs/ner/ner_bert_base_mult.json index e8d9389e4e..a8e9e4b626 100644 --- a/deeppavlov/configs/ner/ner_bert_base_mult.json +++ b/deeppavlov/configs/ner/ner_bert_base_mult.json @@ -9,14 +9,9 @@ "class_name": "data_learning_iterator" }, "chainer": { - "in": ["x_long"], + "in": ["x"], "in_y": ["y"], "pipe": [ - { - "class_name": "sentence_delimiter", - "in": ["x_long"], - "out": ["x"] - }, { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", From ac1511ec82ba8fd6e4884eaaeed138b28ccd6b1b Mon Sep 17 00:00:00 2001 From: vasily Date: Thu, 27 Jul 2023 16:13:29 +0300 Subject: [PATCH 12/39] Create long demo config --- .../configs/ner/ner_mult_long_demo.json | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 deeppavlov/configs/ner/ner_mult_long_demo.json diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json new file mode 100644 index 0000000000..e8d9389e4e --- /dev/null +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -0,0 +1,108 @@ +{ + "dataset_reader": { + "class_name": "conll2003_reader", + "data_path": "{DOWNLOADS_PATH}/conll2003/", + "dataset_name": "conll2003", + "provide_pos": false + }, + "dataset_iterator": { + "class_name": "data_learning_iterator" + }, + "chainer": { + "in": ["x_long"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "sentence_delimiter", + "in": ["x_long"], + "out": ["x"] + }, + { + "class_name": "torch_transformers_ner_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 256, + "in": ["x"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] + }, + { + "id": "tag_vocab", + "class_name": "simple_vocab", + "unk_token": ["O"], + "pad_with_zeros": true, + "save_path": "{MODEL_PATH}/tag.dict", + "load_path": "{MODEL_PATH}/tag.dict", + "fit_on": ["y"], + "in": ["y"], + "out": ["y_ind"] + }, + { + "class_name": "torch_transformers_sequence_tagger", + "n_tags": "#tag_vocab.len", + "pretrained_bert": "{BASE_MODEL}", + "attention_probs_keep_prob": 0.5, + "use_crf": true, + "encoder_layer_ids": [-1], + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 2e-05, + "weight_decay": 1e-06, + "betas": [0.9, 0.999], + "eps": 1e-06 + }, + "clip_norm": 1.0, + "min_learning_rate": 1e-07, + "learning_rate_drop_patience": 20, + "learning_rate_drop_div": 1.5, + "load_before_drop": true, + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "in": ["x_subword_tok_ids", "attention_mask", "startofword_markers"], + "in_y": ["y_ind"], + "out": ["y_pred_ind", "probas"] + }, + { + "ref": "tag_vocab", + "in": ["y_pred_ind"], + "out": ["y_pred"] + } + ], + "out": ["x_tokens", "y_pred"] + }, + "train": { + "batch_size": 128, + "metrics": [ + { + "name": "ner_f1", + "inputs": ["y", "y_pred"] + }, + { + "name": "ner_token_f1", + "inputs": ["y", "y_pred"] + } + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "bert-base-multilingual-cased", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/ner_amazon_massive/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/ner/ner_bert_base_mult.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } +} From ed4270f4f8797c62481c5f745504079117270650 Mon Sep 17 00:00:00 2001 From: Nikolay <99244955+Kolpnick@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:04:19 +0300 Subject: [PATCH 13/39] Updated ner config --- deeppavlov/configs/ner/ner_bert_base_mult.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_mult.json b/deeppavlov/configs/ner/ner_bert_base_mult.json index a8e9e4b626..713f0614f7 100644 --- a/deeppavlov/configs/ner/ner_bert_base_mult.json +++ b/deeppavlov/configs/ner/ner_bert_base_mult.json @@ -16,7 +16,7 @@ "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", "do_lower_case": false, - "max_seq_length": 256, + "max_seq_length": 128, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] }, @@ -65,7 +65,7 @@ "out": ["x_tokens", "y_pred"] }, "train": { - "batch_size": 128, + "batch_size": 256, "metrics": [ { "name": "ner_f1", @@ -91,11 +91,11 @@ "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "{MODELS_PATH}/ner_amazon_massive/{BASE_MODEL}" + "MODEL_PATH": "{MODELS_PATH}/amazon_onto_combined/{BASE_MODEL}" }, "download": [ { - "url": "http://files.deeppavlov.ai/v1/ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/ner_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From f19103ecf5d7e558b0771530a2b99436eebf3073 Mon Sep 17 00:00:00 2001 From: vasily Date: Mon, 31 Jul 2023 12:28:35 +0300 Subject: [PATCH 14/39] Update DEMO NER --- deeppavlov/configs/ner/ner_mult_long_demo.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index e8d9389e4e..8141a4a995 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -15,13 +15,13 @@ { "class_name": "sentence_delimiter", "in": ["x_long"], - "out": ["x"] + "out": ["x"] }, { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", "do_lower_case": false, - "max_seq_length": 256, + "max_seq_length": 128, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] }, @@ -70,7 +70,7 @@ "out": ["x_tokens", "y_pred"] }, "train": { - "batch_size": 128, + "batch_size": 256, "metrics": [ { "name": "ner_f1", @@ -96,11 +96,11 @@ "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "{MODELS_PATH}/ner_amazon_massive/{BASE_MODEL}" + "MODEL_PATH": "{MODELS_PATH}/amazon_onto_combined/{BASE_MODEL}" }, "download": [ { - "url": "http://files.deeppavlov.ai/v1/ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/ner_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From 19262194db0fde7044bbfe2325ae2271c65d5007 Mon Sep 17 00:00:00 2001 From: vasily Date: Mon, 31 Jul 2023 14:02:24 +0300 Subject: [PATCH 15/39] Fix --- deeppavlov/configs/ner/ner_mult_long_demo.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 8141a4a995..e0d4f85479 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -9,20 +9,20 @@ "class_name": "data_learning_iterator" }, "chainer": { - "in": ["x_long"], + "in": ["x"], "in_y": ["y"], "pipe": [ { "class_name": "sentence_delimiter", - "in": ["x_long"], - "out": ["x"] + "in": ["x"], + "out": ["x_short"] }, { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", "do_lower_case": false, "max_seq_length": 128, - "in": ["x"], + "in": ["x_short"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] }, { From 3431e3a53df4ed7b52a88933d29421903934d33f Mon Sep 17 00:00:00 2001 From: vasily Date: Tue, 1 Aug 2023 05:40:01 +0300 Subject: [PATCH 16/39] Add sentence_concatenator --- deeppavlov/core/common/registry.json | 1 + 1 file changed, 1 insertion(+) diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index 5cb03dd9d8..2e7c177c60 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -36,6 +36,7 @@ "line_reader": "deeppavlov.dataset_readers.line_reader:LineReader", "logit_ranker": "deeppavlov.models.doc_retrieval.logit_ranker:LogitRanker", "mask": "deeppavlov.models.preprocessors.mask:Mask", + "sentence_concatenator": "deeppavlov.models.tokenizers.sentence_delimiter:SentenceConcatenator", "sentence_delimiter": "deeppavlov.models.tokenizers.sentence_delimiter:SentenceDelimiter", "morphotagger_dataset_iterator": "deeppavlov.dataset_iterators.morphotagger_iterator:MorphoTaggerDatasetIterator", "morphotagger_dataset_reader": "deeppavlov.dataset_readers.morphotagging_dataset_reader:MorphotaggerDatasetReader", From e371cac8b7aff4d1f0658efc63d67e854be0debe Mon Sep 17 00:00:00 2001 From: vasily Date: Tue, 1 Aug 2023 05:40:17 +0300 Subject: [PATCH 17/39] Add sentence_concatenator --- deeppavlov/models/tokenizers/sentence_delimiter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py index 06a7275acd..fddeb5658d 100644 --- a/deeppavlov/models/tokenizers/sentence_delimiter.py +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -10,5 +10,9 @@ def SentenceDelimiter(x_long): xs = [a for a in seg.segment(x_long[0]) if len(a)>0] return tuple(xs) - - +@register('sentence_concatenator') +def SentenceConcatenator(x_long): + x_short = [] + for sent in x_long: + x_short.extend(sent) + return x_short \ No newline at end of file From 35dea2e42013e9e9add17d67f687d3f1860fd7db Mon Sep 17 00:00:00 2001 From: vasily Date: Tue, 1 Aug 2023 05:40:37 +0300 Subject: [PATCH 18/39] Add sentence_concatenator --- deeppavlov/configs/ner/ner_mult_long_demo.json | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index e0d4f85479..97e88fc4a2 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -65,20 +65,30 @@ "ref": "tag_vocab", "in": ["y_pred_ind"], "out": ["y_pred"] + }, + { + "class_name": "sentence_concatenator", + "in": ["x_tokens"], + "out": ["x_tokens_short"] + }, + { + "class_name": "sentence_concatenator", + "in": ["y_pred"], + "out": ["y_pred_short"] } ], - "out": ["x_tokens", "y_pred"] + "out": ["x_tokens_short", "y_pred_short"] }, "train": { "batch_size": 256, "metrics": [ { "name": "ner_f1", - "inputs": ["y", "y_pred"] + "inputs": ["y", "y_pred_short"] }, { "name": "ner_token_f1", - "inputs": ["y", "y_pred"] + "inputs": ["y", "y_pred_short"] } ], "validation_patience": 5, From 528cc4d7a6d88e01215f01daee76610eb5e5e0de Mon Sep 17 00:00:00 2001 From: vasily Date: Tue, 1 Aug 2023 08:43:34 +0300 Subject: [PATCH 19/39] Fix delimiter --- deeppavlov/configs/ner/ner_mult_long_demo.json | 15 +++++---------- .../models/tokenizers/sentence_delimiter.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 97e88fc4a2..92686966ca 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -68,27 +68,22 @@ }, { "class_name": "sentence_concatenator", - "in": ["x_tokens"], - "out": ["x_tokens_short"] - }, - { - "class_name": "sentence_concatenator", - "in": ["y_pred"], - "out": ["y_pred_short"] + "in": ["x_tokens", "y_pred"], + "out": ["output"] } ], - "out": ["x_tokens_short", "y_pred_short"] + "out": ["output"] }, "train": { "batch_size": 256, "metrics": [ { "name": "ner_f1", - "inputs": ["y", "y_pred_short"] + "inputs": ["y", "y_pred"] }, { "name": "ner_token_f1", - "inputs": ["y", "y_pred_short"] + "inputs": ["y", "y_pred"] } ], "validation_patience": 5, diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py index fddeb5658d..c936e583ec 100644 --- a/deeppavlov/models/tokenizers/sentence_delimiter.py +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -11,8 +11,14 @@ def SentenceDelimiter(x_long): return tuple(xs) @register('sentence_concatenator') -def SentenceConcatenator(x_long): +def SentenceConcatenator(x_long, y_long): x_short = [] + y_short = [] + for sent in x_long: x_short.extend(sent) - return x_short \ No newline at end of file + + for sent in y_long: + y_short.extend(sent) + + return [[x_short, y_short]] \ No newline at end of file From d35967327223e18e47743d5edabcb1449fb29f2f Mon Sep 17 00:00:00 2001 From: Nikolay Date: Tue, 1 Aug 2023 09:15:55 +0300 Subject: [PATCH 20/39] Updated NER configs --- deeppavlov/configs/ner/ner_bert_base_mult.json | 4 ++-- deeppavlov/configs/ner/ner_mult_long_demo.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_mult.json b/deeppavlov/configs/ner/ner_bert_base_mult.json index 713f0614f7..107c5fac8e 100644 --- a/deeppavlov/configs/ner/ner_bert_base_mult.json +++ b/deeppavlov/configs/ner/ner_bert_base_mult.json @@ -15,7 +15,7 @@ { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", - "do_lower_case": false, + "do_lower_case": true, "max_seq_length": 128, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] @@ -95,7 +95,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 92686966ca..7b92400017 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -20,7 +20,7 @@ { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", - "do_lower_case": false, + "do_lower_case": true, "max_seq_length": 128, "in": ["x_short"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] @@ -105,7 +105,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From 3b7680ca930ea682b7ce4f843dbb4bdd5fd74547 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 2 Aug 2023 09:54:15 +0300 Subject: [PATCH 21/39] Fixed NER bug with interpunction --- .../models/preprocessors/torch_transformers_preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py index 165c7909e3..ce8933f18a 100644 --- a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py +++ b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py @@ -510,7 +510,7 @@ def __init__(self, subword_mask_mode: str = "first", return_features: bool = False, **kwargs): - self._re_tokenizer = re.compile(r"[\w']+|[^\w ]") + self._re_tokenizer = re.compile(r"[\d]+[\d\.,]+[\d]+|[\w'\.:@-]+|[^\w ]") self.provide_subword_tags = provide_subword_tags self.mode = kwargs.get('mode') self.max_seq_length = max_seq_length From 998894b1cf6e63279a4500d9a64c3f0ef65096a5 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 2 Aug 2023 10:16:32 +0300 Subject: [PATCH 22/39] Updated model download url --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 7b92400017..db8e256daa 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -105,7 +105,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_mult_langs_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From 108d96e1f1b8ab7405095ba47adeeee8fdeb8872 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 2 Aug 2023 10:32:16 +0300 Subject: [PATCH 23/39] Updated model download url --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index db8e256daa..7b92400017 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -105,7 +105,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_mult_langs_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_base_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From 1dbed3cebae373d007a3f5961f495f9baaa5bf2e Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 10 Aug 2023 09:15:28 +0300 Subject: [PATCH 24/39] Updated NER model download url --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 7b92400017..487d36ff22 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -105,7 +105,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_base_mult.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_cased_mult.tar.gz", "subdir": "{MODEL_PATH}" } ] From e12a869fc39793a2c5e2e878eacee7944ed3e754 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 10 Aug 2023 13:24:11 +0300 Subject: [PATCH 25/39] Updated NER config --- deeppavlov/configs/ner/ner_mult_long_demo.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 487d36ff22..436477eeac 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -20,8 +20,8 @@ { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", - "do_lower_case": true, - "max_seq_length": 128, + "do_lower_case": false, + "max_seq_length": 256, "in": ["x_short"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] }, From 60c806030e32445b6df10ab8b6bfbf779f8c7ad2 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Tue, 5 Sep 2023 09:02:49 +0300 Subject: [PATCH 26/39] Added new intents classification config --- .../classifiers/intents_xlm_roberta_base.json | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 deeppavlov/configs/classifiers/intents_xlm_roberta_base.json diff --git a/deeppavlov/configs/classifiers/intents_xlm_roberta_base.json b/deeppavlov/configs/classifiers/intents_xlm_roberta_base.json new file mode 100644 index 0000000000..f260c0e6ec --- /dev/null +++ b/deeppavlov/configs/classifiers/intents_xlm_roberta_base.json @@ -0,0 +1,106 @@ +{ + "dataset_reader": { + "class_name": "huggingface_dataset_reader", + "path": "AmazonScience/massive", + "name": "all", + "train": "train", + "valid": "validation", + "test": "test" + }, + "dataset_iterator": { + "class_name": "huggingface_dataset_iterator", + "features": "utt", + "label": "intent", + "seed": 42 + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 128, + "in": ["x"], + "out": ["bert_features"] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": ["y"], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": ["y"], + "out": ["y_ids"] + }, + { + "in": ["y_ids"], + "out": ["y_onehot"], + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "torch_transformers_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "pretrained_bert": "{BASE_MODEL}", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 2e-05 + }, + "learning_rate_drop_patience": 3, + "learning_rate_drop_div": 2.0, + "in": ["bert_features"], + "in_y": ["y_ids"], + "out": ["y_pred_probas"] + }, + { + "in": ["y_pred_probas"], + "out": ["y_pred_ids"], + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": ["y_pred_ids"], + "out": ["y_pred_labels"], + "ref": "classes_vocab" + } + ], + "out": ["y_pred_labels"] + }, + "train": { + "batch_size": 128, + "metrics": [ + "accuracy", + "f1_weighted" + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "xlm-roberta-base", + "ROOT_PATH": "~/test_dir", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/intents_classification/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/tmp/classifiers/intents/intents_xlm_roberta_base.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] + } + } + From 64081e1dd88cd621fc562ed0d0a2c1dad0774a92 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Mon, 18 Sep 2023 17:43:36 +0300 Subject: [PATCH 27/39] Moved bug fixes to new PR --- .../preprocessors/torch_transformers_preprocessor.py | 11 +---------- .../torch_bert/torch_transformers_sequence_tagger.py | 2 -- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py index ce8933f18a..effd4388d1 100644 --- a/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py +++ b/deeppavlov/models/preprocessors/torch_transformers_preprocessor.py @@ -510,7 +510,7 @@ def __init__(self, subword_mask_mode: str = "first", return_features: bool = False, **kwargs): - self._re_tokenizer = re.compile(r"[\d]+[\d\.,]+[\d]+|[\w'\.:@-]+|[^\w ]") + self._re_tokenizer = re.compile(r"[\w']+|[^\w ]") self.provide_subword_tags = provide_subword_tags self.mode = kwargs.get('mode') self.max_seq_length = max_seq_length @@ -552,7 +552,6 @@ def __call__(self, self._ner_bert_tokenize(toks, ys, self.tokenizer, - self.max_seq_length, self.max_subword_length, mode=self.mode, subword_mask_mode=self.subword_mask_mode, @@ -580,7 +579,6 @@ def __call__(self, attention_mask, startofword_markers, subword_tags else: nonmasked_tags = [[t for t in ts if t != 'X'] for ts in tags] - nonmasked_tags = nonmasked_tags[:self.max_seq_length] for swts, swids, swms, ts in zip(subword_tokens, subword_tok_ids, startofword_markers, @@ -614,7 +612,6 @@ def __call__(self, def _ner_bert_tokenize(tokens: List[str], tags: List[str], tokenizer: AutoTokenizer, - max_seq_length: int, max_subword_len: int = None, mode: str = None, subword_mask_mode: str = "first", @@ -642,12 +639,6 @@ def _ner_bert_tokenize(tokens: List[str], startofword_markers.extend([token_marker] + [0] * (len(subwords) - 1)) tags_subword.extend([tag] + ['X'] * (len(subwords) - 1)) - if len(tokens_subword) >= max_seq_length: - tokens_subword = tokens_subword[:max_seq_length - 1] - if len(startofword_markers) >= max_seq_length: - startofword_markers = startofword_markers[:max_seq_length - 1] - if len(tags_subword) >= max_seq_length: - tags_subword = tags_subword[:max_seq_length - 1] tokens_subword.append('[SEP]') startofword_markers.append(0) tags_subword.append('X') diff --git a/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py b/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py index f08ca84457..a12fc9f82b 100644 --- a/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py +++ b/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py @@ -205,8 +205,6 @@ def train_on_batch(self, labels=b_labels).loss if self.crf is not None: self.crf(y, y_masks) - if self.is_data_parallel: - loss = loss.mean() self._make_step(loss) return {'loss': loss.item()} From 8e9a4f4b2ff91c6b182a7b5fbabb9c513de6da59 Mon Sep 17 00:00:00 2001 From: Anna Korzanova Date: Tue, 14 May 2024 11:28:21 +0300 Subject: [PATCH 28/39] feat: config for address model --- .../demo/ner_model/ner_anna_config.json | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 regex_for_pii/demo/ner_model/ner_anna_config.json diff --git a/regex_for_pii/demo/ner_model/ner_anna_config.json b/regex_for_pii/demo/ner_model/ner_anna_config.json new file mode 100644 index 0000000000..1ace1bd02b --- /dev/null +++ b/regex_for_pii/demo/ner_model/ner_anna_config.json @@ -0,0 +1,105 @@ +{ + "dataset_reader": { + "class_name": "conll2003_reader", + "data_path": "/home/korzanova/regex_for_pii/demo/train_data/", + "provide_pos": false + }, + "dataset_iterator": { + "class_name": "data_learning_iterator" + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "sentence_delimiter", + "in": ["x"], + "out": ["x_short"] + }, + { + "class_name": "torch_transformers_ner_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 512, + "in": ["x"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] + }, + { + "id": "tag_vocab", + "class_name": "simple_vocab", + "unk_token": ["O"], + "save_path": "{MODEL_PATH}/tag.dict", + "load_path": "{MODEL_PATH}/tag.dict", + "fit_on": ["y"], + "in": ["y"], + "out": ["y_ind"] + }, + { + "class_name": "torch_transformers_sequence_tagger", + "n_tags": "#tag_vocab.len", + "pretrained_bert": "{BASE_MODEL}", + "attention_probs_keep_prob": 0.5, + "use_crf": true, + "encoder_layer_ids": [-1], + "optimizer": "AdamW", + "optimizer_parameters": { + "lr": 2e-05, + "weight_decay": 1e-06, + "betas": [0.9, 0.999], + "eps": 1e-06 + }, + "clip_norm": 1.0, + "min_learning_rate": 1e-07, + "learning_rate_drop_patience": 20, + "learning_rate_drop_div": 1.5, + "load_before_drop": true, + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "in": ["x_subword_tok_ids", "attention_mask", "startofword_markers"], + "in_y": ["y_ind"], + "out": ["y_pred_ind", "probas"] + }, + { + "ref": "tag_vocab", + "in": ["y_pred_ind"], + "out": ["y_pred"] + }, + { + "class_name": "sentence_concatenator", + "in": ["x_tokens", "y_pred"], + "out": ["output"] + } + ], + "out": ["x_tokens", "y_pred"] + }, + "train": { + "batch_size": 128, + "metrics": [ + { + "name": "ner_f1", + "inputs": ["y", "y_pred"] + }, + { + "name": "ner_token_f1", + "inputs": ["y", "y_pred"] + } + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": ["train", "valid"], + "class_name": "torch_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/", + "pytest_max_batches": 2 + }, + "metadata": { + "variables": { + "BASE_MODEL": "bert-base-multilingual-cased", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "/home/korzanova/regex_for_pii/demo/ner_model" + } + } + } \ No newline at end of file From ff1c10b6f3cc2d4bde1064fffe5d132945b93cb6 Mon Sep 17 00:00:00 2001 From: Anna Korzanova Date: Tue, 14 May 2024 12:01:27 +0300 Subject: [PATCH 29/39] fix: address model conifg updated --- .../configs/ner}/ner_anna_config.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) rename {regex_for_pii/demo/ner_model => deeppavlov/configs/ner}/ner_anna_config.json (92%) diff --git a/regex_for_pii/demo/ner_model/ner_anna_config.json b/deeppavlov/configs/ner/ner_anna_config.json similarity index 92% rename from regex_for_pii/demo/ner_model/ner_anna_config.json rename to deeppavlov/configs/ner/ner_anna_config.json index 1ace1bd02b..384ab19301 100644 --- a/regex_for_pii/demo/ner_model/ner_anna_config.json +++ b/deeppavlov/configs/ner/ner_anna_config.json @@ -99,7 +99,13 @@ "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "/home/korzanova/regex_for_pii/demo/ner_model" - } + "MODEL_PATH": "{MODELS_PATH}/ner_address/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/tmp/deeppavlov_demo/ner/dp_demo_ner.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] } } \ No newline at end of file From 44debd21b7d0868ffcc4eb82786473b252803b8a Mon Sep 17 00:00:00 2001 From: Anna Korzanova Date: Tue, 14 May 2024 14:55:01 +0300 Subject: [PATCH 30/39] fix: update address model config download link --- deeppavlov/configs/ner/ner_anna_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_anna_config.json b/deeppavlov/configs/ner/ner_anna_config.json index 384ab19301..5dbfc1efe7 100644 --- a/deeppavlov/configs/ner/ner_anna_config.json +++ b/deeppavlov/configs/ner/ner_anna_config.json @@ -103,7 +103,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/deeppavlov_demo/ner/dp_demo_ner.tar.gz", + "url": "http://files.deeppavlov.ai/v1/ner/dp_demo_ner.tar.gz", "subdir": "{MODEL_PATH}" } ] From 851e10a30d631c2ff4954eabd28a36068ffd2a5d Mon Sep 17 00:00:00 2001 From: Anna Korzanova Date: Fri, 17 May 2024 11:52:47 +0300 Subject: [PATCH 31/39] update: config for address model --- ...config.json => ner_bert_base_address.json} | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) rename deeppavlov/configs/ner/{ner_anna_config.json => ner_bert_base_address.json} (76%) diff --git a/deeppavlov/configs/ner/ner_anna_config.json b/deeppavlov/configs/ner/ner_bert_base_address.json similarity index 76% rename from deeppavlov/configs/ner/ner_anna_config.json rename to deeppavlov/configs/ner/ner_bert_base_address.json index 5dbfc1efe7..fb8d8fb598 100644 --- a/deeppavlov/configs/ner/ner_anna_config.json +++ b/deeppavlov/configs/ner/ner_bert_base_address.json @@ -1,7 +1,7 @@ { "dataset_reader": { "class_name": "conll2003_reader", - "data_path": "/home/korzanova/regex_for_pii/demo/train_data/", + "data_path": "/home/korzanova/DeepPavlov/regex_for_pii/demo/train_data/", "provide_pos": false }, "dataset_iterator": { @@ -11,11 +11,11 @@ "in": ["x"], "in_y": ["y"], "pipe": [ - { - "class_name": "sentence_delimiter", - "in": ["x"], - "out": ["x_short"] - }, + { + "class_name": "sentence_delimiter", + "in": ["x"], + "out": ["x_short"] + }, { "class_name": "torch_transformers_ner_preprocessor", "vocab_file": "{BASE_MODEL}", @@ -65,10 +65,10 @@ "out": ["y_pred"] }, { - "class_name": "sentence_concatenator", - "in": ["x_tokens", "y_pred"], - "out": ["output"] - } + "class_name": "sentence_concatenator", + "in": ["x_tokens", "y_pred"], + "out": ["output"] + } ], "out": ["x_tokens", "y_pred"] }, @@ -94,18 +94,18 @@ "pytest_max_batches": 2 }, "metadata": { - "variables": { - "BASE_MODEL": "bert-base-multilingual-cased", - "ROOT_PATH": "~/.deeppavlov", - "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "{MODELS_PATH}/ner_address/{BASE_MODEL}" - }, - "download": [ - { - "url": "http://files.deeppavlov.ai/v1/ner/dp_demo_ner.tar.gz", - "subdir": "{MODEL_PATH}" - } - ] + "variables": { + "BASE_MODEL": "bert-base-multilingual-cased", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/ner_address/{BASE_MODEL}" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/v1/ner/dp_demo_ner.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] } } \ No newline at end of file From 87a78cff875fff6689d70159543071232d77fe00 Mon Sep 17 00:00:00 2001 From: Anna Korzanova Date: Tue, 21 May 2024 13:42:46 +0300 Subject: [PATCH 32/39] fix: address config --- deeppavlov/configs/ner/ner_bert_base_address.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_address.json b/deeppavlov/configs/ner/ner_bert_base_address.json index fb8d8fb598..45b33e616d 100644 --- a/deeppavlov/configs/ner/ner_bert_base_address.json +++ b/deeppavlov/configs/ner/ner_bert_base_address.json @@ -1,7 +1,7 @@ { "dataset_reader": { "class_name": "conll2003_reader", - "data_path": "/home/korzanova/DeepPavlov/regex_for_pii/demo/train_data/", + "data_path": "{ROOT_PATH}/downloads/", "provide_pos": false }, "dataset_iterator": { From f4ac5f1422a0160f0faeea8913cc3e9018da2a13 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Thu, 30 May 2024 18:20:01 +0300 Subject: [PATCH 33/39] upd demo weights path --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 436477eeac..3d76fec0e4 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -105,7 +105,7 @@ }, "download": [ { - "url": "http://files.deeppavlov.ai/tmp/ner/updated_ner/ner_bert_cased_mult.tar.gz", + "url": "http://files.deeppavlov.ai/v1/ner/ner_bert_base.tar.gz", "subdir": "{MODEL_PATH}" } ] From 0b5eb17834539f7ece093050e8f911c491443216 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Fri, 31 May 2024 10:12:08 +0300 Subject: [PATCH 34/39] upd demo output format --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 3d76fec0e4..48c2caee8a 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -72,7 +72,7 @@ "out": ["output"] } ], - "out": ["output"] + "out": ["x_tokens", "y_pred"] }, "train": { "batch_size": 256, From 6f2cd016b7d2b60b29ded1d7a0d5cb47147c0b21 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Fri, 31 May 2024 14:22:10 +0300 Subject: [PATCH 35/39] changed sentence delimeter output format --- deeppavlov/configs/ner/ner_mult_long_demo.json | 2 +- deeppavlov/models/tokenizers/sentence_delimiter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deeppavlov/configs/ner/ner_mult_long_demo.json b/deeppavlov/configs/ner/ner_mult_long_demo.json index 48c2caee8a..3d76fec0e4 100644 --- a/deeppavlov/configs/ner/ner_mult_long_demo.json +++ b/deeppavlov/configs/ner/ner_mult_long_demo.json @@ -72,7 +72,7 @@ "out": ["output"] } ], - "out": ["x_tokens", "y_pred"] + "out": ["output"] }, "train": { "batch_size": 256, diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py index c936e583ec..5ba840a4aa 100644 --- a/deeppavlov/models/tokenizers/sentence_delimiter.py +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -21,4 +21,4 @@ def SentenceConcatenator(x_long, y_long): for sent in y_long: y_short.extend(sent) - return [[x_short, y_short]] \ No newline at end of file + return [x_short, y_short] \ No newline at end of file From 339132ef82aa943668185ae12506aefe84da8fe1 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Fri, 31 May 2024 15:44:48 +0300 Subject: [PATCH 36/39] revert sentence delimeter output format --- deeppavlov/models/tokenizers/sentence_delimiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/models/tokenizers/sentence_delimiter.py b/deeppavlov/models/tokenizers/sentence_delimiter.py index 5ba840a4aa..c936e583ec 100644 --- a/deeppavlov/models/tokenizers/sentence_delimiter.py +++ b/deeppavlov/models/tokenizers/sentence_delimiter.py @@ -21,4 +21,4 @@ def SentenceConcatenator(x_long, y_long): for sent in y_long: y_short.extend(sent) - return [x_short, y_short] \ No newline at end of file + return [[x_short, y_short]] \ No newline at end of file From a4ce35783f37c84f05a7ff7b69472047bde4f9c5 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Tue, 9 Jul 2024 15:07:51 +0300 Subject: [PATCH 37/39] upd demo model weights --- deeppavlov/configs/ner/ner_bert_base_address.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deeppavlov/configs/ner/ner_bert_base_address.json b/deeppavlov/configs/ner/ner_bert_base_address.json index 45b33e616d..31e9b3643e 100644 --- a/deeppavlov/configs/ner/ner_bert_base_address.json +++ b/deeppavlov/configs/ner/ner_bert_base_address.json @@ -99,11 +99,11 @@ "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "{MODELS_PATH}/ner_address/{BASE_MODEL}" + "MODEL_PATH": "{MODELS_PATH}/ner/{BASE_MODEL}" }, "download": [ { - "url": "http://files.deeppavlov.ai/v1/ner/dp_demo_ner.tar.gz", + "url": "http://files.deeppavlov.ai/tmp/dp_demo.tar.gz", "subdir": "{MODEL_PATH}" } ] From f32ef88db927bdd5179511cbc6f882699de8a582 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Fri, 25 Oct 2024 14:45:03 +0300 Subject: [PATCH 38/39] upd demo ner model --- .../configs/ner/ner_bert_base_address.json | 111 ------------------ .../ner/ner_demo_mdeberta_address.json | 54 +++++++++ 2 files changed, 54 insertions(+), 111 deletions(-) delete mode 100644 deeppavlov/configs/ner/ner_bert_base_address.json create mode 100644 deeppavlov/configs/ner/ner_demo_mdeberta_address.json diff --git a/deeppavlov/configs/ner/ner_bert_base_address.json b/deeppavlov/configs/ner/ner_bert_base_address.json deleted file mode 100644 index 31e9b3643e..0000000000 --- a/deeppavlov/configs/ner/ner_bert_base_address.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "dataset_reader": { - "class_name": "conll2003_reader", - "data_path": "{ROOT_PATH}/downloads/", - "provide_pos": false - }, - "dataset_iterator": { - "class_name": "data_learning_iterator" - }, - "chainer": { - "in": ["x"], - "in_y": ["y"], - "pipe": [ - { - "class_name": "sentence_delimiter", - "in": ["x"], - "out": ["x_short"] - }, - { - "class_name": "torch_transformers_ner_preprocessor", - "vocab_file": "{BASE_MODEL}", - "do_lower_case": false, - "max_seq_length": 512, - "in": ["x"], - "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] - }, - { - "id": "tag_vocab", - "class_name": "simple_vocab", - "unk_token": ["O"], - "save_path": "{MODEL_PATH}/tag.dict", - "load_path": "{MODEL_PATH}/tag.dict", - "fit_on": ["y"], - "in": ["y"], - "out": ["y_ind"] - }, - { - "class_name": "torch_transformers_sequence_tagger", - "n_tags": "#tag_vocab.len", - "pretrained_bert": "{BASE_MODEL}", - "attention_probs_keep_prob": 0.5, - "use_crf": true, - "encoder_layer_ids": [-1], - "optimizer": "AdamW", - "optimizer_parameters": { - "lr": 2e-05, - "weight_decay": 1e-06, - "betas": [0.9, 0.999], - "eps": 1e-06 - }, - "clip_norm": 1.0, - "min_learning_rate": 1e-07, - "learning_rate_drop_patience": 20, - "learning_rate_drop_div": 1.5, - "load_before_drop": true, - "save_path": "{MODEL_PATH}/model", - "load_path": "{MODEL_PATH}/model", - "in": ["x_subword_tok_ids", "attention_mask", "startofword_markers"], - "in_y": ["y_ind"], - "out": ["y_pred_ind", "probas"] - }, - { - "ref": "tag_vocab", - "in": ["y_pred_ind"], - "out": ["y_pred"] - }, - { - "class_name": "sentence_concatenator", - "in": ["x_tokens", "y_pred"], - "out": ["output"] - } - ], - "out": ["x_tokens", "y_pred"] - }, - "train": { - "batch_size": 128, - "metrics": [ - { - "name": "ner_f1", - "inputs": ["y", "y_pred"] - }, - { - "name": "ner_token_f1", - "inputs": ["y", "y_pred"] - } - ], - "validation_patience": 5, - "val_every_n_epochs": 1, - "log_every_n_epochs": 1, - "show_examples": false, - "evaluation_targets": ["train", "valid"], - "class_name": "torch_trainer", - "tensorboard_log_dir": "{MODEL_PATH}/", - "pytest_max_batches": 2 - }, - "metadata": { - "variables": { - "BASE_MODEL": "bert-base-multilingual-cased", - "ROOT_PATH": "~/.deeppavlov", - "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "{MODELS_PATH}/ner/{BASE_MODEL}" - }, - "download": [ - { - "url": "http://files.deeppavlov.ai/tmp/dp_demo.tar.gz", - "subdir": "{MODEL_PATH}" - } - ] - } - } \ No newline at end of file diff --git a/deeppavlov/configs/ner/ner_demo_mdeberta_address.json b/deeppavlov/configs/ner/ner_demo_mdeberta_address.json new file mode 100644 index 0000000000..fa81c15f3c --- /dev/null +++ b/deeppavlov/configs/ner/ner_demo_mdeberta_address.json @@ -0,0 +1,54 @@ +{ + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "class_name": "torch_transformers_ner_preprocessor", + "vocab_file": "{BASE_MODEL}", + "do_lower_case": false, + "max_seq_length": 512, + "in": ["x"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "startofword_markers", "attention_mask", "tokens_offsets"] + }, + { + "id": "tag_vocab", + "class_name": "simple_vocab", + "unk_token": ["O"], + "save_path": "{MODEL_PATH}/tag.dict", + "load_path": "{MODEL_PATH}/tag.dict", + "fit_on": ["y"], + "in": ["y"], + "out": ["y_ind"] + }, + { + "class_name": "torch_transformers_sequence_tagger", + "n_tags": "#tag_vocab.len", + "pretrained_bert": "{BASE_MODEL}", + "attention_probs_keep_prob": 0.5, + "use_crf": false, + "encoder_layer_ids": [-1], + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "in": ["x_subword_tok_ids", "attention_mask", "startofword_markers"], + "in_y": ["y_ind"], + "out": ["y_pred_ind", "probas"] + }, + { + "ref": "tag_vocab", + "in": ["y_pred_ind"], + "out": ["y_pred"] + } + ], + "out": ["x_tokens", "y_pred"] + }, + "metadata": { + "variables": { + "BASE_MODEL": "microsoft/mdeberta-v3-base", + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "/archive/savkin/models/ner/mdeberta_dp_demo_34_labels" + } + } +} From bf83193208a14a08762561b7781d1bcd61b67ef0 Mon Sep 17 00:00:00 2001 From: LogicZMaksimka Date: Fri, 25 Oct 2024 19:06:29 +0300 Subject: [PATCH 39/39] add download --- deeppavlov/configs/ner/ner_demo_mdeberta_address.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deeppavlov/configs/ner/ner_demo_mdeberta_address.json b/deeppavlov/configs/ner/ner_demo_mdeberta_address.json index fa81c15f3c..c8e4222c8d 100644 --- a/deeppavlov/configs/ner/ner_demo_mdeberta_address.json +++ b/deeppavlov/configs/ner/ner_demo_mdeberta_address.json @@ -48,7 +48,13 @@ "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models", - "MODEL_PATH": "/archive/savkin/models/ner/mdeberta_dp_demo_34_labels" - } + "MODEL_PATH": "{MODELS_PATH}/ner/demo/" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/tmp/mdeberta_dp_demo_34_labels.tar.gz", + "subdir": "{MODEL_PATH}" + } + ] } }