diff --git a/README.md b/README.md index f511c19..0cccb25 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,7 @@ Some weights of the model checkpoint at roberta-large were not used when initial | en-sci | allenai/scibert_scivocab_uncased | | zh | bert-base-chinese | | tr | dbmdz/bert-base-turkish-cased | +| ko | kykim/bert-kor-base | | others | bert-base-multilingual-cased | #### Default Layers diff --git a/bert_score/rescale_baseline/ko/kykim/bert-kor-base.tsv b/bert_score/rescale_baseline/ko/kykim/bert-kor-base.tsv new file mode 100644 index 0000000..f7b96d0 --- /dev/null +++ b/bert_score/rescale_baseline/ko/kykim/bert-kor-base.tsv @@ -0,0 +1,14 @@ +LAYER,P,R,F +0,0.25749525,0.2573355,0.25574756 +1,0.32705924,0.32676396,0.3259461 +2,0.3948546,0.39444658,0.39400318 +3,0.37353218,0.37326,0.3727555 +4,0.40359724,0.40326488,0.402877 +5,0.50273615,0.50234073,0.50211596 +6,0.5089314,0.5086513,0.5083372 +7,0.47281617,0.4724911,0.4721551 +8,0.4760624,0.4757873,0.47543192 +9,0.49104717,0.49070883,0.49041948 +10,0.48527616,0.48491594,0.4847094 +11,0.42766362,0.4272024,0.42691776 +12,0.3890542,0.3886027,0.38827258 diff --git a/bert_score/utils.py b/bert_score/utils.py index 8af24be..0add7b5 100644 --- a/bert_score/utils.py +++ b/bert_score/utils.py @@ -33,6 +33,7 @@ "zh": "bert-base-chinese", "tr": "dbmdz/bert-base-turkish-cased", "en-sci": "allenai/scibert_scivocab_uncased", + "ko": "kykim/bert-kor-base" } ) @@ -182,6 +183,20 @@ "microsoft/mdeberta-v3-base": 10, # 0.6778713684091584 "microsoft/deberta-v3-large": 12, # 0.6927693082293821 "khalidalt/DeBERTa-v3-large-mnli": 18, # 0.7428756686018376 + "kykim/bert-kor-base": 8, # 0.6525059442150242 + "klue/bert-base": 8, # 0.610519310980925 + "klue/roberta-small": 5, # 0.6155987371688525 + "klue/roberta-base": 9, # 0.6350758600987101 + "klue/roberta-large": 20, # 0.6632967999039878 + "kakaobank/kf-deberta-base": 8, # 0.6491671887483449 + "beomi/kcbert-base": 11, # 0.5642019923282193 + "lassl/bert-ko-base": 8, # 0.6157721449042898 + "snunlp/KR-BERT-char16424": 9, # 0.6396357376097898 + "snunlp/KR-FinBert-SC": 7, # 0.6092057202033887 + "monologg/distilkobert": 2, # 0.5736902466031919 + "kykim/electra-kor-base": 10, # 0.6393825654827927 + "monologg/koelectra-base-v3-discriminator": 11, # 0.6090487861932803 + "beomi/KcELECTRA-base": 11, # 0.6367047702247479 } @@ -330,7 +345,6 @@ def get_tokenizer(model_type, use_fast=False): else: assert not use_fast, "Fast tokenizer is not available for version < 4.0.0" tokenizer = AutoTokenizer.from_pretrained(model_type) - return tokenizer diff --git a/tune_layers/preprocess.py b/tune_layers/preprocess.py new file mode 100644 index 0000000..06208c7 --- /dev/null +++ b/tune_layers/preprocess.py @@ -0,0 +1,116 @@ +import os +import json +import zipfile +from pathlib import Path +import re # 정규표현식을 위한 import 추가 + +def extract_and_load_json_data(base_path, data_type): + """ + 영-한 키워드가 있는 ZIP 파일을 추출하고 JSON 데이터를 로드하는 함수 + + Args: + base_path (str): 데이터가 있는 기본 경로 + data_type (str): 'training' 또는 'validation' + + Returns: + list: JSON 데이터 리스트 + """ + # 라벨링 데이터 경로 설정 + labeling_path = Path(base_path) / data_type / "02.라벨링데이터" + extract_path = labeling_path / "extracted" + json_data = [] + + # 디렉토리가 존재하는지 확인 + if not labeling_path.exists(): + print(f"경로를 찾을 수 없습니다: {labeling_path}") + return json_data + + # 추출 디렉토리들 생성 + extract_path.mkdir(exist_ok=True) + ht_extract_path = labeling_path / "extracted_ht" # HT 데이터용 새로운 경로 + ht_extract_path.mkdir(exist_ok=True) + + # ZIP 파일 검색 및 압축 해제 + for zip_file in labeling_path.glob("*.zip"): + try: + if re.search(r'평가데이터\(MTPE\)_[^-]+-한', zip_file.name): + # MTPE 데이터 압축 해제 + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(extract_path) + print(f"MTPE 데이터 압축 해제 완료: {zip_file.name}") + elif re.search(r'VL_번역말뭉치\(HT\)_한', zip_file.name): + # HT 데이터 압축 해제 + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(ht_extract_path) + print(f"HT 데이터 압축 해제 완료: {zip_file.name}") + except Exception as e: + print(f"압축 해제 중 에러 발생 ({zip_file.name}): {str(e)}") + continue + + # 압축 해제된 JSON 파일 읽기 + for json_file in extract_path.glob("**/*.json"): + try: + with open(json_file, 'r', encoding='utf-8') as f: + data = json.load(f) + json_data.append(data) + print(f"JSON 파일 처리 완료: {json_file.name}") + except Exception as e: + print(f"JSON 파일 처리 중 에러 발생 ({json_file.name}): {str(e)}") + + return json_data + +def collect_source_sentences(base_path, data_type, output_filename="source_sentences.txt"): + """ + extracted_ht 폴더의 모든 JSON 파일에서 source_sentence를 추출하여 txt 파일로 저장하는 함수 + + Args: + base_path (str): 데이터가 있는 기본 경로 + data_type (str): 'training' 또는 'validation' + output_filename (str): 출력할 txt 파일명 + """ + # 경로 설정 + ht_extract_path = Path(base_path) / data_type / "02.라벨링데이터/extracted_ht" + output_path = Path(base_path) / data_type / "02.라벨링데이터" / output_filename + + # 디렉토리 존재 확인 + if not ht_extract_path.exists(): + print(f"경로를 찾을 수 없습니다: {ht_extract_path}") + return + + # source_sentence 수집 + sentences = [] + for json_file in ht_extract_path.glob("**/*.json"): + try: + with open(json_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, dict) and 'data' in data: + # data 리스트 내의 각 항목에서 source_sentence 추출 + for item in data['data']: + if 'source_sentence' in item: + sentences.append(item['source_sentence']) + print(f"JSON 파일 처리 완료: {json_file.name}") + except Exception as e: + print(f"JSON 파일 처리 중 에러 발생 ({json_file.name}): {str(e)}") + + # 수집된 문장들을 txt 파일로 저장 + try: + with open(output_path, 'w', encoding='utf-8') as f: + for sentence in sentences: + f.write(sentence + '\n') + print(f"문장 추출 완료. 총 {len(sentences)}개의 문장이 {output_path}에 저장되었습니다.") + except Exception as e: + print(f"파일 저장 중 에러 발생: {str(e)}") + +def main(): + # 기본 경로 설정 + base_path = "008.다국어 번역 품질 평가 데이터/3.개방데이터/1.데이터" + + # validation 데이터 처리 + validation_data = extract_and_load_json_data(base_path, "Validation") + print(f"검증 데이터 개수: {len(validation_data)}") + + # source_sentence 수집 및 저장 + collect_source_sentences(base_path, "Validation") + +if __name__ == "__main__": + main()