From 361ab86639104449d6a6bbfd506b82f69fd251dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ali=20h=C3=BCrriyetoglu?= Date: Thu, 27 Jan 2022 23:19:14 +0300 Subject: [PATCH 1/5] convert the data in the benchmarks folder from one format to another --- format_converter.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 84 insertions(+) create mode 100644 format_converter.py create mode 100644 requirements.txt diff --git a/format_converter.py b/format_converter.py new file mode 100644 index 0000000..5166ded --- /dev/null +++ b/format_converter.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from cassis import load_typesystem, load_cas_from_xmi +from zipfile import ZipFile +from glob import glob +import argparse + +def convert_xml2BIO(lang='en', xmi_file_name='teresa'): + + available_languages = [l.split('/')[-1].lower() for l in glob('./benchmarks/*')] + + if lang.lower() not in available_languages: + print(f'{lang} folder does not exists. The available languages are {available_languages}') + return + + + for datafolder in glob('benchmarks/'+lang.upper()+'/xml/*'): + + for zfile in glob(datafolder+'/webanno*.zip'): + + typesystemfile_content, doc = None, None + with ZipFile(zfile) as myzip: + with myzip.open('TypeSystem.xml', 'r') as myfile: + typesystemfile_content = myfile.read().decode() + typesystem = load_typesystem(typesystemfile_content) + + try: + with myzip.open('teresa.xmi') as myfile: + doc = load_cas_from_xmi(myfile.read().decode(), typesystem=typesystem) + except: + continue + + span_list = [] + for sp in doc.select('custom.Span'): + span_list.append({'covered_text':sp.get_covered_text(), 'label':sp.label, 'begin':sp.begin, 'end':sp.end}) + + if len(span_list) == 0: + continue + + tok_list = [] + for tok in doc.select('de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token'): + tok_list.append({'covered_text':tok.get_covered_text(), "begin":tok.begin, "end":tok.end}) + + recent_annot_index = 0 + for tok in tok_list: + recent_span = span_list[recent_annot_index] + if tok['covered_text'] is None or recent_span['label'] is None: + continue + if recent_span['begin'] == tok['begin']: + print(tok['covered_text']+'\t'+'B-'+recent_span['label']) + elif recent_span['begin'] < tok['begin'] and recent_span['end'] >= tok['end']: + print(tok['covered_text']+'\t'+'I-'+recent_span['label']) + else: + print(tok['covered_text']+'\t'+'O') + + if tok['covered_text'] == '.': + print() + + if tok['end'] == recent_span['end']: + left_longest_annot_index = recent_annot_index + while span_list[left_longest_annot_index]['end'] >= span_list[recent_annot_index]['begin']: + if (recent_annot_index + 1) < len(span_list): + recent_annot_index += 1 + else: + break + +if __name__ == "__main__": + + my_parser = argparse.ArgumentParser(description='convert Odeuropa benchmark data from a format to another one.') + my_parser.add_argument('--i', '--iformat', type=str, help='the source format', required=True) + my_parser.add_argument('--o', '--oformat', type=str, help='the target format', required=True) + my_parser.add_argument('--l', '--lang', type=str, help='the language folder', required=True) + + args = my_parser.parse_args() + + # print(args.iformat, args.oformat, args.lang) + + + if args.iformat == 'xml' and args.oformat == 'bio': + convert_xml2BIO(args.lang) + else: + print(f'The conversion from {args.iformat} to {args.oformat} is not defined.') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..caf6845 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dkpro-cassis==0.7.0 From c1df593bd24fa73224575fd0be14aeddbdfe4fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ali=20h=C3=BCrriyetoglu?= Date: Thu, 27 Jan 2022 23:24:26 +0300 Subject: [PATCH 2/5] add info for the format converter in the README. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 0a62b88..a4f085c 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,11 @@ The full list of annotated documents is available in Documents in Benchmark.xlsx | EN | https://drive.google.com/drive/folders/14oA3pT6FmebUWhic6zyxkWSAOPxvOQPg?usp=sharing | | IT | https://drive.google.com/drive/folders/1frkLwHywjDg9SUqxfkymyJNb1DLXJPWZ?usp=sharing | + +## Converting data to additional formats + +The data can be converted to new formats such as BIO. For instance, run the following command for converting xml files to BIO + +>> python format_converter.py --iformat xml --oformat bio --lang en + + From 52aa3399851bc80de5a4879cd18bb349a2bcabc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ali=20h=C3=BCrriyetoglu?= Date: Thu, 27 Jan 2022 23:35:14 +0300 Subject: [PATCH 3/5] fix README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a4f085c..ecb05bd 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ The full list of annotated documents is available in Documents in Benchmark.xlsx ## Converting data to additional formats -The data can be converted to new formats such as BIO. For instance, run the following command for converting xml files to BIO +The data can be converted to new formats such as BIO. For instance, run the following command for converting xml files to BIO: ->> python format_converter.py --iformat xml --oformat bio --lang en +\$ python format_converter.py --iformat xml --oformat bio --lang en From 3315e308f71aab2f76fe7351180e82235d7ac618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ali=20h=C3=BCrriyetoglu?= Date: Sat, 19 Feb 2022 22:56:02 +0300 Subject: [PATCH 4/5] hard coded xmi name was updated --- format_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/format_converter.py b/format_converter.py index 5166ded..102009b 100644 --- a/format_converter.py +++ b/format_converter.py @@ -6,7 +6,7 @@ from glob import glob import argparse -def convert_xml2BIO(lang='en', xmi_file_name='teresa'): +def convert_xml2BIO(lang='en', xmi_file_name='teresa.xmi'): available_languages = [l.split('/')[-1].lower() for l in glob('./benchmarks/*')] @@ -26,7 +26,7 @@ def convert_xml2BIO(lang='en', xmi_file_name='teresa'): typesystem = load_typesystem(typesystemfile_content) try: - with myzip.open('teresa.xmi') as myfile: + with myzip.open(xmi_file_name) as myfile: doc = load_cas_from_xmi(myfile.read().decode(), typesystem=typesystem) except: continue @@ -78,6 +78,6 @@ def convert_xml2BIO(lang='en', xmi_file_name='teresa'): if args.iformat == 'xml' and args.oformat == 'bio': - convert_xml2BIO(args.lang) + convert_xml2BIO(args.lang, ) else: print(f'The conversion from {args.iformat} to {args.oformat} is not defined.') From fdefeacd0f848982e4d971d3b7b8381654437035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ali=20h=C3=BCrriyetoglu?= Date: Sun, 20 Feb 2022 12:26:04 +0300 Subject: [PATCH 5/5] add converter bio to json. and some minor optimizations. --- format_converter.py | 68 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/format_converter.py b/format_converter.py index 102009b..ccc581d 100644 --- a/format_converter.py +++ b/format_converter.py @@ -1,20 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from cassis import load_typesystem, load_cas_from_xmi from zipfile import ZipFile from glob import glob import argparse +import json def convert_xml2BIO(lang='en', xmi_file_name='teresa.xmi'): - + from cassis import load_typesystem, load_cas_from_xmi + available_languages = [l.split('/')[-1].lower() for l in glob('./benchmarks/*')] if lang.lower() not in available_languages: print(f'{lang} folder does not exists. The available languages are {available_languages}') return - for datafolder in glob('benchmarks/'+lang.upper()+'/xml/*'): for zfile in glob(datafolder+'/webanno*.zip'): @@ -41,8 +41,13 @@ def convert_xml2BIO(lang='en', xmi_file_name='teresa.xmi'): tok_list = [] for tok in doc.select('de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token'): tok_list.append({'covered_text':tok.get_covered_text(), "begin":tok.begin, "end":tok.end}) + + sent_list = [] + for snt in doc.select('de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence'): + sent_list.append({'covered_text':snt.get_covered_text(), 'begin':snt.begin, 'end':snt.end}) recent_annot_index = 0 + recent_sent_index = 0 for tok in tok_list: recent_span = span_list[recent_annot_index] if tok['covered_text'] is None or recent_span['label'] is None: @@ -54,8 +59,14 @@ def convert_xml2BIO(lang='en', xmi_file_name='teresa.xmi'): else: print(tok['covered_text']+'\t'+'O') - if tok['covered_text'] == '.': - print() + #if tok['covered_text'] in '.!?': + if tok['end'] == sent_list[recent_sent_index]['end']: + print() # new line between sentences. + recent_sent_index += 1 + if len(sent_list) > recent_sent_index: + print(sent_list[recent_sent_index]['covered_text']) + else: + print('last sent in doc') if tok['end'] == recent_span['end']: left_longest_annot_index = recent_annot_index @@ -64,20 +75,55 @@ def convert_xml2BIO(lang='en', xmi_file_name='teresa.xmi'): recent_annot_index += 1 else: break + +def convert_sentbio2sentJSON(sentbiofile): + with open(sentbiofile) as f: + sentence_chunks = f.read().split('\n\n') + +# print(sentence_chunks[1], end='\n\n') + + testsents = [] + for tok_annots in sentence_chunks: + # print(tok_annots) + + sent_tokens_list = [] + sent_annots_list = [] + + for tk_ann_str in tok_annots.split('\n'): + tk_ann_list = tk_ann_str.split('\t') + + if len(tk_ann_list) != 2: +# print('Skipped:') +# print('tk_ann_str:', tk_ann_str) +# print('tk_ann_list:', tk_ann_list) + continue + + sent_tokens_list.append(tk_ann_list[0]) + sent_annots_list.append(tk_ann_list[1]) + + final_sent_instance = {} + final_sent_instance['text'] = " ".join(sent_tokens_list) + final_sent_instance['label'] = 'smell' if (len(set(sent_annots_list))>1) else 'nonsmell' + + print(json.dumps(final_sent_instance, ensure_ascii=False)) + if __name__ == "__main__": my_parser = argparse.ArgumentParser(description='convert Odeuropa benchmark data from a format to another one.') - my_parser.add_argument('--i', '--iformat', type=str, help='the source format', required=True) - my_parser.add_argument('--o', '--oformat', type=str, help='the target format', required=True) - my_parser.add_argument('--l', '--lang', type=str, help='the language folder', required=True) + my_parser.add_argument('-i', '--iformat', type=str, help='the source format', required=True) + my_parser.add_argument('-o', '--oformat', type=str, help='the target format', required=True) + my_parser.add_argument('-l', '--lang', type=str, help='the language') + my_parser.add_argument('-b', '--sentbiofile', type=str, help='the language') args = my_parser.parse_args() - # print(args.iformat, args.oformat, args.lang) + print('args are:', args) - if args.iformat == 'xml' and args.oformat == 'bio': - convert_xml2BIO(args.lang, ) + if args.iformat == 'xml' and args.oformat == 'sentbio': + convert_xml2BIO(args.lang, 'teresa.xmi') + elif args.iformat == 'sentbio' and args.oformat == 'sentjson': + convert_sentbio2sentJSON(args.sentbiofile) else: print(f'The conversion from {args.iformat} to {args.oformat} is not defined.')