sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-26 07:44:18 +00:00
Code Issues Projects Releases Wiki Activity

Fix

Browse Source
This commit is contained in:
Stephan Porada 2020-05-20 15:01:52 +02:00
parent 3fc6ebff4c
commit 887e814020
1 changed files with 2 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy-nlp
View File

@ -52,7 +52,7 @@ with open(args.i, encoding=encoding) as input_file:
text = input_file.read() text = input_file.read()
# spaCys NLP is limited to strings with maximum 1 million characters at # spaCys NLP is limited to strings with maximum 1 million characters at
# once. So we split it into suitable chunks. # once. So we split it into suitable chunks.
text_chunks = textwrap.wrap(text, 1000, break_long_words=False) text_chunks = textwrap.wrap(text, 1000000, break_long_words=False)
# the text variable potentially occupies a lot of system memory and is no # the text variable potentially occupies a lot of system memory and is no
# longer needed... # longer needed...
del text del text
@ -66,7 +66,6 @@ nlp = spacy.load(SPACY_MODELS[args.language])
# See: http://cwb.sourceforge.net/files/CWB_Encoding_Tutorial/node3.html # See: http://cwb.sourceforge.net/files/CWB_Encoding_Tutorial/node3.html
output_file_original_filename = args.o output_file_original_filename = args.o
output_file_stand_off_filename = args.o.replace('.vrt', '.stand-off.vrt') output_file_stand_off_filename = args.o.replace('.vrt', '.stand-off.vrt')
output_file_tokens_filename = args.o.replace('.vrt', '.tokens.txt')
xml_head = '''<?xml version="1.0" encoding="UTF-8"?>\n\ xml_head = '''<?xml version="1.0" encoding="UTF-8"?>\n\
<corpus>\n\ <corpus>\n\
<text>\n\ <text>\n\
@ -78,12 +77,10 @@ xml_head = '''<?xml version="1.0" encoding="UTF-8"?>\n\
spacy_model=SPACY_MODELS[args.language]) spacy_model=SPACY_MODELS[args.language])
with open(output_file_original_filename, 'w+') as output_file_original, \ with open(output_file_original_filename, 'w+') as output_file_original, \
open(output_file_stand_off_filename, 'w+') as output_file_stand_off, \ open(output_file_stand_off_filename, 'w+') as output_file_stand_off:
open(output_file_tokens_filename, 'w+') as output_file_tokens:
output_file_original.write(xml_head) output_file_original.write(xml_head)
output_file_stand_off.write(xml_head) output_file_stand_off.write(xml_head)
output_file_tokens.write(xml_head)
text_offset = 0 text_offset = 0
for text_chunk in text_chunks: for text_chunk in text_chunks:
doc = nlp(text_chunk) doc = nlp(text_chunk)
@ -110,10 +107,8 @@ with open(output_file_original_filename, 'w+') as output_file_original, \
+ '\t{}'.format(token.pos_) + '\t{}'.format(token.pos_)
+ '\t{}'.format(token.tag_) + '\t{}'.format(token.tag_)
+ '\t{}\n'.format(token.ent_type_ or 'NULL')) + '\t{}\n'.format(token.ent_type_ or 'NULL'))
output_file_tokens.write('{}\n'.format(escape(token.text)))
output_file_original.write('</s>\n') output_file_original.write('</s>\n')
output_file_stand_off.write('</s>\n') output_file_stand_off.write('</s>\n')
text_offset = token_end + 1 text_offset = token_end + 1
output_file_original.write('</metadata>\n</text>\n</corpus>') output_file_original.write('</metadata>\n</text>\n</corpus>')
output_file_stand_off.write('</metadata>\n</text>\n</corpus>') output_file_stand_off.write('</metadata>\n</text>\n</corpus>')
output_file_tokens.write('</metadata>\n</text>\n</corpus>')