mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2024-12-26 08:14:18 +00:00
84 lines
2.7 KiB
Python
Executable File
84 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3.5
|
|
# coding=utf-8
|
|
|
|
from xml.sax.saxutils import escape
|
|
import argparse
|
|
import chardet
|
|
import spacy
|
|
import textwrap
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description=('Tag a text file with spaCy and save it as a verticalized '
|
|
'text file.')
|
|
)
|
|
parser.add_argument('i', metavar='txt-sourcefile')
|
|
parser.add_argument('-l',
|
|
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
|
|
dest='lang',
|
|
required=True)
|
|
parser.add_argument('o', metavar='vrt-destfile')
|
|
parser.add_argument('--check-encoding',
|
|
default=False,
|
|
action='store_true',
|
|
dest='check_encoding'
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
SPACY_MODELS = {'de': 'de_core_news_sm',
|
|
'el': 'el_core_news_sm',
|
|
'en': 'en_core_web_sm',
|
|
'es': 'es_core_news_sm',
|
|
'fr': 'fr_core_news_sm',
|
|
'it': 'it_core_news_sm',
|
|
'nl': 'nl_core_news_sm',
|
|
'pt': 'pt_core_news_sm'}
|
|
|
|
# Set the language model for spacy
|
|
nlp = spacy.load(SPACY_MODELS[args.lang])
|
|
|
|
# Try to determine the encoding of the text in the input file
|
|
if args.check_encoding:
|
|
with open(args.i, "rb") as input_file:
|
|
bytes = input_file.read()
|
|
encoding = chardet.detect(bytes)['encoding']
|
|
else:
|
|
encoding = 'utf-8'
|
|
# Read text from the input file and if neccessary split it into parts with a
|
|
# length of less than 1 million characters.
|
|
with open(args.i, encoding=encoding) as input_file:
|
|
text = input_file.read()
|
|
texts = textwrap.wrap(text, 1000000, break_long_words=False)
|
|
text = None
|
|
|
|
# Create and open the output file
|
|
output_file = open(args.o, 'w+')
|
|
|
|
output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
'<corpus>\n'
|
|
'<text>\n')
|
|
for text in texts:
|
|
# Run spacy nlp over the text (partial string if above 1 million chars)
|
|
doc = nlp(text)
|
|
for sent in doc.sents:
|
|
output_file.write('<s>\n')
|
|
for token in sent:
|
|
# Skip whitespace tokens like "\n" or "\t"
|
|
if token.text.isspace():
|
|
continue
|
|
# Write all information in .vrt style to the output file
|
|
# text, lemma, simple_pos, pos, ner
|
|
output_file.write(
|
|
'{}\t{}\t{}\t{}\t{}\n'.format(
|
|
escape(token.text),
|
|
escape(token.lemma_),
|
|
token.pos_,
|
|
token.tag_,
|
|
token.ent_type_ if token.ent_type_ != '' else 'NULL'
|
|
)
|
|
)
|
|
output_file.write('</s>\n')
|
|
output_file.write('</text>\n'
|
|
'</corpus>')
|
|
|
|
output_file.close()
|