nlp/spacy_nlp

#!/usr/bin/env python3.5
# coding=utf-8

from xml.sax.saxutils import escape
import argparse
import chardet
import spacy
import textwrap

parser = argparse.ArgumentParser(
    description=('Tag a text file with spaCy and save it as a verticalized '
                 'text file.')
)
parser.add_argument('i', metavar='txt-sourcefile')
parser.add_argument('-l',
                    choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
                    dest='lang',
                    required=True)
parser.add_argument('o', metavar='vrt-destfile')
parser.add_argument('--check-encoding',
                    default=False,
                    action='store_true',
                    dest='check_encoding'
                    )
args = parser.parse_args()

SPACY_MODELS = {'de': 'de_core_news_sm',
                'el': 'el_core_news_sm',
                'en': 'en_core_web_sm',
                'es': 'es_core_news_sm',
                'fr': 'fr_core_news_sm',
                'it': 'it_core_news_sm',
                'nl': 'nl_core_news_sm',
                'pt': 'pt_core_news_sm'}

# Set the language model for spacy
nlp = spacy.load(SPACY_MODELS[args.lang])

# Try to determine the encoding of the text in the input file
if args.check_encoding:
    with open(args.i, "rb") as input_file:
        bytes = input_file.read()
        encoding = chardet.detect(bytes)['encoding']
else:
    encoding = 'utf-8'
# Read text from the input file and if neccessary split it into parts with a
# length of less than 1 million characters.
with open(args.i, encoding=encoding) as input_file:
    text = input_file.read()
    texts = textwrap.wrap(text, 1000000, break_long_words=False)
    text = None

# Create and open the output file
output_file = open(args.o, 'w+')

output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n'
                  '<corpus>\n'
                  '<text>\n')
for text in texts:
    # Run spacy nlp over the text (partial string if above 1 million chars)
    doc = nlp(text)
    for sent in doc.sents:
        output_file.write('<s>\n')
        for token in sent:
            # Skip whitespace tokens like "\n" or "\t"
            if token.text.isspace():
                continue
            # Write all information in .vrt style to the output file
            # text, lemma, simple_pos, pos, ner
            output_file.write(
                '{}\t{}\t{}\t{}\t{}\n'.format(
                    escape(token.text),
                    escape(token.lemma_),
                    token.pos_,
                    token.tag_,
                    token.ent_type_ if token.ent_type_ != '' else 'NULL'
                )
            )
        output_file.write('</s>\n')
output_file.write('</text>\n'
                  '</corpus>')

output_file.close()