Add linewrap function and test.py for fun.

spacy_nlp

@@ -5,6 +5,7 @@
 import argparse
 import os
 import spacy
+import textwrap
 
 
 parser = argparse.ArgumentParser(description="Tag a .txt file with spaCy and \
@@ -29,31 +30,33 @@ SPACY_MODELS = {"de": "de_core_news_sm", "en": "en_core_web_sm",
                 "es": "es_core_news_sm", "fr": "fr_core_news_sm",
                 "pt": "pt_core_news_sm"}
 
-
 # Set the language model for spacy
 nlp = spacy.load(SPACY_MODELS[args.lang])
 
-# Read text from the input file
+# Read text from the input file and if neccessary split it into parts with a
+# length of less than 1 million characters.
 with open(args.input) as input_file:
     text = input_file.read()
+    texts = textwrap.wrap(text, 1000000, break_long_words=False)
 
-# Run spacy nlp over the text
-doc = nlp(text)
 
 # Create and open the output file
 output_file = open(args.output, "w+")
 output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n<corpus>\n<text id="' + args.input.rsplit(".", 1)[0] + '">\n')
-for sent in doc.sents:
-    output_file.write('<s>\n')
-    for token in sent:
-        # Skip whitespace tokens like "\n" or "\t"
-        if token.text.isspace():
-            continue
-        # Write all information in .vrt style to the output file
-        # text, lemma, simple_pos, pos, ner
-        output_file.write(token.text + "\t" + token.lemma_ + "\t"
-                          + token.pos_ + "\t" + token.tag_ + "\t"
-                          + (token.ent_type_ if token.ent_type_ != "" else "NULL") + "\n")
-    output_file.write('</s>\n')
+for text in texts:
+    # Run spacy nlp over the text (partial string if above 1 million chars)
+    doc = nlp(text)
+    for sent in doc.sents:
+        output_file.write('<s>\n')
+        for token in sent:
+            # Skip whitespace tokens like "\n" or "\t"
+            if token.text.isspace():
+                continue
+            # Write all information in .vrt style to the output file
+            # text, lemma, simple_pos, pos, ner
+            output_file.write(token.text + "\t" + token.lemma_ + "\t"
+                              + token.pos_ + "\t" + token.tag_ + "\t"
+                              + (token.ent_type_ if token.ent_type_ != "" else "NULL") + "\n")
+        output_file.write('</s>\n')
 output_file.write('</text>\n</corpus>')
 output_file.close()