Add linewrap function and test.py for fun.

spacy_nlp

@@ -5,6 +5,7 @@
 import argparse
 import os
 import spacy
+import textwrap
 
 
 parser = argparse.ArgumentParser(description="Tag a .txt file with spaCy and \
@@ -29,21 +30,23 @@ SPACY_MODELS = {"de": "de_core_news_sm", "en": "en_core_web_sm",
                 "es": "es_core_news_sm", "fr": "fr_core_news_sm",
                 "pt": "pt_core_news_sm"}
 
-
 # Set the language model for spacy
 nlp = spacy.load(SPACY_MODELS[args.lang])
 
-# Read text from the input file
+# Read text from the input file and if neccessary split it into parts with a
+# length of less than 1 million characters.
 with open(args.input) as input_file:
     text = input_file.read()
+    texts = textwrap.wrap(text, 1000000, break_long_words=False)
 
-# Run spacy nlp over the text
-doc = nlp(text)
 
 # Create and open the output file
 output_file = open(args.output, "w+")
 output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n<corpus>\n<text id="' + args.input.rsplit(".", 1)[0] + '">\n')
-for sent in doc.sents:
+for text in texts:
+    # Run spacy nlp over the text (partial string if above 1 million chars)
+    doc = nlp(text)
+    for sent in doc.sents:
         output_file.write('<s>\n')
         for token in sent:
             # Skip whitespace tokens like "\n" or "\t"

test.py

@@ -0,0 +1,36 @@
+import textwrap
+
+
+def limit_text(text, character_limit):
+    """
+    This function checks if a string is below 1000000 (1 Million characters).
+    If it is below that limmit the text will be processed. If it is above the
+    limit, the text will be splitted into parts below 1 million characters.
+    Parts will be as long as possible.
+    Returns a list of strings each below the character limit.
+    """
+    str_list = []
+    if(len(text) > character_limit):
+        cut_off = text.index(" ", character_limit - 10, character_limit)
+        tmp_strings = [text[:cut_off]]
+        tmp_strings.append(text[cut_off:])
+        for string in tmp_strings:
+            if(len(string) < character_limit):
+                str_list.append(string)
+            elif(len(string) > character_limit):
+                tmp_strings = limit_text(string, character_limit)
+                for string in tmp_strings:
+                    str_list.append(string)
+    else:
+        str_list.append(text)
+    return str_list
+
+def main():
+    text = "If true, TextWrapper attempts to detect sentence endings and ensure that sentences are always separated by exactly two spaces. This is generally desired for text in a monospaced font. However, the sentence detection algorithm is imperfect:"
+    texts = limit_text(text, 50)
+    lines = textwrap.wrap(text, 50, break_long_words=False)
+    print("Own version:", texts)
+    print("Lib:", lines)
+
+if __name__ == '__main__':
+    main()