Update to newer Version

spacy-nlp

@@ -4,29 +4,28 @@
 from argparse import ArgumentParser
 from xml.sax.saxutils import escape
 import chardet
+import hashlib
+import os
 import spacy
 import textwrap
-import hashlib
-
-
-SPACY_MODELS = {'de': 'de_core_news_sm',
-                'el': 'el_core_news_sm',
-                'en': 'en_core_web_sm',
-                'es': 'es_core_news_sm',
-                'fr': 'fr_core_news_sm',
-                'it': 'it_core_news_sm',
-                'nl': 'nl_core_news_sm',
-                'pt': 'pt_core_news_sm'}
 
+SPACY_MODELS = {'de': 'de_core_news_lg',
+                'el': 'el_core_news_lg',
+                'en': 'en_core_web_lg',
+                'es': 'es_core_news_lg',
+                'fr': 'fr_core_news_lg',
+                'it': 'it_core_news_lg',
+                'nl': 'nl_core_news_lg',
+                'pt': 'pt_core_news_lg'}
+SPACY_MODELS_VERSION = os.environ.get('SPACY_MODELS_VERSION')
+SPACY_VERSION = os.environ.get('SPACY_VERSION')
 
 # Parse the given arguments
 parser = ArgumentParser(description=('Tag a text file with spaCy and save it '
                                      'as a verticalized text file.'))
 parser.add_argument('-i', '--input', metavar='txt-sourcefile', required=True)
 parser.add_argument('-o', '--output', metavar='vrt-destfile', required=True)
-parser.add_argument('-l', '--language',
-                    choices=SPACY_MODELS.keys(),
-                    required=True)
+parser.add_argument('-l', '--language', choices=SPACY_MODELS.keys(), required=True)  # noqa
 parser.add_argument('--check-encoding', action='store_true')
 args = parser.parse_args()
 
@@ -43,10 +42,10 @@ else:
 
 # hashing in chunks to avoid full RAM with huge files.
 with open(args.input, 'rb') as input_file:
-    md5_hash = hashlib.md5()
-    for chunk in iter(lambda: input_file.read(128 * md5_hash.block_size), b''):
-        md5_hash.update(chunk)
-    md5_hash = md5_hash.hexdigest()
+    source_md5 = hashlib.md5()
+    for chunk in iter(lambda: input_file.read(128 * source_md5.block_size), b''):
+        source_md5.update(chunk)
+    source_md5 = source_md5.hexdigest()
 
 # Load the text contents from the input file
 with open(args.input, encoding=encoding) as input_file:
@@ -60,7 +59,8 @@ with open(args.input, encoding=encoding) as input_file:
 
 
 # Setup the spaCy toolkit by loading the chosen language model
-nlp = spacy.load(SPACY_MODELS[args.language])
+model = SPACY_MODELS[args.language]
+nlp = spacy.load(model)
 
 
 # Create the output file in verticalized text format
@@ -70,11 +70,9 @@ output_file_stand_off_filename = args.output.replace('.vrt', '.stand-off.vrt')
 common_xml = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
               + '<corpus>\n'
               + '<text>\n'
-              + '<nlp name="spaCy"\n'
-              + '     version="{}"\n'.format(spacy.__version__)
-              + '     model="{}"\n'.format(SPACY_MODELS[args.language])
-              + '     model_version="{}"\n'.format(nlp.meta['version'])
-              + '     md5_hash_of_input="{}" />\n'.format(md5_hash))
+              + '<nlp name="spaCy:{}"\n'.format(SPACY_VERSION)
+              + '     model="{}:{}"\n'.format(model, SPACY_MODELS_VERSION)
+              + '     source-md5="{}" />\n'.format(source_md5))
 
 with open(output_file_original_filename, 'w+') as output_file_original, \
      open(output_file_stand_off_filename, 'w+') as output_file_stand_off: