sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-26 09:44:18 +00:00
Code Issues Projects Releases Wiki Activity

Fix check_encoding functionality

Browse Source
This commit is contained in:
Stephan Porada 2020-02-12 14:16:36 +01:00
parent 504861ae07
commit 1a3e4a0a02
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nlp
View File

@ -45,7 +45,7 @@ def parse_arguments():
action='store_true', action='store_true',
default=False, default=False,
dest="check_encoding", dest="check_encoding",
help='''if used the nlp process will know hat the encoding of help='''if used the nlp process will know hat the encoding of
the input files is unkown and thus != utf-8. The process will the input files is unkown and thus != utf-8. The process will
try to determine the encoding of the input files and use this. try to determine the encoding of the input files and use this.
encoding.''' encoding.'''
@ -60,7 +60,7 @@ class NLPWorkflow(WorkflowRunner):
self.n_cores = args.n_cores self.n_cores = args.n_cores
self.output_dir = args.output_dir self.output_dir = args.output_dir
self.zip = args.zip self.zip = args.zip
self.check_encoding self.check_encoding = args.check_encoding
def workflow(self): def workflow(self):
if len(self.jobs) == 0: if len(self.jobs) == 0:
@ -95,8 +95,8 @@ class NLPWorkflow(WorkflowRunner):
cmd = 'spacy_nlp -l "{}" "{}" "{}" "{}"'.format( cmd = 'spacy_nlp -l "{}" "{}" "{}" "{}"'.format(
self.lang, self.lang,
job['path'], job['path'],
os.path.join(job['output_dir'], job['name'] + '.vrt', os.path.join(job['output_dir'], job['name'] + '.vrt'),
if self.check_encoding "--check-encoding" else "") "--check-encoding" if self.check_encoding else ""
) )
nlp_jobs.append( nlp_jobs.append(
self.addTask( self.addTask(

2
spacy_nlp
View File

@ -3,7 +3,7 @@
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
import argparse import argparse
import os import chardet
import spacy import spacy
import textwrap import textwrap