Codestyle

This commit is contained in:
Patrick Jentsch 2019-09-11 16:15:41 +02:00
parent 43717de313
commit af293d6141
2 changed files with 53 additions and 64 deletions

48
nlp
View File

@ -18,33 +18,23 @@ from pyflow import WorkflowRunner
def parse_arguments(): def parse_arguments():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Performs NLP of documents utilizing spaCy. The results are served as verticalized text files.' description=('Performs NLP of documents utilizing spaCy. The results '
) 'are served as verticalized text files.')
parser.add_argument(
'-i',
dest='input_dir',
required=True
) )
parser.add_argument('-i', dest='input_dir', required=True)
parser.add_argument( parser.add_argument(
'-l', '-l',
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'], choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
dest='lang', dest='lang',
required=True required=True
) )
parser.add_argument( parser.add_argument('-o', dest='output_dir', required=True)
'-o', parser.add_argument('--nCores',
dest='output_dir',
required=True
)
parser.add_argument(
'--nCores',
default=min(4, multiprocessing.cpu_count()), default=min(4, multiprocessing.cpu_count()),
dest='n_cores', dest='n_cores',
help='total number of cores available', help='total number of cores available',
required=False, required=False,
type=int type=int)
)
return parser.parse_args() return parser.parse_args()
@ -65,11 +55,11 @@ class NLPWorkflow(WorkflowRunner):
''' '''
create_output_directories_jobs = [] create_output_directories_jobs = []
for index, job in enumerate(self.jobs): for index, job in enumerate(self.jobs):
cmd = 'mkdir -p "%s"' % (job['output_dir']) cmd = 'mkdir -p "{}"'.format(job['output_dir'])
create_output_directories_jobs.append( create_output_directories_jobs.append(
self.addTask( self.addTask(
command=cmd, command=cmd,
label='create_output_directories_job_-_%i' % (index) label='create_output_directories_job_-_{}'.format(index)
) )
) )
@ -84,7 +74,7 @@ class NLPWorkflow(WorkflowRunner):
max(1, int(self.n_cores / len(self.jobs))) max(1, int(self.n_cores / len(self.jobs)))
) )
for index, job in enumerate(self.jobs): for index, job in enumerate(self.jobs):
cmd = 'spacy_nlp -l "%s" "%s" "%s"' % ( cmd = 'spacy_nlp -l "{}" "{}" "{}"'.format(
self.lang, self.lang,
job['path'], job['path'],
os.path.join(job['output_dir'], job['name'] + '.vrt') os.path.join(job['output_dir'], job['name'] + '.vrt')
@ -92,8 +82,10 @@ class NLPWorkflow(WorkflowRunner):
nlp_jobs.append( nlp_jobs.append(
self.addTask( self.addTask(
command=cmd, command=cmd,
dependencies='create_output_directories_job_-_%i' % (index), dependencies='create_output_directories_job_-_{}'.format(
label='nlp_job_-_%i' % (index), index
),
label='nlp_job_-_{}'.format(index),
nCores=nlp_job_n_cores nCores=nlp_job_n_cores
) )
) )
@ -104,19 +96,13 @@ def analyze_jobs(input_dir, output_dir):
for file in os.listdir(input_dir): for file in os.listdir(input_dir):
if os.path.isdir(os.path.join(input_dir, file)): if os.path.isdir(os.path.join(input_dir, file)):
jobs += analyze_jobs( jobs += analyze_jobs(os.path.join(input_dir, file),
os.path.join(input_dir, file), os.path.join(output_dir, file))
os.path.join(output_dir, file),
)
elif file.endswith('.txt'): elif file.endswith('.txt'):
jobs.append( jobs.append({'filename': file,
{
'filename': file,
'name': file.rsplit('.', 1)[0], 'name': file.rsplit('.', 1)[0],
'output_dir': os.path.join(output_dir, file), 'output_dir': os.path.join(output_dir, file),
'path': os.path.join(input_dir, file) 'path': os.path.join(input_dir, file)})
}
)
return jobs return jobs

View File

@ -7,29 +7,25 @@ import spacy
import textwrap import textwrap
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Tag a text file with spaCy and save it as a verticalized text file.' description=('Tag a text file with spaCy and save it as a verticalized '
'text file.')
) )
parser.add_argument( parser.add_argument('i', metavar='txt-sourcefile')
'i', parser.add_argument('-l',
metavar='txt-sourcefile',
)
parser.add_argument(
'-l',
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'], choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
dest='lang', dest='lang',
required=True required=True)
) parser.add_argument('o', metavar='vrt-destfile')
parser.add_argument(
'o',
metavar='vrt-destfile',
)
args = parser.parse_args() args = parser.parse_args()
SPACY_MODELS = { SPACY_MODELS = {'de': 'de_core_news_sm',
'de': 'de_core_news_sm', 'el': 'el_core_news_sm', 'en': 'en_core_web_sm', 'el': 'el_core_news_sm',
'es': 'es_core_news_sm', 'fr': 'fr_core_news_sm', 'it': 'it_core_news_sm', 'en': 'en_core_web_sm',
'nl': 'nl_core_news_sm', 'pt': 'pt_core_news_sm' 'es': 'es_core_news_sm',
} 'fr': 'fr_core_news_sm',
'it': 'it_core_news_sm',
'nl': 'nl_core_news_sm',
'pt': 'pt_core_news_sm'}
# Set the language model for spacy # Set the language model for spacy
nlp = spacy.load(SPACY_MODELS[args.lang]) nlp = spacy.load(SPACY_MODELS[args.lang])
@ -45,9 +41,9 @@ with open(args.i) as input_file:
output_file = open(args.o, 'w+') output_file = open(args.o, 'w+')
output_file.write( output_file.write(
'<?xml version="1.0" encoding="UTF-8"?>\n<corpus>\n<text id="%s">\n' % ( '<?xml version="1.0" encoding="UTF-8"?>\n'
os.path.basename(args.i).rsplit(".", 1)[0] '<corpus>\n'
) '<text id="{}">\n'.format(os.path.basename(args.i).rsplit(".", 1)[0])
) )
for text in texts: for text in texts:
# Run spacy nlp over the text (partial string if above 1 million chars) # Run spacy nlp over the text (partial string if above 1 million chars)
@ -61,11 +57,18 @@ for text in texts:
# Write all information in .vrt style to the output file # Write all information in .vrt style to the output file
# text, lemma, simple_pos, pos, ner # text, lemma, simple_pos, pos, ner
output_file.write( output_file.write(
token.text + '\t' + token.lemma_ + '\t' '{}\t{}\t{}\t{}\t{}\n'.format(
+ token.pos_ + '\t' + token.tag_ + '\t' token.text,
+ (token.ent_type_ if token.ent_type_ != '' else 'NULL') + '\n' token.lemma_,
token.pos_,
token.tag_,
token.ent_type_ if token.ent_type_ != '' else 'NULL'
)
) )
output_file.write('</s>\n') output_file.write('</s>\n')
output_file.write('</text>\n</corpus>') output_file.write(
'</text>\n'
'</corpus>'
)
output_file.close() output_file.close()