mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2024-12-26 20:44:17 +00:00
Codestyle
This commit is contained in:
Patrick Jentsch
parent
43717de313
commit
af293d6141
60
nlp
60
nlp
@ -18,33 +18,23 @@ from pyflow import WorkflowRunner
|
|||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='Performs NLP of documents utilizing spaCy. The results are served as verticalized text files.'
|
description=('Performs NLP of documents utilizing spaCy. The results '
|
||||||
)
|
'are served as verticalized text files.')
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-i',
|
|
||||||
dest='input_dir',
|
|
||||||
required=True
|
|
||||||
)
|
)
|
||||||
|
parser.add_argument('-i', dest='input_dir', required=True)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-l',
|
'-l',
|
||||||
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
|
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
|
||||||
dest='lang',
|
dest='lang',
|
||||||
required=True
|
required=True
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument('-o', dest='output_dir', required=True)
|
||||||
'-o',
|
parser.add_argument('--nCores',
|
||||||
dest='output_dir',
|
default=min(4, multiprocessing.cpu_count()),
|
||||||
required=True
|
dest='n_cores',
|
||||||
)
|
help='total number of cores available',
|
||||||
parser.add_argument(
|
required=False,
|
||||||
'--nCores',
|
type=int)
|
||||||
default=min(4, multiprocessing.cpu_count()),
|
|
||||||
dest='n_cores',
|
|
||||||
help='total number of cores available',
|
|
||||||
required=False,
|
|
||||||
type=int
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@ -65,11 +55,11 @@ class NLPWorkflow(WorkflowRunner):
|
|||||||
'''
|
'''
|
||||||
create_output_directories_jobs = []
|
create_output_directories_jobs = []
|
||||||
for index, job in enumerate(self.jobs):
|
for index, job in enumerate(self.jobs):
|
||||||
cmd = 'mkdir -p "%s"' % (job['output_dir'])
|
cmd = 'mkdir -p "{}"'.format(job['output_dir'])
|
||||||
create_output_directories_jobs.append(
|
create_output_directories_jobs.append(
|
||||||
self.addTask(
|
self.addTask(
|
||||||
command=cmd,
|
command=cmd,
|
||||||
label='create_output_directories_job_-_%i' % (index)
|
label='create_output_directories_job_-_{}'.format(index)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -84,7 +74,7 @@ class NLPWorkflow(WorkflowRunner):
|
|||||||
max(1, int(self.n_cores / len(self.jobs)))
|
max(1, int(self.n_cores / len(self.jobs)))
|
||||||
)
|
)
|
||||||
for index, job in enumerate(self.jobs):
|
for index, job in enumerate(self.jobs):
|
||||||
cmd = 'spacy_nlp -l "%s" "%s" "%s"' % (
|
cmd = 'spacy_nlp -l "{}" "{}" "{}"'.format(
|
||||||
self.lang,
|
self.lang,
|
||||||
job['path'],
|
job['path'],
|
||||||
os.path.join(job['output_dir'], job['name'] + '.vrt')
|
os.path.join(job['output_dir'], job['name'] + '.vrt')
|
||||||
@ -92,8 +82,10 @@ class NLPWorkflow(WorkflowRunner):
|
|||||||
nlp_jobs.append(
|
nlp_jobs.append(
|
||||||
self.addTask(
|
self.addTask(
|
||||||
command=cmd,
|
command=cmd,
|
||||||
dependencies='create_output_directories_job_-_%i' % (index),
|
dependencies='create_output_directories_job_-_{}'.format(
|
||||||
label='nlp_job_-_%i' % (index),
|
index
|
||||||
|
),
|
||||||
|
label='nlp_job_-_{}'.format(index),
|
||||||
nCores=nlp_job_n_cores
|
nCores=nlp_job_n_cores
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@ -104,19 +96,13 @@ def analyze_jobs(input_dir, output_dir):
|
|||||||
|
|
||||||
for file in os.listdir(input_dir):
|
for file in os.listdir(input_dir):
|
||||||
if os.path.isdir(os.path.join(input_dir, file)):
|
if os.path.isdir(os.path.join(input_dir, file)):
|
||||||
jobs += analyze_jobs(
|
jobs += analyze_jobs(os.path.join(input_dir, file),
|
||||||
os.path.join(input_dir, file),
|
os.path.join(output_dir, file))
|
||||||
os.path.join(output_dir, file),
|
|
||||||
)
|
|
||||||
elif file.endswith('.txt'):
|
elif file.endswith('.txt'):
|
||||||
jobs.append(
|
jobs.append({'filename': file,
|
||||||
{
|
'name': file.rsplit('.', 1)[0],
|
||||||
'filename': file,
|
'output_dir': os.path.join(output_dir, file),
|
||||||
'name': file.rsplit('.', 1)[0],
|
'path': os.path.join(input_dir, file)})
|
||||||
'output_dir': os.path.join(output_dir, file),
|
|
||||||
'path': os.path.join(input_dir, file)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return jobs
|
return jobs
|
||||||
|
|
||||||
|
|||||||
57
spacy_nlp
57
spacy_nlp
@ -7,29 +7,25 @@ import spacy
|
|||||||
import textwrap
|
import textwrap
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='Tag a text file with spaCy and save it as a verticalized text file.'
|
description=('Tag a text file with spaCy and save it as a verticalized '
|
||||||
)
|
'text file.')
|
||||||
parser.add_argument(
|
|
||||||
'i',
|
|
||||||
metavar='txt-sourcefile',
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'-l',
|
|
||||||
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
|
|
||||||
dest='lang',
|
|
||||||
required=True
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'o',
|
|
||||||
metavar='vrt-destfile',
|
|
||||||
)
|
)
|
||||||
|
parser.add_argument('i', metavar='txt-sourcefile')
|
||||||
|
parser.add_argument('-l',
|
||||||
|
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
|
||||||
|
dest='lang',
|
||||||
|
required=True)
|
||||||
|
parser.add_argument('o', metavar='vrt-destfile')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
SPACY_MODELS = {
|
SPACY_MODELS = {'de': 'de_core_news_sm',
|
||||||
'de': 'de_core_news_sm', 'el': 'el_core_news_sm', 'en': 'en_core_web_sm',
|
'el': 'el_core_news_sm',
|
||||||
'es': 'es_core_news_sm', 'fr': 'fr_core_news_sm', 'it': 'it_core_news_sm',
|
'en': 'en_core_web_sm',
|
||||||
'nl': 'nl_core_news_sm', 'pt': 'pt_core_news_sm'
|
'es': 'es_core_news_sm',
|
||||||
}
|
'fr': 'fr_core_news_sm',
|
||||||
|
'it': 'it_core_news_sm',
|
||||||
|
'nl': 'nl_core_news_sm',
|
||||||
|
'pt': 'pt_core_news_sm'}
|
||||||
|
|
||||||
# Set the language model for spacy
|
# Set the language model for spacy
|
||||||
nlp = spacy.load(SPACY_MODELS[args.lang])
|
nlp = spacy.load(SPACY_MODELS[args.lang])
|
||||||
@ -45,9 +41,9 @@ with open(args.i) as input_file:
|
|||||||
output_file = open(args.o, 'w+')
|
output_file = open(args.o, 'w+')
|
||||||
|
|
||||||
output_file.write(
|
output_file.write(
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>\n<corpus>\n<text id="%s">\n' % (
|
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||||
os.path.basename(args.i).rsplit(".", 1)[0]
|
'<corpus>\n'
|
||||||
)
|
'<text id="{}">\n'.format(os.path.basename(args.i).rsplit(".", 1)[0])
|
||||||
)
|
)
|
||||||
for text in texts:
|
for text in texts:
|
||||||
# Run spacy nlp over the text (partial string if above 1 million chars)
|
# Run spacy nlp over the text (partial string if above 1 million chars)
|
||||||
@ -61,11 +57,18 @@ for text in texts:
|
|||||||
# Write all information in .vrt style to the output file
|
# Write all information in .vrt style to the output file
|
||||||
# text, lemma, simple_pos, pos, ner
|
# text, lemma, simple_pos, pos, ner
|
||||||
output_file.write(
|
output_file.write(
|
||||||
token.text + '\t' + token.lemma_ + '\t'
|
'{}\t{}\t{}\t{}\t{}\n'.format(
|
||||||
+ token.pos_ + '\t' + token.tag_ + '\t'
|
token.text,
|
||||||
+ (token.ent_type_ if token.ent_type_ != '' else 'NULL') + '\n'
|
token.lemma_,
|
||||||
|
token.pos_,
|
||||||
|
token.tag_,
|
||||||
|
token.ent_type_ if token.ent_type_ != '' else 'NULL'
|
||||||
|
)
|
||||||
)
|
)
|
||||||
output_file.write('</s>\n')
|
output_file.write('</s>\n')
|
||||||
output_file.write('</text>\n</corpus>')
|
output_file.write(
|
||||||
|
'</text>\n'
|
||||||
|
'</corpus>'
|
||||||
|
)
|
||||||
|
|
||||||
output_file.close()
|
output_file.close()
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user