sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-26 22:34:18 +00:00
Code Issues Projects Releases Wiki Activity

Update NLP Pipeline

Browse Source
This commit is contained in:
Patrick Jentsch 2020-04-03 17:35:05 +02:00
parent 41910afb79
commit e061a7426d
5 changed files with 198 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Dockerfile
View File

@ -11,8 +11,10 @@ ENV LANG=C.UTF-8
# Install prerequisites # Install prerequisites
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends \
build-essential \
python2.7 \ python2.7 \
python3.5 \ python3.5 \
python3-dev \
python3-pip \ python3-pip \
zip \ zip \
&& rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/apt/lists/* \
@ -31,30 +33,24 @@ RUN tar -xzf "pyflow-${PYFLOW_VERSION}.tar.gz" \
"pyflow-${PYFLOW_VERSION}" \ "pyflow-${PYFLOW_VERSION}" \
"pyflow-${PYFLOW_VERSION}.tar.gz" "pyflow-${PYFLOW_VERSION}.tar.gz"
ENV SPACY_MODEL_DE=de_core_news_sm \ ENV SPACY_VERSION=2.2.4
SPACY_MODEL_EL=el_core_news_sm \ ENV SPACY_MODELS_VERSION=2.2.5
SPACY_MODEL_EN=en_core_web_sm \
SPACY_MODEL_ES=es_core_news_sm \
SPACY_MODEL_FR=fr_core_news_sm \
SPACY_MODEL_IT=it_core_news_sm \
SPACY_MODEL_NL=nl_core_news_sm \
SPACY_MODEL_PT=pt_core_news_sm \
SPACY_VERSION=2.2.0
RUN pip3 install \ RUN pip3 install \
"spacy==${SPACY_VERSION}" \ "spacy==${SPACY_VERSION}" \
&& python3 -m spacy download "${SPACY_MODEL_DE}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "de_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_EL}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "el_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_EN}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "en_core_web_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_ES}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "es_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_FR}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "fr_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_IT}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "it_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_NL}-${SPACY_VERSION}" --direct \ && python3 -m spacy download "nl_core_news_sm-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "${SPACY_MODEL_PT}-${SPACY_VERSION}" --direct && python3 -m spacy download "pt_core_news_sm-${SPACY_MODELS_VERSION}" --direct
# Install NLP pipeline # Install NLP pipeline
COPY nlp /usr/local/bin COPY nlp /usr/local/bin
COPY spacy_nlp /usr/local/bin COPY spacy-nlp /usr/local/bin
ENTRYPOINT ["nlp"] ENTRYPOINT ["nlp"]

198
nlp
View File

@ -9,147 +9,131 @@ Author: Patrick Jentsch <p.jentsch@uni-bielefeld.de>
""" """
import argparse from argparse import ArgumentParser
from pyflow import WorkflowRunner
import multiprocessing import multiprocessing
import os import os
import sys import sys
from pyflow import WorkflowRunner
def parse_arguments(): SPACY_MODELS = {'de': 'de_core_news_sm',
parser = argparse.ArgumentParser( 'el': 'el_core_news_sm',
description=('Performs NLP of documents utilizing spaCy. The results ' 'en': 'en_core_web_sm',
'are served as verticalized text files.') 'es': 'es_core_news_sm',
) 'fr': 'fr_core_news_sm',
parser.add_argument('-i', dest='input_dir', required=True) 'it': 'it_core_news_sm',
parser.add_argument( 'nl': 'nl_core_news_sm',
'-l', 'pt': 'pt_core_news_sm'}
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
dest='lang',
required=True def parse_args():
) parser = ArgumentParser(description='NLP Pipeline utilizing spaCy.')
parser.add_argument('-o', dest='output_dir', required=True) parser.add_argument('i')
parser.add_argument('--nCores', parser.add_argument('o')
parser.add_argument('-l', '--language', choices=SPACY_MODELS.keys(),
required=True)
parser.add_argument('--n-cores',
default=min(4, multiprocessing.cpu_count()), default=min(4, multiprocessing.cpu_count()),
dest='n_cores', help='total number of cores available', type=int)
help='total number of cores available', parser.add_argument('--check-encoding', action='store_true')
required=False, parser.add_argument('--log-dir')
type=int) parser.add_argument('--zip')
parser.add_argument('--zip',
default='vrt-results',
dest='zip',
type=str,
help='''package result files in zip bundles takes a
string as a filename as an optional paramteer''',
required=False)
parser.add_argument('--check-encoding',
action='store_true',
default=False,
dest="check_encoding",
help='''if used the nlp process will know hat the
encoding of the input files is unkown and
thus != utf-8. The process will try to determine
the encoding of the input files and use this.
encoding.'''
)
return parser.parse_args() return parser.parse_args()
class NLPWorkflow(WorkflowRunner): class NLPPipelineJob:
def __init__(self, args): def __init__(self, file, output_dir):
self.jobs = analyze_jobs(args.input_dir, args.output_dir) self.file = file
self.lang = args.lang self.name = os.path.basename(file).rsplit('.', 1)[0]
self.n_cores = args.n_cores self.output_dir = output_dir
self.output_dir = args.output_dir
self.zip = args.zip
self.check_encoding = args.check_encoding class NLPPipeline(WorkflowRunner):
def __init__(self, check_encoding, jobs, lang, n_cores, output_dir, zip):
self.check_encoding = check_encoding
self.jobs = jobs
self.lang = lang
self.n_cores = n_cores
self.output_dir = output_dir
self.zip = zip
def workflow(self): def workflow(self):
if len(self.jobs) == 0: if not self.jobs:
return return
''' '''
' ################################################## ' ##################################################
' # Create output directories # ' # mkdir_jobs #
' ################################################## ' ##################################################
''' '''
create_output_directories_jobs = [] mkdir_jobs = []
for index, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
cmd = 'mkdir -p "{}"'.format(job['output_dir']) cmd = 'mkdir'
create_output_directories_jobs.append( cmd += ' -p'
self.addTask( cmd += ' "{}"'.format(job.output_dir)
command=cmd, lbl = 'mkdir_job_-_{}'.format(i)
label='create_output_directories_job_-_{}'.format(index) mkdir_jobs.append(self.addTask(command=cmd, label=lbl))
)
)
''' '''
' ################################################## ' ##################################################
' # Natural language processing # ' # spacy_nlp_jobs #
' ################################################## ' ##################################################
''' '''
nlp_jobs = [] spacy_nlp_jobs = []
nlp_job_n_cores = min( n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
self.n_cores, for i, job in enumerate(self.jobs):
max(1, int(self.n_cores / len(self.jobs))) output_file = os.path.join(job.output_dir,
) '{}.vrt'.format(job.name))
for index, job in enumerate(self.jobs): cmd = 'spacy-nlp "{}" "{}"'.format(job.file, output_file)
cmd = 'spacy_nlp -l "{}" "{}" "{}" {}'.format( cmd += ' -l "{}"'.format(self.lang)
self.lang, cmd += ' --check-encoding' if self.check_encoding else ''
job['path'], deps = 'mkdir_job_-_{}'.format(i)
os.path.join(job['output_dir'], job['name'] + '.vrt'), lbl = 'spacy_nlp_job_-_{}'.format(i)
"--check-encoding" if self.check_encoding else "" spacy_nlp_jobs.append(self.addTask(command=cmd, dependencies=deps,
) label=lbl, nCores=n_cores))
nlp_jobs.append(
self.addTask(
command=cmd,
dependencies='create_output_directories_job_-_{}'.format(
index
),
label='nlp_job_-_{}'.format(index),
nCores=nlp_job_n_cores
)
)
if zip: '''
vrt_zip_jobs = [] ' ##################################################
vrt_zip_job_dependencies = nlp_jobs ' # zip_jobs #
cmd = 'cd "%s" && zip -m "%s"-nlp.zip */*.vrt -x "pyflow.data*" && cd -' % ( ' ##################################################
self.output_dir, '''
self.zip zip_jobs = []
) if self.zip is not None:
vrt_zip_jobs.append( cmd = 'cd "{}"'.format(self.output_dir)
self.addTask( cmd += ' && '
command=cmd, cmd += 'zip'
dependencies=vrt_zip_job_dependencies, cmd += ' -m'
label='vrt_zip_job' cmd += ' -r'
) cmd += ' "{}_-_vrt" .'.format(self.zip)
) cmd += ' -x "pyflow.data*"'
cmd += ' -i "*.vrt"'
cmd += ' && '
cmd += 'cd -'
deps = spacy_nlp_jobs
lbl = 'zip_job'
zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
def analyze_jobs(input_dir, output_dir): def collect_jobs(input_dir, output_dir):
jobs = [] jobs = []
for file in os.listdir(input_dir): for file in os.listdir(input_dir):
if os.path.isdir(os.path.join(input_dir, file)): if os.path.isdir(os.path.join(input_dir, file)):
jobs += analyze_jobs(os.path.join(input_dir, file), jobs += collect_jobs(os.path.join(input_dir, file),
os.path.join(output_dir, file)) os.path.join(output_dir, file))
elif file.endswith('.txt'): elif file.endswith('.txt'):
jobs.append({'filename': file, jobs.append(NLPPipelineJob(os.path.join(input_dir, file),
'name': file.rsplit('.', 1)[0], os.path.join(output_dir, file)))
'output_dir': os.path.join(output_dir, file),
'path': os.path.join(input_dir, file)})
return jobs return jobs
def main(): def main():
args = parse_arguments() args = parse_args()
jobs = collect_jobs(args.i, args.o)
wflow = NLPWorkflow(args) nlp_pipeline = NLPPipeline(args.check_encoding, jobs, args.language,
args.n_cores, args.o, args.zip)
retval = wflow.run(dataDirRoot=args.output_dir, nCores=args.n_cores) retval = nlp_pipeline.run(dataDirRoot=(args.log_dir or args.o),
nCores=args.n_cores)
sys.exit(retval) sys.exit(retval)

76
spacy-nlp Executable file
View File

@ -0,0 +1,76 @@
#!/usr/bin/env python3.5
# coding=utf-8
from argparse import ArgumentParser
from xml.sax.saxutils import escape
import chardet
import spacy
import textwrap
SPACY_MODELS = {'de': 'de_core_news_sm',
'el': 'el_core_news_sm',
'en': 'en_core_web_sm',
'es': 'es_core_news_sm',
'fr': 'fr_core_news_sm',
'it': 'it_core_news_sm',
'nl': 'nl_core_news_sm',
'pt': 'pt_core_news_sm'}
# Parse the given arguments
parser = ArgumentParser(description=('Tag a text file with spaCy and save it '
'as a verticalized text file.'))
parser.add_argument('i', metavar='txt-sourcefile')
parser.add_argument('o', metavar='vrt-destfile')
parser.add_argument('-l', '--language', choices=SPACY_MODELS.keys(),
required=True)
parser.add_argument('--check-encoding', action='store_true')
args = parser.parse_args()
# If requested: Check the encoding of the text contents from the input file
# Else: Use utf-8
if args.check_encoding:
with open(args.i, "rb") as input_file:
bytes = input_file.read()
encoding = chardet.detect(bytes)['encoding']
else:
encoding = 'utf-8'
# Load the text contents from the input file
with open(args.i, encoding=encoding) as input_file:
text = input_file.read()
# spaCys NLP is limited to strings with maximum 1 million characters at
# once. So we split it into suitable chunks.
text_chunks = textwrap.wrap(text, 1000000, break_long_words=False)
# the text variable potentially occupies a lot of system memory and is no
# longer needed...
del text
# Setup the spaCy toolkit by loading the chosen language model
nlp = spacy.load(SPACY_MODELS[args.language])
# Create the output file in verticalized text format
# See: http://cwb.sourceforge.net/files/CWB_Encoding_Tutorial/node3.html
output_file = open(args.o, 'w+')
output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n<corpus>\n<text>\n')
for text_chunk in text_chunks:
doc = nlp(text_chunk)
for sent in doc.sents:
output_file.write('<s>\n')
for token in sent:
# Skip whitespace tokens
if token.text.isspace():
continue
output_file.write('{}'.format(escape(token.text))
+ '\t{}'.format(escape(token.lemma_))
+ '\t{}'.format(token.pos_)
+ '\t{}'.format(token.tag_)
+ '\t{}\n'.format(token.ent_type_ or 'NULL'))
output_file.write('</s>\n')
output_file.write('</text>\n</corpus>')
output_file.close()

83
spacy_nlp
View File

@ -1,83 +0,0 @@
#!/usr/bin/env python3.5
# coding=utf-8
from xml.sax.saxutils import escape
import argparse
import chardet
import spacy
import textwrap
parser = argparse.ArgumentParser(
description=('Tag a text file with spaCy and save it as a verticalized '
'text file.')
)
parser.add_argument('i', metavar='txt-sourcefile')
parser.add_argument('-l',
choices=['de', 'el', 'en', 'es', 'fr', 'it', 'nl', 'pt'],
dest='lang',
required=True)
parser.add_argument('o', metavar='vrt-destfile')
parser.add_argument('--check-encoding',
default=False,
action='store_true',
dest='check_encoding'
)
args = parser.parse_args()
SPACY_MODELS = {'de': 'de_core_news_sm',
'el': 'el_core_news_sm',
'en': 'en_core_web_sm',
'es': 'es_core_news_sm',
'fr': 'fr_core_news_sm',
'it': 'it_core_news_sm',
'nl': 'nl_core_news_sm',
'pt': 'pt_core_news_sm'}
# Set the language model for spacy
nlp = spacy.load(SPACY_MODELS[args.lang])
# Try to determine the encoding of the text in the input file
if args.check_encoding:
with open(args.i, "rb") as input_file:
bytes = input_file.read()
encoding = chardet.detect(bytes)['encoding']
else:
encoding = 'utf-8'
# Read text from the input file and if neccessary split it into parts with a
# length of less than 1 million characters.
with open(args.i, encoding=encoding) as input_file:
text = input_file.read()
texts = textwrap.wrap(text, 1000000, break_long_words=False)
text = None
# Create and open the output file
output_file = open(args.o, 'w+')
output_file.write('<?xml version="1.0" encoding="UTF-8"?>\n'
'<corpus>\n'
'<text>\n')
for text in texts:
# Run spacy nlp over the text (partial string if above 1 million chars)
doc = nlp(text)
for sent in doc.sents:
output_file.write('<s>\n')
for token in sent:
# Skip whitespace tokens like "\n" or "\t"
if token.text.isspace():
continue
# Write all information in .vrt style to the output file
# text, lemma, simple_pos, pos, ner
output_file.write(
'{}\t{}\t{}\t{}\t{}\n'.format(
escape(token.text),
escape(token.lemma_),
token.pos_,
token.tag_,
token.ent_type_ if token.ent_type_ != '' else 'NULL'
)
)
output_file.write('</s>\n')
output_file.write('</text>\n'
'</corpus>')
output_file.close()

40
wrapper/nlp
View File

@ -1,35 +1,29 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# coding=utf-8 # coding=utf-8
import argparse from argparse import ArgumentParser
import os import os
import subprocess import subprocess
container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/nlp:latest' CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/nlp:latest'
container_input_dir = '/input' CONTAINER_INPUT_DIR = '/input'
container_output_dir = '/output' CONTAINER_OUTPUT_DIR = '/output'
uid = str(os.getuid()) UID = str(os.getuid())
gid = str(os.getgid()) GID = str(os.getgid())
parser = argparse.ArgumentParser(add_help=False) parser = ArgumentParser(add_help=False)
parser.add_argument('-i', parser.add_argument('-i')
dest='input_dir', parser.add_argument('-o')
required=False)
parser.add_argument('-o',
dest='output_dir',
required=False)
args, remaining_args = parser.parse_known_args() args, remaining_args = parser.parse_known_args()
cmd = ['docker', 'run', '--rm', '-it', '-u', uid + ':' + gid] cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
if args.input_dir is not None: if args.o is not None:
host_input_dir = os.path.abspath(args.input_dir) cmd += ['-v', '{}:{}'.format(os.path.abspath(args.o), CONTAINER_OUTPUT_DIR)]
cmd += ['-v', host_input_dir + ':' + container_input_dir] remaining_args.insert(0, CONTAINER_OUTPUT_DIR)
remaining_args += ['-i', container_input_dir] if args.i is not None:
if args.output_dir is not None: cmd += ['-v', '{}:{}'.format(os.path.abspath(args.i), CONTAINER_INPUT_DIR)]
host_output_dir = os.path.abspath(args.output_dir) remaining_args.insert(0, CONTAINER_INPUT_DIR)
cmd += ['-v', host_output_dir + ':' + container_output_dir] cmd.append(CONTAINER_IMAGE)
remaining_args += ['-o', container_output_dir]
cmd.append(container_image)
cmd += remaining_args cmd += remaining_args
subprocess.run(cmd) subprocess.run(cmd)