sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2025-07-01 05:50:32 +00:00
Code Issues Projects Releases Wiki Activity

fix pipeline

Browse Source
This commit is contained in:
Patrick Jentsch
2020-06-23 15:19:39 +02:00
parent 5980a995e5
commit 5bd0feda5c
4 changed files with 61 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
spacy-nlp
View File

@ -22,9 +22,10 @@ SPACY_MODELS = {'de': 'de_core_news_sm',
# Parse the given arguments
parser = ArgumentParser(description=('Tag a text file with spaCy and save it '
'as a verticalized text file.'))
parser.add_argument('i', metavar='txt-sourcefile')
parser.add_argument('o', metavar='vrt-destfile')
parser.add_argument('-l', '--language', choices=SPACY_MODELS.keys(),
parser.add_argument('-i', '--input', metavar='txt-sourcefile', required=True)
parser.add_argument('-o', '--output', metavar='vrt-destfile', required=True)
parser.add_argument('-l', '--language',
choices=SPACY_MODELS.keys(),
required=True)
parser.add_argument('--check-encoding', action='store_true')
args = parser.parse_args()
@ -33,7 +34,7 @@ args = parser.parse_args()
# If requested: Check the encoding of the text contents from the input file
# Else: Use utf-8
if args.check_encoding:
with open(args.i, "rb") as input_file:
with open(args.input, "rb") as input_file:
bytes = input_file.read()
encoding = chardet.detect(bytes)['encoding']
else:
@ -41,14 +42,14 @@ else:
# hashing in chunks to avoid full RAM with huge files.
with open(args.i, 'rb') as input_file:
with open(args.input, 'rb') as input_file:
md5_hash = hashlib.md5()
for chunk in iter(lambda: input_file.read(128 * md5_hash.block_size), b''):
md5_hash.update(chunk)
md5_hash = md5_hash.hexdigest()
# Load the text contents from the input file
with open(args.i, encoding=encoding) as input_file:
with open(args.input, encoding=encoding) as input_file:
text = input_file.read()
# spaCys NLP is limited to strings with maximum 1 million characters at
# once. So we split it into suitable chunks.
@ -64,8 +65,8 @@ nlp = spacy.load(SPACY_MODELS[args.language])
# Create the output file in verticalized text format
# See: http://cwb.sourceforge.net/files/CWB_Encoding_Tutorial/node3.html
output_file_original_filename = args.o
output_file_stand_off_filename = args.o.replace('.vrt', '.stand-off.vrt')
output_file_original_filename = args.output
output_file_stand_off_filename = args.output.replace('.vrt', '.stand-off.vrt')
common_xml = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
+ '<corpus>\n'
+ '<text>\n'