sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2025-07-01 16:00:33 +00:00
Code Issues Projects Releases Wiki Activity

Bump spaCy version, bugfixes, codestyle

Browse Source
This commit is contained in:
Patrick Jentsch
2022-01-27 16:50:22 +01:00
parent 29ccfac4f6
commit a2e8e72e54
9 changed files with 699 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
vrt-creator
View File

@ -6,31 +6,36 @@ from stand_off_data import StandOffData
import hashlib
import json
parser = ArgumentParser(
description='Convert plain text and JSON stand off to a CWB vrt file'
)
parser.add_argument(
'-s', '--stand-off-data-file',
help='JSON stand off data input file'
)
parser.add_argument(
'-t', '--text-file',
help='Plain text input file'
)
parser.add_argument(
'-o', '--output-file',
help='Output file',
required=True
)
args = parser.parse_args()
def main():
# Parse the given arguments
parser = ArgumentParser(description='Create a vrt from JSON and txt')
parser.add_argument('text', help='Path to txt file')
parser.add_argument('stand_off_data', help='Path to JSON file')
parser.add_argument('output', help='Path to vrt output file')
args = parser.parse_args()
with open(args.stand_off_data_file) as stand_of_data_file:
stand_off_data = StandOffData(json.load(stand_of_data_file))
with open(args.stand_off_data) as stand_of_data_file:
stand_off_data = StandOffData(json.load(stand_of_data_file))
with open(args.text_file, "rb") as text_file:
text_md5 = hashlib.md5()
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''):
text_md5.update(chunk)
if text_md5.hexdigest() != stand_off_data.meta['file']['md5']:
raise Exception('md5 not equal')
with open(args.text, "rb") as text_file:
text_md5 = hashlib.md5()
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''): # noqa
text_md5.update(chunk)
if text_md5.hexdigest() != stand_off_data.meta['file']['md5']:
raise Exception('md5 not equal')
with open(args.text_file, encoding=stand_off_data.meta['file']['encoding']) as text_file: # noqa
text = text_file.read()
with open(args.text, encoding=stand_off_data.meta['file']['encoding']) as text_file:
text = text_file.read()
with open(args.output, 'w') as vrt_file:
vrt_file.write(stand_off_data.to_vrt(text))
if __name__ == '__main__':
main()
with open(args.output_file, 'w') as vrt_file:
vrt_file.write(stand_off_data.to_vrt(text))