sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2025-07-01 15:30:33 +00:00
Code Issues Projects Releases Wiki Activity

Fix problems caused by wrong textwrap.wrap usage

Browse Source
This commit is contained in:
Patrick Jentsch
2021-04-30 09:44:35 +02:00
parent f7b7da2b1f
commit bd5d8ddedb
2 changed files with 63 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
vrt-creator
View File

@ -3,19 +3,13 @@
from argparse import ArgumentParser
from xml.sax.saxutils import escape
import hashlib
import json
# Parse the given arguments
parser = ArgumentParser(description='Create annotations for a given txt file')
parser.add_argument('input', metavar='Path to txt input file')
parser.add_argument('annotations', metavar='Path to JSON annotation file')
parser.add_argument('output', metavar='Path to vrt output file')
args = parser.parse_args()
with open(args.input) as text_file, \
open(args.annotations) as data_file:
text = text_file.read()
stand_off_data = json.load(data_file)
# Two global ressources - Not very elegant but it works for now
stand_off_data = None
text = None
def meta_to_string():
@ -26,7 +20,8 @@ def meta_to_string():
stand_off_data['meta']['generator']['arguments']['check_encoding'],
stand_off_data['meta']['generator']['arguments']['language']
)
string += '<file name="{}" md5="{}"/>\n'.format(
string += '<file encoding="{}" name="{}" md5="{}"/>\n'.format(
stand_off_data['meta']['file']['encoding'],
stand_off_data['meta']['file']['name'],
stand_off_data['meta']['file']['md5']
)
@ -93,15 +88,43 @@ def annotations_to_string(end=float('inf')):
return string
vrt = ''
vrt += '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
vrt += '<corpus>\n'
vrt += '<text>\n'
vrt += meta_to_string()
vrt += tags_to_string()
vrt += annotations_to_string()
vrt += '</text>\n'
vrt += '</corpus>'
def main():
global stand_off_data
global text
with open(args.output, 'w') as vrt_file:
vrt_file.write(vrt)
# Parse the given arguments
parser = ArgumentParser(description='Create a vrt from JSON and txt')
parser.add_argument('text', metavar='Path to txt file')
parser.add_argument('stand_off_data', metavar='Path to JSON file')
parser.add_argument('output', metavar='Path to vrt output file')
args = parser.parse_args()
with open(args.stand_off_data) as stand_of_data_file:
stand_off_data = json.load(stand_of_data_file)
with open(args.text, "rb") as text_file:
text_md5 = hashlib.md5()
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''): # noqa
text_md5.update(chunk)
if text_md5.hexdigest() != stand_off_data['meta']['file']['md5']:
raise Exception('md5 not equal')
with open(args.text, encoding=stand_off_data['meta']['file']['encoding']) as text_file: # noqa
text = text_file.read()
vrt = ''
vrt += '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
vrt += '<corpus>\n'
vrt += '<text>\n'
vrt += meta_to_string()
vrt += tags_to_string()
vrt += annotations_to_string()
vrt += '</text>\n'
vrt += '</corpus>'
with open(args.output, 'w') as vrt_file:
vrt_file.write(vrt)
if __name__ == '__main__':
main()