mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2025-10-31 20:53:14 +00:00
131 lines
5.4 KiB
Python
Executable File
131 lines
5.4 KiB
Python
Executable File
#!/usr/bin/env python3.7
|
|
# coding=utf-8
|
|
|
|
from argparse import ArgumentParser
|
|
from xml.sax.saxutils import escape
|
|
import hashlib
|
|
import json
|
|
|
|
|
|
# Two global ressources - Not very elegant but it works for now
|
|
stand_off_data = None
|
|
text = None
|
|
|
|
|
|
def meta_to_string():
|
|
string = ''
|
|
string += '<generator software="{} ({})" arguments="check_encoding: {}; language: {}"/>\n'.format( # noqa
|
|
stand_off_data['meta']['generator']['name'],
|
|
stand_off_data['meta']['generator']['version'],
|
|
stand_off_data['meta']['generator']['arguments']['check_encoding'],
|
|
stand_off_data['meta']['generator']['arguments']['language']
|
|
)
|
|
string += '<file encoding="{}" name="{}" md5="{}"/>\n'.format(
|
|
stand_off_data['meta']['file']['encoding'],
|
|
stand_off_data['meta']['file']['name'],
|
|
stand_off_data['meta']['file']['md5']
|
|
)
|
|
return string
|
|
|
|
|
|
def tags_to_string():
|
|
return ''
|
|
|
|
|
|
def annotations_to_string(end=float('inf')):
|
|
string = ''
|
|
while stand_off_data['annotations']:
|
|
if stand_off_data['annotations'][0]['start'] >= end:
|
|
break
|
|
annotation = stand_off_data['annotations'].pop(0)
|
|
#######################################################################
|
|
# Check for malformed annotations #
|
|
#######################################################################
|
|
if 'tag' not in annotation:
|
|
raise Exception('Annotation tag is missing')
|
|
|
|
if annotation['tag'] not in stand_off_data['tags']:
|
|
raise Exception('Unknown annotation tag: ' + annotation['tag'])
|
|
|
|
tag_model = stand_off_data['tags'][annotation['tag']]
|
|
if 'properties' in tag_model:
|
|
properties_model = tag_model['properties']
|
|
if properties_model is not None:
|
|
required_properties = filter(lambda x: 'flags' in x and 'required' in x['flags'], properties_model) # noqa
|
|
if required_properties and annotation['properties'] is None:
|
|
raise Exception('There are required properties but the "Properties" attribute is missing') # noqa
|
|
for property in required_properties:
|
|
if property not in annotation['properties']:
|
|
raise Exception('Required property is missing: ' + property) # noqa
|
|
#######################################################################
|
|
# Process tokens ~ cwb's positional attributes #
|
|
#######################################################################
|
|
if annotation['tag'] == 'token':
|
|
string += '{}\t{}\t{}\t{}\t{}\n'.format(
|
|
escape(text[annotation['start']:annotation['end']]),
|
|
escape(annotation['properties']['pos']),
|
|
escape(annotation['properties']['lemma']),
|
|
escape(annotation['properties']['simple_pos']),
|
|
escape(annotation['properties']['ner'] if 'ner' in annotation['properties'] else 'None') # noqa
|
|
)
|
|
#######################################################################
|
|
# Process other tags ~ cwb's structural attributes #
|
|
#######################################################################
|
|
else:
|
|
properties = ''
|
|
if 'properties' in annotation and annotation['properties'] is not None: # noqa
|
|
for property, value in annotation['properties'].items():
|
|
if not value:
|
|
continue
|
|
if properties_model and property in properties_model:
|
|
if 'flags' in properties_model and 'multiple' in properties_model['flags']: # noqa
|
|
properties += ' {}="|{}|"'.format(property, '|'.join(value)) # noqa
|
|
else:
|
|
properties += ' {}="{}"'.format(property, value)
|
|
string += '<' + annotation['tag'] + properties + '>\n'
|
|
string += annotations_to_string(end=min(annotation['end'], end))
|
|
string += '</' + annotation['tag'] + '>\n'
|
|
return string
|
|
|
|
|
|
def main():
|
|
global stand_off_data
|
|
global text
|
|
|
|
# Parse the given arguments
|
|
parser = ArgumentParser(description='Create a vrt from JSON and txt')
|
|
parser.add_argument('text', help='Path to txt file')
|
|
parser.add_argument('stand_off_data', help='Path to JSON file')
|
|
parser.add_argument('output', help='Path to vrt output file')
|
|
args = parser.parse_args()
|
|
|
|
with open(args.stand_off_data) as stand_of_data_file:
|
|
stand_off_data = json.load(stand_of_data_file)
|
|
|
|
with open(args.text, "rb") as text_file:
|
|
text_md5 = hashlib.md5()
|
|
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''): # noqa
|
|
text_md5.update(chunk)
|
|
if text_md5.hexdigest() != stand_off_data['meta']['file']['md5']:
|
|
raise Exception('md5 not equal')
|
|
|
|
with open(args.text, encoding=stand_off_data['meta']['file']['encoding']) as text_file: # noqa
|
|
text = text_file.read()
|
|
|
|
vrt = ''
|
|
vrt += '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
|
|
vrt += '<corpus>\n'
|
|
vrt += '<text>\n'
|
|
vrt += meta_to_string()
|
|
vrt += tags_to_string()
|
|
vrt += annotations_to_string()
|
|
vrt += '</text>\n'
|
|
vrt += '</corpus>'
|
|
|
|
with open(args.output, 'w') as vrt_file:
|
|
vrt_file.write(vrt)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|