mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2025-10-31 12:52:47 +00:00
108 lines
4.7 KiB
Python
Executable File
108 lines
4.7 KiB
Python
Executable File
#!/usr/bin/env python3.7
|
|
# coding=utf-8
|
|
|
|
from argparse import ArgumentParser
|
|
from xml.sax.saxutils import escape
|
|
import json
|
|
|
|
# Parse the given arguments
|
|
parser = ArgumentParser(description='Create annotations for a given txt file')
|
|
parser.add_argument('input', metavar='Path to txt input file')
|
|
parser.add_argument('annotations', metavar='Path to JSON annotation file')
|
|
parser.add_argument('output', metavar='Path to vrt output file')
|
|
args = parser.parse_args()
|
|
|
|
with open(args.input) as text_file, \
|
|
open(args.annotations) as data_file:
|
|
text = text_file.read()
|
|
stand_off_data = json.load(data_file)
|
|
|
|
|
|
def meta_to_string():
|
|
string = ''
|
|
string += '<generator software="{} ({})" arguments="check_encoding: {}; language: {}"/>\n'.format( # noqa
|
|
stand_off_data['meta']['generator']['name'],
|
|
stand_off_data['meta']['generator']['version'],
|
|
stand_off_data['meta']['generator']['arguments']['check_encoding'],
|
|
stand_off_data['meta']['generator']['arguments']['language']
|
|
)
|
|
string += '<file name="{}" md5="{}"/>\n'.format(
|
|
stand_off_data['meta']['file']['name'],
|
|
stand_off_data['meta']['file']['md5']
|
|
)
|
|
return string
|
|
|
|
|
|
def tags_to_string():
|
|
return ''
|
|
|
|
|
|
def annotations_to_string(end=float('inf')):
|
|
string = ''
|
|
while stand_off_data['annotations']:
|
|
if stand_off_data['annotations'][0]['start'] >= end:
|
|
break
|
|
annotation = stand_off_data['annotations'].pop(0)
|
|
#######################################################################
|
|
# Check for malformed annotations #
|
|
#######################################################################
|
|
if 'tag' not in annotation:
|
|
raise Exception('Annotation tag is missing')
|
|
|
|
if annotation['tag'] not in stand_off_data['tags']:
|
|
raise Exception('Unknown annotation tag: ' + annotation['tag'])
|
|
|
|
tag_model = stand_off_data['tags'][annotation['tag']]
|
|
if 'properties' in tag_model:
|
|
properties_model = tag_model['properties']
|
|
if properties_model is not None:
|
|
required_properties = filter(lambda x: 'flags' in x and 'required' in x['flags'], properties_model) # noqa
|
|
if required_properties and annotation['properties'] is None:
|
|
raise Exception('There are required properties but the "Properties" attribute is missing') # noqa
|
|
for property in required_properties:
|
|
if property not in annotation['properties']:
|
|
raise Exception('Required property is missing: ' + property) # noqa
|
|
#######################################################################
|
|
# Process tokens ~ cwb's positional attributes #
|
|
#######################################################################
|
|
if annotation['tag'] == 'token':
|
|
string += '{}\t{}\t{}\t{}\t{}\n'.format(
|
|
escape(text[annotation['start']:annotation['end']]),
|
|
escape(annotation['properties']['pos']),
|
|
escape(annotation['properties']['lemma']),
|
|
escape(annotation['properties']['simple_pos']),
|
|
escape(annotation['properties']['ner'] if 'ner' in annotation['properties'] else 'None') # noqa
|
|
)
|
|
#######################################################################
|
|
# Process other tags ~ cwb's structural attributes #
|
|
#######################################################################
|
|
else:
|
|
properties = ''
|
|
if 'properties' in annotation and annotation['properties'] is not None: # noqa
|
|
for property, value in annotation['properties'].items():
|
|
if not value:
|
|
continue
|
|
if properties_model and property in properties_model:
|
|
if 'flags' in properties_model and 'multiple' in properties_model['flags']: # noqa
|
|
properties += ' {}="|{}|"'.format(property, '|'.join(value)) # noqa
|
|
else:
|
|
properties += ' {}="{}"'.format(property, value)
|
|
string += '<' + annotation['tag'] + properties + '>\n'
|
|
string += annotations_to_string(end=min(annotation['end'], end))
|
|
string += '</' + annotation['tag'] + '>\n'
|
|
return string
|
|
|
|
|
|
vrt = ''
|
|
vrt += '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
|
|
vrt += '<corpus>\n'
|
|
vrt += '<text>\n'
|
|
vrt += meta_to_string()
|
|
vrt += tags_to_string()
|
|
vrt += annotations_to_string()
|
|
vrt += '</text>\n'
|
|
vrt += '</corpus>'
|
|
|
|
with open(args.output, 'w') as vrt_file:
|
|
vrt_file.write(vrt)
|