2021-08-10 12:43:55 +00:00
|
|
|
from xml.sax.saxutils import escape
|
2021-08-06 14:50:22 +00:00
|
|
|
|
|
|
|
|
2021-08-10 12:43:55 +00:00
|
|
|
class StandOffData:
|
2021-08-11 14:47:29 +00:00
|
|
|
def __init__(self, attrs={}):
|
2021-07-22 14:59:29 +00:00
|
|
|
self.meta = attrs.get('meta', {})
|
2021-08-11 14:47:29 +00:00
|
|
|
self.lookup = {}
|
|
|
|
for x in attrs.get('tags', []):
|
|
|
|
self.add_tag_definition(x)
|
|
|
|
self.annotations = [TagAnnotation(x, self.lookup)
|
|
|
|
for x in attrs.get('annotations', [])]
|
|
|
|
|
|
|
|
def add_tag_definition(self, attrs):
|
|
|
|
tag_definition = TagDefinition(attrs)
|
|
|
|
if tag_definition.id in self.lookup:
|
|
|
|
raise Exception('Tag id already in use: {}'.format(self.to_dict()))
|
|
|
|
self.lookup[tag_definition.id] = tag_definition
|
|
|
|
|
|
|
|
def to_dict(self):
|
|
|
|
return {
|
|
|
|
'meta': self.meta,
|
|
|
|
'lookup': {k: v.to_dict() for k, v in self.lookup.items()},
|
|
|
|
'annotations': [x.to_dict() for x in self.annotations]
|
|
|
|
}
|
2021-07-13 14:31:53 +00:00
|
|
|
|
2021-08-10 12:43:55 +00:00
|
|
|
def to_vrt(self, text):
|
|
|
|
# Devide annotations into CWB's verticalized text format (.vrt) logic
|
|
|
|
p_attrs = [] # positional attributes
|
|
|
|
s_attrs = [] # structural attributes
|
|
|
|
for annotation in self.annotations:
|
|
|
|
if annotation.name == 'token':
|
|
|
|
p_attrs.append(annotation)
|
|
|
|
else:
|
|
|
|
s_attrs.append(annotation)
|
|
|
|
# Sort annotations, necessary for the next checks
|
|
|
|
p_attrs.sort()
|
|
|
|
s_attrs.sort()
|
|
|
|
# Check for p_attr<->p_attr overlap
|
|
|
|
for i, p_attr in enumerate(p_attrs[:-1]):
|
|
|
|
next_p_attr = p_attrs[i + 1]
|
|
|
|
# Check if first_p_attr starts/ends within second_p_attr
|
|
|
|
if ((p_attr.start >= next_p_attr.start) and (p_attr.start < next_p_attr.end) # noqa
|
|
|
|
or (p_attr.end > next_p_attr.start) and (p_attr.end <= next_p_attr.end)): # noqa
|
|
|
|
raise Exception(
|
|
|
|
'Positional attribute overlaps another: {}<->{}'.format(p_attr.to_dict(), next_p_attr.to_dict()))
|
|
|
|
# Check for s_attr<->p_attr overlap
|
|
|
|
for i, s_attr in enumerate(s_attrs):
|
|
|
|
for p_attr in p_attrs:
|
|
|
|
# Check if s_attr starts within p_attr
|
|
|
|
if s_attr.start > p_attr.start and s_attr.start < p_attr.end:
|
|
|
|
# Change s_attr start to p_attr's start
|
|
|
|
s_attrs[i].start = p_attr.start
|
|
|
|
# Check if s_attr ends within p_attr
|
|
|
|
if s_attr.end < p_attr.end and s_attr.end > p_attr.start:
|
|
|
|
# Change s_attr end to p_attr's end
|
|
|
|
s_attrs[i].end = p_attr.end
|
|
|
|
# Check if s_attr starts/ends before/after p_attr
|
|
|
|
if p_attr.start >= s_attr.end or p_attr.end <= s_attr.start:
|
|
|
|
# No further Checking needed (just because p_attrs are sorted)
|
|
|
|
break
|
|
|
|
s_attr_start_buffer = {}
|
|
|
|
s_attr_end_buffer = {}
|
|
|
|
for i, s_attr in enumerate(s_attrs):
|
|
|
|
if s_attr.start in s_attr_start_buffer:
|
|
|
|
s_attr_start_buffer[s_attr.start].append(i)
|
|
|
|
else:
|
|
|
|
s_attr_start_buffer[s_attr.start] = [i]
|
|
|
|
if s_attr.end in s_attr_end_buffer:
|
|
|
|
s_attr_end_buffer[s_attr.end].append(i)
|
|
|
|
else:
|
|
|
|
s_attr_end_buffer[s_attr.end] = [i]
|
|
|
|
vrt = ''
|
|
|
|
vrt += '<text>\n'
|
|
|
|
for p_attr in p_attrs:
|
|
|
|
# s_attr_ends
|
|
|
|
for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa
|
|
|
|
s_attr_indexes = s_attr_end_buffer.pop(k)
|
|
|
|
for s_attr_index in s_attr_indexes:
|
|
|
|
s_attr = s_attrs[s_attr_index]
|
|
|
|
vrt += '</{}>\n'.format(escape(s_attr.name))
|
|
|
|
# s_attr_starts
|
|
|
|
for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa
|
|
|
|
s_attr_indexes = s_attr_start_buffer.pop(k)
|
|
|
|
for s_attr_index in s_attr_indexes:
|
|
|
|
s_attr = s_attrs[s_attr_index]
|
|
|
|
foo = ''
|
|
|
|
for property in s_attr.properties:
|
|
|
|
foo += ' {}="{}"'.format(escape(property.name),
|
|
|
|
escape(property.value))
|
|
|
|
vrt += '<{}{}>\n'.format(escape(s_attr.name), foo)
|
|
|
|
foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa
|
|
|
|
for property in p_attr.properties:
|
|
|
|
foo[property.name] = escape(property.value)
|
|
|
|
foo['word'] = escape(text[p_attr.start:p_attr.end])
|
|
|
|
vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(
|
|
|
|
**foo)
|
|
|
|
vrt += '</text>\n'
|
|
|
|
return vrt
|
|
|
|
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
class TagAnnotation:
|
2021-07-22 14:59:29 +00:00
|
|
|
def __init__(self, attrs, lookup):
|
|
|
|
self.lookup = lookup
|
2021-07-13 14:31:53 +00:00
|
|
|
self.tag_id = attrs['tag_id']
|
|
|
|
self.start = attrs['start']
|
|
|
|
self.end = attrs['end']
|
|
|
|
self.properties = [
|
2021-08-11 14:47:29 +00:00
|
|
|
PropertyAnnotation(x, self.lookup[self.tag_id].properties)
|
2021-07-13 14:31:53 +00:00
|
|
|
for x in attrs.get('properties', [])
|
|
|
|
]
|
2021-08-11 14:47:29 +00:00
|
|
|
''' Sanity checks '''
|
|
|
|
if self.tag_id not in self.lookup:
|
|
|
|
raise Exception('Unknown tag: {}'.format(self.to_dict()))
|
|
|
|
if self.end < self.start:
|
|
|
|
raise Exception('Annotation end less then start: '
|
|
|
|
'{}'.format(self.to_dict()))
|
2021-08-10 12:43:55 +00:00
|
|
|
property_ids = [x.property_id for x in self.properties]
|
2021-08-11 14:47:29 +00:00
|
|
|
for required_property_id, required_property in self.lookup[self.tag_id].required_properties.items(): # noqa
|
2021-08-10 12:43:55 +00:00
|
|
|
if required_property_id not in property_ids:
|
2021-08-11 14:47:29 +00:00
|
|
|
raise Exception('Missing required property: '
|
|
|
|
'{}'.format(required_property.to_dict()))
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
@property
|
|
|
|
def name(self):
|
2021-07-22 14:59:29 +00:00
|
|
|
return self.lookup[self.tag_id].name
|
2021-07-13 14:31:53 +00:00
|
|
|
|
2021-08-10 12:43:55 +00:00
|
|
|
def to_dict(self):
|
|
|
|
return {
|
|
|
|
'tag_id': self.tag_id,
|
|
|
|
'start': self.start,
|
|
|
|
'end': self.end,
|
|
|
|
'properties': [x.to_dict() for x in self.properties]
|
|
|
|
}
|
|
|
|
|
2021-07-13 14:31:53 +00:00
|
|
|
def __lt__(self, other):
|
|
|
|
if self.start == other.start:
|
|
|
|
return self.name == 'token' and other.name != 'token'
|
|
|
|
else:
|
|
|
|
return self.start < other.start
|
|
|
|
|
|
|
|
def __le__(self, other):
|
|
|
|
if self.start == other.start:
|
|
|
|
return self.name == 'token' or other.name != 'token'
|
|
|
|
else:
|
|
|
|
return self.start < other.start
|
|
|
|
|
|
|
|
def __eq__(self, other):
|
|
|
|
return self.start == other.start and self.name == other.name
|
|
|
|
|
|
|
|
def __ne__(self, other):
|
|
|
|
return self.start != other.start and self.name != other.name
|
|
|
|
|
|
|
|
def __gt__(self, other):
|
|
|
|
if self.start == other.start:
|
|
|
|
return self.name != 'token' and other.name == 'token'
|
|
|
|
else:
|
|
|
|
return self.start > other.start
|
|
|
|
|
|
|
|
def __ge__(self, other):
|
|
|
|
if self.start == other.start:
|
|
|
|
return self.name != 'token' or other.name == 'token'
|
|
|
|
else:
|
|
|
|
return self.start > other.start
|
|
|
|
|
|
|
|
|
|
|
|
class PropertyAnnotation:
|
2021-07-22 14:59:29 +00:00
|
|
|
def __init__(self, attrs, lookup):
|
|
|
|
self.lookup = lookup
|
|
|
|
self.property_id = attrs['property_id']
|
2021-08-06 14:50:22 +00:00
|
|
|
self.value = attrs['value']
|
2021-07-13 14:31:53 +00:00
|
|
|
# TODO: Process attrs['possibleValues'] as self.labels (no id?)
|
2021-08-11 14:47:29 +00:00
|
|
|
''' Sanity checks '''
|
|
|
|
if self.property_id not in self.lookup:
|
|
|
|
raise Exception('Unknown property: {}'.format(self.to_dict()))
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
@property
|
|
|
|
def name(self):
|
2021-08-11 14:47:29 +00:00
|
|
|
return self.lookup[self.property_id].name
|
2021-07-13 14:31:53 +00:00
|
|
|
|
2021-08-10 12:43:55 +00:00
|
|
|
def to_dict(self):
|
|
|
|
return {
|
|
|
|
'property_id': self.property_id,
|
|
|
|
'tag_id': self.tag_id,
|
|
|
|
'value': self.value
|
|
|
|
}
|
|
|
|
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
class TagDefinition:
|
|
|
|
def __init__(self, attrs):
|
|
|
|
self.id = attrs['id']
|
|
|
|
self.name = attrs['name']
|
|
|
|
self.description = attrs.get('description', '')
|
2021-08-11 14:47:29 +00:00
|
|
|
self.properties = {}
|
|
|
|
for x in attrs.get('properties', []):
|
|
|
|
self.add_property_definition(x)
|
|
|
|
|
|
|
|
def add_property_definition(self, attrs):
|
|
|
|
property_definition = PropertyDefinition(attrs)
|
|
|
|
if property_definition.id in self.properties:
|
|
|
|
raise Exception('Property id already in use: '
|
|
|
|
'{}'.format(property_definition.to_dict()))
|
|
|
|
self.properties[property_definition.id] = property_definition
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
@property
|
|
|
|
def required_properties(self):
|
2021-08-06 14:50:22 +00:00
|
|
|
return {property.id: property for property in self.properties.values()
|
2021-07-13 14:31:53 +00:00
|
|
|
if property.is_required}
|
|
|
|
|
2021-08-11 14:47:29 +00:00
|
|
|
def to_dict(self):
|
|
|
|
return {
|
|
|
|
'id': self.id,
|
|
|
|
'name': self.name,
|
|
|
|
'description': self.description,
|
|
|
|
'properties': {k: v.to_dict() for k, v in self.properties.items()}
|
|
|
|
}
|
|
|
|
|
2021-07-13 14:31:53 +00:00
|
|
|
|
|
|
|
class PropertyDefinition:
|
|
|
|
def __init__(self, attrs):
|
|
|
|
self.id = attrs['id']
|
|
|
|
self.name = attrs['name']
|
|
|
|
self.description = attrs.get('description', '')
|
|
|
|
self.flags = attrs.get('flags', [])
|
|
|
|
self.labels = attrs.get('labels', [])
|
|
|
|
|
|
|
|
@property
|
|
|
|
def is_required(self):
|
|
|
|
return 'required' in self.flags
|
|
|
|
|
|
|
|
@property
|
|
|
|
def has_multiple_values(self):
|
|
|
|
return 'multiple' in self.flags
|
2021-08-11 14:47:29 +00:00
|
|
|
|
|
|
|
def to_dict(self):
|
|
|
|
return {
|
|
|
|
'id': self.id,
|
|
|
|
'name': self.name,
|
|
|
|
'description': self.description,
|
|
|
|
'flags': self.flags,
|
|
|
|
'labels': self.labels
|
|
|
|
}
|