sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2025-07-02 00:30:33 +00:00
Code Issues Projects Releases Wiki Activity

WIP use the new package

Browse Source
This commit is contained in:
Patrick Jentsch
2021-08-06 16:50:22 +02:00
parent a4b2fc3a65
commit 66516eeb89
5 changed files with 219 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
packages/stand-off-data-py/stand_off_data/export.py Normal file
View File

@ -0,0 +1,80 @@
from xml.sax.saxutils import escape
class ExportMixin:
def to_vrt(self, text):
# Devide annotations into CWB's verticalized text format (.vrt) logic
p_attrs = [] # positional attributes
s_attrs = [] # structural attributes
for annotation in self.annotations:
if annotation.name == 'token':
p_attrs.append(annotation)
else:
s_attrs.append(annotation)
# Sort annotations, necessary for the next checks
p_attrs.sort()
s_attrs.sort()
# Check for p_attr<->p_attr overlap
for i, p_attr in enumerate(p_attrs[:-1]):
next_p_attr = p_attrs[i + 1]
# Check if first_p_attr starts/ends within second_p_attr
if ((p_attr.start >= next_p_attr.start) and (p_attr.start <= next_p_attr.end) # noqa
or (p_attr.end >= next_p_attr.start) and (p_attr.end <= next_p_attr.end)): # noqa
raise Exception('Positional attribute overlaps another')
# Check for s_attr<->p_attr overlap
for i, s_attr in enumerate(s_attrs):
for p_attr in p_attrs:
# Check if s_attr starts within p_attr
if s_attr.start > p_attr.start and s_attr.start < p_attr.end:
# Change s_attr start to p_attr's start
s_attrs[i].start = p_attr.start
# Check if s_attr ends within p_attr
if s_attr.end < p_attr.end and s_attr.end > p_attr.start:
# Change s_attr end to p_attr's end
s_attrs[i].end = p_attr.end
# Check if s_attr starts/ends before/after p_attr
if p_attr.start >= s_attr.end or p_attr.end <= s_attr.start:
# No further Checking needed (just because p_attrs are sorted)
break
s_attr_start_buffer = {}
s_attr_end_buffer = {}
for i, s_attr in enumerate(s_attrs):
if s_attr_start_buffer[s_attr.start]:
s_attr_start_buffer[s_attr.start].append(i)
else:
s_attr_start_buffer[s_attr.start] = [i]
if s_attr_end_buffer[s_attr.end]:
s_attr_end_buffer[s_attr.end].append(i)
else:
s_attr_end_buffer[s_attr.end] = [i]
vrt = ''
vrt += '<text>\n'
for p_attr in p_attrs:
# s_attr_starts
for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa
s_attrs = s_attr_start_buffer.pop(k)
for s_attr in s_attrs:
foo = ''
for property in s_attr.properties:
foo += ' {}="{}"'.format(escape(property.name),
escape(property.value))
vrt += '<{}{}>\n'.format(escape(s_attr.name), foo)
for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa
s_attrs = s_attr_end_buffer.pop(k)
for s_attr in s_attrs:
vrt += '</{}>\n'.format(escape(s_attr.name))
# s_attr_ends
foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa
for property in p_attrs.properties:
if property.name == 'lemma':
foo['lemma'] = escape(property.value)
elif property.name == 'ner':
foo['ner'] = escape(property.value)
elif property.name == 'pos':
foo['pos'] = escape(property.value)
elif property.name == 'simple_pos':
foo['simple_pos'] = escape(property.value)
foo['word'] = escape(text[p_attr.start:p_attr.end])
vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(
**foo)
vrt += '</text>\n'

9
packages/stand-off-data-py/stand_off_data/models.py
View File

@ -1,4 +1,7 @@
class StandOffData:
from .export import ExportMixin
class StandOffData(ExportMixin):
def __init__(self, attrs):
self.meta = attrs.get('meta', {})
self.lookup = {tag_definition.id: tag_definition for tag_definition in
@ -69,7 +72,7 @@ class PropertyAnnotation:
self.tag_id = attrs['tag_id']
if self.property_id not in self.lookup[self.tag_id].properties:
raise Exception('Unknown property id: {}'.format(self.property_id))
self.value = property['value']
self.value = attrs['value']
# TODO: Process attrs['possibleValues'] as self.labels (no id?)
@property
@ -91,7 +94,7 @@ class TagDefinition:
@property
def required_properties(self):
return {property.id: property for property in self.properties
return {property.id: property for property in self.properties.values()
if property.is_required}

78
packages/stand-off-data-py/stand_off_data/utils.py
View File

@ -1,78 +0,0 @@
from xml.sax.saxutils import escape
def create_vrt(text, stand_off_data):
# Devide annotations into CWB's verticalized text format (.vrt) logic
p_attrs = [] # positional attributes
s_attrs = [] # structural attributes
for annotation in stand_off_data.annotations:
if annotation.name == 'token':
p_attrs.append(annotation)
else:
s_attrs.append(annotation)
# Sort annotations, necessary for the next checks
p_attrs.sort()
s_attrs.sort()
# Check for p_attr<->p_attr overlap
for i, p_attr in enumerate(p_attrs[:-1]):
next_p_attr = p_attrs[i + 1]
# Check if first_p_attr starts/ends within second_p_attr
if ((p_attr.start >= next_p_attr.start) and (p_attr.start <= next_p_attr.end) # noqa
or (p_attr.end >= next_p_attr.start) and (p_attr.end <= next_p_attr.end)): # noqa
raise Exception('Positional attribute overlaps another')
# Check for s_attr<->p_attr overlap
for i, s_attr in enumerate(s_attrs):
for p_attr in p_attrs:
# Check if s_attr starts within p_attr
if s_attr.start > p_attr.start and s_attr.start < p_attr.end:
# Change s_attr start to p_attr's start
s_attrs[i].start = p_attr.start
# Check if s_attr ends within p_attr
if s_attr.end < p_attr.end and s_attr.end > p_attr.start:
# Change s_attr end to p_attr's end
s_attrs[i].end = p_attr.end
# Check if s_attr starts/ends before/after p_attr
if p_attr.start >= s_attr.end or p_attr.end <= s_attr.start:
# No further Checking needed (just because p_attrs are sorted)
break
s_attr_start_buffer = {}
s_attr_end_buffer = {}
for i, s_attr in enumerate(s_attrs):
if s_attr_start_buffer[s_attr.start]:
s_attr_start_buffer[s_attr.start].append(i)
else:
s_attr_start_buffer[s_attr.start] = [i]
if s_attr_end_buffer[s_attr.end]:
s_attr_end_buffer[s_attr.end].append(i)
else:
s_attr_end_buffer[s_attr.end] = [i]
vrt = ''
vrt += '<text>\n'
for p_attr in p_attrs:
# s_attr_starts
for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa
s_attrs = s_attr_start_buffer.pop(k)
for s_attr in s_attrs:
foo = ''
for property in s_attr.properties:
foo += ' {}="{}"'.format(escape(property.name),
escape(property.value))
vrt += '<{}{}>\n'.format(escape(s_attr.name), foo)
for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa
s_attrs = s_attr_end_buffer.pop(k)
for s_attr in s_attrs:
vrt += '</{}>\n'.format(escape(s_attr.name))
# s_attr_ends
foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa
for property in p_attrs.properties:
if property.name == 'lemma':
foo['lemma'] = escape(property.value)
elif property.name == 'ner':
foo['ner'] = escape(property.value)
elif property.name == 'pos':
foo['pos'] = escape(property.value)
elif property.name == 'simple_pos':
foo['simple_pos'] = escape(property.value)
foo['word'] = escape(text[p_attr.start:p_attr.end])
vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(**foo)
vrt += '</text>\n'