sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2025-07-01 17:10:33 +00:00
Code Issues Projects Releases Wiki Activity

Bump spaCy version, bugfixes, codestyle

Browse Source
This commit is contained in:
Patrick Jentsch
2022-01-27 16:50:22 +01:00
parent 29ccfac4f6
commit a2e8e72e54
9 changed files with 699 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/stand-off-data-py/setup.py
View File

@ -1,14 +1,14 @@
import setuptools
setuptools.setup(
name='stand-off-data',
name='Stand off data',
author='Patrick Jentsch',
author_email='p.jentsch@uni-bielefeld.de',
description='A python library to handle stand off data.',
py_modules=['stand_off_data'],
classifiers=[
'Programming Language :: Python :: 3',
'Operating System :: OS Independent',
],
packages=setuptools.find_packages(),
python_requires='>=3.5'
)

149
packages/stand-off-data-py/stand_off_data/models.py → packages/stand-off-data-py/stand_off_data.py
View File

@ -7,13 +7,15 @@ class StandOffData:
self.lookup = {}
for x in attrs.get('tags', []):
self.add_tag_definition(x)
self.annotations = [TagAnnotation(x, self.lookup)
for x in attrs.get('annotations', [])]
self.annotations = [
TagAnnotation(x, self.lookup)
for x in attrs.get('annotations', [])
]
def add_tag_definition(self, attrs):
tag_definition = TagDefinition(attrs)
if tag_definition.id in self.lookup:
raise Exception('Tag id already in use: {}'.format(self.to_dict()))
raise Exception(f'Tag id already in use: {self.to_dict()}')
self.lookup[tag_definition.id] = tag_definition
def to_dict(self):
@ -42,7 +44,9 @@ class StandOffData:
if ((p_attr.start >= next_p_attr.start) and (p_attr.start < next_p_attr.end) # noqa
or (p_attr.end > next_p_attr.start) and (p_attr.end <= next_p_attr.end)): # noqa
raise Exception(
'Positional attribute overlaps another: {}<->{}'.format(p_attr.to_dict(), next_p_attr.to_dict()))
'Positional attribute overlaps another: '
f'{p_attr.to_dict()}<->{next_p_attr.to_dict()}'
)
# Check for s_attr<->p_attr overlap
for i, s_attr in enumerate(s_attrs):
for p_attr in p_attrs:
@ -56,8 +60,11 @@ class StandOffData:
s_attrs[i].end = p_attr.end
# Check if s_attr starts/ends before/after p_attr
if p_attr.start >= s_attr.end or p_attr.end <= s_attr.start:
# No further Checking needed (just because p_attrs are sorted)
# No further Checking needed (because p_attrs are sorted)
break
p_attr_buffer = {}
for i, p_attr in enumerate(p_attrs):
p_attr_buffer[p_attr.start] = i
s_attr_start_buffer = {}
s_attr_end_buffer = {}
for i, s_attr in enumerate(s_attrs):
@ -66,34 +73,56 @@ class StandOffData:
else:
s_attr_start_buffer[s_attr.start] = [i]
if s_attr.end in s_attr_end_buffer:
s_attr_end_buffer[s_attr.end].append(i)
s_attr_end_buffer[s_attr.end].insert(0, i)
else:
s_attr_end_buffer[s_attr.end] = [i]
vrt = ''
vrt += '<text>\n'
for p_attr in p_attrs:
# s_attr_ends
for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa
s_attr_indexes = s_attr_end_buffer.pop(k)
current_position = 0
text_len = len(text)
# As long as we have something in our buffers we process it
while current_position <= text_len:
# s_attr endings
# for k in {k: v for k, v in s_attr_end_buffer.items() if k <= current_position}: # noqa
if current_position in s_attr_end_buffer:
# s_attr_indexes = s_attr_end_buffer.pop(k)
s_attr_indexes = s_attr_end_buffer.pop(current_position)
for s_attr_index in s_attr_indexes:
s_attr = s_attrs[s_attr_index]
vrt += '</{}>\n'.format(escape(s_attr.name))
# s_attr_starts
for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa
s_attr_indexes = s_attr_start_buffer.pop(k)
vrt += f'</{escape(s_attr.name)}>\n'
# s_attrs starts
# for k in {k: v for k, v in s_attr_start_buffer.items() if k <= current_position}: # noqa
if current_position in s_attr_start_buffer:
# s_attr_indexes = s_attr_start_buffer.pop(k)
s_attr_indexes = s_attr_start_buffer.pop(current_position)
for s_attr_index in s_attr_indexes:
s_attr = s_attrs[s_attr_index]
foo = ''
vrt += f'<{escape(s_attr.name)}'
for property in s_attr.properties:
foo += ' {}="{}"'.format(escape(property.name),
escape(property.value))
vrt += '<{}{}>\n'.format(escape(s_attr.name), foo)
foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa
vrt += f' {escape(property.name)}="{escape(str(property.value))}"' # noqa
vrt += '>\n'
# p_attrs
if current_position not in p_attr_buffer:
current_position += 1
continue
p_attr_index = p_attr_buffer.pop(current_position)
p_attr = p_attrs[p_attr_index]
if text[p_attr.start:p_attr.end].isspace():
current_position = p_attr.end
continue
_p_attr = {
'lemma': 'None',
'pos': 'None',
'simple_pos': 'None',
'word': 'None'
}
for property in p_attr.properties:
foo[property.name] = escape(property.value)
foo['word'] = escape(text[p_attr.start:p_attr.end])
vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(
**foo)
if property.name not in _p_attr:
continue
_p_attr[property.name] = escape(str(property.value))
_p_attr['word'] = escape(text[p_attr.start:p_attr.end])
vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\n'.format(**_p_attr)
current_position = p_attr.end
vrt += '</text>\n'
return vrt
@ -110,15 +139,15 @@ class TagAnnotation:
]
''' Sanity checks '''
if self.tag_id not in self.lookup:
raise Exception('Unknown tag: {}'.format(self.to_dict()))
raise Exception(f'Unknown tag: {self.to_dict()}')
if self.end < self.start:
raise Exception('Annotation end less then start: '
'{}'.format(self.to_dict()))
property_ids = [x.property_id for x in self.properties]
for required_property_id, required_property in self.lookup[self.tag_id].required_properties.items(): # noqa
if required_property_id not in property_ids:
raise Exception('Missing required property: '
'{}'.format(required_property.to_dict()))
raise Exception(f'Annotation end less then start: {self.to_dict()}') # noqa
# property_ids = [x.property_id for x in self.properties]
# for required_property_id, required_property in self.lookup[self.tag_id].required_properties.items(): # noqa
# if required_property_id not in property_ids:
# raise Exception(
# f'Missing required property: {required_property.to_dict()}'
# )
@property
def name(self):
@ -134,33 +163,45 @@ class TagAnnotation:
def __lt__(self, other):
if self.start == other.start:
return self.name == 'token' and other.name != 'token'
if self.name == 'token' and other.name != 'token':
return False
elif self.name != 'token' and other.name == 'token':
return True
else:
return self.end > other.end
else:
return self.start < other.start
def __le__(self, other):
if self.start == other.start:
return self.name == 'token' or other.name != 'token'
if self.name == 'token' and other.name != 'token':
return False
elif self.name != 'token' and other.name == 'token':
return True
else:
return self.end >= other.end
else:
return self.start < other.start
return self.start <= other.start
def __eq__(self, other):
return self.start == other.start and self.name == other.name
if self.start == other.start:
if self.name == 'token' and other.name != 'token':
return False
elif self.name != 'token' and other.name == 'token':
return False
else:
return self.end == other.end
else:
return False
def __ne__(self, other):
return self.start != other.start and self.name != other.name
return not self == other
def __gt__(self, other):
if self.start == other.start:
return self.name != 'token' and other.name == 'token'
else:
return self.start > other.start
return not self <= other
def __ge__(self, other):
if self.start == other.start:
return self.name != 'token' or other.name == 'token'
else:
return self.start > other.start
return not self < other
class PropertyAnnotation:
@ -171,7 +212,7 @@ class PropertyAnnotation:
# TODO: Process attrs['possibleValues'] as self.labels (no id?)
''' Sanity checks '''
if self.property_id not in self.lookup:
raise Exception('Unknown property: {}'.format(self.to_dict()))
raise Exception(f'Unknown property: {self.to_dict()}')
@property
def name(self):
@ -197,14 +238,14 @@ class TagDefinition:
def add_property_definition(self, attrs):
property_definition = PropertyDefinition(attrs)
if property_definition.id in self.properties:
raise Exception('Property id already in use: '
'{}'.format(property_definition.to_dict()))
raise Exception(
f'Property id already in use: {property_definition.to_dict()}')
self.properties[property_definition.id] = property_definition
@property
def required_properties(self):
return {property.id: property for property in self.properties.values()
if property.is_required}
# @property
# def required_properties(self):
# return {property.id: property for property in self.properties.values()
# if property.is_required}
def to_dict(self):
return {
@ -223,9 +264,9 @@ class PropertyDefinition:
self.flags = attrs.get('flags', [])
self.labels = attrs.get('labels', [])
@property
def is_required(self):
return 'required' in self.flags
# @property
# def is_required(self):
# return 'required' in self.flags
@property
def has_multiple_values(self):