From a4b2fc3a65f9b3f837adfad4e067b3da08380b23 Mon Sep 17 00:00:00 2001
From: Patrick Jentsch
Date: Thu, 22 Jul 2021 16:59:29 +0200
Subject: [PATCH] Create package for stand-off-data-py
---
Dockerfile | 9 +++-
packages/stand-off-data-py/setup.py | 14 ++++++
.../stand_off_data/__init__.py | 2 +
.../stand_off_data/models.py | 50 +++++++------------
.../stand-off-data-py/stand_off_data/utils.py | 35 ++++++++++++-
5 files changed, 75 insertions(+), 35 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index bdfddb6..f7d824c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -38,7 +38,7 @@ RUN apt-get install --no-install-recommends --yes \
&& pip3 install "spacy==${SPACY_VERSION}"
-# Only models that include the following components are compatibel:
+# Only models that include the following components are compatibel:
# lemmatizer, ner, parser, senter, tagger,
ENV SPACY_MODELS="de_core_news_md,en_core_web_md,it_core_news_md,nl_core_news_md,pl_core_news_md,zh_core_web_md"
ENV SPACY_MODELS_VERSION=3.0.0
@@ -51,6 +51,13 @@ RUN apt-get install --no-install-recommends --yes \
zip
+COPY packages .
+RUN cd stand-off-data-py \
+ && python3 setup.py build \
+ && python3 setup.py install \
+ && cd -
+
+
## Install Pipeline ##
COPY nlp spacy-nlp vrt-creator /usr/local/bin/
diff --git a/packages/stand-off-data-py/setup.py b/packages/stand-off-data-py/setup.py
index e69de29..2d263ad 100644
--- a/packages/stand-off-data-py/setup.py
+++ b/packages/stand-off-data-py/setup.py
@@ -0,0 +1,14 @@
+import setuptools
+
+setuptools.setup(
+ name='stand-off-data',
+ author='Patrick Jentsch',
+ author_email='p.jentsch@uni-bielefeld.de',
+ description='A python library to handle stand off data.',
+ classifiers=[
+ 'Programming Language :: Python :: 3',
+ 'Operating System :: OS Independent',
+ ],
+ packages=setuptools.find_packages(),
+ python_requires='>=3.5'
+)
diff --git a/packages/stand-off-data-py/stand_off_data/__init__.py b/packages/stand-off-data-py/stand_off_data/__init__.py
index e69de29..9150686 100644
--- a/packages/stand-off-data-py/stand_off_data/__init__.py
+++ b/packages/stand-off-data-py/stand_off_data/__init__.py
@@ -0,0 +1,2 @@
+# flake8: noqa
+from .models import StandOffData
diff --git a/packages/stand-off-data-py/stand_off_data/models.py b/packages/stand-off-data-py/stand_off_data/models.py
index 5d93aad..0e426dd 100644
--- a/packages/stand-off-data-py/stand_off_data/models.py
+++ b/packages/stand-off-data-py/stand_off_data/models.py
@@ -1,33 +1,17 @@
-'''
- 'generator': {
- 'name': 'nopaque NLP service',
- 'version': '1.0.0',
- 'arguments': {
- 'check_encoding': args.check_encoding,
- 'language': args.language
- }
- },
- 'file': {
- 'encoding': encoding,
- 'md5': text_md5.hexdigest(),
- 'name': os.path.basename(args.input)
- }
-'''
-
-
class StandOffData:
def __init__(self, attrs):
- self.tags = {tag_definition.id: tag_definition for tag_definition in
- [TagDefinition(x) for x in attrs.get('tags', [])]}
- self.annotations = [TagAnnotation(x, self.tags) for x in
+ self.meta = attrs.get('meta', {})
+ self.lookup = {tag_definition.id: tag_definition for tag_definition in
+ [TagDefinition(x) for x in attrs.get('tags', [])]}
+ self.annotations = [TagAnnotation(x, self.lookup) for x in
attrs.get('annotations', [])]
class TagAnnotation:
- def __init__(self, attrs, tag_lookup):
+ def __init__(self, attrs, lookup):
+ self.lookup = lookup
self.tag_id = attrs['tag_id']
- self.tag_lookup = tag_lookup
- if self.tag_id not in self.tag_lookup:
+ if self.tag_id not in self.lookup:
raise Exception('Unknown tag id: {}'.format(self.tag_id))
self.start = attrs['start']
self.end = attrs['end']
@@ -35,16 +19,17 @@ class TagAnnotation:
raise Exception('start must be lower then end')
self.description = attrs.get('description', '')
self.properties = [
- PropertyAnnotation(x, self.tag_lookup[self.tag_id].properties)
+ PropertyAnnotation({**x, 'tag_id': self.tag_id}, self.lookup)
for x in attrs.get('properties', [])
]
- for required_property_id in self.tag_lookup[self.tag_id].required_properties:
+ for required_property_id in self.lookup[self.tag_id].required_properties:
if required_property_id not in self.properties:
- raise Exception('Missing required property: {}'.format(required_property_id))
+ raise Exception(
+ 'Missing required property: {}'.format(required_property_id))
@property
def name(self):
- return self.tag_lookup[self.tag_id].name
+ return self.lookup[self.tag_id].name
def __lt__(self, other):
if self.start == other.start:
@@ -78,17 +63,18 @@ class TagAnnotation:
class PropertyAnnotation:
- def __init__(self, attrs, property_lookup):
- self.property_id = property['property_id']
- self.property_lookup = property_lookup
- if self.property_id not in self.property_lookup:
+ def __init__(self, attrs, lookup):
+ self.lookup = lookup
+ self.property_id = attrs['property_id']
+ self.tag_id = attrs['tag_id']
+ if self.property_id not in self.lookup[self.tag_id].properties:
raise Exception('Unknown property id: {}'.format(self.property_id))
self.value = property['value']
# TODO: Process attrs['possibleValues'] as self.labels (no id?)
@property
def name(self):
- return self.property_lookup[self.property_id].name
+ return self.lookup[self.tag_id].properties[self.property_id].name
class TagDefinition:
diff --git a/packages/stand-off-data-py/stand_off_data/utils.py b/packages/stand-off-data-py/stand_off_data/utils.py
index b62fdd5..5a225eb 100644
--- a/packages/stand-off-data-py/stand_off_data/utils.py
+++ b/packages/stand-off-data-py/stand_off_data/utils.py
@@ -1,3 +1,6 @@
+from xml.sax.saxutils import escape
+
+
def create_vrt(text, stand_off_data):
# Devide annotations into CWB's verticalized text format (.vrt) logic
p_attrs = [] # positional attributes
@@ -42,6 +45,34 @@ def create_vrt(text, stand_off_data):
if s_attr_end_buffer[s_attr.end]:
s_attr_end_buffer[s_attr.end].append(i)
else:
- s_attr_end_buffer[s_attr.end] = [1]
+ s_attr_end_buffer[s_attr.end] = [i]
vrt = ''
- # TODO do the work!
+ vrt += '\n'
+ for p_attr in p_attrs:
+ # s_attr_starts
+ for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa
+ s_attrs = s_attr_start_buffer.pop(k)
+ for s_attr in s_attrs:
+ foo = ''
+ for property in s_attr.properties:
+ foo += ' {}="{}"'.format(escape(property.name),
+ escape(property.value))
+ vrt += '<{}{}>\n'.format(escape(s_attr.name), foo)
+ for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa
+ s_attrs = s_attr_end_buffer.pop(k)
+ for s_attr in s_attrs:
+ vrt += '{}>\n'.format(escape(s_attr.name))
+ # s_attr_ends
+ foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa
+ for property in p_attrs.properties:
+ if property.name == 'lemma':
+ foo['lemma'] = escape(property.value)
+ elif property.name == 'ner':
+ foo['ner'] = escape(property.value)
+ elif property.name == 'pos':
+ foo['pos'] = escape(property.value)
+ elif property.name == 'simple_pos':
+ foo['simple_pos'] = escape(property.value)
+ foo['word'] = escape(text[p_attr.start:p_attr.end])
+ vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(**foo)
+ vrt += '\n'