diff --git a/Dockerfile b/Dockerfile index bdfddb6..f7d824c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get install --no-install-recommends --yes \ && pip3 install "spacy==${SPACY_VERSION}" -# Only models that include the following components are compatibel: +# Only models that include the following components are compatibel: # lemmatizer, ner, parser, senter, tagger, ENV SPACY_MODELS="de_core_news_md,en_core_web_md,it_core_news_md,nl_core_news_md,pl_core_news_md,zh_core_web_md" ENV SPACY_MODELS_VERSION=3.0.0 @@ -51,6 +51,13 @@ RUN apt-get install --no-install-recommends --yes \ zip +COPY packages . +RUN cd stand-off-data-py \ + && python3 setup.py build \ + && python3 setup.py install \ + && cd - + + ## Install Pipeline ## COPY nlp spacy-nlp vrt-creator /usr/local/bin/ diff --git a/packages/stand-off-data-py/setup.py b/packages/stand-off-data-py/setup.py index e69de29..2d263ad 100644 --- a/packages/stand-off-data-py/setup.py +++ b/packages/stand-off-data-py/setup.py @@ -0,0 +1,14 @@ +import setuptools + +setuptools.setup( + name='stand-off-data', + author='Patrick Jentsch', + author_email='p.jentsch@uni-bielefeld.de', + description='A python library to handle stand off data.', + classifiers=[ + 'Programming Language :: Python :: 3', + 'Operating System :: OS Independent', + ], + packages=setuptools.find_packages(), + python_requires='>=3.5' +) diff --git a/packages/stand-off-data-py/stand_off_data/__init__.py b/packages/stand-off-data-py/stand_off_data/__init__.py index e69de29..9150686 100644 --- a/packages/stand-off-data-py/stand_off_data/__init__.py +++ b/packages/stand-off-data-py/stand_off_data/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa +from .models import StandOffData diff --git a/packages/stand-off-data-py/stand_off_data/models.py b/packages/stand-off-data-py/stand_off_data/models.py index 5d93aad..0e426dd 100644 --- a/packages/stand-off-data-py/stand_off_data/models.py +++ b/packages/stand-off-data-py/stand_off_data/models.py @@ -1,33 +1,17 @@ -''' - 'generator': { - 'name': 'nopaque NLP service', - 'version': '1.0.0', - 'arguments': { - 'check_encoding': args.check_encoding, - 'language': args.language - } - }, - 'file': { - 'encoding': encoding, - 'md5': text_md5.hexdigest(), - 'name': os.path.basename(args.input) - } -''' - - class StandOffData: def __init__(self, attrs): - self.tags = {tag_definition.id: tag_definition for tag_definition in - [TagDefinition(x) for x in attrs.get('tags', [])]} - self.annotations = [TagAnnotation(x, self.tags) for x in + self.meta = attrs.get('meta', {}) + self.lookup = {tag_definition.id: tag_definition for tag_definition in + [TagDefinition(x) for x in attrs.get('tags', [])]} + self.annotations = [TagAnnotation(x, self.lookup) for x in attrs.get('annotations', [])] class TagAnnotation: - def __init__(self, attrs, tag_lookup): + def __init__(self, attrs, lookup): + self.lookup = lookup self.tag_id = attrs['tag_id'] - self.tag_lookup = tag_lookup - if self.tag_id not in self.tag_lookup: + if self.tag_id not in self.lookup: raise Exception('Unknown tag id: {}'.format(self.tag_id)) self.start = attrs['start'] self.end = attrs['end'] @@ -35,16 +19,17 @@ class TagAnnotation: raise Exception('start must be lower then end') self.description = attrs.get('description', '') self.properties = [ - PropertyAnnotation(x, self.tag_lookup[self.tag_id].properties) + PropertyAnnotation({**x, 'tag_id': self.tag_id}, self.lookup) for x in attrs.get('properties', []) ] - for required_property_id in self.tag_lookup[self.tag_id].required_properties: + for required_property_id in self.lookup[self.tag_id].required_properties: if required_property_id not in self.properties: - raise Exception('Missing required property: {}'.format(required_property_id)) + raise Exception( + 'Missing required property: {}'.format(required_property_id)) @property def name(self): - return self.tag_lookup[self.tag_id].name + return self.lookup[self.tag_id].name def __lt__(self, other): if self.start == other.start: @@ -78,17 +63,18 @@ class TagAnnotation: class PropertyAnnotation: - def __init__(self, attrs, property_lookup): - self.property_id = property['property_id'] - self.property_lookup = property_lookup - if self.property_id not in self.property_lookup: + def __init__(self, attrs, lookup): + self.lookup = lookup + self.property_id = attrs['property_id'] + self.tag_id = attrs['tag_id'] + if self.property_id not in self.lookup[self.tag_id].properties: raise Exception('Unknown property id: {}'.format(self.property_id)) self.value = property['value'] # TODO: Process attrs['possibleValues'] as self.labels (no id?) @property def name(self): - return self.property_lookup[self.property_id].name + return self.lookup[self.tag_id].properties[self.property_id].name class TagDefinition: diff --git a/packages/stand-off-data-py/stand_off_data/utils.py b/packages/stand-off-data-py/stand_off_data/utils.py index b62fdd5..5a225eb 100644 --- a/packages/stand-off-data-py/stand_off_data/utils.py +++ b/packages/stand-off-data-py/stand_off_data/utils.py @@ -1,3 +1,6 @@ +from xml.sax.saxutils import escape + + def create_vrt(text, stand_off_data): # Devide annotations into CWB's verticalized text format (.vrt) logic p_attrs = [] # positional attributes @@ -42,6 +45,34 @@ def create_vrt(text, stand_off_data): if s_attr_end_buffer[s_attr.end]: s_attr_end_buffer[s_attr.end].append(i) else: - s_attr_end_buffer[s_attr.end] = [1] + s_attr_end_buffer[s_attr.end] = [i] vrt = '' - # TODO do the work! + vrt += '\n' + for p_attr in p_attrs: + # s_attr_starts + for k in {k: v for k, v in s_attr_start_buffer.items() if k <= p_attr.start}: # noqa + s_attrs = s_attr_start_buffer.pop(k) + for s_attr in s_attrs: + foo = '' + for property in s_attr.properties: + foo += ' {}="{}"'.format(escape(property.name), + escape(property.value)) + vrt += '<{}{}>\n'.format(escape(s_attr.name), foo) + for k in {k: v for k, v in s_attr_end_buffer.items() if k <= p_attr.start}: # noqa + s_attrs = s_attr_end_buffer.pop(k) + for s_attr in s_attrs: + vrt += '\n'.format(escape(s_attr.name)) + # s_attr_ends + foo = {'lemma': None, 'ner': None, 'pos': None, 'simple_pos': None, 'word': None} # noqa + for property in p_attrs.properties: + if property.name == 'lemma': + foo['lemma'] = escape(property.value) + elif property.name == 'ner': + foo['ner'] = escape(property.value) + elif property.name == 'pos': + foo['pos'] = escape(property.value) + elif property.name == 'simple_pos': + foo['simple_pos'] = escape(property.value) + foo['word'] = escape(text[p_attr.start:p_attr.end]) + vrt += '{word}\t{pos}\t{lemma}\t{simple_pos}\t{ner}\n'.format(**foo) + vrt += '\n'