#!/usr/bin/env python3.7
# coding=utf-8
from argparse import ArgumentParser
import chardet
import hashlib
import json
import os
import spacy
import textwrap
import uuid
def UUIDnopaque(name):
return 'nopaque_{}'.format(
uuid.uuid3(uuid.NAMESPACE_DNS,
'{}@nopaque.sfb1288.uni-bielefeld.de'.format(name))
)
spacy_models = {spacy.info(pipeline)['lang']: pipeline
for pipeline in spacy.info()['pipelines']}
# Parse the given arguments
parser = ArgumentParser(description='Create annotations for a given txt file')
parser.add_argument('input', help='Path to txt input file')
parser.add_argument('output', help='Path to JSON output file')
parser.add_argument('-l', '--language',
choices=spacy_models.keys(),
help='Language of the input (2-character ISO 639-1 language codes)', # noqa
required=True)
parser.add_argument('-c', '--check-encoding',
action='store_true',
help='Check encoding of the input file, UTF-8 is used instead') # noqa
args = parser.parse_args()
with open(args.input, "rb") as text_file:
if args.check_encoding:
encoding = chardet.detect(text_file.read())['encoding']
else:
encoding = 'utf-8'
text_file.seek(0)
text_md5 = hashlib.md5()
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''):
text_md5.update(chunk)
# Load the text contents from the input file
with open(args.input, encoding=encoding) as text_file:
# spaCy NLP is limited to strings with a maximum of 1 million characters at
# once. So we split it into suitable chunks.
text_chunks = textwrap.wrap(
text_file.read(),
1000000,
break_long_words=False,
break_on_hyphens=False,
drop_whitespace=False,
expand_tabs=False,
replace_whitespace=False
)
model = spacy_models[args.language]
nlp = spacy.load(model)
meta = {
'generator': {
'name': 'nopaque NLP service',
'version': '1.0.0',
'arguments': {
'check_encoding': args.check_encoding,
'language': args.language
}
},
'file': {
'encoding': encoding,
'md5': text_md5.hexdigest(),
'name': os.path.basename(args.input)
}
}
tags = [
{
'id': UUIDnopaque('token'),
'name': 'token',
'description': 'An individual token — i.e. a word, punctuation symbol, whitespace, etc.',
'properties': [
{
'id': UUIDnopaque('token.lemma'),
'name': 'lemma',
'description': 'The base form of the word',
'flags': ['required'],
'labels': []
},
{
'id': UUIDnopaque('token.pos'),
'name': 'pos',
'description': 'The detailed part-of-speech tag',
'flags': ['required'],
'labels': [
{
'id': UUIDnopaque('token.pos={}'.format(label)),
'name': label,
'description': spacy.explain(label) or ''
} for label in spacy.info(model)['labels']['tagger']
]
},
{
'id': UUIDnopaque('token.simple_pos'),
'name': 'simple_pos',
'description': 'The simple UPOS part-of-speech tag',
'flags': ['required'],
'labels': [
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'ADJ',
'description': 'adjective'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'ADP',
'description': 'adposition'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'ADV',
'description': 'adverb'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'AUX',
'description': 'auxiliary verb'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'CONJ',
'description': 'coordinating conjunction'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'DET',
'description': 'determiner'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'INTJ',
'description': 'interjection'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'NOUN',
'description': 'noun'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'NUM',
'description': 'numeral'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'PART',
'description': 'particle'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'PRON',
'description': 'pronoun'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'PROPN',
'description': 'proper noun'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'PUNCT',
'description': 'punctuation'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'SCONJ',
'description': 'subordinating conjunction'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'SYM',
'description': 'symbol'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'VERB',
'description': 'verb'
},
{
'id': UUIDnopaque('token.simple_pos=ADJ'),
'name': 'X',
'description': 'other'
}
]
},
{
'id': UUIDnopaque('token.ner'),
'name': 'ner',
'description': 'Label indicating the type of the entity',
'flags': ['required'],
'labels': [
{
'id': UUIDnopaque('token.ner={}'.format(label)),
'name': label,
'description': spacy.explain(label) or ''
} for label in spacy.info(model)['labels']['ner']
]
}
]
},
{
'id': UUIDnopaque('s'),
'name': 's',
'description': 'Encodes the start and end of a sentence',
'properties': []
},
{
'id': UUIDnopaque('ent'),
'name': 'ent',
'description': 'Encodes the start and end of a named entity',
'properties': [
{
'id': UUIDnopaque('ent.type'),
'name': 'type',
'description': 'Label indicating the type of the entity',
'flags': ['required'],
'labels': [
{
'id': UUIDnopaque('ent.type={}'.format(label)),
'name': label,
'description': spacy.explain(label) or ''
} for label in spacy.info(model)['labels']['ner']
]
}
]
}
]
annotations = []
chunk_offset = 0
while text_chunks:
text_chunk = text_chunks.pop(0)
doc = nlp(text_chunk)
for token in doc:
if token.is_space:
continue
if token.is_sent_start:
annotation = {'start': token.sent.start_char + chunk_offset,
'end': token.sent.end_char + chunk_offset,
'tag_id': UUIDnopaque('s'),
'properties': []}
annotations.append(annotation)
# Check if the token is the start of an entity
if token.ent_iob == 3:
for ent_candidate in token.sent.ents:
if ent_candidate.start_char == token.idx:
ent = ent_candidate
annotation = {
'start': ent.start_char + chunk_offset,
'end': ent.end_char + chunk_offset,
'tag_id': UUIDnopaque('ent'),
'properties': [
{
'property_id': UUIDnopaque('ent.type'),
'value': token.ent_type_
}
]
}
annotations.append(annotation)
break
annotation = {
'start': token.idx + chunk_offset,
'end': token.idx + len(token.text) + chunk_offset,
'tag_id': UUIDnopaque('token'),
'properties': [
{
'property_id': UUIDnopaque('token.pos'),
'value': token.tag_
},
{
'property_id': UUIDnopaque('token.lemma'),
'value': token.lemma_
},
{
'property_id': UUIDnopaque('token.simple_pos'),
'value': token.pos_
},
{
'property_id': UUIDnopaque('token.ner'),
'value': token.ent_type_ if token.ent_type_ else 'None'
}
]
}
annotations.append(annotation)
chunk_offset += len(text_chunk)
text_chunk = None
with open(args.output, 'w') as output_file:
json.dump({'meta': meta, 'tags': tags, 'annotations': annotations},
output_file, indent=4)