mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2024-12-26 07:44:18 +00:00
306 lines
11 KiB
Python
Executable File
306 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3.7
|
|
# coding=utf-8
|
|
|
|
from argparse import ArgumentParser
|
|
import chardet
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import spacy
|
|
import textwrap
|
|
import uuid
|
|
|
|
|
|
def UUIDnopaque(name):
|
|
return 'nopaque_{}'.format(
|
|
uuid.uuid3(uuid.NAMESPACE_DNS,
|
|
'{}@nopaque.sfb1288.uni-bielefeld.de'.format(name))
|
|
)
|
|
|
|
|
|
spacy_models = {spacy.info(pipeline)['lang']: pipeline
|
|
for pipeline in spacy.info()['pipelines']}
|
|
|
|
|
|
# Parse the given arguments
|
|
parser = ArgumentParser(description='Create annotations for a given txt file')
|
|
parser.add_argument('input', help='Path to txt input file')
|
|
parser.add_argument('output', help='Path to JSON output file')
|
|
parser.add_argument('-l', '--language',
|
|
choices=spacy_models.keys(),
|
|
help='Language of the input (2-character ISO 639-1 language codes)', # noqa
|
|
required=True)
|
|
parser.add_argument('-c', '--check-encoding',
|
|
action='store_true',
|
|
help='Check encoding of the input file, UTF-8 is used instead') # noqa
|
|
args = parser.parse_args()
|
|
|
|
with open(args.input, "rb") as text_file:
|
|
if args.check_encoding:
|
|
encoding = chardet.detect(text_file.read())['encoding']
|
|
else:
|
|
encoding = 'utf-8'
|
|
text_file.seek(0)
|
|
text_md5 = hashlib.md5()
|
|
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''):
|
|
text_md5.update(chunk)
|
|
|
|
# Load the text contents from the input file
|
|
with open(args.input, encoding=encoding) as text_file:
|
|
# spaCy NLP is limited to strings with a maximum of 1 million characters at
|
|
# once. So we split it into suitable chunks.
|
|
text_chunks = textwrap.wrap(
|
|
text_file.read(),
|
|
1000000,
|
|
break_long_words=False,
|
|
break_on_hyphens=False,
|
|
drop_whitespace=False,
|
|
expand_tabs=False,
|
|
replace_whitespace=False
|
|
)
|
|
|
|
model = spacy_models[args.language]
|
|
nlp = spacy.load(model)
|
|
|
|
meta = {
|
|
'generator': {
|
|
'name': 'nopaque NLP service',
|
|
'version': '1.0.0',
|
|
'arguments': {
|
|
'check_encoding': args.check_encoding,
|
|
'language': args.language
|
|
}
|
|
},
|
|
'file': {
|
|
'encoding': encoding,
|
|
'md5': text_md5.hexdigest(),
|
|
'name': os.path.basename(args.input)
|
|
}
|
|
}
|
|
|
|
tags = [
|
|
{
|
|
'id': UUIDnopaque('token'),
|
|
'name': 'token',
|
|
'description': 'An individual token — i.e. a word, punctuation symbol, whitespace, etc.',
|
|
'properties': [
|
|
{
|
|
'id': UUIDnopaque('token.lemma'),
|
|
'name': 'lemma',
|
|
'description': 'The base form of the word',
|
|
'flags': ['required'],
|
|
'labels': []
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.pos'),
|
|
'name': 'pos',
|
|
'description': 'The detailed part-of-speech tag',
|
|
'flags': ['required'],
|
|
'labels': [
|
|
{
|
|
'id': UUIDnopaque('token.pos={}'.format(label)),
|
|
'name': label,
|
|
'description': spacy.explain(label) or ''
|
|
} for label in spacy.info(model)['labels']['tagger']
|
|
]
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos'),
|
|
'name': 'simple_pos',
|
|
'description': 'The simple UPOS part-of-speech tag',
|
|
'flags': ['required'],
|
|
'labels': [
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'ADJ',
|
|
'description': 'adjective'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'ADP',
|
|
'description': 'adposition'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'ADV',
|
|
'description': 'adverb'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'AUX',
|
|
'description': 'auxiliary verb'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'CONJ',
|
|
'description': 'coordinating conjunction'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'DET',
|
|
'description': 'determiner'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'INTJ',
|
|
'description': 'interjection'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'NOUN',
|
|
'description': 'noun'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'NUM',
|
|
'description': 'numeral'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'PART',
|
|
'description': 'particle'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'PRON',
|
|
'description': 'pronoun'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'PROPN',
|
|
'description': 'proper noun'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'PUNCT',
|
|
'description': 'punctuation'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'SCONJ',
|
|
'description': 'subordinating conjunction'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'SYM',
|
|
'description': 'symbol'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'VERB',
|
|
'description': 'verb'
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.simple_pos=ADJ'),
|
|
'name': 'X',
|
|
'description': 'other'
|
|
}
|
|
]
|
|
},
|
|
{
|
|
'id': UUIDnopaque('token.ner'),
|
|
'name': 'ner',
|
|
'description': 'Label indicating the type of the entity',
|
|
'flags': ['required'],
|
|
'labels': [
|
|
{
|
|
'id': UUIDnopaque('token.ner={}'.format(label)),
|
|
'name': label,
|
|
'description': spacy.explain(label) or ''
|
|
} for label in spacy.info(model)['labels']['ner']
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
'id': UUIDnopaque('s'),
|
|
'name': 's',
|
|
'description': 'Encodes the start and end of a sentence',
|
|
'properties': []
|
|
},
|
|
{
|
|
'id': UUIDnopaque('ent'),
|
|
'name': 'ent',
|
|
'description': 'Encodes the start and end of a named entity',
|
|
'properties': [
|
|
{
|
|
'id': UUIDnopaque('ent.type'),
|
|
'name': 'type',
|
|
'description': 'Label indicating the type of the entity',
|
|
'flags': ['required'],
|
|
'labels': [
|
|
{
|
|
'id': UUIDnopaque('ent.type={}'.format(label)),
|
|
'name': label,
|
|
'description': spacy.explain(label) or ''
|
|
} for label in spacy.info(model)['labels']['ner']
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
|
|
annotations = []
|
|
|
|
chunk_offset = 0
|
|
while text_chunks:
|
|
text_chunk = text_chunks.pop(0)
|
|
doc = nlp(text_chunk)
|
|
for token in doc:
|
|
if token.is_space:
|
|
continue
|
|
if token.is_sent_start:
|
|
annotation = {'start': token.sent.start_char + chunk_offset,
|
|
'end': token.sent.end_char + chunk_offset,
|
|
'tag_id': UUIDnopaque('s'),
|
|
'properties': []}
|
|
annotations.append(annotation)
|
|
# Check if the token is the start of an entity
|
|
if token.ent_iob == 3:
|
|
for ent_candidate in token.sent.ents:
|
|
if ent_candidate.start_char == token.idx:
|
|
ent = ent_candidate
|
|
annotation = {
|
|
'start': ent.start_char + chunk_offset,
|
|
'end': ent.end_char + chunk_offset,
|
|
'tag_id': UUIDnopaque('ent'),
|
|
'properties': [
|
|
{
|
|
'property_id': UUIDnopaque('ent.type'),
|
|
'value': token.ent_type_
|
|
}
|
|
]
|
|
}
|
|
annotations.append(annotation)
|
|
break
|
|
annotation = {
|
|
'start': token.idx + chunk_offset,
|
|
'end': token.idx + len(token.text) + chunk_offset,
|
|
'tag_id': UUIDnopaque('token'),
|
|
'properties': [
|
|
{
|
|
'property_id': UUIDnopaque('token.pos'),
|
|
'value': token.tag_
|
|
},
|
|
{
|
|
'property_id': UUIDnopaque('token.lemma'),
|
|
'value': token.lemma_
|
|
},
|
|
{
|
|
'property_id': UUIDnopaque('token.simple_pos'),
|
|
'value': token.pos_
|
|
},
|
|
{
|
|
'property_id': UUIDnopaque('token.ner'),
|
|
'value': token.ent_type_ if token.ent_type_ else 'None'
|
|
}
|
|
]
|
|
}
|
|
annotations.append(annotation)
|
|
chunk_offset += len(text_chunk)
|
|
text_chunk = None
|
|
|
|
with open(args.output, 'w') as output_file:
|
|
json.dump({'meta': meta, 'tags': tags, 'annotations': annotations},
|
|
output_file, indent=4)
|