mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git
synced 2025-07-01 05:40:33 +00:00
Use JSON files for stand-off annotations.
This commit is contained in:
Patrick Jentsch
213
spacy-nlp
213
spacy-nlp
@ -2,56 +2,39 @@
|
||||
# coding=utf-8
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from xml.sax.saxutils import escape
|
||||
import chardet
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import spacy
|
||||
import textwrap
|
||||
|
||||
|
||||
SPACY_MODELS = {'da': 'da_core_news_md',
|
||||
'de': 'de_core_news_md',
|
||||
'el': 'el_core_news_md',
|
||||
'en': 'en_core_web_md',
|
||||
'es': 'es_core_news_md',
|
||||
'fr': 'fr_core_news_md',
|
||||
'it': 'it_core_news_md',
|
||||
'nl': 'nl_core_news_md',
|
||||
'pt': 'pt_core_news_md',
|
||||
'ru': 'ru_core_news_md',
|
||||
'zh': 'zh_core_web_md'}
|
||||
spacy_models = {spacy.info(pipeline)['lang']: pipeline
|
||||
for pipeline in spacy.info()['pipelines']}
|
||||
|
||||
|
||||
SPACY_MODELS_VERSION = os.environ.get('SPACY_MODELS_VERSION')
|
||||
SPACY_VERSION = os.environ.get('SPACY_VERSION')
|
||||
|
||||
# Parse the given arguments
|
||||
parser = ArgumentParser(description=('Tag a text file with spaCy and save it '
|
||||
'as a verticalized text file.'))
|
||||
parser.add_argument('-i', '--input', metavar='txt-sourcefile', required=True)
|
||||
parser.add_argument('-o', '--output', metavar='vrt-destfile', required=True)
|
||||
parser.add_argument('-l', '--language', choices=SPACY_MODELS.keys(), required=True) # noqa
|
||||
parser.add_argument('--check-encoding', action='store_true')
|
||||
parser = ArgumentParser(description='Create annotations for a given txt file')
|
||||
parser.add_argument('input', metavar='Path to txt input file')
|
||||
parser.add_argument('output', metavar='Path to JSON output file')
|
||||
parser.add_argument('-l', '--language',
|
||||
choices=spacy_models.keys(),
|
||||
required=True)
|
||||
parser.add_argument('-c', '--check-encoding', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
# If requested: Check the encoding of the text contents from the input file
|
||||
# Else: Use utf-8
|
||||
if args.check_encoding:
|
||||
with open(args.input, "rb") as input_file:
|
||||
bytes = input_file.read()
|
||||
encoding = chardet.detect(bytes)['encoding']
|
||||
else:
|
||||
encoding = 'utf-8'
|
||||
|
||||
|
||||
# hashing in chunks to avoid full RAM with huge files.
|
||||
with open(args.input, 'rb') as input_file:
|
||||
source_md5 = hashlib.md5()
|
||||
for chunk in iter(lambda: input_file.read(128 * source_md5.block_size), b''):
|
||||
source_md5.update(chunk)
|
||||
source_md5 = source_md5.hexdigest()
|
||||
with open(args.input, "rb") as input_file:
|
||||
if args.check_encoding:
|
||||
encoding = chardet.detect(input_file.read())['encoding']
|
||||
else:
|
||||
encoding = 'utf-8'
|
||||
text_md5 = hashlib.md5()
|
||||
for chunk in iter(lambda: input_file.read(128 * text_md5.block_size), b''):
|
||||
text_md5.update(chunk)
|
||||
|
||||
# Load the text contents from the input file
|
||||
with open(args.input, encoding=encoding) as input_file:
|
||||
@ -63,57 +46,119 @@ with open(args.input, encoding=encoding) as input_file:
|
||||
# longer needed...
|
||||
del text
|
||||
|
||||
|
||||
# Setup the spaCy toolkit by loading the chosen language model
|
||||
model = SPACY_MODELS[args.language]
|
||||
model = spacy_models[args.language]
|
||||
nlp = spacy.load(model)
|
||||
|
||||
meta = {
|
||||
'generator': {
|
||||
'name': 'nopaque NLP service',
|
||||
'version': '1.0.0',
|
||||
'arguments': {
|
||||
'check_encoding': args.check_encoding,
|
||||
'language': args.language
|
||||
}
|
||||
},
|
||||
'file': {
|
||||
'md5': text_md5.hexdigest(),
|
||||
'name': os.path.basename(args.input)
|
||||
}
|
||||
}
|
||||
|
||||
# Create the output file in verticalized text format
|
||||
# See: http://cwb.sourceforge.net/files/CWB_Encoding_Tutorial/node3.html
|
||||
output_file_original_filename = args.output
|
||||
output_file_stand_off_filename = args.output.replace('.vrt', '.stand-off.vrt')
|
||||
common_xml = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
|
||||
+ '<corpus>\n'
|
||||
+ '<text>\n'
|
||||
+ '<nlp name="spaCy:{}"\n'.format(SPACY_VERSION)
|
||||
+ ' model="{}:{}"\n'.format(model, SPACY_MODELS_VERSION)
|
||||
+ ' source-md5="{}" />\n'.format(source_md5))
|
||||
|
||||
with open(output_file_original_filename, 'w+') as output_file_original, \
|
||||
open(output_file_stand_off_filename, 'w+') as output_file_stand_off:
|
||||
tags = {
|
||||
'token': {
|
||||
'description': '',
|
||||
'properties': {
|
||||
'lemma': {
|
||||
'description': 'The base form of the word',
|
||||
'flags': ['required'],
|
||||
'tagset': None
|
||||
},
|
||||
'pos': {
|
||||
'description': 'The detailed part-of-speech tag',
|
||||
'flags': ['required'],
|
||||
'tagset': {label: spacy.explain(label) for label in spacy.info(model)['labels']['tagger']} # noqa
|
||||
},
|
||||
'simple_pos': {
|
||||
'description': 'The simple UPOS part-of-speech tag',
|
||||
'flags': ['required'],
|
||||
'tagset': {
|
||||
'ADJ': 'adjective',
|
||||
'ADP': 'adposition',
|
||||
'ADV': 'adverb',
|
||||
'AUX': 'auxiliary verb',
|
||||
'CONJ': 'coordinating conjunction',
|
||||
'DET': 'determiner',
|
||||
'INTJ': 'interjection',
|
||||
'NOUN': 'noun',
|
||||
'NUM': 'numeral',
|
||||
'PART': 'particle',
|
||||
'PRON': 'pronoun',
|
||||
'PROPN': 'proper noun',
|
||||
'PUNCT': 'punctuation',
|
||||
'SCONJ': 'subordinating conjunction',
|
||||
'SYM': 'symbol',
|
||||
'VERB': 'verb',
|
||||
'X': 'other'
|
||||
}
|
||||
},
|
||||
'ner': {
|
||||
'description': 'Label indicating the type of the entity',
|
||||
'tagset': {label: spacy.explain(label) for label in spacy.info(model)['labels']['ner']} # noqa
|
||||
}
|
||||
}
|
||||
},
|
||||
's': {
|
||||
'description': 'Encodes the start and end of a sentence',
|
||||
'properties': None
|
||||
},
|
||||
'ent': {
|
||||
'description': 'Encodes the start and end of a named entity',
|
||||
'properties': {
|
||||
'type': {
|
||||
'description': 'Label indicating the type of the entity',
|
||||
'flags': ['required'],
|
||||
'tagset': {label: spacy.explain(label) for label in spacy.info(model)['labels']['ner']} # noqa
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output_file_original.write(common_xml)
|
||||
output_file_stand_off.write(common_xml)
|
||||
text_offset = 0
|
||||
for text_chunk in text_chunks:
|
||||
doc = nlp(text_chunk)
|
||||
for sent in doc.sents:
|
||||
output_file_original.write('<s>\n')
|
||||
output_file_stand_off.write('<s>\n')
|
||||
space_flag = False
|
||||
# Skip whitespace tokens
|
||||
sent_no_space = [token for token in sent
|
||||
if not token.text.isspace()]
|
||||
# No space variant for cwb original .vrt file input.
|
||||
for token in sent_no_space:
|
||||
output_file_original.write('{}'.format(escape(token.text))
|
||||
+ '\t{}'.format(escape(token.lemma_))
|
||||
+ '\t{}'.format(token.pos_)
|
||||
+ '\t{}'.format(token.tag_)
|
||||
+ '\t{}\n'.format(token.ent_type_ or 'NULL'))
|
||||
# Stand off variant with spaces.
|
||||
for token in sent:
|
||||
token_start = token.idx + text_offset
|
||||
token_end = token.idx + len(token.text) + text_offset
|
||||
output_file_stand_off.write('{}:{}'.format(token_start,
|
||||
token_end)
|
||||
+ '\t{}'.format(escape(token.lemma_))
|
||||
+ '\t{}'.format(token.pos_)
|
||||
+ '\t{}'.format(token.tag_)
|
||||
+ '\t{}\n'.format(token.ent_type_ or 'NULL'))
|
||||
output_file_original.write('</s>\n')
|
||||
output_file_stand_off.write('</s>\n')
|
||||
text_offset = token_end + 1
|
||||
output_file_original.write('</text>\n</corpus>')
|
||||
output_file_stand_off.write('</text>\n</corpus>')
|
||||
annotations = []
|
||||
|
||||
chunk_offset = 0
|
||||
for text_chunk in text_chunks:
|
||||
doc = nlp(text_chunk)
|
||||
for token in doc:
|
||||
if token.is_space:
|
||||
continue
|
||||
if token.is_sent_start:
|
||||
annotation = {'start': token.sent.start_char + chunk_offset,
|
||||
'end': token.sent.end_char + chunk_offset,
|
||||
'tag': 's'}
|
||||
annotations.append(annotation)
|
||||
# Check if the token is the start of an entity
|
||||
if token.ent_iob == 3:
|
||||
for ent_candidate in token.sent.ents:
|
||||
if ent_candidate.start_char == token.idx:
|
||||
ent = ent_candidate
|
||||
break
|
||||
annotation = {'start': ent.start_char + chunk_offset,
|
||||
'end': ent.end_char + chunk_offset,
|
||||
'tag': 'ent',
|
||||
'properties': {'type': token.ent_type_}}
|
||||
annotations.append(annotation)
|
||||
annotation = {'start': token.idx + chunk_offset,
|
||||
'end': token.idx + len(token.text) + chunk_offset,
|
||||
'tag': 'token',
|
||||
'properties': {'pos': token.tag_,
|
||||
'lemma': token.lemma_,
|
||||
'simple_pos': token.pos_}}
|
||||
if token.ent_type_:
|
||||
annotation['properties']['ner'] = token.ent_type_
|
||||
annotations.append(annotation)
|
||||
chunk_offset = len(text_chunk)
|
||||
|
||||
with open(args.output, 'w') as output_file:
|
||||
json.dump({'meta': meta, 'tags': tags, 'annotations': annotations},
|
||||
output_file, indent=4)
|
||||
|
||||
Reference in New Issue
Block a user