Make models in package work

This commit is contained in:
Patrick Jentsch
2024-03-06 13:20:02 +01:00
parent a1e5bd61e0
commit 55a62053b0
8 changed files with 56 additions and 1861 deletions

@@ -1,2 +1,18 @@
from .avatar import Avatar
from .corpus_file import CorpusFile
from .avatar import *
from .corpus_file import *
from .corpus_follower_association import *
from .corpus_follower_role import *
from .corpus import *
from .job_input import *
from .job_result import *
from .job import *
from .role import *
from .spacy_nlp_pipeline_model import *
from .tesseract_ocr_pipeline_model import *
from .token import *
from .user import *
@login.user_loader
def load_user(user_id):
return User.query.get(int(user_id))

@@ -0,0 +1,231 @@
- title: 'Catalan'
description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.2.0/ca_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'ca_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'German'
description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.2.0/de_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'de_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Greek'
description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.2.0/el_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'el_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'English'
description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.2.0/en_core_web_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.2.0'
publishing_year: 2021
pipeline_name: 'en_core_web_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Spanish'
description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.2.0/es_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'es_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'French'
description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.2.0/fr_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'fr_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Italian'
description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.2.0/it_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'it_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Polish'
description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.2.0/pl_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'pl_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Russian'
description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.2.0/ru_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'ru_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Chinese'
description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.2.0/zh_core_web_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.2.0'
publishing_year: 2021
pipeline_name: 'zh_core_web_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Catalan'
description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.4.0/ca_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'ca_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'German'
description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'de_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Greek'
description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.4.0/el_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'el_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'English'
description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.4.1'
publishing_year: 2022
pipeline_name: 'en_core_web_md'
version: '3.4.1'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Spanish'
description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.4.0/es_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'es_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'French'
description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.4.0/fr_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'fr_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Italian'
description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner'
url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'it_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Polish'
description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), tagger, senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.4.0/pl_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'pl_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Russian'
description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.4.0/ru_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'ru_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Chinese'
description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.4.0/zh_core_web_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.4.0'
publishing_year: 2022
pipeline_name: 'zh_core_web_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'

File diff suppressed because it is too large Load Diff

@@ -2,24 +2,39 @@ from datetime import datetime
from enum import Enum
from app import db, mail, socketio
from app.email import create_message
from .corpus import Corpus
from .corpus_file import CorpusFile
from .corpus_follower_association import CorpusFollowerAssociation
from .job import Job, JobStatus
from .corpus import Corpus
from .job_input import JobInput
from .job_result import JobResult
from .job import Job, JobStatus
from .spacy_nlp_pipeline_model import SpaCyNLPPipelineModel
from .tesseract_ocr_pipeline_model import TesseractOCRPipelineModel
from .user import UserSettingJobStatusMailNotificationLevel
@db.event.listens_for(Corpus, 'after_delete')
@db.event.listens_for(CorpusFile, 'after_delete')
@db.event.listens_for(Job, 'after_delete')
@db.event.listens_for(JobInput, 'after_delete')
@db.event.listens_for(JobResult, 'after_delete')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_delete')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_delete')
def register_event_listeners():
resources = [
Corpus,
CorpusFile,
Job,
JobInput,
JobResult,
SpaCyNLPPipelineModel,
TesseractOCRPipelineModel
]
for resource in resources:
db.event.listen(resource, 'after_delete', resource_after_delete)
db.event.listen(resource, 'after_insert', resource_after_insert)
db.event.listen(resource, 'after_update', resource_after_update)
db.event.listen(CorpusFollowerAssociation, 'after_delete', cfa_after_delete)
db.event.listen(CorpusFollowerAssociation, 'after_insert', cfa_after_insert)
db.event.listen(Job, 'after_update', job_after_update)
def resource_after_delete(mapper, connection, resource):
jsonpatch = [
{
@@ -31,8 +46,7 @@ def resource_after_delete(mapper, connection, resource):
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(CorpusFollowerAssociation, 'after_delete')
def cfa_after_delete_handler(mapper, connection, cfa):
def cfa_after_delete(mapper, connection, cfa):
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
{
@@ -44,14 +58,7 @@ def cfa_after_delete_handler(mapper, connection, cfa):
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Corpus, 'after_insert')
@db.event.listens_for(CorpusFile, 'after_insert')
@db.event.listens_for(Job, 'after_insert')
@db.event.listens_for(JobInput, 'after_insert')
@db.event.listens_for(JobResult, 'after_insert')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_insert')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_insert')
def resource_after_insert_handler(mapper, connection, resource):
def resource_after_insert(mapper, connection, resource):
jsonpatch_value = resource.to_json_serializeable()
for attr in mapper.relationships:
jsonpatch_value[attr.key] = {}
@@ -66,8 +73,7 @@ def resource_after_insert_handler(mapper, connection, resource):
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(CorpusFollowerAssociation, 'after_insert')
def cfa_after_insert_handler(mapper, connection, cfa):
def cfa_after_insert(mapper, connection, cfa):
jsonpatch_value = cfa.to_json_serializeable()
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
@@ -81,14 +87,7 @@ def cfa_after_insert_handler(mapper, connection, cfa):
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Corpus, 'after_update')
@db.event.listens_for(CorpusFile, 'after_update')
@db.event.listens_for(Job, 'after_update')
@db.event.listens_for(JobInput, 'after_update')
@db.event.listens_for(JobResult, 'after_update')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_update')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_update')
def resource_after_update_handler(mapper, connection, resource):
def resource_after_update(mapper, connection, resource):
jsonpatch = []
for attr in db.inspect(resource).attrs:
if attr.key in mapper.relationships:
@@ -114,8 +113,7 @@ def resource_after_update_handler(mapper, connection, resource):
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Job, 'after_update')
def job_after_update_handler(mapper, connection, job):
def job_after_update(mapper, connection, job):
for attr in db.inspect(job).attrs:
if attr.key != 'status':
continue

@@ -59,7 +59,8 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
nopaque_user = User.query.filter_by(username='nopaque').first()
defaults_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'SpaCyNLPPipelineModel.defaults.yml'
'default_records',
'spacy_nlp_pipeline_model.yml'
)
with open(defaults_file, 'r') as f:
defaults = yaml.safe_load(f)

@@ -57,7 +57,8 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
nopaque_user = User.query.filter_by(username='nopaque').first()
defaults_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'TesseractOCRPipelineModel.defaults.yml'
'default_records',
'tesseract_ocr_pipeline_model.yml'
)
with open(defaults_file, 'r') as f:
defaults = yaml.safe_load(f)