nopaque/app/models/spacy_nlp_pipeline_model.py

from flask import current_app, url_for
from flask_hashids import HashidMixin
from tqdm import tqdm
from pathlib import Path
import requests
import yaml
from app import db
from app.extensions.sqlalchemy_extras import ContainerColumn
from .file_mixin import FileMixin
from .user import User


class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
    __tablename__ = 'spacy_nlp_pipeline_models'
    # Primary key
    id = db.Column(db.Integer, primary_key=True)
    # Foreign keys
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    # Fields
    title = db.Column(db.String(64))
    description = db.Column(db.String(255))
    version = db.Column(db.String(16))
    compatible_service_versions = db.Column(ContainerColumn(list, 255))
    publisher = db.Column(db.String(128))
    publisher_url = db.Column(db.String(512))
    publishing_url = db.Column(db.String(512))
    publishing_year = db.Column(db.Integer)
    pipeline_name = db.Column(db.String(64))
    is_public = db.Column(db.Boolean, default=False)
    # Relationships
    user = db.relationship('User', back_populates='spacy_nlp_pipeline_models')

    @property
    def path(self) -> Path:
        return self.user.path / 'spacy_nlp_pipeline_models' / f'{self.id}'

    @property
    def jsonpatch_path(self):
        return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}'

    @property
    def url(self):
        return url_for(
            'contributions.spacy_nlp_pipeline_model',
            spacy_nlp_pipeline_model_id=self.id
        )

    @property
    def user_hashid(self):
        return self.user.hashid

    @staticmethod
    def insert_defaults(force_download=False):
        nopaque_user = User.query.filter_by(username='nopaque').first()
        default_records_file = Path(__file__).parent / 'default_records' / 'spacy_nlp_pipeline_model.yml'
        with default_records_file.open('r') as f:
            default_records = yaml.safe_load(f)
        for m in default_records:
            model = SpaCyNLPPipelineModel.query.filter_by(title=m['title'], version=m['version']).first()  # noqa
            if model is not None:
                model.compatible_service_versions = m['compatible_service_versions']
                model.description = m['description']
                model.filename = m['url'].split('/')[-1]
                model.publisher = m['publisher']
                model.publisher_url = m['publisher_url']
                model.publishing_url = m['publishing_url']
                model.publishing_year = m['publishing_year']
                model.is_public = True
                model.title = m['title']
                model.version = m['version']
                model.pipeline_name = m['pipeline_name']
            else:
                model = SpaCyNLPPipelineModel(
                    compatible_service_versions=m['compatible_service_versions'],
                    description=m['description'],
                    filename=m['url'].split('/')[-1],
                    publisher=m['publisher'],
                    publisher_url=m['publisher_url'],
                    publishing_url=m['publishing_url'],
                    publishing_year=m['publishing_year'],
                    is_public=True,
                    title=m['title'],
                    user=nopaque_user,
                    version=m['version'],
                    pipeline_name=m['pipeline_name']
                )
                db.session.add(model)
                db.session.flush(objects=[model])
                db.session.refresh(model)
            if not model.path.exists() or force_download:
                r = requests.get(m['url'], stream=True)
                pbar = tqdm(
                    desc=f'{model.title} ({model.filename})',
                    unit="B",
                    unit_scale=True,
                    unit_divisor=1024,
                    total=int(r.headers['Content-Length'])
                )
                pbar.clear()
                with open(model.path, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=1024):
                        if chunk:  # filter out keep-alive new chunks
                            pbar.update(len(chunk))
                            f.write(chunk)
                    pbar.close()
        db.session.commit()
    
    def delete(self):
        try:
            self.path.unlink(missing_ok=True)
        except OSError as e:
            current_app.logger.error(e)
            raise
        db.session.delete(self)

    def to_json_serializeable(self, backrefs=False, relationships=False):
        json_serializeable = {
            'id': self.hashid,
            'compatible_service_versions': self.compatible_service_versions,
            'description': self.description,
            'publisher': self.publisher,
            'publisher_url': self.publisher_url,
            'publishing_url': self.publishing_url,
            'publishing_year': self.publishing_year,
            'pipeline_name': self.pipeline_name,
            'is_public': self.is_public,
            'title': self.title,
            'version': self.version,
            **self.file_mixin_to_json_serializeable()
        }
        if backrefs:
            json_serializeable['user'] = \
                self.user.to_json_serializeable(backrefs=True)
        if relationships:
            pass
        return json_serializeable