57 Commits

Author SHA1 Message Date
a65b1ff578 remove manual modal 2024-05-27 16:58:51 +02:00
fe0fcb0e10 fix job status notifications 2024-05-27 09:24:40 +02:00
32fa632961 move anonymous user to seperate file 2024-05-27 09:23:08 +02:00
562b8d5ce0 Move clearfix to helpers.scss 2024-05-27 08:58:04 +02:00
cbd0a41bce style and compatibility update 2024-05-21 10:29:12 +02:00
c68286e010 more flexible navbar code in base template 2024-05-08 14:01:01 +02:00
4a29a52f2a Remove implementation of FileSize validator 2024-05-04 17:17:06 +02:00
991810cff5 Use HTML comments in navbar.html.j2 2024-05-04 17:04:25 +02:00
6025a4a606 add API back 2024-05-04 15:14:21 +02:00
e1cfd394fa move socketio decorators 2024-05-04 14:55:05 +02:00
882987ba68 fixes 2024-05-04 14:41:40 +02:00
a03b5918d9 More Streamlining 2024-05-03 15:08:57 +02:00
43b38b2216 better large screen handling 2024-04-30 16:21:23 +02:00
543276d766 A lot of generalization for better scaling and overview 2024-04-30 16:00:06 +02:00
485a0155c6 Bump dependencies. Some parts needed to be deactivated for that. They need to be reimplemented.
- breadcrumbs (!flask-breadcrumbs)
- manual modal button
- api blueprint (!flask-marshmallow/!marshmallow-sqlalchemy)
2024-04-30 08:41:29 +02:00
c29c50feb9 new event system first draft 2024-04-18 15:37:17 +02:00
c191e7bd4a rename extension extras 2024-04-18 15:35:41 +02:00
8f960cf359 explicitly set permissions to false for anonymous users 2024-04-11 15:46:58 +02:00
ccf484c9bc make is_administrator a property, add back db events 2024-04-11 14:33:47 +02:00
d0d2a8abd6 Move external static content into external directory 2024-04-11 14:13:04 +02:00
03876f6a39 Self host external css/fonts/js 2024-04-11 11:08:37 +02:00
cdf6f9fcfd declutter by using default filenames 2024-04-10 13:52:33 +02:00
268da220d2 Enhance code structure 2024-04-10 13:34:48 +02:00
84e1755a57 Remove version declaration in docker compose files. 2024-04-10 13:34:14 +02:00
82d6f6003f Restructure Dockerfile for better caching 2024-03-13 12:58:39 +01:00
9da74c1c6f Use pathlib where possible 2024-03-07 15:49:04 +01:00
ec23bd94ee add missing import 2024-03-06 14:46:25 +01:00
55a62053b0 Make models in package work 2024-03-06 13:20:02 +01:00
a1e5bd61e0 move models in seperate modules 2024-03-05 16:02:23 +01:00
cf8c164d60 allow .whl files for spacy nlp pipeline contributions 2024-02-14 14:24:21 +01:00
05ab204e5a Merge branch 'master' into development 2024-02-14 13:43:45 +01:00
9f188afd16 Bump nopaque version 2024-02-14 13:42:20 +01:00
dc77ac7b76 Add new spaCy NLP Pipeline version 2024-02-14 13:40:49 +01:00
84276af322 Merge branch 'development' 2024-01-23 14:38:24 +01:00
d9d4067536 Set new version in config 2024-01-23 14:38:10 +01:00
ba65cf5911 Merge branch 'development' 2024-01-23 14:19:06 +01:00
69a1edc51e fix eventlet version 2024-01-23 13:31:15 +01:00
32ad8c7359 News + user avatar fix 2024-01-22 10:58:52 +01:00
8c0843d2d0 Merge branch 'development' 2023-12-21 14:28:10 +01:00
d4c9ab5821 Add user reste cli command 2023-12-21 14:27:50 +01:00
518a245133 Merge branch 'development' 2023-12-21 13:54:25 +01:00
b6864b355a Bug fixes 2023-12-21 13:03:58 +01:00
0a45e1bb65 Bug fixes 2023-12-21 12:48:50 +01:00
08ca938333 Corrections in Terms of Use 2023-12-21 09:31:42 +01:00
cfdef8d1fa New terms of use + privacy statement 2023-12-20 15:37:59 +01:00
9ac626c64d Merge branch 'development' 2023-07-26 20:44:54 +02:00
c9ad538bee Merge branch 'development' 2023-07-26 11:35:39 +02:00
baf70750e8 Merge branch 'development' 2023-06-07 15:14:57 +02:00
525723818e Merge branch 'development' 2023-02-15 11:37:09 +01:00
20c0678d3e Merge branch 'master' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque 2023-02-06 15:51:41 +01:00
c323c53f37 Merge branch 'development' 2023-02-06 15:51:13 +01:00
2d8cef64e8 Merge branch 'development' 2023-02-06 12:40:07 +01:00
9b9edf501d Merge branch 'development' 2022-11-25 10:47:15 +01:00
903310c17f Merge branch 'development' 2022-11-24 12:29:28 +01:00
bc92fd249f Merge branch 'development' 2022-11-18 16:02:52 +01:00
422415065d Merge branch 'development' 2022-10-28 13:21:29 +02:00
07ec01ae2e Merge branch 'development' 2022-10-11 11:34:19 +02:00
411 changed files with 1564137 additions and 4143 deletions

View File

@ -5,9 +5,9 @@
!app
!migrations
!tests
!.flaskenv
!boot.sh
!config.py
!docker-nopaque-entrypoint.sh
!nopaque.py
!requirements.txt
!requirements.freezed.txt
!wsgi.py

View File

@ -1 +0,0 @@
FLASK_APP=nopaque.py

View File

@ -4,11 +4,13 @@ FROM python:3.10.13-slim-bookworm
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
# Set environment variables
ENV LANG="C.UTF-8"
ENV PYTHONDONTWRITEBYTECODE="1"
ENV PYTHONUNBUFFERED="1"
# Install system dependencies
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
@ -17,37 +19,40 @@ RUN apt-get update \
&& rm --recursive /var/lib/apt/lists/*
# Create a non-root user
RUN useradd --create-home --no-log-init nopaque \
&& groupadd docker \
&& usermod --append --groups docker nopaque
USER nopaque
WORKDIR /home/nopaque
# Create a Python virtual environment
ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv"
RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}"
ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}"
# Install Python dependencies
COPY --chown=nopaque:nopaque requirements.freezed.txt requirements.freezed.txt
RUN python3 -m pip install --requirement requirements.freezed.txt \
&& rm requirements.freezed.txt
# Install the application
COPY docker-nopaque-entrypoint.sh /usr/local/bin/
COPY --chown=nopaque:nopaque app app
COPY --chown=nopaque:nopaque migrations migrations
COPY --chown=nopaque:nopaque tests tests
COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py requirements.txt ./
RUN python3 -m pip install --requirement requirements.txt \
&& mkdir logs
USER root
COPY docker-nopaque-entrypoint.sh /usr/local/bin/
COPY --chown=nopaque:nopaque boot.sh config.py wsgi.py ./
RUN mkdir logs
EXPOSE 5000
USER root
ENTRYPOINT ["docker-nopaque-entrypoint.sh"]

View File

@ -4,7 +4,6 @@ from docker import DockerClient
from flask import Flask
from flask_apscheduler import APScheduler
from flask_assets import Environment
from flask_breadcrumbs import Breadcrumbs, default_breadcrumb_root
from flask_login import LoginManager
from flask_mail import Mail
from flask_marshmallow import Marshmallow
@ -17,7 +16,6 @@ from flask_hashids import Hashids
apifairy = APIFairy()
assets = Environment()
breadcrumbs = Breadcrumbs()
db = SQLAlchemy()
docker_client = DockerClient()
hashids = Hashids()
@ -33,6 +31,9 @@ scheduler = APScheduler()
socketio = SocketIO()
# TODO: Create export for lemmatized corpora
def create_app(config: Config = Config) -> Flask:
''' Creates an initialized Flask (WSGI Application) object. '''
app = Flask(__name__)
@ -46,7 +47,6 @@ def create_app(config: Config = Config) -> Flask:
apifairy.init_app(app)
assets.init_app(app)
breadcrumbs.init_app(app)
db.init_app(app)
hashids.init_app(app)
login.init_app(app)
@ -57,24 +57,23 @@ def create_app(config: Config = Config) -> Flask:
scheduler.init_app(app)
socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) # noqa
from .models.event_listeners import register_event_listeners
register_event_listeners()
from .admin import bp as admin_blueprint
default_breadcrumb_root(admin_blueprint, '.admin')
app.register_blueprint(admin_blueprint, url_prefix='/admin')
from .api import bp as api_blueprint
app.register_blueprint(api_blueprint, url_prefix='/api')
from .auth import bp as auth_blueprint
default_breadcrumb_root(auth_blueprint, '.')
app.register_blueprint(auth_blueprint)
from .contributions import bp as contributions_blueprint
default_breadcrumb_root(contributions_blueprint, '.contributions')
app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
from .corpora import bp as corpora_blueprint
from .corpora.cqi_over_sio import CQiNamespace
default_breadcrumb_root(corpora_blueprint, '.corpora')
app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
socketio.on_namespace(CQiNamespace('/cqi_over_sio'))
@ -82,24 +81,19 @@ def create_app(config: Config = Config) -> Flask:
app.register_blueprint(errors_bp)
from .jobs import bp as jobs_blueprint
default_breadcrumb_root(jobs_blueprint, '.jobs')
app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
from .main import bp as main_blueprint
default_breadcrumb_root(main_blueprint, '.')
app.register_blueprint(main_blueprint, cli_group=None)
from .services import bp as services_blueprint
default_breadcrumb_root(services_blueprint, '.services')
app.register_blueprint(services_blueprint, url_prefix='/services')
from .settings import bp as settings_blueprint
default_breadcrumb_root(settings_blueprint, '.settings')
app.register_blueprint(settings_blueprint, url_prefix='/settings')
from .users import bp as users_blueprint
default_breadcrumb_root(users_blueprint, '.users')
app.register_blueprint(users_blueprint, url_prefix='/users')
app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
from .workshops import bp as workshops_blueprint
app.register_blueprint(workshops_blueprint, url_prefix='/workshops')

49
app/admin/events.py Normal file
View File

@ -0,0 +1,49 @@
from flask_login import current_user
from flask_socketio import disconnect, Namespace
from app import db, hashids
from app.decorators import socketio_admin_required
from app.models import User
class AdminNamespace(Namespace):
def on_connect(self):
# Check if the user is authenticated and is an administrator
if not (current_user.is_authenticated and current_user.is_administrator):
disconnect()
@socketio_admin_required
def on_set_user_confirmed(self, user_hashid: str, confirmed_value: bool):
# Decode the user hashid
user_id = hashids.decode(user_hashid)
# Validate user_id
if not isinstance(user_id, int):
return {
'code': 400,
'body': 'user_id is invalid'
}
# Validate confirmed_value
if not isinstance(confirmed_value, bool):
return {
'code': 400,
'body': 'confirmed_value is invalid'
}
# Load user from database
user = User.query.get(user_id)
if user is None:
return {
'code': 404,
'body': 'User not found'
}
# Update user confirmed status
user.confirmed = confirmed_value
db.session.commit()
return {
'code': 200,
'body': f'User "{user.username}" is now {"confirmed" if confirmed_value else "unconfirmed"}'
}

View File

@ -1,6 +1,6 @@
from flask import abort, request
from app import db
from app.decorators import content_negotiation
from app import db
from app.models import User
from . import bp

View File

@ -1,5 +1,4 @@
from flask import abort, flash, redirect, render_template, url_for
from flask_breadcrumbs import register_breadcrumb
from app import db, hashids
from app.models import Avatar, Corpus, Role, User
from app.users.settings.forms import (
@ -11,14 +10,9 @@ from app.users.settings.forms import (
)
from . import bp
from .forms import UpdateUserForm
from app.users.utils import (
user_endpoint_arguments_constructor as user_eac,
user_dynamic_list_constructor as user_dlc
)
@bp.route('')
@register_breadcrumb(bp, '.', '<i class="material-icons left">admin_panel_settings</i>Administration')
def admin():
return render_template(
'admin/admin.html.j2',
@ -27,7 +21,6 @@ def admin():
@bp.route('/corpora')
@register_breadcrumb(bp, '.corpora', 'Corpora')
def corpora():
corpora = Corpus.query.all()
return render_template(
@ -38,7 +31,6 @@ def corpora():
@bp.route('/users')
@register_breadcrumb(bp, '.users', '<i class="material-icons left">group</i>Users')
def users():
users = User.query.all()
return render_template(
@ -49,7 +41,6 @@ def users():
@bp.route('/users/<hashid:user_id>')
@register_breadcrumb(bp, '.users.entity', '', dynamic_list_constructor=user_dlc)
def user(user_id):
user = User.query.get_or_404(user_id)
corpora = Corpus.query.filter(Corpus.user == user).all()
@ -62,7 +53,6 @@ def user(user_id):
@bp.route('/users/<hashid:user_id>/settings', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.users.entity.settings', '<i class="material-icons left">settings</i>Settings')
def user_settings(user_id):
user = User.query.get_or_404(user_id)
update_account_information_form = UpdateAccountInformationForm(user)

View File

@ -5,8 +5,8 @@ from flask import abort, Blueprint
from werkzeug.exceptions import InternalServerError
from app import db, hashids
from app.models import Job, JobInput, JobStatus, TesseractOCRPipelineModel
from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
from .auth import auth_error_responses, token_auth
from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
bp = Blueprint('jobs', __name__)
@ -77,7 +77,7 @@ def delete_job(job_id):
job = Job.query.get(job_id)
if job is None:
abort(404)
if not (job.user == current_user or current_user.is_administrator()):
if not (job.user == current_user or current_user.is_administrator):
abort(403)
try:
job.delete()
@ -97,6 +97,6 @@ def get_job(job_id):
job = Job.query.get(job_id)
if job is None:
abort(404)
if not (job.user == current_user or current_user.is_administrator()):
if not (job.user == current_user or current_user.is_administrator):
abort(403)
return job

View File

@ -3,11 +3,11 @@ from apifairy import authenticate, body, response
from apifairy.decorators import other_responses
from flask import abort, Blueprint
from werkzeug.exceptions import InternalServerError
from app import db
from app.email import create_message, send
from app import db
from app.models import User
from .schemas import EmptySchema, UserSchema
from .auth import auth_error_responses, token_auth
from .schemas import EmptySchema, UserSchema
bp = Blueprint('users', __name__)
@ -60,7 +60,7 @@ def delete_user(user_id):
user = User.query.get(user_id)
if user is None:
abort(404)
if not (user == current_user or current_user.is_administrator()):
if not (user == current_user or current_user.is_administrator):
abort(403)
user.delete()
db.session.commit()
@ -78,7 +78,7 @@ def get_user(user_id):
user = User.query.get(user_id)
if user is None:
abort(404)
if not (user == current_user or current_user.is_administrator()):
if not (user == current_user or current_user.is_administrator):
abort(403)
return user
@ -94,6 +94,6 @@ def get_user_by_username(username):
user = User.query.filter(User.username == username).first()
if user is None:
abort(404)
if not (user == current_user or current_user.is_administrator()):
if not (user == current_user or current_user.is_administrator):
abort(403)
return user

View File

@ -1,5 +1,4 @@
from flask import abort, flash, redirect, render_template, request, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user, login_user, login_required, logout_user
from app import db
from app.email import create_message, send
@ -19,18 +18,21 @@ def before_request():
Checks if a user is unconfirmed when visiting specific sites. Redirects to
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
current_user.ping()
db.session.commit()
if (not current_user.confirmed
and request.endpoint
and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
if not current_user.is_authenticated:
return
current_user.ping()
db.session.commit()
if (not current_user.confirmed
and request.endpoint
and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
if not current_user.terms_of_use_accepted:
return redirect(url_for('main.terms_of_use'))
@bp.route('/register', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.register', 'Register')
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
@ -67,7 +69,6 @@ def register():
@bp.route('/login', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.login', 'Login')
def login():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
@ -98,7 +99,6 @@ def logout():
@bp.route('/unconfirmed')
@register_breadcrumb(bp, '.unconfirmed', 'Unconfirmed')
@login_required
def unconfirmed():
if current_user.confirmed:
@ -141,7 +141,6 @@ def confirm(token):
@bp.route('/reset-password-request', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.reset_password_request', 'Password Reset')
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
@ -171,7 +170,6 @@ def reset_password_request():
@bp.route('/reset-password/<token>', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.reset_password', 'Password Reset')
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))

View File

@ -1,9 +1,7 @@
from flask import redirect, url_for
from flask_breadcrumbs import register_breadcrumb
from . import bp
@bp.route('')
@register_breadcrumb(bp, '.', '<i class="material-icons left">new_label</i>My Contributions')
def contributions():
return redirect(url_for('main.dashboard', _anchor='contributions'))

View File

@ -16,8 +16,8 @@ class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
)
def validate_spacy_model_file(self, field):
if not field.data.filename.lower().endswith('.tar.gz'):
raise ValidationError('.tar.gz files only!')
if not field.data.filename.lower().endswith(('.tar.gz', ('.whl'))):
raise ValidationError('.tar.gz or .whl files only!')
def __init__(self, *args, **kwargs):
if 'prefix' not in kwargs:

View File

@ -4,7 +4,7 @@ from threading import Thread
from app import db
from app.decorators import content_negotiation, permission_required
from app.models import SpaCyNLPPipelineModel
from .. import bp
from . import bp
@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
@ -17,7 +17,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
db.session.commit()
snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
if not (snpm.user == current_user or current_user.is_administrator()):
if not (snpm.user == current_user or current_user.is_administrator):
abort(403)
thread = Thread(
target=_delete_spacy_model,
@ -39,7 +39,7 @@ def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
if not isinstance(is_public, bool):
abort(400)
snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
if not (snpm.user == current_user or current_user.is_administrator()):
if not (snpm.user == current_user or current_user.is_administrator):
abort(403)
snpm.is_public = is_public
db.session.commit()

View File

@ -1,5 +1,4 @@
from flask import abort, flash, redirect, render_template, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user
from app import db
from app.models import SpaCyNLPPipelineModel
@ -8,13 +7,9 @@ from .forms import (
CreateSpaCyNLPPipelineModelForm,
UpdateSpaCyNLPPipelineModelForm
)
from .utils import (
spacy_nlp_pipeline_model_dlc as spacy_nlp_pipeline_model_dlc
)
@bp.route('/spacy-nlp-pipeline-models')
@register_breadcrumb(bp, '.spacy_nlp_pipeline_models', 'SpaCy NLP Pipeline Models')
def spacy_nlp_pipeline_models():
return render_template(
'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
@ -23,7 +18,6 @@ def spacy_nlp_pipeline_models():
@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.create', 'Create')
def create_spacy_nlp_pipeline_model():
form = CreateSpaCyNLPPipelineModelForm()
if form.is_submitted():
@ -57,10 +51,9 @@ def create_spacy_nlp_pipeline_model():
@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.entity', '', dynamic_list_constructor=spacy_nlp_pipeline_model_dlc)
def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
if not (snpm.user == current_user or current_user.is_administrator()):
if not (snpm.user == current_user or current_user.is_administrator):
abort(403)
form = UpdateSpaCyNLPPipelineModelForm(data=snpm.to_json_serializeable())
if form.validate_on_submit():

View File

@ -1,13 +0,0 @@
from flask import request, url_for
from app.models import SpaCyNLPPipelineModel
def spacy_nlp_pipeline_model_dlc():
snpm_id = request.view_args['spacy_nlp_pipeline_model_id']
snpm = SpaCyNLPPipelineModel.query.get_or_404(snpm_id)
return [
{
'text': f'{snpm.title} {snpm.version}',
'url': url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=snpm_id)
}
]

View File

@ -17,7 +17,7 @@ def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
db.session.commit()
topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
if not (topm.user == current_user or current_user.is_administrator()):
if not (topm.user == current_user or current_user.is_administrator):
abort(403)
thread = Thread(
target=_delete_tesseract_ocr_pipeline_model,
@ -39,7 +39,7 @@ def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_i
if not isinstance(is_public, bool):
abort(400)
topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
if not (topm.user == current_user or current_user.is_administrator()):
if not (topm.user == current_user or current_user.is_administrator):
abort(403)
topm.is_public = is_public
db.session.commit()

View File

@ -1,5 +1,4 @@
from flask import abort, flash, redirect, render_template, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user
from app import db
from app.models import TesseractOCRPipelineModel
@ -8,13 +7,9 @@ from .forms import (
CreateTesseractOCRPipelineModelForm,
UpdateTesseractOCRPipelineModelForm
)
from .utils import (
tesseract_ocr_pipeline_model_dlc as tesseract_ocr_pipeline_model_dlc
)
@bp.route('/tesseract-ocr-pipeline-models')
@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models', 'Tesseract OCR Pipeline Models')
def tesseract_ocr_pipeline_models():
return render_template(
'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
@ -23,7 +18,6 @@ def tesseract_ocr_pipeline_models():
@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.create', 'Create')
def create_tesseract_ocr_pipeline_model():
form = CreateTesseractOCRPipelineModelForm()
if form.is_submitted():
@ -56,10 +50,9 @@ def create_tesseract_ocr_pipeline_model():
@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.entity', '', dynamic_list_constructor=tesseract_ocr_pipeline_model_dlc)
def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
if not (topm.user == current_user or current_user.is_administrator()):
if not (topm.user == current_user or current_user.is_administrator):
abort(403)
form = UpdateTesseractOCRPipelineModelForm(data=topm.to_json_serializeable())
if form.validate_on_submit():

View File

@ -1,13 +0,0 @@
from flask import request, url_for
from app.models import TesseractOCRPipelineModel
def tesseract_ocr_pipeline_model_dlc():
topm_id = request.view_args['tesseract_ocr_pipeline_model_id']
topm = TesseractOCRPipelineModel.query.get_or_404(topm_id)
return [
{
'text': f'{topm.title} {topm.version}',
'url': url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=topm_id)
}
]

View File

@ -1,81 +1,70 @@
from datetime import datetime
from flask import current_app
from pathlib import Path
from typing import Dict, List
import json
import shutil
from app import db
from app.models import User, Corpus, CorpusFile
from datetime import datetime
import json
import os
import shutil
class SandpaperConverter:
def __init__(self, json_db_file, data_dir):
def __init__(self, json_db_file: Path, data_dir: Path):
self.json_db_file = json_db_file
self.data_dir = data_dir
def run(self):
with open(self.json_db_file, 'r') as f:
json_db = json.loads(f.read())
with self.json_db_file.open('r') as f:
json_db: List[Dict] = json.load(f)
for json_user in json_db:
if not json_user['confirmed']:
current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
continue
user_dir = os.path.join(self.data_dir, str(json_user['id']))
user_dir = self.data_dir / f'{json_user["id"]}'
self.convert_user(json_user, user_dir)
db.session.commit()
def convert_user(self, json_user, user_dir):
def convert_user(self, json_user: Dict, user_dir: Path):
current_app.logger.info(f'Create User {json_user["username"]}...')
user = User(
confirmed=json_user['confirmed'],
email=json_user['email'],
last_seen=datetime.fromtimestamp(json_user['last_seen']),
member_since=datetime.fromtimestamp(json_user['member_since']),
password_hash=json_user['password_hash'], # TODO: Needs to be added manually
username=json_user['username']
)
db.session.add(user)
db.session.flush(objects=[user])
db.session.refresh(user)
try:
user.makedirs()
except OSError as e:
current_app.logger.error(e)
db.session.rollback()
user = User.create(
confirmed=json_user['confirmed'],
email=json_user['email'],
last_seen=datetime.fromtimestamp(json_user['last_seen']),
member_since=datetime.fromtimestamp(json_user['member_since']),
password_hash=json_user['password_hash'], # TODO: Needs to be added manually
username=json_user['username']
)
except OSError:
raise Exception('Internal Server Error')
for json_corpus in json_user['corpora'].values():
if not json_corpus['files'].values():
current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
continue
corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
self.convert_corpus(json_corpus, user, corpus_dir)
current_app.logger.info('Done')
def convert_corpus(self, json_corpus, user, corpus_dir):
def convert_corpus(self, json_corpus: Dict, user: User, corpus_dir: Path):
current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
corpus = Corpus(
user=user,
creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
description=json_corpus['description'],
title=json_corpus['title']
)
db.session.add(corpus)
db.session.flush(objects=[corpus])
db.session.refresh(corpus)
try:
corpus.makedirs()
except OSError as e:
current_app.logger.error(e)
db.session.rollback()
corpus = Corpus.create(
user=user,
creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
description=json_corpus['description'],
title=json_corpus['title']
)
except OSError:
raise Exception('Internal Server Error')
for json_corpus_file in json_corpus['files'].values():
self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
current_app.logger.info('Done')
def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
def convert_corpus_file(self, json_corpus_file: Dict, corpus: Corpus, corpus_dir: Path):
current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
corpus_file = CorpusFile(
corpus=corpus,
@ -99,13 +88,13 @@ class SandpaperConverter:
db.session.refresh(corpus_file)
try:
shutil.copy2(
os.path.join(corpus_dir, json_corpus_file['filename']),
corpus_dir / json_corpus_file['filename'],
corpus_file.path
)
except:
current_app.logger.warning(
'Can not convert corpus file: '
f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
f'{corpus_dir / json_corpus_file["filename"]}'
' -> '
f'{corpus_file.path}'
)

View File

@ -1,7 +1,7 @@
from app.models import Corpus, CorpusStatus
import os
from flask import current_app
import shutil
from app import db
from app.models import Corpus, CorpusStatus
from . import bp
@ -18,10 +18,17 @@ def reset():
]
for corpus in [x for x in Corpus.query.all() if x.status in status]:
print(f'Resetting corpus {corpus}')
shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
os.mkdir(os.path.join(corpus.path, 'cwb'))
os.mkdir(os.path.join(corpus.path, 'cwb', 'data'))
os.mkdir(os.path.join(corpus.path, 'cwb', 'registry'))
corpus_cwb_dir = corpus.path / 'cwb'
corpus_cwb_data_dir = corpus_cwb_dir / 'data'
corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
try:
shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
corpus_cwb_dir.mkdir()
corpus_cwb_data_dir.mkdir()
corpus_cwb_registry_dir.mkdir()
except OSError as e:
current_app.logger.error(e)
raise
corpus.status = CorpusStatus.UNPREPARED
corpus.num_analysis_sessions = 0
db.session.commit()

View File

@ -99,7 +99,7 @@ class CQiNamespace(Namespace):
return {'code': 404, 'msg': 'Not Found'}
if not (db_corpus.user == current_user
or current_user.is_following_corpus(db_corpus)
or current_user.is_administrator()):
or current_user.is_administrator):
return {'code': 403, 'msg': 'Forbidden'}
if db_corpus.status not in [
CorpusStatus.BUILT,

View File

@ -12,7 +12,6 @@ from typing import Dict, List
import gzip
import json
import math
import os
from app import db
from app.models import Corpus
from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
@ -42,9 +41,9 @@ def ext_corpus_static_data(corpus: str) -> Dict:
db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
db_corpus: Corpus = Corpus.query.get(db_corpus_id)
static_data_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
if os.path.exists(static_data_file_path):
with open(static_data_file_path, 'rb') as f:
static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
if static_data_file_path.exists():
with static_data_file_path.open('rb') as f:
return f.read()
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']

View File

@ -10,7 +10,7 @@ def corpus_follower_permission_required(*permissions):
def decorated_function(*args, **kwargs):
corpus_id = kwargs.get('corpus_id')
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
if not (corpus.user == current_user or current_user.is_administrator):
cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
if cfa is None:
abort(403)
@ -26,7 +26,7 @@ def corpus_owner_or_admin_required(f):
def decorated_function(*args, **kwargs):
corpus_id = kwargs.get('corpus_id')
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
if not (corpus.user == current_user or current_user.is_administrator):
abort(403)
return f(*args, **kwargs)
return decorated_function

View File

@ -15,7 +15,7 @@ def get_corpus(corpus_hashid):
if not (
corpus.is_public
or corpus.user == current_user
or current_user.is_administrator()
or current_user.is_administrator
):
return {'options': {'status': 403, 'statusText': 'Forbidden'}}
return {
@ -38,7 +38,7 @@ def subscribe_corpus(corpus_hashid):
if not (
corpus.is_public
or corpus.user == current_user
or current_user.is_administrator()
or current_user.is_administrator
):
return {'options': {'status': 403, 'statusText': 'Forbidden'}}
join_room(f'/corpora/{corpus.hashid}')

View File

@ -1,7 +1,7 @@
from flask import abort, current_app
from flask import current_app
from threading import Thread
from app import db
from app.decorators import content_negotiation
from app import db
from app.models import CorpusFile
from ..decorators import corpus_follower_permission_required
from . import bp

View File

@ -6,25 +6,19 @@ from flask import (
send_from_directory,
url_for
)
from flask_breadcrumbs import register_breadcrumb
import os
from app import db
from app.models import Corpus, CorpusFile, CorpusStatus
from ..decorators import corpus_follower_permission_required
from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
from . import bp
from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
from .utils import corpus_file_dynamic_list_constructor as corpus_file_dlc
@bp.route('/<hashid:corpus_id>/files')
@register_breadcrumb(bp, '.entity.files', 'Files', endpoint_arguments_constructor=corpus_eac)
def corpus_files(corpus_id):
return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.entity.files.create', 'Create', endpoint_arguments_constructor=corpus_eac)
@corpus_follower_permission_required('MANAGE_FILES')
def create_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
@ -66,7 +60,6 @@ def create_corpus_file(corpus_id):
@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.entity.files.entity', '', dynamic_list_constructor=corpus_file_dlc)
@corpus_follower_permission_required('MANAGE_FILES')
def corpus_file(corpus_id, corpus_file_id):
corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
@ -92,9 +85,9 @@ def corpus_file(corpus_id, corpus_file_id):
def download_corpus_file(corpus_id, corpus_file_id):
corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
return send_from_directory(
os.path.dirname(corpus_file.path),
os.path.basename(corpus_file.path),
corpus_file.path.parent,
corpus_file.path.name,
as_attachment=True,
attachment_filename=corpus_file.filename,
download_name=corpus_file.filename,
mimetype=corpus_file.mimetype
)

View File

@ -1,15 +0,0 @@
from flask import request, url_for
from app.models import CorpusFile
from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
def corpus_file_dynamic_list_constructor():
corpus_id = request.view_args['corpus_id']
corpus_file_id = request.view_args['corpus_file_id']
corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
return [
{
'text': f'{corpus_file.author}: {corpus_file.title} ({corpus_file.publishing_year})',
'url': url_for('.corpus_file', corpus_id=corpus_id, corpus_file_id=corpus_file_id)
}
]

View File

@ -58,7 +58,7 @@ def delete_corpus_follower(corpus_id, follower_id):
current_user.id == follower_id
or current_user == cfa.corpus.user
or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
or current_user.is_administrator()):
or current_user.is_administrator):
abort(403)
if current_user.id == follower_id:
flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')

View File

@ -1,5 +1,4 @@
from flask import abort, flash, redirect, render_template, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user
from app import db
from app.models import (
@ -11,20 +10,14 @@ from app.models import (
from . import bp
from .decorators import corpus_follower_permission_required
from .forms import CreateCorpusForm
from .utils import (
corpus_endpoint_arguments_constructor as corpus_eac,
corpus_dynamic_list_constructor as corpus_dlc
)
@bp.route('')
@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">I</i>My Corpora')
def corpora():
return redirect(url_for('main.dashboard', _anchor='corpora'))
@bp.route('/create', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.create', 'Create')
def create_corpus():
form = CreateCorpusForm()
if form.validate_on_submit():
@ -47,7 +40,6 @@ def create_corpus():
@bp.route('/<hashid:corpus_id>')
@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=corpus_dlc)
def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
cfrs = CorpusFollowerRole.query.all()
@ -55,13 +47,13 @@ def corpus(corpus_id):
users = User.query.filter(User.is_public == True, User.id != current_user.id, User.id != corpus.user.id, User.role_id < 4).all()
cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
if cfa is None:
if corpus.user == current_user or current_user.is_administrator():
if corpus.user == current_user or current_user.is_administrator:
cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
else:
cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
else:
cfr = cfa.role
if corpus.user == current_user or current_user.is_administrator():
if corpus.user == current_user or current_user.is_administrator:
return render_template(
'corpora/corpus.html.j2',
title=corpus.title,
@ -87,7 +79,6 @@ def corpus(corpus_id):
@bp.route('/<hashid:corpus_id>/analysis')
@corpus_follower_permission_required('VIEW')
@register_breadcrumb(bp, '.entity.analysis', 'Analysis', endpoint_arguments_constructor=corpus_eac)
def analysis(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
return render_template(
@ -97,24 +88,22 @@ def analysis(corpus_id):
)
# @bp.route('/<hashid:corpus_id>/follow/<token>')
# def follow_corpus(corpus_id, token):
# corpus = Corpus.query.get_or_404(corpus_id)
# if current_user.follow_corpus_by_token(token):
# db.session.commit()
# flash(f'You are following "{corpus.title}" now', category='corpus')
# return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
# abort(403)
@bp.route('/<hashid:corpus_id>/follow/<token>')
def follow_corpus(corpus_id, token):
corpus = Corpus.query.get_or_404(corpus_id)
if current_user.follow_corpus_by_token(token):
db.session.commit()
flash(f'You are following "{corpus.title}" now', category='corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
abort(403)
@bp.route('/import', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.import', 'Import')
def import_corpus():
abort(503)
@bp.route('/<hashid:corpus_id>/export')
@corpus_follower_permission_required('VIEW')
@register_breadcrumb(bp, '.entity.export', 'Export', endpoint_arguments_constructor=corpus_eac)
def export_corpus(corpus_id):
abort(503)

View File

@ -1,17 +0,0 @@
from flask import request, url_for
from app.models import Corpus
def corpus_endpoint_arguments_constructor():
return {'corpus_id': request.view_args['corpus_id']}
def corpus_dynamic_list_constructor():
corpus_id = request.view_args['corpus_id']
corpus = Corpus.query.get_or_404(corpus_id)
return [
{
'text': f'<i class="material-icons left">book</i>{corpus.title}',
'url': url_for('.corpus', corpus_id=corpus_id)
}
]

View File

@ -1,7 +1,6 @@
from flask import abort, current_app, request
from flask import abort, request
from flask_login import current_user
from functools import wraps
from threading import Thread
from typing import List, Union
from werkzeug.exceptions import NotAcceptable
from app.models import Permission
@ -24,22 +23,21 @@ def admin_required(f):
def socketio_login_required(f):
@wraps(f)
def decorated_function(*args, **kwargs):
def wrapper(*args, **kwargs):
if current_user.is_authenticated:
return f(*args, **kwargs)
else:
return {'code': 401, 'msg': 'Unauthorized'}
return decorated_function
return {'code': 401, 'body': 'Unauthorized'}
return wrapper
def socketio_permission_required(permission):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
def wrapper(*args, **kwargs):
if not current_user.can(permission):
return {'code': 403, 'msg': 'Forbidden'}
return {'code': 403, 'body': 'Forbidden'}
return f(*args, **kwargs)
return decorated_function
return wrapper
return decorator
@ -47,24 +45,6 @@ def socketio_admin_required(f):
return socketio_permission_required(Permission.ADMINISTRATE)(f)
def background(f):
'''
' This decorator executes a function in a Thread.
' Decorated functions need to be executed within a code block where an
' app context exists.
'
' NOTE: An app object is passed as a keyword argument to the decorated
' function.
'''
@wraps(f)
def wrapped(*args, **kwargs):
kwargs['app'] = current_app._get_current_object()
thread = Thread(target=f, args=args, kwargs=kwargs)
thread.start()
return thread
return wrapped
def content_negotiation(
produces: Union[str, List[str], None] = None,
consumes: Union[str, List[str], None] = None

View File

@ -0,0 +1,2 @@
from .types import ContainerColumn
from .types import IntEnumColumn

View File

@ -0,0 +1,42 @@
import json
from app import db
class ContainerColumn(db.TypeDecorator):
impl = db.String
def __init__(self, container_type, *args, **kwargs):
super().__init__(*args, **kwargs)
self.container_type = container_type
def process_bind_param(self, value, dialect):
if isinstance(value, self.container_type):
return json.dumps(value)
elif isinstance(value, str) and isinstance(json.loads(value), self.container_type):
return value
else:
return TypeError()
def process_result_value(self, value, dialect):
return json.loads(value)
class IntEnumColumn(db.TypeDecorator):
impl = db.Integer
def __init__(self, enum_type, *args, **kwargs):
super().__init__(*args, **kwargs)
self.enum_type = enum_type
def process_bind_param(self, value, dialect):
if isinstance(value, self.enum_type) and isinstance(value.value, int):
return value.value
elif isinstance(value, int):
return self.enum_type(value).value
elif isinstance(value, str):
return self.enum_type[value].value
else:
return TypeError()
def process_result_value(self, value, dialect):
return self.enum_type(value)

138
app/jobs/events.py Normal file
View File

@ -0,0 +1,138 @@
from flask import current_app
from flask_login import current_user
from flask_socketio import Namespace
from app import db, hashids, socketio
from app.extensions.flask_socketio import admin_required, login_required
from app.models import Job, JobStatus
class JobsNamespace(Namespace):
@login_required
def on_delete(self, job_hashid: str):
# Decode the job hashid
job_id = hashids.decode(job_hashid)
# Validate job_id
if not isinstance(job_id, int):
return {
'code': 400,
'body': 'job_id is invalid'
}
# Load job from database
job = Job.query.get(job_id)
if job is None:
return {
'code': 404,
'body': 'Job not found'
}
# Check if the current user is allowed to delete the job
if not (job.user == current_user or current_user.is_administrator):
return {
'code': 403,
'body': 'Forbidden'
}
# TODO: This should be a method in the Job model
def _delete_job(app, job_id):
with app.app_context():
job = Job.query.get(job_id)
job.delete()
db.session.commit()
# Delete the job in a background task
socketio.start_background_task(
target=_delete_job,
app=current_app._get_current_object(),
job_id=job_id
)
return {
'code': 202,
'body': f'Job "{job.title}" marked for deletion'
}
@admin_required
def on_get_log(self, job_hashid: str):
# Decode the job hashid
job_id = hashids.decode(job_hashid)
# Validate job_id
if not isinstance(job_id, int):
return {
'code': 400,
'body': 'job_id is invalid'
}
# Load job from database
job = Job.query.get(job_id)
if job is None:
return {
'code': 404,
'body': 'Job not found'
}
# Check if the job is already processed
if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
return {
'code': 409,
'body': 'Job is not done processing'
}
# Read the log file
with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
job_log = log_file.read()
return {
'code': 200,
'body': job_log
}
@login_required
def on_restart(self, job_hashid: str):
# Decode the job hashid
job_id = hashids.decode(job_hashid)
# Validate job_id
if not isinstance(job_id, int):
return {
'code': 400,
'body': 'job_id is invalid'
}
# Load job from database
job = Job.query.get(job_id)
if job is None:
return {
'code': 404,
'body': 'Job not found'
}
# Check if the current user is allowed to restart the job
if not (job.user == current_user or current_user.is_administrator):
return {
'code': 403,
'body': 'Forbidden'
}
# TODO: This should be a method in the Job model
def _restart_job(app, job_id):
with app.app_context():
job = Job.query.get(job_id)
job.restart()
db.session.commit()
# Restart the job in a background task
socketio.start_background_task(
target=_restart_job,
app=current_app._get_current_object(),
job_id=job_id
)
return {
'code': 202,
'body': f'Job "{job.title}" restarted'
}

View File

@ -1,7 +1,6 @@
from flask import abort, current_app
from flask_login import current_user
from threading import Thread
import os
from app import db
from app.decorators import admin_required, content_negotiation
from app.models import Job, JobStatus
@ -18,7 +17,7 @@ def delete_job(job_id):
db.session.commit()
job = Job.query.get_or_404(job_id)
if not (job.user == current_user or current_user.is_administrator()):
if not (job.user == current_user or current_user.is_administrator):
abort(403)
thread = Thread(
target=_delete_job,
@ -39,7 +38,7 @@ def job_log(job_id):
if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
response = {'errors': {'message': 'Job status is not completed or failed'}}
return response, 409
with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
log = log_file.read()
response_data = {
'jobLog': log
@ -57,7 +56,7 @@ def restart_job(job_id):
db.session.commit()
job = Job.query.get_or_404(job_id)
if not (job.user == current_user or current_user.is_administrator()):
if not (job.user == current_user or current_user.is_administrator):
abort(403)
if job.status == JobStatus.FAILED:
response = {'errors': {'message': 'Job status is not "failed"'}}

View File

@ -5,25 +5,20 @@ from flask import (
send_from_directory,
url_for
)
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user
import os
from app.models import Job, JobInput, JobResult
from . import bp
from .utils import job_dynamic_list_constructor as job_dlc
@bp.route('')
@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">J</i>My Jobs')
def corpora():
return redirect(url_for('main.dashboard', _anchor='jobs'))
@bp.route('/<hashid:job_id>')
@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=job_dlc)
def job(job_id):
job = Job.query.get_or_404(job_id)
if not (job.user == current_user or current_user.is_administrator()):
if not (job.user == current_user or current_user.is_administrator):
abort(403)
return render_template(
'jobs/job.html.j2',
@ -35,13 +30,13 @@ def job(job_id):
@bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
def download_job_input(job_id, job_input_id):
job_input = JobInput.query.filter_by(job_id=job_id, id=job_input_id).first_or_404()
if not (job_input.job.user == current_user or current_user.is_administrator()):
if not (job_input.job.user == current_user or current_user.is_administrator):
abort(403)
return send_from_directory(
os.path.dirname(job_input.path),
os.path.basename(job_input.path),
job_input.path.parent,
job_input.path.name,
as_attachment=True,
attachment_filename=job_input.filename,
download_name=job_input.filename,
mimetype=job_input.mimetype
)
@ -49,12 +44,12 @@ def download_job_input(job_id, job_input_id):
@bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
def download_job_result(job_id, job_result_id):
job_result = JobResult.query.filter_by(job_id=job_id, id=job_result_id).first_or_404()
if not (job_result.job.user == current_user or current_user.is_administrator()):
if not (job_result.job.user == current_user or current_user.is_administrator):
abort(403)
return send_from_directory(
os.path.dirname(job_result.path),
os.path.basename(job_result.path),
job_result.path.parent,
job_result.path.name,
as_attachment=True,
attachment_filename=job_result.filename,
download_name=job_result.filename,
mimetype=job_result.mimetype
)

View File

@ -1,13 +0,0 @@
from flask import request, url_for
from app.models import Job
def job_dynamic_list_constructor():
job_id = request.view_args['job_id']
job = Job.query.get_or_404(job_id)
return [
{
'text': f'<i class="nopaque-icons left service-icons" data-service="{job.service}"></i>{job.title}',
'url': url_for('.job', job_id=job_id)
}
]

View File

@ -1,6 +1,7 @@
from flask import current_app
from flask_migrate import upgrade
import os
from pathlib import Path
from typing import List
from app.models import (
CorpusFollowerRole,
Role,
@ -17,16 +18,15 @@ def deploy():
# Make default directories
print('Make default directories')
base_dir = current_app.config['NOPAQUE_DATA_DIR']
default_dirs = [
os.path.join(base_dir, 'tmp'),
os.path.join(base_dir, 'users')
default_dirs: List[Path] = [
base_dir / 'tmp',
base_dir / 'users'
]
for dir in default_dirs:
if os.path.exists(dir):
if not os.path.isdir(dir):
raise NotADirectoryError(f'{dir} is not a directory')
else:
os.mkdir(dir)
for default_dir in default_dirs:
if not default_dir.exists():
default_dir.mkdir()
if not default_dir.is_dir():
raise NotADirectoryError(f'{default_dir} is not a directory')
# migrate database to latest revision
print('Migrate database to latest revision')

View File

@ -1,14 +1,11 @@
from flask import flash, redirect, render_template, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user, login_required, login_user
from app.auth.forms import LoginForm
from app.models import Corpus, User
from sqlalchemy import or_
from . import bp
@bp.route('/', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.', '<i class="material-icons">home</i>')
def index():
form = LoginForm()
if form.validate_on_submit():
@ -27,7 +24,6 @@ def index():
@bp.route('/faq')
@register_breadcrumb(bp, '.faq', 'Frequently Asked Questions')
def faq():
return render_template(
'main/faq.html.j2',
@ -36,7 +32,6 @@ def faq():
@bp.route('/dashboard')
@register_breadcrumb(bp, '.dashboard', '<i class="material-icons left">dashboard</i>Dashboard')
@login_required
def dashboard():
return render_template(
@ -45,8 +40,15 @@ def dashboard():
)
@bp.route('/manual')
def manual():
return render_template(
'main/manual.html.j2',
title='Manual'
)
@bp.route('/news')
@register_breadcrumb(bp, '.news', '<i class="material-icons left">email</i>News')
def news():
return render_template(
'main/news.html.j2',
@ -55,7 +57,6 @@ def news():
@bp.route('/privacy_policy')
@register_breadcrumb(bp, '.privacy_policy', 'Private statement (GDPR)')
def privacy_policy():
return render_template(
'main/privacy_policy.html.j2',
@ -64,7 +65,6 @@ def privacy_policy():
@bp.route('/terms_of_use')
@register_breadcrumb(bp, '.terms_of_use', 'Terms of Use')
def terms_of_use():
return render_template(
'main/terms_of_use.html.j2',
@ -73,7 +73,6 @@ def terms_of_use():
@bp.route('/social-area')
@register_breadcrumb(bp, '.social_area', '<i class="material-icons left">group</i>Social Area')
@login_required
def social_area():
print('test')

File diff suppressed because it is too large Load Diff

150
app/models/__init__.py Normal file
View File

@ -0,0 +1,150 @@
from enum import Enum
from app import db, login, mail, socketio
from app.email import create_message
from .anonymous_user import *
from .avatar import *
from .corpus_file import *
from .corpus_follower_association import *
from .corpus_follower_role import *
from .corpus import *
from .job_input import *
from .job_result import *
from .job import *
from .role import *
from .spacy_nlp_pipeline_model import *
from .tesseract_ocr_pipeline_model import *
from .token import *
from .user import *
@db.event.listens_for(Corpus, 'after_delete')
@db.event.listens_for(CorpusFile, 'after_delete')
@db.event.listens_for(Job, 'after_delete')
@db.event.listens_for(JobInput, 'after_delete')
@db.event.listens_for(JobResult, 'after_delete')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_delete')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_delete')
def resource_after_delete(mapper, connection, resource):
print('[START] resource_after_delete')
jsonpatch = [
{
'op': 'remove',
'path': resource.jsonpatch_path
}
]
room = f'/users/{resource.user_hashid}'
print('[EMIT] PATCH', jsonpatch)
socketio.emit('PATCH', jsonpatch, room=room)
print('[END] resource_after_delete')
@db.event.listens_for(CorpusFollowerAssociation, 'after_delete')
def cfa_after_delete_handler(mapper, connection, cfa):
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
{
'op': 'remove',
'path': jsonpatch_path
}
]
room = f'/users/{cfa.corpus.user.hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Corpus, 'after_insert')
@db.event.listens_for(CorpusFile, 'after_insert')
@db.event.listens_for(Job, 'after_insert')
@db.event.listens_for(JobInput, 'after_insert')
@db.event.listens_for(JobResult, 'after_insert')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_insert')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_insert')
def resource_after_insert_handler(mapper, connection, resource):
jsonpatch_value = resource.to_json_serializeable()
for attr in mapper.relationships:
jsonpatch_value[attr.key] = {}
jsonpatch = [
{
'op': 'add',
'path': resource.jsonpatch_path,
'value': jsonpatch_value
}
]
room = f'/users/{resource.user_hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(CorpusFollowerAssociation, 'after_insert')
def cfa_after_insert_handler(mapper, connection, cfa):
jsonpatch_value = cfa.to_json_serializeable()
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
{
'op': 'add',
'path': jsonpatch_path,
'value': jsonpatch_value
}
]
room = f'/users/{cfa.corpus.user.hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Corpus, 'after_update')
@db.event.listens_for(CorpusFile, 'after_update')
@db.event.listens_for(Job, 'after_update')
@db.event.listens_for(JobInput, 'after_update')
@db.event.listens_for(JobResult, 'after_update')
@db.event.listens_for(SpaCyNLPPipelineModel, 'after_update')
@db.event.listens_for(TesseractOCRPipelineModel, 'after_update')
def resource_after_update_handler(mapper, connection, resource):
jsonpatch = []
for attr in db.inspect(resource).attrs:
if attr.key in mapper.relationships:
continue
if not attr.load_history().has_changes():
continue
jsonpatch_path = f'{resource.jsonpatch_path}/{attr.key}'
if isinstance(attr.value, datetime):
jsonpatch_value = f'{attr.value.isoformat()}Z'
elif isinstance(attr.value, Enum):
jsonpatch_value = attr.value.name
else:
jsonpatch_value = attr.value
jsonpatch.append(
{
'op': 'replace',
'path': jsonpatch_path,
'value': jsonpatch_value
}
)
if jsonpatch:
room = f'/users/{resource.user_hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
@db.event.listens_for(Job, 'after_update')
def job_after_update_handler(mapper, connection, job):
for attr in db.inspect(job).attrs:
if attr.key != 'status':
continue
if not attr.load_history().has_changes():
return
if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:
return
if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:
if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
return
msg = create_message(
job.user.email,
f'Status update for your Job "{job.title}"',
'tasks/email/notification',
job=job
)
mail.send(msg)
login.anonymous_user = AnonymousUser
@login.user_loader
def load_user(user_id):
return User.query.get(int(user_id))

View File

@ -0,0 +1,10 @@
from flask_login import AnonymousUserMixin
class AnonymousUser(AnonymousUserMixin):
def can(self, permissions):
return False
@property
def is_administrator(self):
return False

40
app/models/avatar.py Normal file
View File

@ -0,0 +1,40 @@
from flask import current_app
from flask_hashids import HashidMixin
from pathlib import Path
from app import db
from .file_mixin import FileMixin
class Avatar(HashidMixin, FileMixin, db.Model):
__tablename__ = 'avatars'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Relationships
user = db.relationship('User', back_populates='avatar')
@property
def path(self) -> Path:
return self.user.path / 'avatar'
# return os.path.join(self.user.path, 'avatar')
def delete(self):
try:
self.path.unlink(missing_ok=True)
except OSError as e:
current_app.logger.error(e)
raise
db.session.delete(self)
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
**self.file_mixin_to_json_serializeable()
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

200
app/models/corpus.py Normal file
View File

@ -0,0 +1,200 @@
from datetime import datetime
from enum import IntEnum
from flask import current_app, url_for
from flask_hashids import HashidMixin
from sqlalchemy.ext.associationproxy import association_proxy
from typing import Union
from pathlib import Path
import shutil
import xml.etree.ElementTree as ET
from app import db
from app.converters.vrt import normalize_vrt_file
from app.extensions.sqlalchemy_extras import IntEnumColumn
from .corpus_follower_association import CorpusFollowerAssociation
class CorpusStatus(IntEnum):
UNPREPARED = 1
SUBMITTED = 2
QUEUED = 3
BUILDING = 4
BUILT = 5
FAILED = 6
STARTING_ANALYSIS_SESSION = 7
RUNNING_ANALYSIS_SESSION = 8
CANCELING_ANALYSIS_SESSION = 9
@staticmethod
def get(corpus_status: Union['CorpusStatus', int, str]) -> 'CorpusStatus':
if isinstance(corpus_status, CorpusStatus):
return corpus_status
if isinstance(corpus_status, int):
return CorpusStatus(corpus_status)
if isinstance(corpus_status, str):
return CorpusStatus[corpus_status]
raise TypeError('corpus_status must be CorpusStatus, int, or str')
class Corpus(HashidMixin, db.Model):
'''
Class to define a corpus.
'''
__tablename__ = 'corpora'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
description = db.Column(db.String(255))
status = db.Column(
IntEnumColumn(CorpusStatus),
default=CorpusStatus.UNPREPARED
)
title = db.Column(db.String(32))
num_analysis_sessions = db.Column(db.Integer, default=0)
num_tokens = db.Column(db.Integer, default=0)
is_public = db.Column(db.Boolean, default=False)
# Relationships
files = db.relationship(
'CorpusFile',
back_populates='corpus',
lazy='dynamic',
cascade='all, delete-orphan'
)
corpus_follower_associations = db.relationship(
'CorpusFollowerAssociation',
back_populates='corpus',
cascade='all, delete-orphan'
)
followers = association_proxy(
'corpus_follower_associations',
'follower',
creator=lambda u: CorpusFollowerAssociation(follower=u)
)
user = db.relationship('User', back_populates='corpora')
# "static" attributes
max_num_tokens = 2_147_483_647
def __repr__(self):
return f'<Corpus {self.title}>'
@property
def analysis_url(self):
return url_for('corpora.analysis', corpus_id=self.id)
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/corpora/{self.hashid}'
@property
def path(self) -> Path:
return self.user.path / 'corpora' / f'{self.id}'
@property
def url(self):
return url_for('corpora.corpus', corpus_id=self.id)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def create(**kwargs):
corpus = Corpus(**kwargs)
db.session.add(corpus)
db.session.flush(objects=[corpus])
db.session.refresh(corpus)
corpus_files_dir = corpus.path / 'files'
corpus_cwb_dir = corpus.path / 'cwb'
corpus_cwb_data_dir = corpus_cwb_dir / 'data'
corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
try:
corpus.path.mkdir()
corpus_files_dir.mkdir()
corpus_cwb_dir.mkdir()
corpus_cwb_data_dir.mkdir()
corpus_cwb_registry_dir.mkdir()
except OSError as e:
# TODO: Potential leftover cleanup
current_app.logger.error(e)
db.session.rollback()
raise
return corpus
def build(self):
corpus_cwb_dir = self.path / 'cwb'
corpus_cwb_data_dir = corpus_cwb_dir / 'data'
corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
try:
shutil.rmtree(corpus_cwb_dir, ignore_errors=True)
corpus_cwb_dir.mkdir()
corpus_cwb_data_dir.mkdir()
corpus_cwb_registry_dir.mkdir()
except OSError as e:
current_app.logger.error(e)
self.status = CorpusStatus.FAILED
raise
corpus_element = ET.fromstring('<corpus>\n</corpus>')
for corpus_file in self.files:
normalized_vrt_path = corpus_cwb_dir / f'{corpus_file.id}.norm.vrt'
try:
normalize_vrt_file(corpus_file.path, normalized_vrt_path)
except:
self.status = CorpusStatus.FAILED
return
element_tree = ET.parse(normalized_vrt_path)
text_element = element_tree.getroot()
text_element.set('author', corpus_file.author)
text_element.set('title', corpus_file.title)
text_element.set(
'publishing_year',
f'{corpus_file.publishing_year}'
)
text_element.set('address', corpus_file.address or 'NULL')
text_element.set('booktitle', corpus_file.booktitle or 'NULL')
text_element.set('chapter', corpus_file.chapter or 'NULL')
text_element.set('editor', corpus_file.editor or 'NULL')
text_element.set('institution', corpus_file.institution or 'NULL')
text_element.set('journal', corpus_file.journal or 'NULL')
text_element.set('pages', f'{corpus_file.pages}' or 'NULL')
text_element.set('publisher', corpus_file.publisher or 'NULL')
text_element.set('school', corpus_file.school or 'NULL')
text_element.tail = '\n'
# corpus_element.insert(1, text_element)
corpus_element.append(text_element)
ET.ElementTree(corpus_element).write(
corpus_cwb_dir / 'corpus.vrt',
encoding='utf-8'
)
self.status = CorpusStatus.SUBMITTED
def delete(self):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'creation_date': f'{self.creation_date.isoformat()}Z',
'description': self.description,
'max_num_tokens': self.max_num_tokens,
'num_analysis_sessions': self.num_analysis_sessions,
'num_tokens': self.num_tokens,
'status': self.status.name,
'title': self.title,
'is_public': self.is_public
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
json_serializeable['corpus_follower_associations'] = {
x.hashid: x.to_json_serializeable()
for x in self.corpus_follower_associations
}
json_serializeable['files'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.files
}
return json_serializeable

102
app/models/corpus_file.py Normal file
View File

@ -0,0 +1,102 @@
from flask import current_app, url_for
from flask_hashids import HashidMixin
from pathlib import Path
from app import db
from .corpus import CorpusStatus
from .file_mixin import FileMixin
class CorpusFile(FileMixin, HashidMixin, db.Model):
__tablename__ = 'corpus_files'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
# Fields
author = db.Column(db.String(255))
description = db.Column(db.String(255))
publishing_year = db.Column(db.Integer)
title = db.Column(db.String(255))
address = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
editor = db.Column(db.String(255))
institution = db.Column(db.String(255))
journal = db.Column(db.String(255))
pages = db.Column(db.String(255))
publisher = db.Column(db.String(255))
school = db.Column(db.String(255))
# Relationships
corpus = db.relationship(
'Corpus',
back_populates='files'
)
@property
def download_url(self):
return url_for(
'corpora.download_corpus_file',
corpus_id=self.corpus_id,
corpus_file_id=self.id
)
@property
def jsonpatch_path(self):
return f'{self.corpus.jsonpatch_path}/files/{self.hashid}'
@property
def path(self) -> Path:
return self.corpus.path / 'files' / f'{self.id}'
@property
def url(self):
return url_for(
'corpora.corpus_file',
corpus_id=self.corpus_id,
corpus_file_id=self.id
)
@property
def user_hashid(self):
return self.corpus.user.hashid
@property
def user_id(self):
return self.corpus.user_id
def delete(self):
try:
self.path.unlink(missing_ok=True)
except OSError as e:
current_app.logger.error(e)
raise
db.session.delete(self)
self.corpus.status = CorpusStatus.UNPREPARED
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'address': self.address,
'author': self.author,
'description': self.description,
'booktitle': self.booktitle,
'chapter': self.chapter,
'editor': self.editor,
'institution': self.institution,
'journal': self.journal,
'pages': self.pages,
'publisher': self.publisher,
'publishing_year': self.publishing_year,
'school': self.school,
'title': self.title,
**self.file_mixin_to_json_serializeable(
backrefs=backrefs,
relationships=relationships
)
}
if backrefs:
json_serializeable['corpus'] = \
self.corpus.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

View File

@ -0,0 +1,47 @@
from flask_hashids import HashidMixin
from app import db
from .corpus_follower_role import CorpusFollowerRole
class CorpusFollowerAssociation(HashidMixin, db.Model):
__tablename__ = 'corpus_follower_associations'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
follower_id = db.Column(db.Integer, db.ForeignKey('users.id'))
role_id = db.Column(db.Integer, db.ForeignKey('corpus_follower_roles.id'))
# Relationships
corpus = db.relationship(
'Corpus',
back_populates='corpus_follower_associations'
)
follower = db.relationship(
'User',
back_populates='corpus_follower_associations'
)
role = db.relationship(
'CorpusFollowerRole',
back_populates='corpus_follower_associations'
)
def __init__(self, **kwargs):
if 'role' not in kwargs:
kwargs['role'] = CorpusFollowerRole.query.filter_by(default=True).first()
super().__init__(**kwargs)
def __repr__(self):
return f'<CorpusFollowerAssociation {self.follower.__repr__()} ~ {self.role.__repr__()} ~ {self.corpus.__repr__()}>'
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'corpus': self.corpus.to_json_serializeable(backrefs=True),
'follower': self.follower.to_json_serializeable(),
'role': self.role.to_json_serializeable()
}
if backrefs:
pass
if relationships:
pass
return json_serializeable

View File

@ -0,0 +1,107 @@
from flask_hashids import HashidMixin
from enum import IntEnum
from typing import Union
from app import db
class CorpusFollowerPermission(IntEnum):
VIEW = 1
MANAGE_FILES = 2
MANAGE_FOLLOWERS = 4
MANAGE_CORPUS = 8
@staticmethod
def get(corpus_follower_permission: Union['CorpusFollowerPermission', int, str]) -> 'CorpusFollowerPermission':
if isinstance(corpus_follower_permission, CorpusFollowerPermission):
return corpus_follower_permission
if isinstance(corpus_follower_permission, int):
return CorpusFollowerPermission(corpus_follower_permission)
if isinstance(corpus_follower_permission, str):
return CorpusFollowerPermission[corpus_follower_permission]
raise TypeError('corpus_follower_permission must be CorpusFollowerPermission, int, or str')
class CorpusFollowerRole(HashidMixin, db.Model):
__tablename__ = 'corpus_follower_roles'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Fields
name = db.Column(db.String(64), unique=True)
default = db.Column(db.Boolean, default=False, index=True)
permissions = db.Column(db.Integer, default=0)
# Relationships
corpus_follower_associations = db.relationship(
'CorpusFollowerAssociation',
back_populates='role'
)
def __repr__(self):
return f'<CorpusFollowerRole {self.name}>'
def has_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
perm = CorpusFollowerPermission.get(permission)
return self.permissions & perm.value == perm.value
def add_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
perm = CorpusFollowerPermission.get(permission)
if not self.has_permission(perm):
self.permissions += perm.value
def remove_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
perm = CorpusFollowerPermission.get(permission)
if self.has_permission(perm):
self.permissions -= perm.value
def reset_permissions(self):
self.permissions = 0
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'default': self.default,
'name': self.name,
'permissions': [
x.name
for x in CorpusFollowerPermission
if self.has_permission(x)
]
}
if backrefs:
pass
if relationships:
json_serializeable['corpus_follower_association'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.corpus_follower_association
}
return json_serializeable
@staticmethod
def insert_defaults():
roles = {
'Anonymous': [],
'Viewer': [
CorpusFollowerPermission.VIEW
],
'Contributor': [
CorpusFollowerPermission.VIEW,
CorpusFollowerPermission.MANAGE_FILES
],
'Administrator': [
CorpusFollowerPermission.VIEW,
CorpusFollowerPermission.MANAGE_FILES,
CorpusFollowerPermission.MANAGE_FOLLOWERS,
CorpusFollowerPermission.MANAGE_CORPUS
]
}
default_role_name = 'Viewer'
for role_name, permissions in roles.items():
role = CorpusFollowerRole.query.filter_by(name=role_name).first()
if role is None:
role = CorpusFollowerRole(name=role_name)
role.reset_permissions()
for permission in permissions:
role.add_permission(permission)
role.default = role.name == default_role_name
db.session.add(role)
db.session.commit()

View File

@ -120,6 +120,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'German'
description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz'
@ -131,6 +132,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Greek'
description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.4.0/el_core_news_md-3.4.0.tar.gz'
@ -142,6 +144,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'English'
description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1.tar.gz'
@ -153,6 +156,7 @@
version: '3.4.1'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Spanish'
description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.4.0/es_core_news_md-3.4.0.tar.gz'
@ -164,6 +168,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'French'
description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.4.0/fr_core_news_md-3.4.0.tar.gz'
@ -175,6 +180,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Italian'
description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner'
url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz'
@ -186,6 +192,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Polish'
description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), tagger, senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.4.0/pl_core_news_md-3.4.0.tar.gz'
@ -197,6 +204,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Russian'
description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.4.0/ru_core_news_md-3.4.0.tar.gz'
@ -208,6 +216,7 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'
- title: 'Chinese'
description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.4.0/zh_core_web_md-3.4.0.tar.gz'
@ -219,3 +228,4 @@
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- '0.1.2'

View File

@ -0,0 +1,133 @@
from datetime import datetime
from enum import Enum
from app import db, mail, socketio
from app.email import create_message
from .corpus_file import CorpusFile
from .corpus_follower_association import CorpusFollowerAssociation
from .corpus import Corpus
from .job_input import JobInput
from .job_result import JobResult
from .job import Job, JobStatus
from .spacy_nlp_pipeline_model import SpaCyNLPPipelineModel
from .tesseract_ocr_pipeline_model import TesseractOCRPipelineModel
from .user import UserSettingJobStatusMailNotificationLevel
def register_event_listeners():
resources = [
Corpus,
CorpusFile,
Job,
JobInput,
JobResult,
SpaCyNLPPipelineModel,
TesseractOCRPipelineModel
]
for resource in resources:
db.event.listen(resource, 'after_delete', resource_after_delete)
db.event.listen(resource, 'after_insert', resource_after_insert)
db.event.listen(resource, 'after_update', resource_after_update)
db.event.listen(CorpusFollowerAssociation, 'after_delete', cfa_after_delete)
db.event.listen(CorpusFollowerAssociation, 'after_insert', cfa_after_insert)
db.event.listen(Job, 'after_update', job_after_update)
def resource_after_delete(mapper, connection, resource):
jsonpatch = [
{
'op': 'remove',
'path': resource.jsonpatch_path
}
]
room = f'/users/{resource.user_hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
def cfa_after_delete(mapper, connection, cfa):
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
{
'op': 'remove',
'path': jsonpatch_path
}
]
room = f'/users/{cfa.corpus.user.hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
def resource_after_insert(mapper, connection, resource):
jsonpatch_value = resource.to_json_serializeable()
for attr in mapper.relationships:
jsonpatch_value[attr.key] = {}
jsonpatch = [
{
'op': 'add',
'path': resource.jsonpatch_path,
'value': jsonpatch_value
}
]
room = f'/users/{resource.user_hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
def cfa_after_insert(mapper, connection, cfa):
jsonpatch_value = cfa.to_json_serializeable()
jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
jsonpatch = [
{
'op': 'add',
'path': jsonpatch_path,
'value': jsonpatch_value
}
]
room = f'/users/{cfa.corpus.user.hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
def resource_after_update(mapper, connection, resource):
jsonpatch = []
for attr in db.inspect(resource).attrs:
if attr.key in mapper.relationships:
continue
if not attr.load_history().has_changes():
continue
jsonpatch_path = f'{resource.jsonpatch_path}/{attr.key}'
if isinstance(attr.value, datetime):
jsonpatch_value = f'{attr.value.isoformat()}Z'
elif isinstance(attr.value, Enum):
jsonpatch_value = attr.value.name
else:
jsonpatch_value = attr.value
jsonpatch.append(
{
'op': 'replace',
'path': jsonpatch_path,
'value': jsonpatch_value
}
)
if jsonpatch:
room = f'/users/{resource.user_hashid}'
socketio.emit('PATCH', jsonpatch, room=room)
def job_after_update(mapper, connection, job):
for attr in db.inspect(job).attrs:
if attr.key != 'status':
continue
if not attr.load_history().has_changes():
return
if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:
return
if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:
if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
return
msg = create_message(
job.user.email,
f'Status update for your Job "{job.title}"',
'tasks/email/notification',
job=job
)
mail.send(msg)

40
app/models/file_mixin.py Normal file
View File

@ -0,0 +1,40 @@
from datetime import datetime
from flask import current_app
from werkzeug.utils import secure_filename
from app import db
class FileMixin:
'''
Mixin for db.Model classes. All file related models should use this.
'''
creation_date = db.Column(db.DateTime, default=datetime.utcnow)
filename = db.Column(db.String(255))
mimetype = db.Column(db.String(255))
def file_mixin_to_json_serializeable(self, backrefs=False, relationships=False):
return {
'creation_date': f'{self.creation_date.isoformat()}Z',
'filename': self.filename,
'mimetype': self.mimetype
}
@classmethod
def create(cls, file_storage, **kwargs):
filename = kwargs.pop('filename', file_storage.filename)
mimetype = kwargs.pop('mimetype', file_storage.mimetype)
obj = cls(
filename=secure_filename(filename),
mimetype=mimetype,
**kwargs
)
db.session.add(obj)
db.session.flush(objects=[obj])
db.session.refresh(obj)
try:
file_storage.save(obj.path)
except (AttributeError, OSError) as e:
current_app.logger.error(e)
db.session.rollback()
raise e
return obj

172
app/models/job.py Normal file
View File

@ -0,0 +1,172 @@
from datetime import datetime
from enum import IntEnum
from flask import current_app, url_for
from flask_hashids import HashidMixin
from time import sleep
from typing import Union
from pathlib import Path
import shutil
from app import db
from app.extensions.sqlalchemy_extras import ContainerColumn, IntEnumColumn
class JobStatus(IntEnum):
INITIALIZING = 1
SUBMITTED = 2
QUEUED = 3
RUNNING = 4
CANCELING = 5
CANCELED = 6
COMPLETED = 7
FAILED = 8
@staticmethod
def get(job_status: Union['JobStatus', int, str]) -> 'JobStatus':
if isinstance(job_status, JobStatus):
return job_status
if isinstance(job_status, int):
return JobStatus(job_status)
if isinstance(job_status, str):
return JobStatus[job_status]
raise TypeError('job_status must be JobStatus, int, or str')
class Job(HashidMixin, db.Model):
'''
Class to define Jobs.
'''
__tablename__ = 'jobs'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
creation_date = \
db.Column(db.DateTime(), default=datetime.utcnow)
description = db.Column(db.String(255))
end_date = db.Column(db.DateTime())
service = db.Column(db.String(64))
service_args = db.Column(ContainerColumn(dict, 255))
service_version = db.Column(db.String(16))
status = db.Column(
IntEnumColumn(JobStatus),
default=JobStatus.INITIALIZING
)
title = db.Column(db.String(32))
# Relationships
inputs = db.relationship(
'JobInput',
back_populates='job',
cascade='all, delete-orphan',
lazy='dynamic'
)
results = db.relationship(
'JobResult',
back_populates='job',
cascade='all, delete-orphan',
lazy='dynamic'
)
user = db.relationship(
'User',
back_populates='jobs'
)
def __repr__(self):
return f'<Job {self.title}>'
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/jobs/{self.hashid}'
@property
def path(self) -> Path:
return self.user.path / 'jobs' / f'{self.id}'
@property
def url(self):
return url_for('jobs.job', job_id=self.id)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def create(**kwargs):
job = Job(**kwargs)
db.session.add(job)
db.session.flush(objects=[job])
db.session.refresh(job)
job_inputs_dir = job.path / 'inputs'
job_pipeline_data_dir = job.path / 'pipeline_data'
job_results_dir = job.path / 'results'
try:
job.path.mkdir()
job_inputs_dir.mkdir()
job_pipeline_data_dir.mkdir()
job_results_dir.mkdir()
except OSError as e:
# TODO: Potential leftover cleanup
current_app.logger.error(e)
db.session.rollback()
raise
return job
def delete(self):
''' Delete the job and its inputs and results from the database. '''
if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: # noqa
self.status = JobStatus.CANCELING
db.session.commit()
while self.status != JobStatus.CANCELED:
# In case the daemon handled a job in any way
if self.status != JobStatus.CANCELING:
self.status = JobStatus.CANCELING
db.session.commit()
sleep(1)
db.session.refresh(self)
try:
shutil.rmtree(self.path)
except OSError as e:
current_app.logger.error(e)
db.session.rollback()
raise e
db.session.delete(self)
def restart(self):
''' Restart a job - only if the status is failed '''
if self.status != JobStatus.FAILED:
raise Exception('Job status is not "failed"')
shutil.rmtree(self.path / 'results', ignore_errors=True)
shutil.rmtree(self.path / 'pyflow.data', ignore_errors=True)
for result in self.results:
db.session.delete(result)
self.end_date = None
self.status = JobStatus.SUBMITTED
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'creation_date': f'{self.creation_date.isoformat()}Z',
'description': self.description,
'end_date': (
None if self.end_date is None
else f'{self.end_date.isoformat()}Z'
),
'service': self.service,
'service_args': self.service_args,
'service_version': self.service_version,
'status': self.status.name,
'title': self.title
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
json_serializeable['inputs'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.inputs
}
json_serializeable['results'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.results
}
return json_serializeable

65
app/models/job_input.py Normal file
View File

@ -0,0 +1,65 @@
from flask import url_for
from flask_hashids import HashidMixin
from pathlib import Path
from app import db
from .file_mixin import FileMixin
class JobInput(FileMixin, HashidMixin, db.Model):
__tablename__ = 'job_inputs'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Relationships
job = db.relationship(
'Job',
back_populates='inputs'
)
def __repr__(self):
return f'<JobInput {self.filename}>'
@property
def content_url(self):
return url_for(
'jobs.download_job_input',
job_id=self.job.id,
job_input_id=self.id
)
@property
def jsonpatch_path(self):
return f'{self.job.jsonpatch_path}/inputs/{self.hashid}'
@property
def path(self) -> Path:
return self.job.path / 'inputs' / f'{self.id}'
@property
def url(self):
return url_for(
'jobs.job',
job_id=self.job_id,
_anchor=f'job-{self.job.hashid}-input-{self.hashid}'
)
@property
def user_hashid(self):
return self.job.user.hashid
@property
def user_id(self):
return self.job.user.id
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
**self.file_mixin_to_json_serializeable()
}
if backrefs:
json_serializeable['job'] = \
self.job.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

71
app/models/job_result.py Normal file
View File

@ -0,0 +1,71 @@
from flask import url_for
from flask_hashids import HashidMixin
from pathlib import Path
from app import db
from .file_mixin import FileMixin
class JobResult(FileMixin, HashidMixin, db.Model):
__tablename__ = 'job_results'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
description = db.Column(db.String(255))
# Relationships
job = db.relationship(
'Job',
back_populates='results'
)
def __repr__(self):
return f'<JobResult {self.filename}>'
@property
def download_url(self):
return url_for(
'jobs.download_job_result',
job_id=self.job_id,
job_result_id=self.id
)
@property
def jsonpatch_path(self):
return f'{self.job.jsonpatch_path}/results/{self.hashid}'
@property
def path(self) -> Path:
return self.job.path / 'results' / f'{self.id}'
@property
def url(self):
return url_for(
'jobs.job',
job_id=self.job_id,
_anchor=f'job-{self.job.hashid}-result-{self.hashid}'
)
@property
def user_hashid(self):
return self.job.user.hashid
@property
def user_id(self):
return self.job.user.id
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'description': self.description,
**self.file_mixin_to_json_serializeable(
backrefs=backrefs,
relationships=relationships
)
}
if backrefs:
json_serializeable['job'] = \
self.job.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

100
app/models/role.py Normal file
View File

@ -0,0 +1,100 @@
from enum import IntEnum
from flask_hashids import HashidMixin
from typing import Union
from app import db
class Permission(IntEnum):
'''
Defines User permissions as integers by the power of 2. User permission
can be evaluated using the bitwise operator &.
'''
ADMINISTRATE = 1
CONTRIBUTE = 2
USE_API = 4
@staticmethod
def get(permission: Union['Permission', int, str]) -> 'Permission':
if isinstance(permission, Permission):
return permission
if isinstance(permission, int):
return Permission(permission)
if isinstance(permission, str):
return Permission[permission]
raise TypeError('permission must be Permission, int, or str')
class Role(HashidMixin, db.Model):
__tablename__ = 'roles'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Fields
name = db.Column(db.String(64), unique=True)
default = db.Column(db.Boolean, default=False, index=True)
permissions = db.Column(db.Integer, default=0)
# Relationships
users = db.relationship('User', back_populates='role', lazy='dynamic')
def __repr__(self):
return f'<Role {self.name}>'
def has_permission(self, permission: Union[Permission, int, str]):
p = Permission.get(permission)
return self.permissions & p.value == p.value
def add_permission(self, permission: Union[Permission, int, str]):
p = Permission.get(permission)
if not self.has_permission(p):
self.permissions += p.value
def remove_permission(self, permission: Union[Permission, int, str]):
p = Permission.get(permission)
if self.has_permission(p):
self.permissions -= p.value
def reset_permissions(self):
self.permissions = 0
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'default': self.default,
'name': self.name,
'permissions': [
x.name for x in Permission
if self.has_permission(x.value)
]
}
if backrefs:
pass
if relationships:
json_serializeable['users'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.users
}
return json_serializeable
@staticmethod
def insert_defaults():
roles = {
'User': [],
'API user': [Permission.USE_API],
'Contributor': [Permission.CONTRIBUTE],
'Administrator': [
Permission.ADMINISTRATE,
Permission.CONTRIBUTE,
Permission.USE_API
],
'System user': []
}
default_role_name = 'User'
for role_name, permissions in roles.items():
role = Role.query.filter_by(name=role_name).first()
if role is None:
role = Role(name=role_name)
role.reset_permissions()
for permission in permissions:
role.add_permission(permission)
role.default = role.name == default_role_name
db.session.add(role)
db.session.commit()

View File

@ -0,0 +1,136 @@
from flask import current_app, url_for
from flask_hashids import HashidMixin
from tqdm import tqdm
from pathlib import Path
import requests
import yaml
from app import db
from app.extensions.sqlalchemy_extras import ContainerColumn
from .file_mixin import FileMixin
from .user import User
class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
__tablename__ = 'spacy_nlp_pipeline_models'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
title = db.Column(db.String(64))
description = db.Column(db.String(255))
version = db.Column(db.String(16))
compatible_service_versions = db.Column(ContainerColumn(list, 255))
publisher = db.Column(db.String(128))
publisher_url = db.Column(db.String(512))
publishing_url = db.Column(db.String(512))
publishing_year = db.Column(db.Integer)
pipeline_name = db.Column(db.String(64))
is_public = db.Column(db.Boolean, default=False)
# Relationships
user = db.relationship('User', back_populates='spacy_nlp_pipeline_models')
@property
def path(self) -> Path:
return self.user.path / 'spacy_nlp_pipeline_models' / f'{self.id}'
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}'
@property
def url(self):
return url_for(
'contributions.spacy_nlp_pipeline_model',
spacy_nlp_pipeline_model_id=self.id
)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def insert_defaults(force_download=False):
nopaque_user = User.query.filter_by(username='nopaque').first()
default_records_file = Path(__file__).parent / 'default_records' / 'spacy_nlp_pipeline_model.yml'
with default_records_file.open('r') as f:
default_records = yaml.safe_load(f)
for m in default_records:
model = SpaCyNLPPipelineModel.query.filter_by(title=m['title'], version=m['version']).first() # noqa
if model is not None:
model.compatible_service_versions = m['compatible_service_versions']
model.description = m['description']
model.filename = m['url'].split('/')[-1]
model.publisher = m['publisher']
model.publisher_url = m['publisher_url']
model.publishing_url = m['publishing_url']
model.publishing_year = m['publishing_year']
model.is_public = True
model.title = m['title']
model.version = m['version']
model.pipeline_name = m['pipeline_name']
else:
model = SpaCyNLPPipelineModel(
compatible_service_versions=m['compatible_service_versions'],
description=m['description'],
filename=m['url'].split('/')[-1],
publisher=m['publisher'],
publisher_url=m['publisher_url'],
publishing_url=m['publishing_url'],
publishing_year=m['publishing_year'],
is_public=True,
title=m['title'],
user=nopaque_user,
version=m['version'],
pipeline_name=m['pipeline_name']
)
db.session.add(model)
db.session.flush(objects=[model])
db.session.refresh(model)
if not model.path.exists() or force_download:
r = requests.get(m['url'], stream=True)
pbar = tqdm(
desc=f'{model.title} ({model.filename})',
unit="B",
unit_scale=True,
unit_divisor=1024,
total=int(r.headers['Content-Length'])
)
pbar.clear()
with open(model.path, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
pbar.update(len(chunk))
f.write(chunk)
pbar.close()
db.session.commit()
def delete(self):
try:
self.path.unlink(missing_ok=True)
except OSError as e:
current_app.logger.error(e)
raise
db.session.delete(self)
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'compatible_service_versions': self.compatible_service_versions,
'description': self.description,
'publisher': self.publisher,
'publisher_url': self.publisher_url,
'publishing_url': self.publishing_url,
'publishing_year': self.publishing_year,
'pipeline_name': self.pipeline_name,
'is_public': self.is_public,
'title': self.title,
'version': self.version,
**self.file_mixin_to_json_serializeable()
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

View File

@ -0,0 +1,132 @@
from flask import current_app, url_for
from flask_hashids import HashidMixin
from tqdm import tqdm
from pathlib import Path
import requests
import yaml
from app import db
from app.extensions.sqlalchemy_extras import ContainerColumn
from .file_mixin import FileMixin
from .user import User
class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
__tablename__ = 'tesseract_ocr_pipeline_models'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
title = db.Column(db.String(64))
description = db.Column(db.String(255))
version = db.Column(db.String(16))
compatible_service_versions = db.Column(ContainerColumn(list, 255))
publisher = db.Column(db.String(128))
publisher_url = db.Column(db.String(512))
publishing_url = db.Column(db.String(512))
publishing_year = db.Column(db.Integer)
is_public = db.Column(db.Boolean, default=False)
# Relationships
user = db.relationship('User', back_populates='tesseract_ocr_pipeline_models')
@property
def path(self) -> Path:
return self.user.path / 'tesseract_ocr_pipeline_models' / f'{self.id}'
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/tesseract_ocr_pipeline_models/{self.hashid}'
@property
def url(self):
return url_for(
'contributions.tesseract_ocr_pipeline_model',
tesseract_ocr_pipeline_model_id=self.id
)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def insert_defaults(force_download=False):
nopaque_user = User.query.filter_by(username='nopaque').first()
default_records_file = Path(__file__).parent / 'default_records' / 'tesseract_ocr_pipeline_model.yml'
with default_records_file.open('r') as f:
default_records = yaml.safe_load(f)
for m in default_records:
model = TesseractOCRPipelineModel.query.filter_by(title=m['title'], version=m['version']).first() # noqa
if model is not None:
model.compatible_service_versions = m['compatible_service_versions']
model.description = m['description']
model.filename = f'{model.id}.traineddata'
model.publisher = m['publisher']
model.publisher_url = m['publisher_url']
model.publishing_url = m['publishing_url']
model.publishing_year = m['publishing_year']
model.is_public = True
model.title = m['title']
model.version = m['version']
else:
model = TesseractOCRPipelineModel(
compatible_service_versions=m['compatible_service_versions'],
description=m['description'],
publisher=m['publisher'],
publisher_url=m['publisher_url'],
publishing_url=m['publishing_url'],
publishing_year=m['publishing_year'],
is_public=True,
title=m['title'],
user=nopaque_user,
version=m['version']
)
db.session.add(model)
db.session.flush(objects=[model])
db.session.refresh(model)
model.filename = f'{model.id}.traineddata'
if not model.path.exists() or force_download:
r = requests.get(m['url'], stream=True)
pbar = tqdm(
desc=f'{model.title} ({model.filename})',
unit="B",
unit_scale=True,
unit_divisor=1024,
total=int(r.headers['Content-Length'])
)
pbar.clear()
with open(model.path, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
pbar.update(len(chunk))
f.write(chunk)
pbar.close()
db.session.commit()
def delete(self):
try:
self.path.unlink(missing_ok=True)
except OSError as e:
current_app.logger.error(e)
raise
db.session.delete(self)
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'compatible_service_versions': self.compatible_service_versions,
'description': self.description,
'publisher': self.publisher,
'publisher_url': self.publisher_url,
'publishing_url': self.publishing_url,
'publishing_year': self.publishing_year,
'is_public': self.is_public,
'title': self.title,
'version': self.version,
**self.file_mixin_to_json_serializeable()
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable

48
app/models/token.py Normal file
View File

@ -0,0 +1,48 @@
from datetime import datetime, timedelta
from app import db
class Token(db.Model):
__tablename__ = 'tokens'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
access_token = db.Column(db.String(64), index=True)
access_expiration = db.Column(db.DateTime)
refresh_token = db.Column(db.String(64), index=True)
refresh_expiration = db.Column(db.DateTime)
# Relationships
user = db.relationship('User', back_populates='tokens')
def expire(self):
self.access_expiration = datetime.utcnow()
self.refresh_expiration = datetime.utcnow()
def to_json_serializeable(self, backrefs=False, relationships=False):
json_serializeable = {
'id': self.hashid,
'access_token': self.access_token,
'access_expiration': (
None if self.access_expiration is None
else f'{self.access_expiration.isoformat()}Z'
),
'refresh_token': self.refresh_token,
'refresh_expiration': (
None if self.refresh_expiration is None
else f'{self.refresh_expiration.isoformat()}Z'
)
}
if backrefs:
json_serializeable['user'] = \
self.user.to_json_serializeable(backrefs=True)
if relationships:
pass
return json_serializeable
@staticmethod
def clean():
"""Remove any tokens that have been expired for more than a day."""
yesterday = datetime.utcnow() - timedelta(days=1)
Token.query.filter(Token.refresh_expiration < yesterday).delete()

454
app/models/user.py Normal file
View File

@ -0,0 +1,454 @@
from datetime import datetime, timedelta
from enum import IntEnum
from flask import current_app, url_for
from flask_hashids import HashidMixin
from flask_login import UserMixin
from sqlalchemy.ext.associationproxy import association_proxy
from pathlib import Path
from typing import Union
from werkzeug.security import generate_password_hash, check_password_hash
import jwt
import re
import secrets
import shutil
from app import db, hashids
from app.extensions.sqlalchemy_extras import IntEnumColumn
from .corpus import Corpus
from .corpus_follower_association import CorpusFollowerAssociation
from .corpus_follower_role import CorpusFollowerRole
from .role import Permission, Role
from .token import Token
class ProfilePrivacySettings(IntEnum):
SHOW_EMAIL = 1
SHOW_LAST_SEEN = 2
SHOW_MEMBER_SINCE = 4
@staticmethod
def get(profile_privacy_setting: Union['ProfilePrivacySettings', int, str]) -> 'ProfilePrivacySettings':
if isinstance(profile_privacy_setting, ProfilePrivacySettings):
return profile_privacy_setting
if isinstance(profile_privacy_setting, int):
return ProfilePrivacySettings(profile_privacy_setting)
if isinstance(profile_privacy_setting, str):
return ProfilePrivacySettings[profile_privacy_setting]
raise TypeError('profile_privacy_setting must be ProfilePrivacySettings, int, or str')
class UserSettingJobStatusMailNotificationLevel(IntEnum):
NONE = 1
END = 2
ALL = 3
class User(HashidMixin, UserMixin, db.Model):
__tablename__ = 'users'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
# Fields
email = db.Column(db.String(254), index=True, unique=True)
username = db.Column(db.String(64), index=True, unique=True)
username_pattern = re.compile(r'^[A-Za-zÄÖÜäöüß0-9_.]*$')
password_hash = db.Column(db.String(128))
confirmed = db.Column(db.Boolean, default=False)
terms_of_use_accepted = db.Column(db.Boolean, default=False)
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
setting_job_status_mail_notification_level = db.Column(
IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
default=UserSettingJobStatusMailNotificationLevel.END
)
last_seen = db.Column(db.DateTime())
full_name = db.Column(db.String(64))
about_me = db.Column(db.String(256))
location = db.Column(db.String(64))
website = db.Column(db.String(128))
organization = db.Column(db.String(128))
is_public = db.Column(db.Boolean, default=False)
profile_privacy_settings = db.Column(db.Integer(), default=0)
# Relationships
avatar = db.relationship(
'Avatar',
back_populates='user',
cascade='all, delete-orphan',
uselist=False
)
corpora = db.relationship(
'Corpus',
back_populates='user',
cascade='all, delete-orphan',
lazy='dynamic'
)
corpus_follower_associations = db.relationship(
'CorpusFollowerAssociation',
back_populates='follower',
cascade='all, delete-orphan'
)
followed_corpora = association_proxy(
'corpus_follower_associations',
'corpus',
creator=lambda c: CorpusFollowerAssociation(corpus=c)
)
jobs = db.relationship(
'Job',
back_populates='user',
cascade='all, delete-orphan',
lazy='dynamic'
)
role = db.relationship(
'Role',
back_populates='users'
)
spacy_nlp_pipeline_models = db.relationship(
'SpaCyNLPPipelineModel',
back_populates='user',
cascade='all, delete-orphan',
lazy='dynamic'
)
tesseract_ocr_pipeline_models = db.relationship(
'TesseractOCRPipelineModel',
back_populates='user',
cascade='all, delete-orphan',
lazy='dynamic'
)
tokens = db.relationship(
'Token',
back_populates='user',
cascade='all, delete-orphan',
lazy='dynamic'
)
def __init__(self, **kwargs):
if 'role' not in kwargs:
kwargs['role'] = (
Role.query.filter_by(name='Administrator').first()
if kwargs['email'] == current_app.config['NOPAQUE_ADMIN']
else Role.query.filter_by(default=True).first()
)
super().__init__(**kwargs)
def __repr__(self):
return f'<User {self.username}>'
@property
def is_administrator(self):
return self.can(Permission.ADMINISTRATE)
@property
def jsonpatch_path(self):
return f'/users/{self.hashid}'
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
#pbkdf2:sha256
self.password_hash = generate_password_hash(password, method='pbkdf2')
@property
def path(self) -> Path:
return current_app.config.get('NOPAQUE_DATA_DIR') / 'users' / f'{self.id}'
@staticmethod
def create(**kwargs):
user = User(**kwargs)
db.session.add(user)
db.session.flush(objects=[user])
db.session.refresh(user)
user_spacy_nlp_pipeline_models_dir = user.path / 'spacy_nlp_pipeline_models'
user_tesseract_ocr_pipeline_models_dir = user.path / 'tesseract_ocr_pipeline_models'
user_corpora_dir = user.path / 'corpora'
user_jobs_dir = user.path / 'jobs'
try:
user.path.mkdir()
user_spacy_nlp_pipeline_models_dir.mkdir()
user_tesseract_ocr_pipeline_models_dir.mkdir()
user_corpora_dir.mkdir()
user_jobs_dir.mkdir()
except OSError as e:
# TODO: Potential leftover cleanup
current_app.logger.error(e)
db.session.rollback()
raise
return user
@staticmethod
def insert_defaults():
nopaque_user = User.query.filter_by(username='nopaque').first()
system_user_role = Role.query.filter_by(name='System user').first()
if nopaque_user is None:
nopaque_user = User.create(
username='nopaque',
role=system_user_role
)
db.session.add(nopaque_user)
elif nopaque_user.role != system_user_role:
nopaque_user.role = system_user_role
db.session.commit()
@staticmethod
def reset_password(token, new_password):
try:
payload = jwt.decode(
token,
current_app.config['SECRET_KEY'],
algorithms=['HS256'],
issuer=current_app.config['SERVER_NAME'],
options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
)
except jwt.PyJWTError:
return False
if payload.get('purpose') != 'User.reset_password':
return False
user_hashid = payload.get('sub')
user_id = hashids.decode(user_hashid)
user = User.query.get(user_id)
if user is None:
return False
user.password = new_password
db.session.add(user)
return True
@staticmethod
def verify_access_token(access_token, refresh_token=None):
token = Token.query.filter(Token.access_token == access_token).first()
if token is not None:
if token.access_expiration > datetime.utcnow():
token.user.ping()
db.session.commit()
if token.user.role.name != 'System user':
return token.user
@staticmethod
def verify_refresh_token(refresh_token, access_token):
token = Token.query.filter((Token.refresh_token == refresh_token) & (Token.access_token == access_token)).first()
if token is not None:
if token.refresh_expiration > datetime.utcnow():
return token
# someone tried to refresh with an expired token
# revoke all tokens from this user as a precaution
token.user.revoke_auth_tokens()
db.session.commit()
def can(self, permission):
return self.role is not None and self.role.has_permission(permission)
def confirm(self, confirmation_token):
try:
payload = jwt.decode(
confirmation_token,
current_app.config['SECRET_KEY'],
algorithms=['HS256'],
issuer=current_app.config['SERVER_NAME'],
options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
)
except jwt.PyJWTError:
return False
if payload.get('purpose') != 'user.confirm':
return False
if payload.get('sub') != self.hashid:
return False
self.confirmed = True
db.session.add(self)
return True
def delete(self):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def generate_auth_token(self):
return Token(
access_token=secrets.token_urlsafe(),
access_expiration=datetime.utcnow() + timedelta(minutes=15),
refresh_token=secrets.token_urlsafe(),
refresh_expiration=datetime.utcnow() + timedelta(days=7),
user=self
)
def generate_confirm_token(self, expiration=3600):
now = datetime.utcnow()
payload = {
'exp': now + timedelta(seconds=expiration),
'iat': now,
'iss': current_app.config['SERVER_NAME'],
'purpose': 'user.confirm',
'sub': self.hashid
}
return jwt.encode(
payload,
current_app.config['SECRET_KEY'],
algorithm='HS256'
)
def generate_reset_password_token(self, expiration=3600):
now = datetime.utcnow()
payload = {
'exp': now + timedelta(seconds=expiration),
'iat': now,
'iss': current_app.config['SERVER_NAME'],
'purpose': 'User.reset_password',
'sub': self.hashid
}
return jwt.encode(
payload,
current_app.config['SECRET_KEY'],
algorithm='HS256'
)
def ping(self):
self.last_seen = datetime.utcnow()
def revoke_auth_tokens(self):
for token in self.tokens:
db.session.delete(token)
def verify_password(self, password):
if self.role.name == 'System user':
return False
return check_password_hash(self.password_hash, password)
#region Profile Privacy settings
def has_profile_privacy_setting(self, setting):
s = ProfilePrivacySettings.get(setting)
return self.profile_privacy_settings & s.value == s.value
def add_profile_privacy_setting(self, setting):
s = ProfilePrivacySettings.get(setting)
if not self.has_profile_privacy_setting(s):
self.profile_privacy_settings += s.value
def remove_profile_privacy_setting(self, setting):
s = ProfilePrivacySettings.get(setting)
if self.has_profile_privacy_setting(s):
self.profile_privacy_settings -= s.value
def reset_profile_privacy_settings(self):
self.profile_privacy_settings = 0
#endregion Profile Privacy settings
def follow_corpus(self, corpus, role=None):
if role is None:
cfr = CorpusFollowerRole.query.filter_by(default=True).first()
else:
cfr = role
if self.is_following_corpus(corpus):
cfa = CorpusFollowerAssociation.query.filter_by(corpus=corpus, follower=self).first()
if cfa.role != cfr:
cfa.role = cfr
else:
cfa = CorpusFollowerAssociation(corpus=corpus, role=cfr, follower=self)
db.session.add(cfa)
def unfollow_corpus(self, corpus):
if not self.is_following_corpus(corpus):
return
self.followed_corpora.remove(corpus)
def is_following_corpus(self, corpus):
return corpus in self.followed_corpora
def generate_follow_corpus_token(self, corpus_hashid, role_name, expiration=7):
now = datetime.utcnow()
payload = {
'exp': expiration,
'iat': now,
'iss': current_app.config['SERVER_NAME'],
'purpose': 'User.follow_corpus',
'role_name': role_name,
'sub': corpus_hashid
}
return jwt.encode(
payload,
current_app.config['SECRET_KEY'],
algorithm='HS256'
)
def follow_corpus_by_token(self, token):
try:
payload = jwt.decode(
token,
current_app.config['SECRET_KEY'],
algorithms=['HS256'],
issuer=current_app.config['SERVER_NAME'],
options={'require': ['exp', 'iat', 'iss', 'purpose', 'role_name', 'sub']}
)
except jwt.PyJWTError:
return False
if payload.get('purpose') != 'User.follow_corpus':
return False
corpus_hashid = payload.get('sub')
corpus_id = hashids.decode(corpus_hashid)
corpus = Corpus.query.get_or_404(corpus_id)
if corpus is None:
return False
role_name = payload.get('role_name')
role = CorpusFollowerRole.query.filter_by(name=role_name).first()
if role is None:
return False
self.follow_corpus(corpus, role)
# db.session.add(self)
return True
def to_json_serializeable(self, backrefs=False, relationships=False, filter_by_privacy_settings=False):
json_serializeable = {
'id': self.hashid,
'confirmed': self.confirmed,
'avatar': url_for('users.user_avatar', user_id=self.id),
'email': self.email,
'last_seen': (
None if self.last_seen is None
else f'{self.last_seen.isoformat()}Z'
),
'member_since': f'{self.member_since.isoformat()}Z',
'username': self.username,
'full_name': self.full_name,
'about_me': self.about_me,
'website': self.website,
'location': self.location,
'organization': self.organization,
'job_status_mail_notification_level': \
self.setting_job_status_mail_notification_level.name,
'profile_privacy_settings': {
'is_public': self.is_public,
'show_email': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL),
'show_last_seen': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN),
'show_member_since': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE)
}
}
if backrefs:
json_serializeable['role'] = \
self.role.to_json_serializeable(backrefs=True)
if relationships:
json_serializeable['corpus_follower_associations'] = {
x.hashid: x.to_json_serializeable()
for x in self.corpus_follower_associations
}
json_serializeable['corpora'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.corpora
}
json_serializeable['jobs'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.jobs
}
json_serializeable['tesseract_ocr_pipeline_models'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.tesseract_ocr_pipeline_models
}
json_serializeable['spacy_nlp_pipeline_models'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.spacy_nlp_pipeline_models
}
if filter_by_privacy_settings:
if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL):
json_serializeable.pop('email')
if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN):
json_serializeable.pop('last_seen')
if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE):
json_serializeable.pop('member_since')
return json_serializeable

View File

@ -1,12 +1,11 @@
from flask import Blueprint
from flask_login import login_required
import os
from pathlib import Path
import yaml
services_file = \
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'services.yml')
with open(services_file, 'r') as f:
services_file = Path(__file__).parent / 'services.yml'
with services_file.open('r') as f:
SERVICES = yaml.safe_load(f)
bp = Blueprint('services', __name__)

View File

@ -1,5 +1,4 @@
from flask import abort, current_app, flash, Markup, redirect, render_template, request, url_for
from flask_breadcrumbs import register_breadcrumb
from flask import abort, current_app, flash, redirect, render_template, request, url_for
from flask_login import current_user
import requests
from app import db, hashids
@ -20,13 +19,11 @@ from .forms import (
@bp.route('/services')
@register_breadcrumb(bp, '.', 'Services')
def services():
return redirect(url_for('main.dashboard'))
@bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.file_setup_pipeline', '<i class="nopaque-icons service-icons left" data-service="file-setup-pipeline"></i>File Setup')
def file_setup_pipeline():
service = 'file-setup-pipeline'
service_manifest = SERVICES[service]
@ -56,7 +53,7 @@ def file_setup_pipeline():
abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
message = f'Job "<a href="{job.url}">{job.title}</a>" created'
flash(message, 'job')
return {}, 201, {'Location': job.url}
return render_template(
@ -67,7 +64,6 @@ def file_setup_pipeline():
@bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.tesseract_ocr_pipeline', '<i class="nopaque-icons service-icons left" data-service="tesseract-ocr-pipeline"></i>Tesseract OCR Pipeline')
def tesseract_ocr_pipeline():
service_name = 'tesseract-ocr-pipeline'
service_manifest = SERVICES[service_name]
@ -100,7 +96,7 @@ def tesseract_ocr_pipeline():
abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
message = f'Job "<a href="{job.url}">{job.title}</a>" created'
flash(message, 'job')
return {}, 201, {'Location': job.url}
tesseract_ocr_pipeline_models = [
@ -118,7 +114,6 @@ def tesseract_ocr_pipeline():
@bp.route('/transkribus-htr-pipeline', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.transkribus_htr_pipeline', '<i class="nopaque-icons service-icons left" data-service="transkribus-htr-pipeline"></i>Transkribus HTR Pipeline')
def transkribus_htr_pipeline():
if not current_app.config.get('NOPAQUE_TRANSKRIBUS_ENABLED'):
abort(404)
@ -164,7 +159,7 @@ def transkribus_htr_pipeline():
abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
message = f'Job "<a href="{job.url}">{job.title}</a>" created'
flash(message, 'job')
return {}, 201, {'Location': job.url}
return render_template(
@ -176,7 +171,6 @@ def transkribus_htr_pipeline():
@bp.route('/spacy-nlp-pipeline', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.spacy_nlp_pipeline', '<i class="nopaque-icons service-icons left" data-service="spacy-nlp-pipeline"></i>SpaCy NLP Pipeline')
def spacy_nlp_pipeline():
service = 'spacy-nlp-pipeline'
service_manifest = SERVICES[service]
@ -210,7 +204,7 @@ def spacy_nlp_pipeline():
abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
message = f'Job "<a href="{job.url}">{job.title}</a>" created'
flash(message, 'job')
return {}, 201, {'Location': job.url}
return render_template(
@ -223,7 +217,6 @@ def spacy_nlp_pipeline():
@bp.route('/corpus-analysis')
@register_breadcrumb(bp, '.corpus_analysis', '<i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus Analysis')
def corpus_analysis():
return render_template(
'services/corpus_analysis.html.j2',

View File

@ -59,3 +59,8 @@ spacy-nlp-pipeline:
- 'encoding_detection'
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.1'
0.1.2:
methods:
- 'encoding_detection'
publishing_year: 2024
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2'

View File

@ -1,12 +1,10 @@
from flask import g, url_for
from flask_breadcrumbs import register_breadcrumb
from flask_login import current_user
from app.users.settings.routes import settings as settings_route
from . import bp
@bp.route('/settings', methods=['GET', 'POST'])
@register_breadcrumb(bp, '.', '<i class="material-icons left">settings</i>Settings')
def settings():
g._nopaque_redirect_location_on_post = url_for('.settings')
return settings_route(current_user.id)

View File

@ -1,290 +0,0 @@
/// Map deep get
/// @author Kitty Giraudel
/// @access public
/// @param {Map} $map - Map
/// @param {Arglist} $keys - Key chain
/// @return {*} - Desired value
@function map-deep-get($map, $keys...) {
@each $key in $keys {
$map: map-get($map, $key);
}
@return $map;
}
$color: (
"baseline": (
"primary": #00426f,
"primary-variant": #1a5c89,
"secondary": #00426f,
"secondary-variant": #1a5c89,
"background": #ffffff,
"surface": #ffffff,
"error": #b00020
),
"social-area": (
"base": #d6ae86,
"darken": #C98536,
"lighten": #EAE2DB
),
"service": (
"corpus-analysis": (
"base": #aa9cc9,
"darken": #6b3f89,
"lighten": #ebe8f6
),
"file-setup-pipeline": (
"base": #d5dc95,
"darken": #a1b300,
"lighten": #f2f3e1
),
"spacy-nlp-pipeline": (
"base": #98acd2,
"darken": #0064a3,
"lighten": #e5e8f5
),
"tesseract-ocr-pipeline": (
"base": #a9d8c8,
"darken": #00a58b,
"lighten": #e7f4f1
),
"transkribus-htr-pipeline": (
"base": #607d8b,
"darken": #37474f,
"lighten": #cfd8dc
)
),
"status": (
"corpus": (
"UNPREPARED": #9e9e9e,
"QUEUED": #2196f3,
"BUILDING": #ffc107,
"BUILT": #4caf50,
"FAILED": #f44336,
"STARTING_ANALYSIS_SESSION": #2196f3,
"RUNNING_ANALYSIS_SESSION": #4caf50,
"CANCELING_ANALYSIS_SESSION": #ff5722
),
"job": (
"INITIALIZING": #9e9e9e,
"SUBMITTED": #9e9e9e,
"QUEUED": #2196f3,
"RUNNING": #ffc107,
"CANCELING": #ff5722,
"CANCELED": #ff5722,
"COMPLETED": #4caf50,
"FAILED": #f44336
)
),
"s-attr": (
"PERSON": #a6e22d,
"PER": #a6e22d,
"NORP": #ef60b4,
"FACILITY": #43c6fc,
"ORG": #43c6fc,
"GPE": #fd9720,
"LOC": #fd9720,
"PRODUCT": #a99dfb,
"MISC": #a99dfb,
"EVENT": #fc0,
"WORK_OF_ART": #fc0,
"LANGUAGE": #fc0,
"DATE": #2fbbab,
"TIME": #2fbbab,
"PERCENT": #bbb,
"MONEY": #bbb,
"QUANTITY": #bbb,
"ORDINAL": #bbb,
"CARDINAL": #bbb
)
);
@each $key, $color-code in map-get($color, "baseline") {
.#{$key}-color {
background-color: $color-code !important;
}
.#{$key}-color-border {
border-color: $color-code !important;
}
.#{$key}-color-text {
color: $color-code !important;
}
}
@each $key, $color-code in map-get($color, "social-area") {
.social-area-color-#{$key} {
background-color: $color-code !important;
}
.social-area-color-border-#{$key} {
border-color: $color-code !important;
}
}
@each $service-name, $color-palette in map-get($color, "service") {
.service-color[data-service="#{$service-name}"] {
background-color: map-get($color-palette, "base") !important;
&.darken {
background-color: map-get($color-palette, "darken") !important;
}
&.lighten {
background-color: map-get($color-palette, "lighten") !important;
}
}
.service-color-border[data-service="#{$service-name}"] {
border-color: map-get($color-palette, "base") !important;
&.border-darken {
border-color: map-get($color-palette, "darken") !important;
}
&.border-lighten {
border-color: map-get($color-palette, "lighten") !important;
}
}
.service-color-text[data-service="#{$service-name}"] {
color: map-get($color-palette, "base") !important;
&.text-darken {
color: map-get($color-palette, "darken") !important;
}
&.text-lighten {
color: map-get($color-palette, "lighten") !important;
}
}
.service-scheme[data-service="#{$service-name}"] {
background-color: map-get($color-palette, "lighten");
.btn, .btn-small, .btn-large, .btn-floating {
background-color: map-get($color-palette, "darken");
&:hover {
background-color: map-get($color-palette, "base");
}
}
.pagination {
li.active {
background-color: map-get($color-palette, "darken");
}
}
.table-of-contents {
a.active, a:hover {
border-color: map-get($color-palette, "darken");
}
}
.tabs {
.tab {
&.disabled {
a {
color: inherit;
&:hover {
color: change-color(map-get($color-palette, "darken"), $alpha: 0.15);
}
}
}
a {
color: inherit;
&:focus, &:hover, &.active {
color: map-get($color-palette, "darken");
}
&:focus, &.active, &.active:focus {
background-color: change-color(map-get($color-palette, "darken"), $alpha: 0.15);
}
}
}
.indicator {
background-color: map-get($color-palette, "darken");
}
}
}
}
@each $ressource-name, $color-palette in map-get($color, "status") {
@each $key, $color-code in $color-palette {
.#{$ressource-name}-status-color[data-status="#{$key}"] {
background-color: $color-code !important;
}
.#{$ressource-name}-status-color-border[data-status="#{$key}"] {
border-color: $color-code !important;
}
.#{$ressource-name}-status-color-text[data-status="#{$key}"] {
color: $color-code !important;
}
}
}
@each $key, $color-code in map-get($color, "s-attr") {
.chip.s-attr[data-s-attr-type="ent"][data-s-attr-ent-type="#{$key}"] {
background-color: $color-code !important;
}
}
main {
.btn, .btn-small, .btn-large, .btn-floating {
background-color: map-deep-get($color, "baseline", "secondary");
&:hover {
background-color: map-deep-get($color, "baseline", "secondary-variant");
}
}
.pagination {
li.active {
background-color: map-deep-get($color, "baseline", "secondary");
}
}
.table-of-contents {
a.active, a:hover {
border-color: map-deep-get($color, "baseline", "secondary");
}
}
.tabs {
.tab {
&.disabled {
a {
color: inherit;
&:hover {
color: change-color(map-deep-get($color, "baseline", "secondary"), $alpha: 0.15);
}
}
}
a {
color: inherit;
&:focus, &:hover, &.active {
color: map-deep-get($color, "baseline", "secondary");
}
&:focus, &.active, &.active:focus {
background-color: change-color(map-deep-get($color, "baseline", "secondary"), $alpha: 0.15);
}
}
}
.indicator {
background-color: map-deep-get($color, "baseline", "secondary");
}
}
}

View File

@ -1,8 +0,0 @@
.parallax-container .parallax {
z-index: 0;
}
.autocomplete-content {
width: 100% !important;
left: 0 !important;
}

View File

@ -1,12 +0,0 @@
/*
* The sidenav-fixed class is used which causes the sidenav to be fixed and open
* on large screens and hides to the regular functionality on smaller screens.
* In order to prevent the sidenav to overlap the content, the content (in our
* case header, main and footer) gets an offset equal to the width of the
* sidenav.
*/
@media only screen and (min-width : 993px) {
header, main, footer {padding-left: 300px;}
.modal:not(.bottom-sheet) {left: 300px;}
.navbar-fixed > nav {width: calc(100% - 300px)}
}

View File

@ -1,67 +0,0 @@
/* Change navbar height bacause an extended and fixed navbar is used */
.navbar-fixed {
height: 112px;
}
/* Change placholdertext color of file uplaod fields */
::placeholder {
color: #9e9e9e;
opacity: 1;
}
/*
* changes preoloader size etc. to fit visually better with the chip status
* indicator of jobs
*/
.status-spinner {
margin-bottom: -10px;
width: 30px !important;
height: 30px !important;
}
#manual-modal .manual-chapter-title {
display: none;
}
.show-if-only-child:not(:only-child) {
display: none !important;
}
.btn-scale-x2 {
transform: scale(2);
}
.btn-scale-x2 .nopaque-icons.service-icons {
font-size: 2.5rem;
}
/* Fix material icon vertical alignment when nested in various elements */
h1 .nopaque-icons, h2 .nopaque-icons, h3 .nopaque-icons, h4 .nopaque-icons, .tab .nopaque-icons, .tab .material-icons {
line-height: inherit;
}
.corpus-status-text, .job-status-text {text-transform: lowercase;}
.corpus-status-text[data-status]:empty::before, .job-status-text[data-status]:empty::before {content: attr(data-status);}
.service-scheme[data-service="file-setup-pipeline"] .nopaque-icons.service-icons[data-service="inherit"]:empty::before {content: "E";}
.service-scheme[data-service="tesseract-ocr-pipeline"] .nopaque-icons.service-icons[data-service="inherit"]:empty::before {content: "F";}
.service-scheme[data-service="transkribus-htr-pipeline"] .nopaque-icons.service-icons[data-service="inherit"]:empty::before {content: "F";}
.service-scheme[data-service="spacy-nlp-pipeline"] .nopaque-icons.service-icons[data-service="inherit"]:empty::before {content: "G";}
.service-scheme[data-service="corpus-analysis"] .nopaque-icons.service-icons[data-service="inherit"]:empty::before {content: "H";}
.nopaque-icons.service-icons[data-service="file-setup-pipeline"]:empty::before {content: "E";}
.nopaque-icons.service-icons[data-service="tesseract-ocr-pipeline"]:empty::before {content: "F";}
.nopaque-icons.service-icons[data-service="transkribus-htr-pipeline"]:empty::before {content: "F";}
.nopaque-icons.service-icons[data-service="spacy-nlp-pipeline"]:empty::before {content: "G";}
.nopaque-icons.service-icons[data-service="corpus-analysis"]:empty::before {content: "H";}
[draggable="true"] {cursor: move !important;}
.clickable {cursor: pointer !important;}
.chip.s-attr .chip.p-attr {background-color: inherit;}
.width-25 {width: 25%;}
.width-50 {width: 50%;}
.width-75 {width: 75%;}
.width-100 {width: 100%;}

22
app/static/external/JSON-Patch/LICENSE vendored Normal file
View File

@ -0,0 +1,22 @@
(The MIT License)
Copyright (c) 2013, 2014, 2020 Joachim Wester
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -0,0 +1 @@
3.1.1

View File

@ -0,0 +1,928 @@
/*! fast-json-patch, version: 3.1.1 */
var jsonpatch =
/******/ (function(modules) { // webpackBootstrap
/******/ // The module cache
/******/ var installedModules = {};
/******/
/******/ // The require function
/******/ function __webpack_require__(moduleId) {
/******/
/******/ // Check if module is in cache
/******/ if(installedModules[moduleId]) {
/******/ return installedModules[moduleId].exports;
/******/ }
/******/ // Create a new module (and put it into the cache)
/******/ var module = installedModules[moduleId] = {
/******/ i: moduleId,
/******/ l: false,
/******/ exports: {}
/******/ };
/******/
/******/ // Execute the module function
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/
/******/ // Flag the module as loaded
/******/ module.l = true;
/******/
/******/ // Return the exports of the module
/******/ return module.exports;
/******/ }
/******/
/******/
/******/ // expose the modules object (__webpack_modules__)
/******/ __webpack_require__.m = modules;
/******/
/******/ // expose the module cache
/******/ __webpack_require__.c = installedModules;
/******/
/******/ // define getter function for harmony exports
/******/ __webpack_require__.d = function(exports, name, getter) {
/******/ if(!__webpack_require__.o(exports, name)) {
/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter });
/******/ }
/******/ };
/******/
/******/ // define __esModule on exports
/******/ __webpack_require__.r = function(exports) {
/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) {
/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
/******/ }
/******/ Object.defineProperty(exports, '__esModule', { value: true });
/******/ };
/******/
/******/ // create a fake namespace object
/******/ // mode & 1: value is a module id, require it
/******/ // mode & 2: merge all properties of value into the ns
/******/ // mode & 4: return value when already ns object
/******/ // mode & 8|1: behave like require
/******/ __webpack_require__.t = function(value, mode) {
/******/ if(mode & 1) value = __webpack_require__(value);
/******/ if(mode & 8) return value;
/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value;
/******/ var ns = Object.create(null);
/******/ __webpack_require__.r(ns);
/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value });
/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key));
/******/ return ns;
/******/ };
/******/
/******/ // getDefaultExport function for compatibility with non-harmony modules
/******/ __webpack_require__.n = function(module) {
/******/ var getter = module && module.__esModule ?
/******/ function getDefault() { return module['default']; } :
/******/ function getModuleExports() { return module; };
/******/ __webpack_require__.d(getter, 'a', getter);
/******/ return getter;
/******/ };
/******/
/******/ // Object.prototype.hasOwnProperty.call
/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); };
/******/
/******/ // __webpack_public_path__
/******/ __webpack_require__.p = "";
/******/
/******/
/******/ // Load entry module and return exports
/******/ return __webpack_require__(__webpack_require__.s = 2);
/******/ })
/************************************************************************/
/******/ ([
/* 0 */
/***/ (function(module, exports) {
/*!
* https://github.com/Starcounter-Jack/JSON-Patch
* (c) 2017-2022 Joachim Wester
* MIT licensed
*/
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
var _hasOwnProperty = Object.prototype.hasOwnProperty;
function hasOwnProperty(obj, key) {
return _hasOwnProperty.call(obj, key);
}
exports.hasOwnProperty = hasOwnProperty;
function _objectKeys(obj) {
if (Array.isArray(obj)) {
var keys_1 = new Array(obj.length);
for (var k = 0; k < keys_1.length; k++) {
keys_1[k] = "" + k;
}
return keys_1;
}
if (Object.keys) {
return Object.keys(obj);
}
var keys = [];
for (var i in obj) {
if (hasOwnProperty(obj, i)) {
keys.push(i);
}
}
return keys;
}
exports._objectKeys = _objectKeys;
;
/**
* Deeply clone the object.
* https://jsperf.com/deep-copy-vs-json-stringify-json-parse/25 (recursiveDeepCopy)
* @param {any} obj value to clone
* @return {any} cloned obj
*/
function _deepClone(obj) {
switch (typeof obj) {
case "object":
return JSON.parse(JSON.stringify(obj)); //Faster than ES5 clone - http://jsperf.com/deep-cloning-of-objects/5
case "undefined":
return null; //this is how JSON.stringify behaves for array items
default:
return obj; //no need to clone primitives
}
}
exports._deepClone = _deepClone;
//3x faster than cached /^\d+$/.test(str)
function isInteger(str) {
var i = 0;
var len = str.length;
var charCode;
while (i < len) {
charCode = str.charCodeAt(i);
if (charCode >= 48 && charCode <= 57) {
i++;
continue;
}
return false;
}
return true;
}
exports.isInteger = isInteger;
/**
* Escapes a json pointer path
* @param path The raw pointer
* @return the Escaped path
*/
function escapePathComponent(path) {
if (path.indexOf('/') === -1 && path.indexOf('~') === -1)
return path;
return path.replace(/~/g, '~0').replace(/\//g, '~1');
}
exports.escapePathComponent = escapePathComponent;
/**
* Unescapes a json pointer path
* @param path The escaped pointer
* @return The unescaped path
*/
function unescapePathComponent(path) {
return path.replace(/~1/g, '/').replace(/~0/g, '~');
}
exports.unescapePathComponent = unescapePathComponent;
function _getPathRecursive(root, obj) {
var found;
for (var key in root) {
if (hasOwnProperty(root, key)) {
if (root[key] === obj) {
return escapePathComponent(key) + '/';
}
else if (typeof root[key] === 'object') {
found = _getPathRecursive(root[key], obj);
if (found != '') {
return escapePathComponent(key) + '/' + found;
}
}
}
}
return '';
}
exports._getPathRecursive = _getPathRecursive;
function getPath(root, obj) {
if (root === obj) {
return '/';
}
var path = _getPathRecursive(root, obj);
if (path === '') {
throw new Error("Object not found in root");
}
return "/" + path;
}
exports.getPath = getPath;
/**
* Recursively checks whether an object has any undefined values inside.
*/
function hasUndefined(obj) {
if (obj === undefined) {
return true;
}
if (obj) {
if (Array.isArray(obj)) {
for (var i_1 = 0, len = obj.length; i_1 < len; i_1++) {
if (hasUndefined(obj[i_1])) {
return true;
}
}
}
else if (typeof obj === "object") {
var objKeys = _objectKeys(obj);
var objKeysLength = objKeys.length;
for (var i = 0; i < objKeysLength; i++) {
if (hasUndefined(obj[objKeys[i]])) {
return true;
}
}
}
}
return false;
}
exports.hasUndefined = hasUndefined;
function patchErrorMessageFormatter(message, args) {
var messageParts = [message];
for (var key in args) {
var value = typeof args[key] === 'object' ? JSON.stringify(args[key], null, 2) : args[key]; // pretty print
if (typeof value !== 'undefined') {
messageParts.push(key + ": " + value);
}
}
return messageParts.join('\n');
}
var PatchError = /** @class */ (function (_super) {
__extends(PatchError, _super);
function PatchError(message, name, index, operation, tree) {
var _newTarget = this.constructor;
var _this = _super.call(this, patchErrorMessageFormatter(message, { name: name, index: index, operation: operation, tree: tree })) || this;
_this.name = name;
_this.index = index;
_this.operation = operation;
_this.tree = tree;
Object.setPrototypeOf(_this, _newTarget.prototype); // restore prototype chain, see https://stackoverflow.com/a/48342359
_this.message = patchErrorMessageFormatter(message, { name: name, index: index, operation: operation, tree: tree });
return _this;
}
return PatchError;
}(Error));
exports.PatchError = PatchError;
/***/ }),
/* 1 */
/***/ (function(module, exports, __webpack_require__) {
Object.defineProperty(exports, "__esModule", { value: true });
var helpers_js_1 = __webpack_require__(0);
exports.JsonPatchError = helpers_js_1.PatchError;
exports.deepClone = helpers_js_1._deepClone;
/* We use a Javascript hash to store each
function. Each hash entry (property) uses
the operation identifiers specified in rfc6902.
In this way, we can map each patch operation
to its dedicated function in efficient way.
*/
/* The operations applicable to an object */
var objOps = {
add: function (obj, key, document) {
obj[key] = this.value;
return { newDocument: document };
},
remove: function (obj, key, document) {
var removed = obj[key];
delete obj[key];
return { newDocument: document, removed: removed };
},
replace: function (obj, key, document) {
var removed = obj[key];
obj[key] = this.value;
return { newDocument: document, removed: removed };
},
move: function (obj, key, document) {
/* in case move target overwrites an existing value,
return the removed value, this can be taxing performance-wise,
and is potentially unneeded */
var removed = getValueByPointer(document, this.path);
if (removed) {
removed = helpers_js_1._deepClone(removed);
}
var originalValue = applyOperation(document, { op: "remove", path: this.from }).removed;
applyOperation(document, { op: "add", path: this.path, value: originalValue });
return { newDocument: document, removed: removed };
},
copy: function (obj, key, document) {
var valueToCopy = getValueByPointer(document, this.from);
// enforce copy by value so further operations don't affect source (see issue #177)
applyOperation(document, { op: "add", path: this.path, value: helpers_js_1._deepClone(valueToCopy) });
return { newDocument: document };
},
test: function (obj, key, document) {
return { newDocument: document, test: _areEquals(obj[key], this.value) };
},
_get: function (obj, key, document) {
this.value = obj[key];
return { newDocument: document };
}
};
/* The operations applicable to an array. Many are the same as for the object */
var arrOps = {
add: function (arr, i, document) {
if (helpers_js_1.isInteger(i)) {
arr.splice(i, 0, this.value);
}
else { // array props
arr[i] = this.value;
}
// this may be needed when using '-' in an array
return { newDocument: document, index: i };
},
remove: function (arr, i, document) {
var removedList = arr.splice(i, 1);
return { newDocument: document, removed: removedList[0] };
},
replace: function (arr, i, document) {
var removed = arr[i];
arr[i] = this.value;
return { newDocument: document, removed: removed };
},
move: objOps.move,
copy: objOps.copy,
test: objOps.test,
_get: objOps._get
};
/**
* Retrieves a value from a JSON document by a JSON pointer.
* Returns the value.
*
* @param document The document to get the value from
* @param pointer an escaped JSON pointer
* @return The retrieved value
*/
function getValueByPointer(document, pointer) {
if (pointer == '') {
return document;
}
var getOriginalDestination = { op: "_get", path: pointer };
applyOperation(document, getOriginalDestination);
return getOriginalDestination.value;
}
exports.getValueByPointer = getValueByPointer;
/**
* Apply a single JSON Patch Operation on a JSON document.
* Returns the {newDocument, result} of the operation.
* It modifies the `document` and `operation` objects - it gets the values by reference.
* If you would like to avoid touching your values, clone them:
* `jsonpatch.applyOperation(document, jsonpatch._deepClone(operation))`.
*
* @param document The document to patch
* @param operation The operation to apply
* @param validateOperation `false` is without validation, `true` to use default jsonpatch's validation, or you can pass a `validateOperation` callback to be used for validation.
* @param mutateDocument Whether to mutate the original document or clone it before applying
* @param banPrototypeModifications Whether to ban modifications to `__proto__`, defaults to `true`.
* @return `{newDocument, result}` after the operation
*/
function applyOperation(document, operation, validateOperation, mutateDocument, banPrototypeModifications, index) {
if (validateOperation === void 0) { validateOperation = false; }
if (mutateDocument === void 0) { mutateDocument = true; }
if (banPrototypeModifications === void 0) { banPrototypeModifications = true; }
if (index === void 0) { index = 0; }
if (validateOperation) {
if (typeof validateOperation == 'function') {
validateOperation(operation, 0, document, operation.path);
}
else {
validator(operation, 0);
}
}
/* ROOT OPERATIONS */
if (operation.path === "") {
var returnValue = { newDocument: document };
if (operation.op === 'add') {
returnValue.newDocument = operation.value;
return returnValue;
}
else if (operation.op === 'replace') {
returnValue.newDocument = operation.value;
returnValue.removed = document; //document we removed
return returnValue;
}
else if (operation.op === 'move' || operation.op === 'copy') { // it's a move or copy to root
returnValue.newDocument = getValueByPointer(document, operation.from); // get the value by json-pointer in `from` field
if (operation.op === 'move') { // report removed item
returnValue.removed = document;
}
return returnValue;
}
else if (operation.op === 'test') {
returnValue.test = _areEquals(document, operation.value);
if (returnValue.test === false) {
throw new exports.JsonPatchError("Test operation failed", 'TEST_OPERATION_FAILED', index, operation, document);
}
returnValue.newDocument = document;
return returnValue;
}
else if (operation.op === 'remove') { // a remove on root
returnValue.removed = document;
returnValue.newDocument = null;
return returnValue;
}
else if (operation.op === '_get') {
operation.value = document;
return returnValue;
}
else { /* bad operation */
if (validateOperation) {
throw new exports.JsonPatchError('Operation `op` property is not one of operations defined in RFC-6902', 'OPERATION_OP_INVALID', index, operation, document);
}
else {
return returnValue;
}
}
} /* END ROOT OPERATIONS */
else {
if (!mutateDocument) {
document = helpers_js_1._deepClone(document);
}
var path = operation.path || "";
var keys = path.split('/');
var obj = document;
var t = 1; //skip empty element - http://jsperf.com/to-shift-or-not-to-shift
var len = keys.length;
var existingPathFragment = undefined;
var key = void 0;
var validateFunction = void 0;
if (typeof validateOperation == 'function') {
validateFunction = validateOperation;
}
else {
validateFunction = validator;
}
while (true) {
key = keys[t];
if (key && key.indexOf('~') != -1) {
key = helpers_js_1.unescapePathComponent(key);
}
if (banPrototypeModifications &&
(key == '__proto__' ||
(key == 'prototype' && t > 0 && keys[t - 1] == 'constructor'))) {
throw new TypeError('JSON-Patch: modifying `__proto__` or `constructor/prototype` prop is banned for security reasons, if this was on purpose, please set `banPrototypeModifications` flag false and pass it to this function. More info in fast-json-patch README');
}
if (validateOperation) {
if (existingPathFragment === undefined) {
if (obj[key] === undefined) {
existingPathFragment = keys.slice(0, t).join('/');
}
else if (t == len - 1) {
existingPathFragment = operation.path;
}
if (existingPathFragment !== undefined) {
validateFunction(operation, 0, document, existingPathFragment);
}
}
}
t++;
if (Array.isArray(obj)) {
if (key === '-') {
key = obj.length;
}
else {
if (validateOperation && !helpers_js_1.isInteger(key)) {
throw new exports.JsonPatchError("Expected an unsigned base-10 integer value, making the new referenced value the array element with the zero-based index", "OPERATION_PATH_ILLEGAL_ARRAY_INDEX", index, operation, document);
} // only parse key when it's an integer for `arr.prop` to work
else if (helpers_js_1.isInteger(key)) {
key = ~~key;
}
}
if (t >= len) {
if (validateOperation && operation.op === "add" && key > obj.length) {
throw new exports.JsonPatchError("The specified index MUST NOT be greater than the number of elements in the array", "OPERATION_VALUE_OUT_OF_BOUNDS", index, operation, document);
}
var returnValue = arrOps[operation.op].call(operation, obj, key, document); // Apply patch
if (returnValue.test === false) {
throw new exports.JsonPatchError("Test operation failed", 'TEST_OPERATION_FAILED', index, operation, document);
}
return returnValue;
}
}
else {
if (t >= len) {
var returnValue = objOps[operation.op].call(operation, obj, key, document); // Apply patch
if (returnValue.test === false) {
throw new exports.JsonPatchError("Test operation failed", 'TEST_OPERATION_FAILED', index, operation, document);
}
return returnValue;
}
}
obj = obj[key];
// If we have more keys in the path, but the next value isn't a non-null object,
// throw an OPERATION_PATH_UNRESOLVABLE error instead of iterating again.
if (validateOperation && t < len && (!obj || typeof obj !== "object")) {
throw new exports.JsonPatchError('Cannot perform operation at the desired path', 'OPERATION_PATH_UNRESOLVABLE', index, operation, document);
}
}
}
}
exports.applyOperation = applyOperation;
/**
* Apply a full JSON Patch array on a JSON document.
* Returns the {newDocument, result} of the patch.
* It modifies the `document` object and `patch` - it gets the values by reference.
* If you would like to avoid touching your values, clone them:
* `jsonpatch.applyPatch(document, jsonpatch._deepClone(patch))`.
*
* @param document The document to patch
* @param patch The patch to apply
* @param validateOperation `false` is without validation, `true` to use default jsonpatch's validation, or you can pass a `validateOperation` callback to be used for validation.
* @param mutateDocument Whether to mutate the original document or clone it before applying
* @param banPrototypeModifications Whether to ban modifications to `__proto__`, defaults to `true`.
* @return An array of `{newDocument, result}` after the patch
*/
function applyPatch(document, patch, validateOperation, mutateDocument, banPrototypeModifications) {
if (mutateDocument === void 0) { mutateDocument = true; }
if (banPrototypeModifications === void 0) { banPrototypeModifications = true; }
if (validateOperation) {
if (!Array.isArray(patch)) {
throw new exports.JsonPatchError('Patch sequence must be an array', 'SEQUENCE_NOT_AN_ARRAY');
}
}
if (!mutateDocument) {
document = helpers_js_1._deepClone(document);
}
var results = new Array(patch.length);
for (var i = 0, length_1 = patch.length; i < length_1; i++) {
// we don't need to pass mutateDocument argument because if it was true, we already deep cloned the object, we'll just pass `true`
results[i] = applyOperation(document, patch[i], validateOperation, true, banPrototypeModifications, i);
document = results[i].newDocument; // in case root was replaced
}
results.newDocument = document;
return results;
}
exports.applyPatch = applyPatch;
/**
* Apply a single JSON Patch Operation on a JSON document.
* Returns the updated document.
* Suitable as a reducer.
*
* @param document The document to patch
* @param operation The operation to apply
* @return The updated document
*/
function applyReducer(document, operation, index) {
var operationResult = applyOperation(document, operation);
if (operationResult.test === false) { // failed test
throw new exports.JsonPatchError("Test operation failed", 'TEST_OPERATION_FAILED', index, operation, document);
}
return operationResult.newDocument;
}
exports.applyReducer = applyReducer;
/**
* Validates a single operation. Called from `jsonpatch.validate`. Throws `JsonPatchError` in case of an error.
* @param {object} operation - operation object (patch)
* @param {number} index - index of operation in the sequence
* @param {object} [document] - object where the operation is supposed to be applied
* @param {string} [existingPathFragment] - comes along with `document`
*/
function validator(operation, index, document, existingPathFragment) {
if (typeof operation !== 'object' || operation === null || Array.isArray(operation)) {
throw new exports.JsonPatchError('Operation is not an object', 'OPERATION_NOT_AN_OBJECT', index, operation, document);
}
else if (!objOps[operation.op]) {
throw new exports.JsonPatchError('Operation `op` property is not one of operations defined in RFC-6902', 'OPERATION_OP_INVALID', index, operation, document);
}
else if (typeof operation.path !== 'string') {
throw new exports.JsonPatchError('Operation `path` property is not a string', 'OPERATION_PATH_INVALID', index, operation, document);
}
else if (operation.path.indexOf('/') !== 0 && operation.path.length > 0) {
// paths that aren't empty string should start with "/"
throw new exports.JsonPatchError('Operation `path` property must start with "/"', 'OPERATION_PATH_INVALID', index, operation, document);
}
else if ((operation.op === 'move' || operation.op === 'copy') && typeof operation.from !== 'string') {
throw new exports.JsonPatchError('Operation `from` property is not present (applicable in `move` and `copy` operations)', 'OPERATION_FROM_REQUIRED', index, operation, document);
}
else if ((operation.op === 'add' || operation.op === 'replace' || operation.op === 'test') && operation.value === undefined) {
throw new exports.JsonPatchError('Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)', 'OPERATION_VALUE_REQUIRED', index, operation, document);
}
else if ((operation.op === 'add' || operation.op === 'replace' || operation.op === 'test') && helpers_js_1.hasUndefined(operation.value)) {
throw new exports.JsonPatchError('Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)', 'OPERATION_VALUE_CANNOT_CONTAIN_UNDEFINED', index, operation, document);
}
else if (document) {
if (operation.op == "add") {
var pathLen = operation.path.split("/").length;
var existingPathLen = existingPathFragment.split("/").length;
if (pathLen !== existingPathLen + 1 && pathLen !== existingPathLen) {
throw new exports.JsonPatchError('Cannot perform an `add` operation at the desired path', 'OPERATION_PATH_CANNOT_ADD', index, operation, document);
}
}
else if (operation.op === 'replace' || operation.op === 'remove' || operation.op === '_get') {
if (operation.path !== existingPathFragment) {
throw new exports.JsonPatchError('Cannot perform the operation at a path that does not exist', 'OPERATION_PATH_UNRESOLVABLE', index, operation, document);
}
}
else if (operation.op === 'move' || operation.op === 'copy') {
var existingValue = { op: "_get", path: operation.from, value: undefined };
var error = validate([existingValue], document);
if (error && error.name === 'OPERATION_PATH_UNRESOLVABLE') {
throw new exports.JsonPatchError('Cannot perform the operation from a path that does not exist', 'OPERATION_FROM_UNRESOLVABLE', index, operation, document);
}
}
}
}
exports.validator = validator;
/**
* Validates a sequence of operations. If `document` parameter is provided, the sequence is additionally validated against the object document.
* If error is encountered, returns a JsonPatchError object
* @param sequence
* @param document
* @returns {JsonPatchError|undefined}
*/
function validate(sequence, document, externalValidator) {
try {
if (!Array.isArray(sequence)) {
throw new exports.JsonPatchError('Patch sequence must be an array', 'SEQUENCE_NOT_AN_ARRAY');
}
if (document) {
//clone document and sequence so that we can safely try applying operations
applyPatch(helpers_js_1._deepClone(document), helpers_js_1._deepClone(sequence), externalValidator || true);
}
else {
externalValidator = externalValidator || validator;
for (var i = 0; i < sequence.length; i++) {
externalValidator(sequence[i], i, document, undefined);
}
}
}
catch (e) {
if (e instanceof exports.JsonPatchError) {
return e;
}
else {
throw e;
}
}
}
exports.validate = validate;
// based on https://github.com/epoberezkin/fast-deep-equal
// MIT License
// Copyright (c) 2017 Evgeny Poberezkin
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
function _areEquals(a, b) {
if (a === b)
return true;
if (a && b && typeof a == 'object' && typeof b == 'object') {
var arrA = Array.isArray(a), arrB = Array.isArray(b), i, length, key;
if (arrA && arrB) {
length = a.length;
if (length != b.length)
return false;
for (i = length; i-- !== 0;)
if (!_areEquals(a[i], b[i]))
return false;
return true;
}
if (arrA != arrB)
return false;
var keys = Object.keys(a);
length = keys.length;
if (length !== Object.keys(b).length)
return false;
for (i = length; i-- !== 0;)
if (!b.hasOwnProperty(keys[i]))
return false;
for (i = length; i-- !== 0;) {
key = keys[i];
if (!_areEquals(a[key], b[key]))
return false;
}
return true;
}
return a !== a && b !== b;
}
exports._areEquals = _areEquals;
;
/***/ }),
/* 2 */
/***/ (function(module, exports, __webpack_require__) {
var core = __webpack_require__(1);
Object.assign(exports, core);
var duplex = __webpack_require__(3);
Object.assign(exports, duplex);
var helpers = __webpack_require__(0);
exports.JsonPatchError = helpers.PatchError;
exports.deepClone = helpers._deepClone;
exports.escapePathComponent = helpers.escapePathComponent;
exports.unescapePathComponent = helpers.unescapePathComponent;
/***/ }),
/* 3 */
/***/ (function(module, exports, __webpack_require__) {
Object.defineProperty(exports, "__esModule", { value: true });
/*!
* https://github.com/Starcounter-Jack/JSON-Patch
* (c) 2017-2021 Joachim Wester
* MIT license
*/
var helpers_js_1 = __webpack_require__(0);
var core_js_1 = __webpack_require__(1);
var beforeDict = new WeakMap();
var Mirror = /** @class */ (function () {
function Mirror(obj) {
this.observers = new Map();
this.obj = obj;
}
return Mirror;
}());
var ObserverInfo = /** @class */ (function () {
function ObserverInfo(callback, observer) {
this.callback = callback;
this.observer = observer;
}
return ObserverInfo;
}());
function getMirror(obj) {
return beforeDict.get(obj);
}
function getObserverFromMirror(mirror, callback) {
return mirror.observers.get(callback);
}
function removeObserverFromMirror(mirror, observer) {
mirror.observers.delete(observer.callback);
}
/**
* Detach an observer from an object
*/
function unobserve(root, observer) {
observer.unobserve();
}
exports.unobserve = unobserve;
/**
* Observes changes made to an object, which can then be retrieved using generate
*/
function observe(obj, callback) {
var patches = [];
var observer;
var mirror = getMirror(obj);
if (!mirror) {
mirror = new Mirror(obj);
beforeDict.set(obj, mirror);
}
else {
var observerInfo = getObserverFromMirror(mirror, callback);
observer = observerInfo && observerInfo.observer;
}
if (observer) {
return observer;
}
observer = {};
mirror.value = helpers_js_1._deepClone(obj);
if (callback) {
observer.callback = callback;
observer.next = null;
var dirtyCheck = function () {
generate(observer);
};
var fastCheck = function () {
clearTimeout(observer.next);
observer.next = setTimeout(dirtyCheck);
};
if (typeof window !== 'undefined') { //not Node
window.addEventListener('mouseup', fastCheck);
window.addEventListener('keyup', fastCheck);
window.addEventListener('mousedown', fastCheck);
window.addEventListener('keydown', fastCheck);
window.addEventListener('change', fastCheck);
}
}
observer.patches = patches;
observer.object = obj;
observer.unobserve = function () {
generate(observer);
clearTimeout(observer.next);
removeObserverFromMirror(mirror, observer);
if (typeof window !== 'undefined') {
window.removeEventListener('mouseup', fastCheck);
window.removeEventListener('keyup', fastCheck);
window.removeEventListener('mousedown', fastCheck);
window.removeEventListener('keydown', fastCheck);
window.removeEventListener('change', fastCheck);
}
};
mirror.observers.set(callback, new ObserverInfo(callback, observer));
return observer;
}
exports.observe = observe;
/**
* Generate an array of patches from an observer
*/
function generate(observer, invertible) {
if (invertible === void 0) { invertible = false; }
var mirror = beforeDict.get(observer.object);
_generate(mirror.value, observer.object, observer.patches, "", invertible);
if (observer.patches.length) {
core_js_1.applyPatch(mirror.value, observer.patches);
}
var temp = observer.patches;
if (temp.length > 0) {
observer.patches = [];
if (observer.callback) {
observer.callback(temp);
}
}
return temp;
}
exports.generate = generate;
// Dirty check if obj is different from mirror, generate patches and update mirror
function _generate(mirror, obj, patches, path, invertible) {
if (obj === mirror) {
return;
}
if (typeof obj.toJSON === "function") {
obj = obj.toJSON();
}
var newKeys = helpers_js_1._objectKeys(obj);
var oldKeys = helpers_js_1._objectKeys(mirror);
var changed = false;
var deleted = false;
//if ever "move" operation is implemented here, make sure this test runs OK: "should not generate the same patch twice (move)"
for (var t = oldKeys.length - 1; t >= 0; t--) {
var key = oldKeys[t];
var oldVal = mirror[key];
if (helpers_js_1.hasOwnProperty(obj, key) && !(obj[key] === undefined && oldVal !== undefined && Array.isArray(obj) === false)) {
var newVal = obj[key];
if (typeof oldVal == "object" && oldVal != null && typeof newVal == "object" && newVal != null && Array.isArray(oldVal) === Array.isArray(newVal)) {
_generate(oldVal, newVal, patches, path + "/" + helpers_js_1.escapePathComponent(key), invertible);
}
else {
if (oldVal !== newVal) {
changed = true;
if (invertible) {
patches.push({ op: "test", path: path + "/" + helpers_js_1.escapePathComponent(key), value: helpers_js_1._deepClone(oldVal) });
}
patches.push({ op: "replace", path: path + "/" + helpers_js_1.escapePathComponent(key), value: helpers_js_1._deepClone(newVal) });
}
}
}
else if (Array.isArray(mirror) === Array.isArray(obj)) {
if (invertible) {
patches.push({ op: "test", path: path + "/" + helpers_js_1.escapePathComponent(key), value: helpers_js_1._deepClone(oldVal) });
}
patches.push({ op: "remove", path: path + "/" + helpers_js_1.escapePathComponent(key) });
deleted = true; // property has been deleted
}
else {
if (invertible) {
patches.push({ op: "test", path: path, value: mirror });
}
patches.push({ op: "replace", path: path, value: obj });
changed = true;
}
}
if (!deleted && newKeys.length == oldKeys.length) {
return;
}
for (var t = 0; t < newKeys.length; t++) {
var key = newKeys[t];
if (!helpers_js_1.hasOwnProperty(mirror, key) && obj[key] !== undefined) {
patches.push({ op: "add", path: path + "/" + helpers_js_1.escapePathComponent(key), value: helpers_js_1._deepClone(obj[key]) });
}
}
}
/**
* Create an array of patches from the differences in two objects
*/
function compare(tree1, tree2, invertible) {
if (invertible === void 0) { invertible = false; }
var patches = [];
_generate(tree1, tree2, patches, '', invertible);
return patches;
}
exports.compare = compare;
/***/ })
/******/ ]);

File diff suppressed because one or more lines are too long

21
app/static/external/list.js/LICENSE vendored Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2011-2018 Jonny Strömberg, jonnystromberg.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

1
app/static/external/list.js/VERSION vendored Normal file
View File

@ -0,0 +1 @@
2.3.1

2020
app/static/external/list.js/js/list.js vendored Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"version":3,"file":"list.min.js","sources":["webpack://List/list.min.js"],"mappings":"AAAA","sourceRoot":""}

View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1 @@
https://github.com/google/material-design-icons/tree/f7bd4f25f3764883717c09a1fd867f560c9a9581

View File

@ -0,0 +1,37 @@
@font-face {
font-family: "Material Icons";
font-style: normal;
font-weight: 400;
/* For IE6-8 */
/* src: url("../font/MaterialIcons-Regular.eot"); */
src: local("Material Icons"),
local("MaterialIcons-Regular"),
/* url("../font/MaterialIcons-Regular.woff2") format('woff2'), */
/* url("../font/MaterialIcons-Regular.woff") format('woff'), */
url("../font/MaterialIcons-Regular.ttf") format("truetype");
}
.material-icons {
font-family: 'Material Icons';
font-weight: normal;
font-style: normal;
font-size: 24px; /* Preferred icon size */
display: inline-block;
line-height: 1;
text-transform: none;
letter-spacing: normal;
word-wrap: normal;
white-space: nowrap;
direction: ltr;
/* Support for all WebKit browsers. */
-webkit-font-smoothing: antialiased;
/* Support for Safari and Chrome. */
text-rendering: optimizeLegibility;
/* Support for Firefox. */
-moz-osx-font-smoothing: grayscale;
/* Support for IE. */
font-feature-settings: 'liga';
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

21
app/static/external/materialize/LICENSE vendored Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014-2018 Materialize
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1 @@
1.0.0

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,55 @@
// Badges
span.badge {
min-width: 3rem;
padding: 0 6px;
margin-left: 14px;
text-align: center;
font-size: 1rem;
line-height: $badge-height;
height: $badge-height;
color: color('grey', 'darken-1');
float: right;
box-sizing: border-box;
&.new {
font-weight: 300;
font-size: 0.8rem;
color: #fff;
background-color: $badge-bg-color;
border-radius: 2px;
}
&.new:after {
content: " new";
}
&[data-badge-caption]::after {
content: " " attr(data-badge-caption);
}
}
// Special cases
nav ul a span.badge {
display: inline-block;
float: none;
margin-left: 4px;
line-height: $badge-height;
height: $badge-height;
-webkit-font-smoothing: auto;
}
// Line height centering
.collection-item span.badge {
margin-top: calc(#{$collection-line-height / 2} - #{$badge-height / 2});
}
.collapsible span.badge {
margin-left: auto;
}
.sidenav span.badge {
margin-top: calc(#{$sidenav-line-height / 2} - #{$badge-height / 2});
}
table span.badge {
display: inline-block;
float: none;
margin-left: auto;
}

View File

@ -0,0 +1,322 @@
// shared styles
.btn,
.btn-flat {
border: $button-border;
border-radius: $button-radius;
display: inline-block;
height: $button-height;
line-height: $button-height;
padding: $button-padding;
text-transform: uppercase;
vertical-align: middle;
-webkit-tap-highlight-color: transparent; // Gets rid of tap active state
}
// Disabled shared style
.btn.disabled,
.btn-floating.disabled,
.btn-large.disabled,
.btn-small.disabled,
.btn-flat.disabled,
.btn:disabled,
.btn-floating:disabled,
.btn-large:disabled,
.btn-small:disabled,
.btn-flat:disabled,
.btn[disabled],
.btn-floating[disabled],
.btn-large[disabled],
.btn-small[disabled],
.btn-flat[disabled] {
pointer-events: none;
background-color: $button-disabled-background !important;
box-shadow: none;
color: $button-disabled-color !important;
cursor: default;
&:hover {
background-color: $button-disabled-background !important;
color: $button-disabled-color !important;
}
}
// Shared icon styles
.btn,
.btn-floating,
.btn-large,
.btn-small,
.btn-flat {
font-size: $button-font-size;
outline: 0;
i {
font-size: $button-icon-font-size;
line-height: inherit;
}
}
// Shared focus button style
.btn,
.btn-floating {
&:focus {
background-color: darken($button-raised-background, 10%);
}
}
// Raised Button
.btn {
text-decoration: none;
color: $button-raised-color;
background-color: $button-raised-background;
text-align: center;
letter-spacing: .5px;
@extend .z-depth-1;
transition: background-color .2s ease-out;
cursor: pointer;
&:hover {
background-color: $button-raised-background-hover;
@extend .z-depth-1-half;
}
}
// Floating button
.btn-floating {
&:hover {
background-color: $button-floating-background-hover;
@extend .z-depth-1-half;
}
&:before {
border-radius: 0;
}
&.btn-large {
&.halfway-fab {
bottom: -$button-floating-large-size / 2;
}
width: $button-floating-large-size;
height: $button-floating-large-size;
padding: 0;
i {
line-height: $button-floating-large-size;
}
}
&.btn-small {
&.halfway-fab {
bottom: -$button-floating-small-size / 2;
}
width: $button-floating-small-size;
height: $button-floating-small-size;
i {
line-height: $button-floating-small-size;
}
}
&.halfway-fab {
&.left {
right: auto;
left: 24px;
}
position: absolute;
right: 24px;
bottom: -$button-floating-size / 2;
}
display: inline-block;
color: $button-floating-color;
position: relative;
overflow: hidden;
z-index: 1;
width: $button-floating-size;
height: $button-floating-size;
line-height: $button-floating-size;
padding: 0;
background-color: $button-floating-background;
border-radius: $button-floating-radius;
@extend .z-depth-1;
transition: background-color .3s;
cursor: pointer;
vertical-align: middle;
i {
width: inherit;
display: inline-block;
text-align: center;
color: $button-floating-color;
font-size: $button-large-icon-font-size;
line-height: $button-floating-size;
}
}
// button fix
button.btn-floating {
border: $button-border;
}
// Fixed Action Button
.fixed-action-btn {
&.active {
ul {
visibility: visible;
}
}
// Directions
&.direction-left,
&.direction-right {
padding: 0 0 0 15px;
ul {
text-align: right;
right: 64px;
top: 50%;
transform: translateY(-50%);
height: 100%;
left: auto;
/*width 100% only goes to width of button container */
width: 500px;
li {
display: inline-block;
margin: 7.5px 15px 0 0;
}
}
}
&.direction-right {
padding: 0 15px 0 0;
ul {
text-align: left;
direction: rtl;
left: 64px;
right: auto;
li {
margin: 7.5px 0 0 15px;
}
}
}
&.direction-bottom {
padding: 0 0 15px 0;
ul {
top: 64px;
bottom: auto;
display: flex;
flex-direction: column-reverse;
li {
margin: 15px 0 0 0;
}
}
}
&.toolbar {
&.active {
&>a i {
opacity: 0;
}
}
padding: 0;
height: $button-floating-large-size;
ul {
display: flex;
top: 0;
bottom: 0;
z-index: 1;
li {
flex: 1;
display: inline-block;
margin: 0;
height: 100%;
transition: none;
a {
display: block;
overflow: hidden;
position: relative;
width: 100%;
height: 100%;
background-color: transparent;
box-shadow: none;
color: #fff;
line-height: $button-floating-large-size;
z-index: 1;
i {
line-height: inherit;
}
}
}
}
}
position: fixed;
right: 23px;
bottom: 23px;
padding-top: 15px;
margin-bottom: 0;
z-index: 997;
ul {
left: 0;
right: 0;
text-align: center;
position: absolute;
bottom: 64px;
margin: 0;
visibility: hidden;
li {
margin-bottom: 15px;
}
a.btn-floating {
opacity: 0;
}
}
.fab-backdrop {
position: absolute;
top: 0;
left: 0;
z-index: -1;
width: $button-floating-size;
height: $button-floating-size;
background-color: $button-floating-background;
border-radius: $button-floating-radius;
transform: scale(0);
}
}
// Flat button
.btn-flat {
box-shadow: none;
background-color: transparent;
color: $button-flat-color;
cursor: pointer;
transition: background-color .2s;
&:focus,
&:hover {
box-shadow: none;
}
&:focus {
background-color: rgba(0, 0, 0, .1);
}
&.disabled,
&.btn-flat[disabled] {
background-color: transparent !important;
color: $button-flat-disabled-color !important;
cursor: default;
}
}
// Large button
.btn-large {
@extend .btn;
height: $button-large-height;
line-height: $button-large-height;
font-size: $button-large-font-size;
padding: 0 28px;
i {
font-size: $button-large-icon-font-size;
}
}
// Small button
.btn-small {
@extend .btn;
height: $button-small-height;
line-height: $button-small-height;
font-size: $button-small-font-size;
i {
font-size: $button-small-icon-font-size;
}
}
// Block button
.btn-block {
display: block;
}

View File

@ -0,0 +1,195 @@
.card-panel {
transition: box-shadow .25s;
padding: $card-padding;
margin: $element-top-margin 0 $element-bottom-margin 0;
border-radius: 2px;
@extend .z-depth-1;
background-color: $card-bg-color;
}
.card {
position: relative;
margin: $element-top-margin 0 $element-bottom-margin 0;
background-color: $card-bg-color;
transition: box-shadow .25s;
border-radius: 2px;
@extend .z-depth-1;
.card-title {
font-size: 24px;
font-weight: 300;
&.activator {
cursor: pointer;
}
}
// Card Sizes
&.small, &.medium, &.large {
position: relative;
.card-image {
max-height: 60%;
overflow: hidden;
}
.card-image + .card-content {
max-height: 40%;
}
.card-content {
max-height: 100%;
overflow: hidden;
}
.card-action {
position: absolute;
bottom: 0;
left: 0;
right: 0;
}
}
&.small {
height: 300px;
}
&.medium {
height: 400px;
}
&.large {
height: 500px;
}
// Horizontal Cards
&.horizontal {
&.small, &.medium, &.large {
.card-image {
height: 100%;
max-height: none;
overflow: visible;
img {
height: 100%;
}
}
}
display: flex;
.card-image {
max-width: 50%;
img {
border-radius: 2px 0 0 2px;
max-width: 100%;
width: auto;
}
}
.card-stacked {
display: flex;
flex-direction: column;
flex: 1;
position: relative;
.card-content {
flex-grow: 1;
}
}
}
// Sticky Action Section
&.sticky-action {
.card-action {
z-index: 2;
}
.card-reveal {
z-index: 1;
padding-bottom: 64px;
}
}
.card-image {
position: relative;
// Image background for content
img {
display: block;
border-radius: 2px 2px 0 0;
position: relative;
left: 0;
right: 0;
top: 0;
bottom: 0;
width: 100%;
}
.card-title {
color: $card-bg-color;
position: absolute;
bottom: 0;
left: 0;
max-width: 100%;
padding: $card-padding;
}
}
.card-content {
padding: $card-padding;
border-radius: 0 0 2px 2px;
p {
margin: 0;
}
.card-title {
display: block;
line-height: 32px;
margin-bottom: 8px;
i {
line-height: 32px;
}
}
}
.card-action {
&:last-child {
border-radius: 0 0 2px 2px;
}
background-color: inherit; // Use inherit to inherit color classes
border-top: 1px solid rgba(160,160,160,.2);
position: relative;
padding: 16px $card-padding;
a:not(.btn):not(.btn-large):not(.btn-floating) {
color: $card-link-color;
margin-right: $card-padding;
transition: color .3s ease;
text-transform: uppercase;
&:hover { color: $card-link-color-light; }
}
}
.card-reveal {
padding: $card-padding;
position: absolute;
background-color: $card-bg-color;
width: 100%;
overflow-y: auto;
left: 0;
top: 100%;
height: 100%;
z-index: 3;
display: none;
.card-title {
cursor: pointer;
display: block;
}
}
}

View File

@ -0,0 +1,90 @@
.carousel {
&.carousel-slider {
top: 0;
left: 0;
.carousel-fixed-item {
&.with-indicators {
bottom: 68px;
}
position: absolute;
left: 0;
right: 0;
bottom: 20px;
z-index: 1;
}
.carousel-item {
width: 100%;
height: 100%;
min-height: $carousel-height;
position: absolute;
top: 0;
left: 0;
h2 {
font-size: 24px;
font-weight: 500;
line-height: 32px;
}
p {
font-size: 15px;
}
}
}
overflow: hidden;
position: relative;
width: 100%;
height: $carousel-height;
perspective: 500px;
transform-style: preserve-3d;
transform-origin: 0% 50%;
.carousel-item {
visibility: hidden;
width: $carousel-item-width;
height: $carousel-item-height;
position: absolute;
top: 0;
left: 0;
& > img {
width: 100%;
}
}
.indicators {
position: absolute;
text-align: center;
left: 0;
right: 0;
bottom: 0;
margin: 0;
.indicator-item {
&.active {
background-color: #fff;
}
display: inline-block;
position: relative;
cursor: pointer;
height: 8px;
width: 8px;
margin: 24px 4px;
background-color: rgba(255,255,255,.5);
transition: background-color .3s;
border-radius: 50%;
}
}
// Materialbox compatibility
&.scrolling .carousel-item .materialboxed,
.carousel-item:not(.active) .materialboxed {
pointer-events: none;
}
}

Some files were not shown because too many files have changed in this diff Show More