move blueprints in dedicated folder

app/blueprints/admin/__init__.py

@@ -0,0 +1,20 @@
+from flask import Blueprint
+from flask_login import login_required
+from app.decorators import admin_required
+
+
+bp = Blueprint('admin', __name__)
+
+
+@bp.before_request
+@login_required
+@admin_required
+def before_request():
+    '''
+    Ensures that the routes in this package can be visited only by users with
+    administrator privileges (login_required and admin_required).
+    '''
+    pass
+
+
+from . import json_routes, routes

app/blueprints/admin/events.py

@@ -0,0 +1,49 @@
+from flask_login import current_user
+from flask_socketio import disconnect, Namespace
+from app import db, hashids
+from app.decorators import socketio_admin_required
+from app.models import User
+
+
+class AdminNamespace(Namespace):
+    def on_connect(self):
+        # Check if the user is authenticated and is an administrator
+        if not (current_user.is_authenticated and current_user.is_administrator):
+            disconnect()
+
+
+    @socketio_admin_required
+    def on_set_user_confirmed(self, user_hashid: str, confirmed_value: bool):
+        # Decode the user hashid
+        user_id = hashids.decode(user_hashid)
+
+        # Validate user_id
+        if not isinstance(user_id, int):
+            return {
+                'code': 400,
+                'body': 'user_id is invalid'
+            }
+
+        # Validate confirmed_value
+        if not isinstance(confirmed_value, bool):
+            return {
+                'code': 400,
+                'body': 'confirmed_value is invalid'
+            }
+
+        # Load user from database
+        user = User.query.get(user_id)
+        if user is None:
+            return {
+                'code': 404,
+                'body': 'User not found'
+            }
+
+        # Update user confirmed status
+        user.confirmed = confirmed_value
+        db.session.commit()
+
+        return {
+            'code': 200,
+            'body': f'User "{user.username}" is now {"confirmed" if confirmed_value else "unconfirmed"}'
+        }

app/blueprints/admin/forms.py

@@ -0,0 +1,16 @@
+from flask_wtf import FlaskForm
+from wtforms import SelectField, SubmitField
+from app.models import Role
+
+
+class UpdateUserForm(FlaskForm):
+    role = SelectField('Role')
+    submit = SubmitField()
+
+    def __init__(self, user, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = {'role': user.role.hashid}
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-user-form'
+        super().__init__(*args, **kwargs)
+        self.role.choices = [(x.hashid, x.name) for x in Role.query.all()]

app/blueprints/admin/json_routes.py

@@ -0,0 +1,23 @@
+from flask import abort, request
+from app.decorators import content_negotiation
+from app import db
+from app.models import User
+from . import bp
+
+
+@bp.route('/users/<hashid:user_id>/confirmed', methods=['PUT'])
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_user_role(user_id):
+    confirmed = request.json
+    if not isinstance(confirmed, bool):
+        abort(400)
+    user = User.query.get_or_404(user_id)
+    user.confirmed = confirmed
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'User "{user.username}" is now '
+            f'{"confirmed" if confirmed else "unconfirmed"}'
+        )
+    }
+    return response_data, 200

app/blueprints/admin/routes.py

@@ -0,0 +1,136 @@
+from flask import abort, flash, redirect, render_template, url_for
+from app import db, hashids
+from app.models import Avatar, Corpus, Role, User
+from app.blueprints.users.settings.forms import (
+    UpdateAvatarForm,
+    UpdatePasswordForm,
+    UpdateNotificationsForm,
+    UpdateAccountInformationForm,
+    UpdateProfileInformationForm
+)
+from . import bp
+from .forms import UpdateUserForm
+
+
+@bp.route('')
+def admin():
+    return render_template(
+        'admin/admin.html.j2',
+        title='Administration'
+    )
+
+
+@bp.route('/corpora')
+def corpora():
+    corpora = Corpus.query.all()
+    return render_template(
+        'admin/corpora.html.j2',
+        title='Corpora',
+        corpora=corpora
+    )
+
+
+@bp.route('/users')
+def users():
+    users = User.query.all()
+    return render_template(
+        'admin/users.html.j2',
+        title='Users',
+        users=users
+    )
+
+
+@bp.route('/users/<hashid:user_id>')
+def user(user_id):
+    user = User.query.get_or_404(user_id)
+    corpora = Corpus.query.filter(Corpus.user == user).all()
+    return render_template(
+        'admin/user.html.j2',
+        title=user.username,
+        user=user,
+        corpora=corpora
+    )
+
+
+@bp.route('/users/<hashid:user_id>/settings', methods=['GET', 'POST'])
+def user_settings(user_id):
+    user = User.query.get_or_404(user_id)
+    update_account_information_form = UpdateAccountInformationForm(user)
+    update_profile_information_form = UpdateProfileInformationForm(user)
+    update_avatar_form = UpdateAvatarForm()
+    update_password_form = UpdatePasswordForm(user)
+    update_notifications_form = UpdateNotificationsForm(user)
+    update_user_form = UpdateUserForm(user)
+
+    # region handle update profile information form
+    if update_profile_information_form.submit.data and update_profile_information_form.validate():
+        user.about_me = update_profile_information_form.about_me.data
+        user.location = update_profile_information_form.location.data
+        user.organization = update_profile_information_form.organization.data
+        user.website = update_profile_information_form.website.data
+        user.full_name = update_profile_information_form.full_name.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update profile information form
+
+    # region handle update avatar form
+    if update_avatar_form.submit.data and update_avatar_form.validate():
+        try:
+            Avatar.create(
+                update_avatar_form.avatar.data,
+                user=user
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update avatar form
+
+    # region handle update account information form
+    if update_account_information_form.submit.data and update_account_information_form.validate():
+        user.email = update_account_information_form.email.data
+        user.username = update_account_information_form.username.data
+        db.session.commit()
+        flash('Profile settings updated')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update account information form
+
+    # region handle update password form
+    if update_password_form.submit.data and update_password_form.validate():
+        user.password = update_password_form.new_password.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update password form
+
+    # region handle update notifications form
+    if update_notifications_form.submit.data and update_notifications_form.validate():
+        user.setting_job_status_mail_notification_level = \
+            update_notifications_form.job_status_mail_notification_level.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update notifications form
+
+    # region handle update user form
+    if update_user_form.submit.data and update_user_form.validate():
+        role_id = hashids.decode(update_user_form.role.data)
+        user.role = Role.query.get(role_id)
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update user form
+
+    return render_template(
+        'admin/user_settings.html.j2',
+        title='Settings',
+        update_account_information_form=update_account_information_form,
+        update_avatar_form=update_avatar_form,
+        update_notifications_form=update_notifications_form,
+        update_password_form=update_password_form,
+        update_profile_information_form=update_profile_information_form,
+        update_user_form=update_user_form,
+        user=user
+    )

app/blueprints/api/__init__.py

@@ -0,0 +1,14 @@
+from flask import Blueprint
+
+
+bp = Blueprint('api', __name__)
+
+
+from .tokens import bp as tokens_blueprint
+bp.register_blueprint(tokens_blueprint, url_prefix='/tokens')
+
+from .users import bp as users_blueprint
+bp.register_blueprint(users_blueprint, url_prefix='/users')
+
+from .jobs import bp as jobs_blueprint
+bp.register_blueprint(jobs_blueprint, url_prefix='/jobs')

app/blueprints/api/auth.py

@@ -0,0 +1,48 @@
+from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth
+from werkzeug.exceptions import Forbidden, Unauthorized
+from app.models import User
+
+
+basic_auth = HTTPBasicAuth()
+token_auth = HTTPTokenAuth()
+auth_error_responses = {
+    Unauthorized.code: Unauthorized.description,
+    Forbidden.code: Forbidden.description
+}
+
+@basic_auth.verify_password
+def verify_password(email_or_username, password):
+    user = User.query.filter((User.email == email_or_username.lower()) | (User.username == email_or_username)).first()
+    if user is not None and user.verify_password(password):
+        return user
+
+
+@basic_auth.error_handler
+def basic_auth_error(status):
+    error = (Forbidden if status == 403 else Unauthorized)()
+    return {
+        'code': error.code,
+        'message': error.name,
+        'description': error.description,
+    }, error.code, {'WWW-Authenticate': 'Form'}
+
+
+@token_auth.verify_token
+def verify_token(token):
+    return User.verify_access_token(token) if token else None
+
+
+@token_auth.error_handler
+def token_auth_error(status):
+    error = (Forbidden if status == 403 else Unauthorized)()
+    return {
+        'code': error.code,
+        'message': error.name,
+        'description': error.description,
+    }, error.code
+
+
+@basic_auth.get_user_roles
+@token_auth.get_user_roles
+def get_user_roles(user):
+    return [user.role.name]

app/blueprints/api/jobs.py

@@ -0,0 +1,102 @@
+
+from apifairy import authenticate, response
+from apifairy.decorators import body, other_responses
+from flask import abort, Blueprint
+from werkzeug.exceptions import InternalServerError
+from app import db, hashids
+from app.models import Job, JobInput, JobStatus, TesseractOCRPipelineModel
+from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
+
+
+bp = Blueprint('jobs', __name__)
+job_schema = JobSchema()
+jobs_schema = JobSchema(many=True)
+spacy_nlp_pipeline_job_schema = SpaCyNLPPipelineJobSchema()
+tesseract_ocr_pipeline_job_schema = TesseractOCRPipelineJobSchema()
+tesseract_ocr_pipeline_model_schema = TesseractOCRPipelineModelSchema()
+tesseract_ocr_pipeline_models_schema = TesseractOCRPipelineModelSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(jobs_schema)
+@other_responses(auth_error_responses)
+def get_jobs():
+    """Get all jobs"""
+    return Job.query.all()
+
+
+@bp.route('/tesseract-ocr-pipeline', methods=['POST'])
+@authenticate(token_auth)
+@body(tesseract_ocr_pipeline_job_schema, location='form')
+@response(job_schema)
+@other_responses({**auth_error_responses, InternalServerError.code: InternalServerError.description})
+def create_tesseract_ocr_pipeline_job(args):
+    """Create a new Tesseract OCR Pipeline job"""
+    current_user = token_auth.current_user()
+    try:
+        job = Job.create(
+            title=args['title'],
+            description=args['description'],
+            service='tesseract-ocr-pipeline',
+            service_args={
+                'model': hashids.decode(args['model_id']),
+                'binarization': args['binarization']
+            },
+            service_version=args['service_version'],
+            user=current_user
+        )
+    except OSError:
+        abort(500)
+    try:
+        JobInput.create(args['pdf'], job=job)
+    except OSError:
+        abort(500)
+    job.status = JobStatus.SUBMITTED
+    db.session.commit()
+    return job, 201
+
+
+@bp.route('/tesseract-ocr-pipeline/models', methods=['GET'])
+@authenticate(token_auth)
+@response(tesseract_ocr_pipeline_models_schema)
+@other_responses(auth_error_responses)
+def get_tesseract_ocr_models():
+    """Get all Tesseract OCR Models"""
+    return TesseractOCRPipelineModel.query.all()
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_job(job_id):
+    """Delete a job by id"""
+    current_user = token_auth.current_user()
+    job = Job.query.get(job_id)
+    if job is None:
+        abort(404)
+    if not (job.user == current_user or current_user.is_administrator):
+        abort(403)
+    try:
+        job.delete()
+    except OSError as e:
+        abort(500)
+    db.session.commit()
+    return {}, 204
+
+
+@bp.route('/<hashid:job_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(job_schema)
+@other_responses(auth_error_responses)
+def get_job(job_id):
+    """Get a job by id"""
+    current_user = token_auth.current_user()
+    job = Job.query.get(job_id)
+    if job is None:
+        abort(404)
+    if not (job.user == current_user or current_user.is_administrator):
+        abort(403)
+    return job

app/blueprints/api/schemas.py

@@ -0,0 +1,173 @@
+from apifairy.fields import FileField
+from marshmallow import validate, validates, ValidationError
+from marshmallow.decorators import post_dump
+from app import ma
+from app.models import (
+    Job,
+    JobStatus,
+    TesseractOCRPipelineModel,
+    Token,
+    User,
+    UserSettingJobStatusMailNotificationLevel
+)
+from app.blueprints.services import SERVICES
+
+
+
+class EmptySchema(ma.Schema):
+    pass
+
+
+class TokenSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = Token
+        ordered = True
+
+    access_token = ma.String(required=True)
+    refresh_token = ma.String()
+
+
+class TesseractOCRPipelineModelSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = TesseractOCRPipelineModel
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    user_hashid = ma.String(data_key='user_id', dump_only=True)
+    title = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=64)
+    )
+    description = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    version = ma.String(
+        required=True,
+        validate=validate.Length(min=1, max=16)
+    )
+    compatible_service_versions = ma.List(
+        ma.String(required=True, validate=validate.Length(min=1, max=16)),
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    publisher = ma.String(
+        required=True,
+        validate=validate.Length(min=1, max=128)
+    )
+    publisher_url = ma.String(
+        validate=[validate.URL(), validate.Length(min=1, max=512)]
+    )
+    publishing_url = ma.String(
+        required=True,
+        validate=[validate.URL(), validate.Length(min=1, max=512)]
+    )
+    publishing_year = ma.Int(
+        required=True
+    )
+    is_public = ma.Boolean(required=True)
+
+
+class JobSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = Job
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    user_hashid = ma.String(data_key='user_id', dump_only=True)
+    title = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=32)
+    )
+    description = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    creation_date = ma.auto_field(dump_only=True)
+    end_date = ma.auto_field(dump_only=True)
+    service = ma.String(
+        dump_only=True,
+        validate=validate.OneOf(SERVICES.keys())
+    )
+    service_args = ma.Dict(dump_only=True)
+    service_version = ma.String(dump_only=True)
+    status = ma.String(
+        dump_only=True,
+        validate=validate.OneOf(list(JobStatus.__members__.keys()))
+    )
+
+    @post_dump(pass_original=True)
+    def post_dump(self, serialized_job, job, **kwargs):
+        serialized_job['status'] = job.status.name
+        return serialized_job
+
+
+class TesseractOCRPipelineJobSchema(JobSchema):
+    binarization = ma.Boolean(load_only=True, missing=False)
+    model_id = ma.String(required=True, load_only=True)
+    service_version = ma.auto_field(
+        required=True,
+        validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+    )
+    pdf = FileField()
+
+    @validates('pdf')
+    def validate_pdf(self, value):
+        if value.mimetype != 'application/pdf':
+            raise ValidationError('PDF files only!')
+
+
+class SpaCyNLPPipelineJobSchema(JobSchema):
+    binarization = ma.Boolean(load_only=True, missing=False)
+    model_id = ma.String(required=True, load_only=True)
+    service_version = ma.auto_field(
+        required=True,
+        validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+    )
+    txt = FileField(required=True)
+
+    @validates('txt')
+    def validate_txt(self, value):
+        if value.mimetype != 'text/plain':
+            raise ValidationError('Plain text files only!')
+
+
+class UserSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = User
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    username = ma.auto_field(
+        validate=[
+            validate.Length(min=1, max=64),
+            validate.Regexp(
+                User.username_pattern,
+                error='Usernames must have only letters, numbers, dots or underscores'
+            )
+        ]
+    )
+    email = ma.auto_field(validate=validate.Email())
+    member_since = ma.auto_field(dump_only=True)
+    last_seen = ma.auto_field(dump_only=True)
+    password = ma.String(load_only=True)
+    last_seen = ma.auto_field(dump_only=True)
+    setting_job_status_mail_notification_level = ma.String(
+        validate=validate.OneOf(list(UserSettingJobStatusMailNotificationLevel.__members__.keys()))
+    )
+
+    @validates('email')
+    def validate_email(self, email):
+        if User.query.filter(User.email == email).first():
+            raise ValidationError('Email already registered')
+
+    @validates('username')
+    def validate_username(self, username):
+        if User.query.filter(User.username == username).first():
+            raise ValidationError('Username already in use')
+
+    @post_dump(pass_original=True)
+    def post_dump(self, serialized_user, user, **kwargs):
+        serialized_user['setting_job_status_mail_notification_level'] = \
+            user.setting_job_status_mail_notification_level.name
+        return serialized_user

app/blueprints/api/tokens.py

@@ -0,0 +1,58 @@
+from apifairy import authenticate, body, response, other_responses
+from flask import Blueprint, request, abort
+from app import db
+from app.models import Token, User
+from .auth import basic_auth
+from .schemas import EmptySchema, TokenSchema
+
+
+bp = Blueprint('tokens', __name__)
+token_schema = TokenSchema()
+
+
+@bp.route('', methods=['DELETE'])
+@response(EmptySchema, status_code=204, description='Token revoked')
+@other_responses({401: 'Invalid access token'})
+def delete_token():
+    """Revoke an access token"""
+    access_token = request.headers['Authorization'].split()[1]
+    token = Token.query.filter(Token.access_token == access_token).first()
+    if token is None:  # pragma: no cover
+        abort(401)
+    token.expire()
+    db.session.commit()
+    return {}
+
+
+@bp.route('', methods=['POST'])
+@authenticate(basic_auth)
+@response(token_schema)
+@other_responses({401: 'Invalid username or password'})
+def create_token():
+    """Create new access and refresh tokens"""
+    user = basic_auth.current_user()
+    token = user.generate_auth_token()
+    db.session.add(token)
+    Token.clean()  # keep token table clean of old tokens
+    db.session.commit()
+    return token, 200
+
+
+@bp.route('', methods=['PUT'])
+@body(token_schema)
+@response(token_schema, description='Newly issued access and refresh tokens')
+@other_responses({401: 'Invalid access or refresh token'})
+def refresh_token(args):
+    """Refresh an access token"""
+    access_token = args.get('access_token')
+    refresh_token = args.get('refresh_token')
+    if access_token is None or refresh_token is None:
+        abort(401)
+    token = User.verify_refresh_token(refresh_token, access_token)
+    if token is None:
+        abort(401)
+    token.expire()
+    new_token = token.user.generate_auth_token()
+    db.session.add_all([token, new_token])
+    db.session.commit()
+    return new_token, 200

app/blueprints/api/users.py

@@ -0,0 +1,99 @@
+
+from apifairy import authenticate, body, response
+from apifairy.decorators import other_responses
+from flask import abort, Blueprint
+from werkzeug.exceptions import InternalServerError
+from app.email import create_message, send
+from app import db
+from app.models import User
+from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, UserSchema
+
+
+bp = Blueprint('users', __name__)
+user_schema = UserSchema()
+users_schema = UserSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(users_schema)
+@other_responses(auth_error_responses)
+def get_users():
+    """Get all users"""
+    return User.query.all()
+
+
+@bp.route('', methods=['POST'])
+@body(user_schema)
+@response(user_schema, 201)
+@other_responses({InternalServerError.code: InternalServerError.description})
+def create_user(args):
+    """Create a new user"""
+    try:
+        user = User.create(
+            email=args['email'].lower(),
+            password=args['password'],
+            username=args['username']
+        )
+    except OSError:
+        abort(500)
+    msg = create_message(
+        user.email,
+        'Confirm Your Account',
+        'auth/email/confirm',
+        token=user.generate_confirm_token(),
+        user=user
+    )
+    send(msg)
+    db.session.commit()
+    return user, 201
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_user(user_id):
+    """Delete a user by id"""
+    current_user = token_auth.current_user()
+    user = User.query.get(user_id)
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    user.delete()
+    db.session.commit()
+    return {}, 204
+
+
+@bp.route('/<hashid:user_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user(user_id):
+    """Retrieve a user by id"""
+    current_user = token_auth.current_user()
+    user = User.query.get(user_id)
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    return user
+
+
+@bp.route('/<username>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user_by_username(username):
+    """Retrieve a user by username"""
+    current_user = token_auth.current_user()
+    user = User.query.filter(User.username == username).first()
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    return user

app/blueprints/auth/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('auth', __name__)
+from . import routes

app/blueprints/auth/forms.py

@@ -0,0 +1,108 @@
+from flask_wtf import FlaskForm
+from wtforms import (
+    BooleanField,
+    PasswordField,
+    StringField,
+    SubmitField,
+    ValidationError
+)
+from wtforms.validators import InputRequired, Email, EqualTo, Length, Regexp
+from app.models import User
+
+
+class RegistrationForm(FlaskForm):
+    email = StringField(
+        'Email',
+        validators=[InputRequired(), Email(), Length(max=254)]
+    )
+    username = StringField(
+        'Username',
+        validators=[
+            InputRequired(),
+            Length(max=64),
+            Regexp(
+                User.username_pattern,
+                message=(
+                    'Usernames must have only letters, numbers, dots or '
+                    'underscores'
+                )
+            )
+       ]
+    )
+    password = PasswordField(
+        'Password',
+        validators=[
+            InputRequired(),
+            EqualTo('password_2', message='Passwords must match')
+        ]
+    )
+    password_2 = PasswordField(
+        'Password confirmation',
+        validators=[
+            InputRequired(),
+            EqualTo('password', message='Passwords must match')
+        ]
+    )
+    terms_of_use_accepted = BooleanField(
+        'I have read and accept the terms of use',
+        validators=[InputRequired()]
+    )
+    submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'registration-form'
+        super().__init__(*args, **kwargs)
+
+    def validate_email(self, field):
+        if User.query.filter_by(email=field.data.lower()).first():
+            raise ValidationError('Email already registered')
+
+    def validate_username(self, field):
+        if User.query.filter_by(username=field.data).first():
+            raise ValidationError('Username already in use')
+
+
+class LoginForm(FlaskForm):
+    user = StringField('Email or username', validators=[InputRequired()])
+    password = PasswordField('Password', validators=[InputRequired()])
+    remember_me = BooleanField('Keep me logged in')
+    submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'login-form'
+        super().__init__(*args, **kwargs)
+
+
+class ResetPasswordRequestForm(FlaskForm):
+    email = StringField('Email', validators=[InputRequired(), Email()])
+    submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-request-form'
+        super().__init__(*args, **kwargs)
+
+
+class ResetPasswordForm(FlaskForm):
+    password = PasswordField(
+        'New password',
+        validators=[
+            InputRequired(),
+            EqualTo('password_2', message='Passwords must match')
+        ]
+    )
+    password_2 = PasswordField(
+        'New password confirmation',
+        validators=[
+            InputRequired(),
+            EqualTo('password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-form'
+        super().__init__(*args, **kwargs)

app/blueprints/auth/routes.py

@@ -0,0 +1,188 @@
+from flask import abort, flash, redirect, render_template, request, url_for
+from flask_login import current_user, login_user, login_required, logout_user
+from app import db
+from app.email import create_message, send
+from app.models import User
+from . import bp
+from .forms import (
+    LoginForm,
+    ResetPasswordForm,
+    ResetPasswordRequestForm,
+    RegistrationForm
+)
+
+
+@bp.before_app_request
+def before_request():
+    """
+    Checks if a user is unconfirmed when visiting specific sites. Redirects to
+    unconfirmed view if user is unconfirmed.
+    """
+    if not current_user.is_authenticated:
+        return
+
+    current_user.ping()
+    db.session.commit()
+    if (not current_user.confirmed
+            and request.endpoint
+            and request.blueprint != 'auth'
+            and request.endpoint != 'static'):
+        return redirect(url_for('auth.unconfirmed'))
+    if not current_user.terms_of_use_accepted:
+        return redirect(url_for('main.terms_of_use'))
+
+
+@bp.route('/register', methods=['GET', 'POST'])
+def register():
+    if current_user.is_authenticated:
+        return redirect(url_for('main.dashboard'))
+    form = RegistrationForm()
+    if form.validate_on_submit():
+        try:
+            user = User.create(
+                email=form.email.data.lower(),
+                password=form.password.data,
+                username=form.username.data,
+                terms_of_use_accepted=form.terms_of_use_accepted.data
+            )
+        except OSError:
+            flash('Internal Server Error', category='error')
+            abort(500)
+        flash(f'User "{user.username}" created')
+        token = user.generate_confirm_token()
+        msg = create_message(
+            user.email,
+            'Confirm Your Account',
+            'auth/email/confirm',
+            token=token,
+            user=user
+        )
+        send(msg)
+        flash('A confirmation email has been sent to you by email')
+        db.session.commit()
+        return redirect(url_for('.login'))
+    return render_template(
+        'auth/register.html.j2',
+        title='Register',
+        form=form
+    )
+
+
+@bp.route('/login', methods=['GET', 'POST'])
+def login():
+    if current_user.is_authenticated:
+        return redirect(url_for('main.dashboard'))
+    form = LoginForm()
+    if form.validate_on_submit():
+        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+        if user and user.verify_password(form.password.data):
+            login_user(user, form.remember_me.data)
+            next = request.args.get('next')
+            if next is None or not next.startswith('/'):
+                next = url_for('main.dashboard')
+            flash('You have been logged in')
+            return redirect(next)
+        flash('Invalid email/username or password', category='error')
+    return render_template(
+        'auth/login.html.j2',
+        title='Log in',
+        form=form
+    )
+
+
+@bp.route('/logout')
+@login_required
+def logout():
+    logout_user()
+    flash('You have been logged out')
+    return redirect(url_for('main.index'))
+
+
+@bp.route('/unconfirmed')
+@login_required
+def unconfirmed():
+    if current_user.confirmed:
+        return redirect(url_for('main.dashboard'))
+    return render_template(
+        'auth/unconfirmed.html.j2',
+        title='Unconfirmed'
+    )
+
+
+@bp.route('/confirm-request')
+@login_required
+def confirm_request():
+    if current_user.confirmed:
+        return redirect(url_for('main.dashboard'))
+    token = current_user.generate_confirm_token()
+    msg = create_message(
+        current_user.email,
+        'Confirm Your Account',
+        'auth/email/confirm',
+        token=token,
+        user=current_user
+    )
+    send(msg)
+    flash('A new confirmation email has been sent to you by email')
+    return redirect(url_for('.unconfirmed'))
+
+
+@bp.route('/confirm/<token>')
+@login_required
+def confirm(token):
+    if current_user.confirmed:
+        return redirect(url_for('main.dashboard'))
+    if current_user.confirm(token):
+        db.session.commit()
+        flash('You have confirmed your account')
+        return redirect(url_for('main.dashboard'))
+    flash('The confirmation link is invalid or has expired', category='error')
+    return redirect(url_for('.unconfirmed'))
+
+
+@bp.route('/reset-password-request', methods=['GET', 'POST'])
+def reset_password_request():
+    if current_user.is_authenticated:
+        return redirect(url_for('main.dashboard'))
+    form = ResetPasswordRequestForm()
+    if form.validate_on_submit():
+        user = User.query.filter_by(email=form.email.data.lower()).first()
+        if user is not None:
+            token = user.generate_reset_password_token()
+            msg = create_message(
+                user.email,
+                'Reset Your Password',
+                'auth/email/reset_password',
+                token=token,
+                user=user
+            )
+            send(msg)
+        flash(
+            'An email with instructions to reset your password has been sent '
+            'to you'
+        )
+        return redirect(url_for('.login'))
+    return render_template(
+        'auth/reset_password_request.html.j2',
+        title='Password Reset',
+        form=form
+    )
+
+
+@bp.route('/reset-password/<token>', methods=['GET', 'POST'])
+def reset_password(token):
+    if current_user.is_authenticated:
+        return redirect(url_for('main.dashboard'))
+    form = ResetPasswordForm()
+    if form.validate_on_submit():
+        if User.reset_password(token, form.password.data):
+            db.session.commit()
+            flash('Your password has been updated')
+            return redirect(url_for('.login'))
+        return redirect(url_for('main.index'))
+    return render_template(
+        'auth/reset_password.html.j2',
+        title='Password Reset',
+        form=form,
+        token=token
+    )

app/blueprints/contributions/__init__.py

@@ -0,0 +1,23 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('contributions', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import (
+    routes,
+    spacy_nlp_pipeline_models,
+    tesseract_ocr_pipeline_models,
+    transkribus_htr_pipeline_models
+)

app/blueprints/contributions/forms.py

@@ -0,0 +1,47 @@
+from flask_wtf import FlaskForm
+from wtforms import (
+    StringField,
+    SubmitField,
+    SelectMultipleField,
+    IntegerField
+)
+from wtforms.validators import InputRequired, Length
+
+
+class ContributionBaseForm(FlaskForm):
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=64)]
+    )
+    description = StringField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    version = StringField(
+        'Version',
+        validators=[InputRequired(), Length(max=16)]
+    )
+    publisher = StringField(
+        'Publisher',
+        validators=[InputRequired(), Length(max=128)]
+    )
+    publisher_url = StringField(
+        'Publisher URL',
+        validators=[InputRequired(), Length(max=512)]
+    )
+    publishing_url = StringField(
+        'Publishing URL',
+        validators=[InputRequired(), Length(max=512)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    compatible_service_versions = SelectMultipleField(
+        'Compatible service versions'
+    )
+    submit = SubmitField()
+
+
+class UpdateContributionBaseForm(ContributionBaseForm):
+    pass

app/blueprints/contributions/routes.py

@@ -0,0 +1,7 @@
+from flask import redirect, url_for
+from . import bp
+
+
+@bp.route('')
+def contributions():
+    return redirect(url_for('main.dashboard', _anchor='contributions'))

app/blueprints/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/contributions/spacy_nlp_pipeline_models/forms.py

@@ -0,0 +1,48 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import StringField, ValidationError
+from wtforms.validators import InputRequired, Length
+from app.blueprints.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
+    spacy_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+
+    def validate_spacy_model_file(self, field):
+        if not field.data.filename.lower().endswith(('.tar.gz', ('.whl'))):
+            raise ValidationError('.tar.gz or .whl files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateSpaCyNLPPipelineModelForm(UpdateContributionBaseForm):
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/blueprints/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -0,0 +1,52 @@
+from flask import abort, current_app, request
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import SpaCyNLPPipelineModel
+from . import bp
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_spacy_model(spacy_nlp_pipeline_model_id):
+    def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
+        with app.app_context():
+            snpm = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
+            snpm.delete()
+            db.session.commit()
+    
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_spacy_model,
+        args=(current_app._get_current_object(), snpm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'SpaCy NLP Pipeline Model "{snpm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    snpm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'SpaCy NLP Pipeline Model "{snpm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/blueprints/contributions/spacy_nlp_pipeline_models/routes.py

@@ -0,0 +1,70 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import SpaCyNLPPipelineModel
+from . import bp
+from .forms import (
+    CreateSpaCyNLPPipelineModelForm,
+    UpdateSpaCyNLPPipelineModelForm
+)
+
+
+@bp.route('/spacy-nlp-pipeline-models')
+def spacy_nlp_pipeline_models():
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
+        title='SpaCy NLP Pipeline Models'
+    )
+
+
+@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
+def create_spacy_nlp_pipeline_model():
+    form = CreateSpaCyNLPPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            snpm = SpaCyNLPPipelineModel.create(
+                form.spacy_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                pipeline_name=form.pipeline_name.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
+        return {}, 201, {'Location': url_for('.spacy_nlp_pipeline_models')}
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/create.html.j2',
+        title='Create SpaCy NLP Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    form = UpdateSpaCyNLPPipelineModelForm(data=snpm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(snpm)
+        if db.session.is_modified(snpm):
+            flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.spacy_nlp_pipeline_models'))
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_model.html.j2',
+        title=f'{snpm.title} {snpm.version}',
+        form=form,
+        spacy_nlp_pipeline_model=snpm
+    )

app/blueprints/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/contributions/tesseract_ocr_pipeline_models/forms.py

@@ -0,0 +1,39 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import ValidationError
+from app.blueprints.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
+    tesseract_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+
+    def validate_tesseract_model_file(self, field):
+        if not field.data.filename.lower().endswith('.traineddata'):
+            raise ValidationError('traineddata files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateTesseractOCRPipelineModelForm(UpdateContributionBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/blueprints/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -0,0 +1,52 @@
+from flask import abort, current_app, request
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import TesseractOCRPipelineModel
+from . import bp
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
+    def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
+        with app.app_context():
+            topm = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
+            topm.delete()
+            db.session.commit()
+
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_tesseract_ocr_pipeline_model,
+        args=(current_app._get_current_object(), topm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'Tesseract OCR Pipeline Model "{topm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    topm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Tesseract OCR Pipeline Model "{topm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/blueprints/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -0,0 +1,69 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import TesseractOCRPipelineModel
+from . import bp
+from .forms import (
+    CreateTesseractOCRPipelineModelForm,
+    UpdateTesseractOCRPipelineModelForm
+)
+
+
+@bp.route('/tesseract-ocr-pipeline-models')
+def tesseract_ocr_pipeline_models():
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
+        title='Tesseract OCR Pipeline Models'
+    )
+
+
+@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
+def create_tesseract_ocr_pipeline_model():
+    form = CreateTesseractOCRPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            topm = TesseractOCRPipelineModel.create(
+                form.tesseract_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
+        return {}, 201, {'Location': url_for('.tesseract_ocr_pipeline_models')}
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/create.html.j2',
+        title='Create Tesseract OCR Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    form = UpdateTesseractOCRPipelineModelForm(data=topm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(topm)
+        if db.session.is_modified(topm):
+            flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.tesseract_ocr_pipeline_models'))
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_model.html.j2',
+        title=f'{topm.title} {topm.version}',
+        form=form,
+        tesseract_ocr_pipeline_model=topm
+    )

app/blueprints/contributions/transkribus_htr_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import routes

app/blueprints/contributions/transkribus_htr_pipeline_models/routes.py

@@ -0,0 +1,7 @@
+from flask import abort
+from . import bp
+
+
+@bp.route('/transkribus_htr_pipeline_models')
+def transkribus_htr_pipeline_models():
+    return abort(503)

app/blueprints/corpora/__init__.py

@@ -0,0 +1,19 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('corpora', __name__)
+bp.cli.short_help = 'Corpus commands.'
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import cli, files, followers, routes, json_routes

app/blueprints/corpora/cli.py

@@ -0,0 +1,34 @@
+from flask import current_app
+import shutil
+from app import db
+from app.models import Corpus, CorpusStatus
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset built corpora. '''
+    status = [
+        CorpusStatus.QUEUED,
+        CorpusStatus.BUILDING,
+        CorpusStatus.BUILT,
+        CorpusStatus.STARTING_ANALYSIS_SESSION,
+        CorpusStatus.RUNNING_ANALYSIS_SESSION,
+        CorpusStatus.CANCELING_ANALYSIS_SESSION
+    ]
+    for corpus in [x for x in Corpus.query.all() if x.status in status]:
+        print(f'Resetting corpus {corpus}')
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        corpus.status = CorpusStatus.UNPREPARED
+        corpus.num_analysis_sessions = 0
+    db.session.commit()

app/blueprints/corpora/cqi_over_sio/__init__.py

@@ -0,0 +1,206 @@
+from cqi import CQiClient
+from cqi.errors import CQiException
+from cqi.status import CQiStatus
+from docker.models.containers import Container
+from flask import current_app, session
+from flask_login import current_user
+from flask_socketio import Namespace
+from inspect import signature
+from threading import Lock
+from typing import Callable
+from app import db, docker_client, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus, CorpusStatus
+from . import extensions
+
+
+'''
+This package tunnels the Corpus Query interface (CQi) protocol through
+Socket.IO (SIO) by tunneling CQi API calls through an event called "exec".
+
+Basic concept:
+1. A client connects to the namespace.
+2. The client emits the "init" event and provides a corpus id for the corpus
+   that should be analysed in this session.
+     1.1 The analysis session counter of the corpus is incremented.
+     1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
+     1.3 Wait until the CQP server is running.
+     1.4 Connect the CQiClient to the server.
+     1.5 Save the CQiClient, the Lock and the corpus id in the session for
+         subsequential use.
+3. The client emits "exec" events, within which it provides the name of a CQi
+   API function and the corresponding arguments.
+     3.1 The "exec" event handler will execute the function, make sure that
+         the result is serializable and returns the result back to the client.
+4. The client disconnects from the namespace
+     4.1 The analysis session counter of the corpus is decremented.
+     4.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
+'''
+
+
+CQI_API_FUNCTION_NAMES: list[str] = [
+    'ask_feature_cl_2_3',
+    'ask_feature_cqi_1_0',
+    'ask_feature_cqp_2_3',
+    'cl_alg2cpos',
+    'cl_attribute_size',
+    'cl_cpos2alg',
+    'cl_cpos2id',
+    'cl_cpos2lbound',
+    'cl_cpos2rbound',
+    'cl_cpos2str',
+    'cl_cpos2struc',
+    'cl_drop_attribute',
+    'cl_id2cpos',
+    'cl_id2freq',
+    'cl_id2str',
+    'cl_idlist2cpos',
+    'cl_lexicon_size',
+    'cl_regex2id',
+    'cl_str2id',
+    'cl_struc2cpos',
+    'cl_struc2str',
+    'corpus_alignment_attributes',
+    'corpus_charset',
+    'corpus_drop_corpus',
+    'corpus_full_name',
+    'corpus_info',
+    'corpus_list_corpora',
+    'corpus_positional_attributes',
+    'corpus_properties',
+    'corpus_structural_attribute_has_values',
+    'corpus_structural_attributes',
+    'cqp_drop_subcorpus',
+    'cqp_dump_subcorpus',
+    'cqp_fdist_1',
+    'cqp_fdist_2',
+    'cqp_list_subcorpora',
+    'cqp_query',
+    'cqp_subcorpus_has_field',
+    'cqp_subcorpus_size',
+    'ctrl_bye',
+    'ctrl_connect',
+    'ctrl_last_general_error',
+    'ctrl_ping',
+    'ctrl_user_abort'
+]
+
+
+class CQiOverSocketIO(Namespace):
+    @socketio_login_required
+    def on_connect(self):
+        pass
+
+    @socketio_login_required
+    def on_init(self, db_corpus_hashid: str):
+        db_corpus_id: int = hashids.decode(db_corpus_hashid)
+        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
+        if db_corpus is None:
+            return {'code': 404, 'msg': 'Not Found'}
+        if not (db_corpus.user == current_user
+                or current_user.is_following_corpus(db_corpus)
+                or current_user.is_administrator):
+            return {'code': 403, 'msg': 'Forbidden'}
+        if db_corpus.status not in [
+            CorpusStatus.BUILT,
+            CorpusStatus.STARTING_ANALYSIS_SESSION,
+            CorpusStatus.RUNNING_ANALYSIS_SESSION,
+            CorpusStatus.CANCELING_ANALYSIS_SESSION
+        ]:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if db_corpus.num_analysis_sessions is None:
+            db_corpus.num_analysis_sessions = 0
+            db.session.commit()
+        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+        db.session.commit()
+        retry_counter: int = 20
+        while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+            if retry_counter == 0:
+                db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+                db.session.commit()
+                return {'code': 408, 'msg': 'Request Timeout'}
+            socketio.sleep(3)
+            retry_counter -= 1
+            db.session.refresh(db_corpus)
+        # cqi_client: CQiClient = CQiClient(f'cqpserver_{db_corpus_id}')
+        cqpserver_container_name: str = f'cqpserver_{db_corpus_id}'
+        cqpserver_container: Container = docker_client.containers.get(cqpserver_container_name)
+        cqpserver_host: str = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
+        cqi_client: CQiClient = CQiClient(cqpserver_host)
+        session['cqi_over_sio'] = {
+            'cqi_client': cqi_client,
+            'cqi_client_lock': Lock(),
+            'db_corpus_id': db_corpus_id
+        }
+        return {'code': 200, 'msg': 'OK'}
+
+    @socketio_login_required
+    def on_exec(self, fn_name: str, fn_args: dict = {}):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+        except KeyError:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if fn_name in CQI_API_FUNCTION_NAMES:
+            fn: Callable = getattr(cqi_client.api, fn_name)
+        elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
+            fn: Callable = getattr(extensions, fn_name)
+        else:
+            return {'code': 400, 'msg': 'Bad Request'}
+        for param in signature(fn).parameters.values():
+            if param.default is param.empty:
+                if param.name not in fn_args:
+                    return {'code': 400, 'msg': 'Bad Request'}
+            else:
+                if param.name not in fn_args:
+                    continue
+            if type(fn_args[param.name]) is not param.annotation:
+                return {'code': 400, 'msg': 'Bad Request'}
+        cqi_client_lock.acquire()
+        try:
+            fn_return_value = fn(**fn_args)
+        except BrokenPipeError as e:
+            return {'code': 500, 'msg': 'Internal Server Error'}
+        except CQiException as e:
+            return {
+                'code': 502,
+                'msg': 'Bad Gateway',
+                'payload': {
+                    'code': e.code,
+                    'desc': e.description,
+                    'msg': e.__class__.__name__
+                }
+            }
+        finally:
+            cqi_client_lock.release()
+        if isinstance(fn_return_value, CQiStatus):
+            payload = {
+                'code': fn_return_value.code,
+                'msg': fn_return_value.__class__.__name__
+            }
+        else:
+            payload = fn_return_value
+        return {'code': 200, 'msg': 'OK', 'payload': payload}
+
+    def on_disconnect(self):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+            db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+        except KeyError:
+            return
+        cqi_client_lock.acquire()
+        try:
+            session.pop('cqi_over_sio')
+        except KeyError:
+            pass
+        try:
+            cqi_client.api.ctrl_bye()
+        except (BrokenPipeError, CQiException):
+            pass
+        cqi_client_lock.release()
+        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
+        if db_corpus is None:
+            return
+        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+        db.session.commit()

app/blueprints/corpora/cqi_over_sio/extensions.py

@@ -0,0 +1,286 @@
+from collections import Counter
+from cqi import CQiClient
+from cqi.models.corpora import Corpus as CQiCorpus
+from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
+from cqi.models.attributes import (
+    PositionalAttribute as CQiPositionalAttribute,
+    StructuralAttribute as CQiStructuralAttribute
+)
+from cqi.status import StatusOk as CQiStatusOk
+from flask import session
+import gzip
+import json
+import math
+from app import db
+from app.models import Corpus
+from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
+
+
+CQI_EXTENSION_FUNCTION_NAMES: list[str] = [
+    'ext_corpus_update_db',
+    'ext_corpus_static_data',
+    'ext_corpus_paginate_corpus',
+    'ext_cqp_paginate_subcorpus',
+    'ext_cqp_partial_export_subcorpus',
+    'ext_cqp_export_subcorpus',
+]
+
+
+def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    db_corpus.num_tokens = cqi_corpus.size
+    db.session.commit()
+    return CQiStatusOk()
+
+
+def ext_corpus_static_data(corpus: str) -> dict:
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+
+    static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
+    if static_data_file_path.exists():
+        with static_data_file_path.open('rb') as f:
+            return f.read()
+
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    cqi_p_attrs: list[CQiPositionalAttribute] = cqi_corpus.positional_attributes.list()
+    cqi_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list()
+
+    static_data = {
+        'corpus': {
+            'bounds': [0, cqi_corpus.size - 1],
+            'freqs': {}
+        },
+        'p_attrs': {},
+        's_attrs': {},
+        'values': {'p_attrs': {}, 's_attrs': {}}
+    }
+
+    for p_attr in cqi_p_attrs:
+        print(f'corpus.freqs.{p_attr.name}')
+        static_data['corpus']['freqs'][p_attr.name] = []
+        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        static_data['corpus']['freqs'][p_attr.name].extend(p_attr.freqs_by_ids(p_attr_id_list))
+        del p_attr_id_list
+
+        print(f'p_attrs.{p_attr.name}')
+        static_data['p_attrs'][p_attr.name] = []
+        cpos_list: list[int] = list(range(cqi_corpus.size))
+        static_data['p_attrs'][p_attr.name].extend(p_attr.ids_by_cpos(cpos_list))
+        del cpos_list
+
+        print(f'values.p_attrs.{p_attr.name}')
+        static_data['values']['p_attrs'][p_attr.name] = []
+        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        static_data['values']['p_attrs'][p_attr.name].extend(p_attr.values_by_ids(p_attr_id_list))
+        del p_attr_id_list
+
+    for s_attr in cqi_s_attrs:
+        if s_attr.has_values:
+            continue
+
+        static_data['s_attrs'][s_attr.name] = {'lexicon': [], 'values': None}
+
+        if s_attr.name in ['s', 'ent']:
+            ##############################################################
+            # A faster way to get cpos boundaries for smaller s_attrs    #
+            # Note: Needs more testing, don't use it in production       #
+            ##############################################################
+            cqi_corpus.query('Last', f'<{s_attr.name}> []* </{s_attr.name}>;')
+            cqi_subcorpus: CQiSubcorpus = cqi_corpus.subcorpora.get('Last')
+            first_match: int = 0
+            last_match: int = cqi_subcorpus.size - 1
+            match_boundaries = zip(
+                range(first_match, last_match + 1),
+                cqi_subcorpus.dump(
+                    cqi_subcorpus.fields['match'],
+                    first_match,
+                    last_match
+                ),
+                cqi_subcorpus.dump(
+                    cqi_subcorpus.fields['matchend'],
+                    first_match,
+                    last_match
+                )
+            )
+            cqi_subcorpus.drop()
+            del cqi_subcorpus, first_match, last_match
+            for id, lbound, rbound in match_boundaries:
+                static_data['s_attrs'][s_attr.name]['lexicon'].append({})
+                print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+                static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
+            del match_boundaries
+
+        if s_attr.name != 'text':
+            continue
+
+        for id in range(0, s_attr.size):
+            static_data['s_attrs'][s_attr.name]['lexicon'].append({})
+            # This is a very slow operation, thats why we only use it for
+            # the text attribute
+            lbound, rbound = s_attr.cpos_by_id(id)
+            print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+            static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
+            static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
+            cpos_list: list[int] = list(range(lbound, rbound + 1))
+            for p_attr in cqi_p_attrs:
+                p_attr_ids: list[int] = []
+                p_attr_ids.extend(p_attr.ids_by_cpos(cpos_list))
+                print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
+                static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids))
+                del p_attr_ids
+            del cpos_list
+
+        sub_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
+        print(f's_attrs.{s_attr.name}.values')
+        static_data['s_attrs'][s_attr.name]['values'] = [
+            sub_s_attr.name[(len(s_attr.name) + 1):]
+            for sub_s_attr in sub_s_attrs
+        ]
+        s_attr_id_list: list[int] = list(range(s_attr.size))
+        sub_s_attr_values: list[str] = []
+        for sub_s_attr in sub_s_attrs:
+            tmp = []
+            tmp.extend(sub_s_attr.values_by_ids(s_attr_id_list))
+            sub_s_attr_values.append(tmp)
+            del tmp
+        del s_attr_id_list
+        print(f'values.s_attrs.{s_attr.name}')
+        static_data['values']['s_attrs'][s_attr.name] = [
+            {
+                s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id]
+                for s_attr_value_name_idx, s_attr_value_name in enumerate(
+                    static_data['s_attrs'][s_attr.name]['values']
+                )
+            } for s_attr_id in range(0, s_attr.size)
+        ]
+        del sub_s_attr_values
+    print('Saving static data to file')
+    with gzip.open(static_data_file_path, 'wt') as f:
+        json.dump(static_data, f)
+    del static_data
+    print('Sending static data to client')
+    with open(static_data_file_path, 'rb') as f:
+        return f.read()
+
+
+def ext_corpus_paginate_corpus(
+    corpus: str,
+    page: int = 1,
+    per_page: int = 20
+) -> dict:
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_corpus.size > 0
+            and page > math.ceil(cqi_corpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    first_cpos = (page - 1) * per_page
+    last_cpos = min(cqi_corpus.size, first_cpos + per_page)
+    cpos_list = [*range(first_cpos, last_cpos)]
+    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
+    payload = {}
+    # the items for the current page
+    payload['items'] = [cpos_list]
+    # the lookups for the items
+    payload['lookups'] = lookups
+    # the total number of items matching the query
+    payload['total'] = cqi_corpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_paginate_subcorpus(
+    subcorpus: str,
+    context: int = 50,
+    page: int = 1,
+    per_page: int = 20
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_subcorpus.size > 0
+            and page > math.ceil(cqi_subcorpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    offset = (page - 1) * per_page
+    cutoff = per_page
+    cqi_results_export = export_subcorpus(
+        cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
+    payload = {}
+    # the items for the current page
+    payload['items'] = cqi_results_export.pop('matches')
+    # the lookups for the items
+    payload['lookups'] = cqi_results_export
+    # the total number of items matching the query
+    payload['total'] = cqi_subcorpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_partial_export_subcorpus(
+    subcorpus: str,
+    match_id_list: list,
+    context: int = 50
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
+    return cqi_subcorpus_partial_export
+
+
+def ext_cqp_export_subcorpus(
+    subcorpus: str,
+    context: int = 50
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
+    return cqi_subcorpus_export

app/blueprints/corpora/cqi_over_sio/utils.py

@@ -0,0 +1,130 @@
+from cqi.models.corpora import Corpus as CQiCorpus
+from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
+
+
+def lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
+    lookups = {}
+    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
+    for attr in corpus.positional_attributes.list():
+        cpos_attr_values: list[str] = attr.values_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
+    for attr in corpus.structural_attributes.list():
+        # We only want to iterate over non subattributes, identifiable by
+        # attr.has_values == False
+        if attr.has_values:
+            continue
+        cpos_attr_ids: list[int] = attr.ids_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            if cpos_attr_ids[i] == -1:
+                continue
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
+        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
+        if len(occured_attr_ids) == 0:
+            continue
+        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
+        if len(subattrs) == 0:
+            continue
+        lookup_name: str = f'{attr.name}_lookup'
+        lookups[lookup_name] = {}
+        for attr_id in occured_attr_ids:
+            lookups[lookup_name][attr_id] = {}
+        for subattr in subattrs:
+            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
+            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
+                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
+    return lookups
+
+
+def partial_export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    match_id_list: list[int],
+    context: int = 25
+) -> dict:
+    if subcorpus.size == 0:
+        return {"matches": []}
+    match_boundaries = []
+    for match_id in match_id_list:
+        if match_id < 0 or match_id >= subcorpus.size:
+            continue
+        match_boundaries.append(
+            (
+                match_id,
+                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
+                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
+            )
+        )
+    cpos_set = set()
+    matches = []
+    for match_boundary in match_boundaries:
+        match_num, match_start, match_end = match_boundary
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}
+
+
+def export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    context: int = 25,
+    cutoff: float = float('inf'),
+    offset: int = 0
+) -> dict:
+    if subcorpus.size == 0:
+        return {"matches": []}
+    first_match = max(0, offset)
+    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
+    match_boundaries = zip(
+        range(first_match, last_match + 1),
+        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
+        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
+    )
+    cpos_set = set()
+    matches = []
+    for match_num, match_start, match_end in match_boundaries:
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}

app/blueprints/corpora/decorators.py

@@ -0,0 +1,33 @@
+from flask import abort
+from flask_login import current_user
+from functools import wraps
+from app.models import Corpus, CorpusFollowerAssociation
+
+
+def corpus_follower_permission_required(*permissions):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            corpus_id = kwargs.get('corpus_id')
+            corpus = Corpus.query.get_or_404(corpus_id)
+            if not (corpus.user == current_user or current_user.is_administrator):
+                cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+                if cfa is None:
+                    abort(403)
+                if not all([cfa.role.has_permission(p) for p in permissions]):
+                    abort(403)
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator
+
+
+def corpus_owner_or_admin_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        corpus_id = kwargs.get('corpus_id')
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if not (corpus.user == current_user or current_user.is_administrator):
+            abort(403)
+        return f(*args, **kwargs)
+    return decorated_function
+

app/blueprints/corpora/events.py

@@ -0,0 +1,45 @@
+from flask_login import current_user
+from flask_socketio import join_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus
+
+
+@socketio.on('GET /corpora/<corpus_id>')
+@socketio_login_required
+def get_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    return {
+        'body': corpus.to_json_serializable(),
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /corpora/<corpus_id>')
+@socketio_login_required
+def subscribe_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    join_room(f'/corpora/{corpus.hashid}')
+    return {'options': {'status': 200, 'statusText': 'OK'}}

app/blueprints/corpora/files/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/corpora/files/forms.py

@@ -0,0 +1,54 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired
+from wtforms import (
+    StringField,
+    SubmitField,
+    ValidationError,
+    IntegerField
+)
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusFileBaseForm(FlaskForm):
+    author = StringField(
+        'Author',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    address = StringField('Adress', validators=[Length(max=255)])
+    booktitle = StringField('Booktitle', validators=[Length(max=255)])
+    chapter = StringField('Chapter', validators=[Length(max=255)])
+    editor = StringField('Editor', validators=[Length(max=255)])
+    institution = StringField('Institution', validators=[Length(max=255)])
+    journal = StringField('Journal', validators=[Length(max=255)])
+    pages = StringField('Pages', validators=[Length(max=255)])
+    publisher = StringField('Publisher', validators=[Length(max=255)])
+    school = StringField('School', validators=[Length(max=255)])
+    submit = SubmitField()
+
+
+class CreateCorpusFileForm(CorpusFileBaseForm):
+    vrt = FileField('File', validators=[FileRequired()])
+
+    def validate_vrt(self, field):
+        if not field.data.filename.lower().endswith('.vrt'):
+            raise ValidationError('VRT files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-file-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusFileForm(CorpusFileBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-file-form'
+        super().__init__(*args, **kwargs)

app/blueprints/corpora/files/json_routes.py

@@ -0,0 +1,30 @@
+from flask import current_app
+from threading import Thread
+from app.decorators import content_negotiation
+from app import db
+from app.models import CorpusFile
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def delete_corpus_file(corpus_id, corpus_file_id):
+    def _delete_corpus_file(app, corpus_file_id):
+        with app.app_context():
+            corpus_file = CorpusFile.query.get(corpus_file_id)
+            corpus_file.delete()
+            db.session.commit()
+
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    thread = Thread(
+        target=_delete_corpus_file,
+        args=(current_app._get_current_object(), corpus_file.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus File "{corpus_file.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 202

app/blueprints/corpora/files/routes.py

@@ -0,0 +1,93 @@
+from flask import (
+    abort,
+    flash,
+    redirect,
+    render_template,
+    send_from_directory,
+    url_for
+)
+from app import db
+from app.models import Corpus, CorpusFile, CorpusStatus
+from ..decorators import corpus_follower_permission_required
+from . import bp
+from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
+
+
+@bp.route('/<hashid:corpus_id>/files')
+def corpus_files(corpus_id):
+    return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
+
+
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def create_corpus_file(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    form = CreateCorpusFileForm()
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            corpus_file = CorpusFile.create(
+                form.vrt.data,
+                address=form.address.data,
+                author=form.author.data,
+                booktitle=form.booktitle.data,
+                chapter=form.chapter.data,
+                editor=form.editor.data,
+                institution=form.institution.data,
+                journal=form.journal.data,
+                pages=form.pages.data,
+                publisher=form.publisher.data,
+                publishing_year=form.publishing_year.data,
+                school=form.school.data,
+                title=form.title.data,
+                mimetype='application/vrt+xml',
+                corpus=corpus
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        corpus.status = CorpusStatus.UNPREPARED
+        db.session.commit()
+        flash(f'Corpus File "{corpus_file.filename}" added', category='corpus')
+        return '', 201, {'Location': corpus.url}
+    return render_template(
+        'corpora/files/create.html.j2',
+        title='Add corpus file',
+        form=form,
+        corpus=corpus
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(corpus_file)
+        if db.session.is_modified(corpus_file):
+            corpus_file.corpus.status = CorpusStatus.UNPREPARED
+            db.session.commit()
+            flash(f'Corpus file "{corpus_file.filename}" updated', category='corpus')
+        return redirect(corpus_file.corpus.url)
+    return render_template(
+        'corpora/files/corpus_file.html.j2',
+        title='Edit corpus file',
+        form=form,
+        corpus=corpus_file.corpus,
+        corpus_file=corpus_file
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
+@corpus_follower_permission_required('VIEW')
+def download_corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    return send_from_directory(
+        corpus_file.path.parent,
+        corpus_file.path.name,
+        as_attachment=True,
+        download_name=corpus_file.filename,
+        mimetype=corpus_file.mimetype
+    )

app/blueprints/corpora/followers/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes

app/blueprints/corpora/followers/json_routes.py

@@ -0,0 +1,76 @@
+from flask import abort, flash, jsonify, make_response, request
+from flask_login import current_user
+from app import db
+from app.decorators import content_negotiation
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def create_corpus_followers(corpus_id):
+    usernames = request.json
+    if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    for username in usernames:
+        user = User.query.filter_by(username=username, is_public=True).first_or_404()
+        user.follow_corpus(corpus)
+    db.session.commit()
+    response_data = {
+        'message': f'Users are now following "{corpus.title}"',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_follower_role(corpus_id, follower_id):
+    role_name = request.json
+    if not isinstance(role_name, str):
+        abort(400)
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    cfa.role = cfr
+    db.session.commit()
+    response_data = {
+        'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
+def delete_corpus_follower(corpus_id, follower_id):
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    if not (
+        current_user.id == follower_id
+        or current_user == cfa.corpus.user 
+        or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
+        or current_user.is_administrator):
+        abort(403)
+    if current_user.id == follower_id:
+        flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
+        response = make_response()
+        response.status_code = 204
+    else:
+        response_data = {
+            'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
+            'category': 'corpus'
+        }
+        response = jsonify(response_data)
+        response.status_code = 200
+    cfa.follower.unfollow_corpus(cfa.corpus)
+    db.session.commit()
+    return response

app/blueprints/corpora/forms.py

@@ -0,0 +1,33 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, SubmitField, TextAreaField
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusBaseForm(FlaskForm):
+    description = TextAreaField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    title = StringField('Title', validators=[InputRequired(), Length(max=32)])
+    submit = SubmitField()
+
+
+class CreateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class ImportCorpusForm(FlaskForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'import-corpus-form'
+        super().__init__(*args, **kwargs)

app/blueprints/corpora/json_routes.py

@@ -0,0 +1,125 @@
+from datetime import datetime
+from flask import abort, current_app, request, url_for
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation
+from app.models import Corpus, CorpusFollowerRole
+from . import bp
+from .decorators import corpus_follower_permission_required, corpus_owner_or_admin_required
+import nltk
+from string import punctuation
+
+
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+@corpus_owner_or_admin_required
+@content_negotiation(produces='application/json')
+def delete_corpus(corpus_id):
+    def _delete_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.delete()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    thread = Thread(
+        target=_delete_corpus,
+        args=(current_app._get_current_object(), corpus.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def build_corpus(corpus_id):
+    def _build_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.build()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if len(corpus.files.all()) == 0:
+        abort(409)
+    thread = Thread(
+        target=_build_corpus,
+        args=(current_app._get_current_object(), corpus_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for building',
+        'category': 'corpus'
+    }
+    return response_data, 202
+
+@bp.route('/stopwords')
+@content_negotiation(produces='application/json')
+def get_stopwords():
+    nltk.download('stopwords', quiet=True)
+    languages = ["german", "english", "catalan", "greek", "spanish", "french", "italian", "russian", "chinese"]
+    stopwords = {}
+    for language in languages:
+        stopwords[language] = nltk.corpus.stopwords.words(language)
+    stopwords['punctuation'] = list(punctuation) + ['—', '|', '–', '“', '„', '--']
+    stopwords['user_stopwords'] = []
+    response_data = stopwords
+    return response_data, 202
+
+@bp.route('/<hashid:corpus_id>/generate-share-link', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def generate_corpus_share_link(corpus_id):
+    data = request.json
+    if not isinstance(data, dict):
+        abort(400)
+    expiration = data.get('expiration')
+    if not isinstance(expiration, str):
+        abort(400)
+    role_name = data.get('role')
+    if not isinstance(role_name, str):
+        abort(400)
+    expiration_date = datetime.strptime(expiration, '%b %d, %Y')
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date)
+    corpus_share_link = url_for(
+        'corpora.follow_corpus',
+        corpus_id=corpus_id,
+        token=token,
+        _external=True
+    )
+    response_data = {
+        'message': 'Corpus share link generated',
+        'category': 'corpus',
+        'corpusShareLink': corpus_share_link
+    }
+    return response_data, 200
+    
+
+
+@bp.route('/<hashid:corpus_id>/is_public', methods=['PUT'])
+@corpus_owner_or_admin_required
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_is_public(corpus_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    corpus.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Corpus "{corpus.title}" is now'
+            f' {"public" if is_public else "private"}'
+        ),
+        'category': 'corpus'
+    }
+    return response_data, 200

app/blueprints/corpora/routes.py

@@ -0,0 +1,109 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from . import bp
+from .decorators import corpus_follower_permission_required
+from .forms import CreateCorpusForm
+
+
+@bp.route('')
+def corpora():
+    return redirect(url_for('main.dashboard', _anchor='corpora'))
+
+
+@bp.route('/create', methods=['GET', 'POST'])
+def create_corpus():
+    form = CreateCorpusForm()
+    if form.validate_on_submit():
+        try:
+            corpus = Corpus.create(
+                title=form.title.data,
+                description=form.description.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'Corpus "{corpus.title}" created', 'corpus')
+        return redirect(corpus.url)
+    return render_template(
+        'corpora/create.html.j2',
+        title='Create corpus',
+        form=form
+    )
+
+
+@bp.route('/<hashid:corpus_id>')
+def corpus(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    cfrs = CorpusFollowerRole.query.all()
+    # TODO: Better solution for filtering admin
+    users = User.query.filter(User.is_public == True, User.id != current_user.id, User.id != corpus.user.id, User.role_id < 4).all()
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+    if cfa is None:
+        if corpus.user == current_user or current_user.is_administrator:
+            cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
+        else:
+            cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
+    else:
+        cfr = cfa.role
+    if corpus.user == current_user or current_user.is_administrator:
+        return render_template(
+            'corpora/corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfr=cfr,
+            cfrs=cfrs,
+            users=users
+        )
+    if (current_user.is_following_corpus(corpus) or corpus.is_public):
+        cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all()
+        return render_template(
+            'corpora/public_corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfrs=cfrs,
+            cfr=cfr,
+            cfas=cfas,
+            users=users
+        )
+    abort(403)
+
+
+
+@bp.route('/<hashid:corpus_id>/analysis')
+@corpus_follower_permission_required('VIEW')
+def analysis(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    return render_template(
+        'corpora/analysis.html.j2',
+        corpus=corpus,
+        title=f'Analyse Corpus {corpus.title}'
+    )
+
+
+@bp.route('/<hashid:corpus_id>/follow/<token>')
+def follow_corpus(corpus_id, token):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if current_user.follow_corpus_by_token(token):
+        db.session.commit()
+        flash(f'You are following "{corpus.title}" now', category='corpus')
+        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    abort(403)
+
+
+@bp.route('/import', methods=['GET', 'POST'])
+def import_corpus():
+    abort(503)
+
+
+@bp.route('/<hashid:corpus_id>/export')
+@corpus_follower_permission_required('VIEW')
+def export_corpus(corpus_id):
+    abort(503)

app/blueprints/errors/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('errors', __name__)
+from . import handlers

app/blueprints/errors/handlers.py

@@ -0,0 +1,14 @@
+from flask import jsonify, render_template, request
+from werkzeug.exceptions import HTTPException
+from . import bp
+
+
+@bp.app_errorhandler(HTTPException)
+def handle_http_exception(error):
+    ''' Generic HTTP exception handler '''
+    accept_json = request.accept_mimetypes.accept_json
+    accept_html = request.accept_mimetypes.accept_html
+    if accept_json and not accept_html:
+        response = jsonify(str(error))
+        return response, error.code
+    return render_template('errors/error.html.j2', error=error), error.code

app/blueprints/jobs/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('jobs', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/jobs/events.py

@@ -0,0 +1,138 @@
+from flask import current_app
+from flask_login import current_user
+from flask_socketio import Namespace
+from app import db, hashids, socketio
+from app.extensions.flask_socketio import admin_required, login_required
+from app.models import Job, JobStatus
+
+
+class JobsNamespace(Namespace):
+    @login_required
+    def on_delete(self, job_hashid: str):
+        # Decode the job hashid
+        job_id = hashids.decode(job_hashid)
+
+        # Validate job_id
+        if not isinstance(job_id, int):
+            return {
+                'code': 400,
+                'body': 'job_id is invalid'
+            }
+
+        # Load job from database
+        job = Job.query.get(job_id)
+        if job is None:
+            return {
+                'code': 404,
+                'body': 'Job not found'
+            }
+
+        # Check if the current user is allowed to delete the job
+        if not (job.user == current_user or current_user.is_administrator):
+            return {
+                'code': 403,
+                'body': 'Forbidden'
+            }
+
+        # TODO: This should be a method in the Job model
+        def _delete_job(app, job_id):
+            with app.app_context():
+                job = Job.query.get(job_id)
+                job.delete()
+                db.session.commit()
+
+        # Delete the job in a background task
+        socketio.start_background_task(
+            target=_delete_job,
+            app=current_app._get_current_object(),
+            job_id=job_id
+        )
+
+        return {
+            'code': 202,
+            'body': f'Job "{job.title}" marked for deletion'
+        }
+
+
+    @admin_required
+    def on_get_log(self, job_hashid: str):
+        # Decode the job hashid
+        job_id = hashids.decode(job_hashid)
+
+        # Validate job_id
+        if not isinstance(job_id, int):
+            return {
+                'code': 400,
+                'body': 'job_id is invalid'
+            }
+
+        # Load job from database
+        job = Job.query.get(job_id)
+        if job is None:
+            return {
+                'code': 404,
+                'body': 'Job not found'
+            }
+
+        # Check if the job is already processed
+        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+            return {
+                'code': 409,
+                'body': 'Job is not done processing'
+            }
+
+        # Read the log file
+        with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
+            job_log = log_file.read()
+
+        return {
+            'code': 200,
+            'body': job_log
+        }
+
+
+    @login_required
+    def on_restart(self, job_hashid: str):
+        # Decode the job hashid
+        job_id = hashids.decode(job_hashid)
+
+        # Validate job_id
+        if not isinstance(job_id, int):
+            return {
+                'code': 400,
+                'body': 'job_id is invalid'
+            }
+
+        # Load job from database
+        job = Job.query.get(job_id)
+        if job is None:
+            return {
+                'code': 404,
+                'body': 'Job not found'
+            }
+
+        # Check if the current user is allowed to restart the job
+        if not (job.user == current_user or current_user.is_administrator):
+            return {
+                'code': 403,
+                'body': 'Forbidden'
+            }
+
+        # TODO: This should be a method in the Job model
+        def _restart_job(app, job_id):
+            with app.app_context():
+                job = Job.query.get(job_id)
+                job.restart()
+                db.session.commit()
+
+        # Restart the job in a background task
+        socketio.start_background_task(
+            target=_restart_job,
+            app=current_app._get_current_object(),
+            job_id=job_id
+        )
+
+        return {
+            'code': 202,
+            'body': f'Job "{job.title}" restarted'
+        }

app/blueprints/jobs/json_routes.py

@@ -0,0 +1,72 @@
+from flask import abort, current_app
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import admin_required, content_negotiation
+from app.models import Job, JobStatus
+from . import bp
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_job(job_id):
+    def _delete_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.delete()
+            db.session.commit()
+
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_job,
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Job "{job.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/<hashid:job_id>/log')
+@admin_required
+@content_negotiation(produces='application/json')
+def job_log(job_id):
+    job = Job.query.get_or_404(job_id)
+    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+        response = {'errors': {'message': 'Job status is not completed or failed'}}
+        return response, 409
+    with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
+        log = log_file.read()
+    response_data = {
+        'jobLog': log
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:job_id>/restart', methods=['POST'])
+@content_negotiation(produces='application/json')
+def restart_job(job_id):
+    def _restart_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.restart()
+            db.session.commit()
+
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator):
+        abort(403)
+    if job.status == JobStatus.FAILED:
+        response = {'errors': {'message': 'Job status is not "failed"'}}
+        return response, 409
+    thread = Thread(
+        target=_restart_job,
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Job "{job.title}" marked for restarting'
+    }
+    return response_data, 202

app/blueprints/jobs/routes.py

@@ -0,0 +1,55 @@
+from flask import (
+    abort,
+    redirect,
+    render_template,
+    send_from_directory,
+    url_for
+)
+from flask_login import current_user
+from app.models import Job, JobInput, JobResult
+from . import bp
+
+
+@bp.route('')
+def jobs():
+    return redirect(url_for('main.dashboard', _anchor='jobs'))
+
+
+@bp.route('/<hashid:job_id>')
+def job(job_id):
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator):
+        abort(403)
+    return render_template(
+        'jobs/job.html.j2',
+        title='Job',
+        job=job
+    )
+
+
+@bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
+def download_job_input(job_id, job_input_id):
+    job_input = JobInput.query.filter_by(job_id=job_id, id=job_input_id).first_or_404()
+    if not (job_input.job.user == current_user or current_user.is_administrator):
+        abort(403)
+    return send_from_directory(
+        job_input.path.parent,
+        job_input.path.name,
+        as_attachment=True,
+        download_name=job_input.filename,
+        mimetype=job_input.mimetype
+    )
+
+
+@bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
+def download_job_result(job_id, job_result_id):
+    job_result = JobResult.query.filter_by(job_id=job_id, id=job_result_id).first_or_404()
+    if not (job_result.job.user == current_user or current_user.is_administrator):
+        abort(403)
+    return send_from_directory(
+        job_result.path.parent,
+        job_result.path.name,
+        as_attachment=True,
+        download_name=job_result.filename,
+        mimetype=job_result.mimetype
+    )

app/blueprints/main/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('main', __name__, cli_group=None)
+from . import cli, routes

app/blueprints/main/cli.py

@@ -0,0 +1,51 @@
+from flask import current_app
+from flask_migrate import upgrade
+from pathlib import Path
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerRole,
+    Role,
+    SpaCyNLPPipelineModel,
+    TesseractOCRPipelineModel,
+    User
+)
+from . import bp
+
+
+@bp.cli.command('deploy')
+def deploy():
+    ''' Run deployment tasks. '''
+
+    print('Make default directories')
+    base_dir = current_app.config['NOPAQUE_DATA_DIR']
+    default_dirs: list[Path] = [
+        base_dir / 'tmp',
+        base_dir / 'users'
+    ]
+    for default_dir in default_dirs:
+        if not default_dir.exists():
+            default_dir.mkdir()
+        if not default_dir.is_dir():
+            raise NotADirectoryError(f'{default_dir} is not a directory')
+
+    print('Migrate database to latest revision')
+    upgrade()
+
+    print('Insert/Update default Roles')
+    Role.insert_defaults()
+    print('Insert/Update default Users')
+    User.insert_defaults()
+    print('Insert/Update default CorpusFollowerRoles')
+    CorpusFollowerRole.insert_defaults()
+    print('Insert/Update default SpaCyNLPPipelineModels')
+    SpaCyNLPPipelineModel.insert_defaults()
+    print('Insert/Update default TesseractOCRPipelineModels')
+    TesseractOCRPipelineModel.insert_defaults()
+
+    print('Stop running analysis sessions')
+    for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0).all():
+        corpus.num_analysis_sessions = 0
+    db.session.commit()
+
+    # TODO: Implement checks for if the nopaque network exists

app/blueprints/main/routes.py

@@ -0,0 +1,87 @@
+from flask import flash, redirect, render_template, url_for
+from flask_login import current_user, login_required, login_user
+from app.blueprints.auth.forms import LoginForm
+from app.models import Corpus, User
+from . import bp
+
+
+@bp.route('/', methods=['GET', 'POST'])
+def index():
+    form = LoginForm()
+    if form.validate_on_submit():
+        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+        if user and user.verify_password(form.password.data):
+            login_user(user, form.remember_me.data)
+            flash('You have been logged in')
+            return redirect(url_for('.dashboard'))
+        flash('Invalid email/username or password', category='error')
+        redirect(url_for('.index'))
+    return render_template(
+        'main/index.html.j2',
+        title='nopaque',
+        form=form
+    )
+
+
+@bp.route('/faq')
+def faq():
+    return render_template(
+        'main/faq.html.j2',
+        title='Frequently Asked Questions'
+    )
+
+
+@bp.route('/dashboard')
+@login_required
+def dashboard():
+    return render_template(
+        'main/dashboard.html.j2',
+        title='Dashboard'
+    )
+
+
+@bp.route('/manual')
+def manual():
+    return render_template(
+        'main/manual.html.j2',
+        title='Manual'
+    )
+
+
+@bp.route('/news')
+def news():
+    return render_template(
+        'main/news.html.j2',
+        title='News'
+    )
+
+
+@bp.route('/privacy_policy')
+def privacy_policy():
+    return render_template(
+        'main/privacy_policy.html.j2',
+        title='Privacy statement (GDPR)'
+    )
+
+
+@bp.route('/terms_of_use')
+def terms_of_use():
+    return render_template(
+        'main/terms_of_use.html.j2',
+        title='Terms of Use'
+    )
+
+
+@bp.route('/social-area')
+@login_required
+def social_area():
+    print('test')
+    corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
+    print(corpora)
+    users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
+    return render_template(
+        'main/social_area.html.j2',
+        title='Social Area',
+        corpora=corpora,
+        users=users
+    )

app/blueprints/services/__init__.py

@@ -0,0 +1,24 @@
+from flask import Blueprint
+from flask_login import login_required
+from pathlib import Path
+import yaml
+
+
+services_file = Path(__file__).parent / 'services.yml'
+with services_file.open('r') as f:
+    SERVICES = yaml.safe_load(f)
+
+bp = Blueprint('services', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes  # noqa

app/blueprints/services/forms.py

@@ -0,0 +1,192 @@
+from flask_wtf import FlaskForm
+from flask_login import current_user
+from flask_wtf.file import FileField, FileRequired
+from wtforms import (
+    BooleanField,
+    DecimalRangeField,
+    MultipleFileField,
+    SelectField,
+    StringField,
+    SubmitField,
+    ValidationError
+)
+from wtforms.validators import InputRequired, Length
+from app.models import SpaCyNLPPipelineModel, TesseractOCRPipelineModel
+from . import SERVICES
+
+
+class CreateJobBaseForm(FlaskForm):
+    description = StringField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=32)]
+    )
+    version = SelectField('Version', validators=[InputRequired()])
+    submit = SubmitField()
+
+
+class CreateFileSetupPipelineJobForm(CreateJobBaseForm):
+    images = MultipleFileField('File(s)', validators=[InputRequired()])
+
+    def validate_images(form, field):
+        valid_mimetypes = ['image/jpeg', 'image/png', 'image/tiff']
+        for image in field.data:
+            if image.mimetype not in valid_mimetypes:
+                raise ValidationError('JPEG, PNG and TIFF files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-file-setup-pipeline-job-form'
+        service_manifest = SERVICES['file-setup-pipeline']
+        version = kwargs.pop('version', service_manifest['latest_version'])
+        super().__init__(*args, **kwargs)
+        self.version.choices = [(x, x) for x in service_manifest['versions']]
+        self.version.data = version
+        self.version.default = service_manifest['latest_version']
+
+
+class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
+    binarization = BooleanField('Binarization')
+    pdf = FileField('File', validators=[FileRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
+    ocropus_nlbin_threshold = DecimalRangeField(
+        render_kw={'min': 0, 'max': 1, 'step': 0.1, 'start': [0.5], 'disabled': True}
+    )
+
+    def validate_binarization(self, field):
+        service_info = SERVICES['tesseract-ocr-pipeline']['versions'][self.version.data]
+        if field.data:
+            if not('methods' in service_info and 'binarization' in service_info['methods']):
+                raise ValidationError('Binarization is not available')
+              
+    def validate_pdf(self, field):
+        if field.data.mimetype != 'application/pdf':
+            raise ValidationError('PDF files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-job-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        version = kwargs.pop('version', service_manifest['latest_version'])
+        super().__init__(*args, **kwargs)
+        service_info = service_manifest['versions'][version]
+        if self.binarization.render_kw is None:
+            self.binarization.render_kw = {}
+        self.binarization.render_kw['disabled'] = True
+        if self.ocropus_nlbin_threshold.render_kw is None:
+            self.ocropus_nlbin_threshold.render_kw = {}
+        self.ocropus_nlbin_threshold.render_kw['disabled'] = True
+        if 'methods' in service_info:
+            if 'binarization' in service_info['methods']:
+                del self.binarization.render_kw['disabled']
+                if 'ocropus_nlbin_threshold' in service_info['methods']:
+                    del self.ocropus_nlbin_threshold.render_kw['disabled']
+        user_models = [
+            x for x in current_user.tesseract_ocr_pipeline_models.order_by(TesseractOCRPipelineModel.title).all()
+        ]
+        models = [
+            x for x in TesseractOCRPipelineModel.query.order_by(TesseractOCRPipelineModel.title).all()
+            if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
+        ]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
+        self.model.default = ''
+        self.version.choices = [(x, x) for x in service_manifest['versions']]
+        self.version.data = version
+        self.version.default = service_manifest['latest_version']
+
+
+class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
+    binarization = BooleanField('Binarization')
+    pdf = FileField('File', validators=[FileRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
+
+    def validate_binarization(self, field):
+        service_info = SERVICES['transkribus-htr-pipeline']['versions'][self.version.data]
+        if field.data:
+            if(
+                'methods' not in service_info
+                or 'binarization' not in service_info['methods']
+            ):
+                raise ValidationError('Binarization is not available')
+
+    def validate_pdf(self, field):
+        if field.data.mimetype != 'application/pdf':
+            raise ValidationError('PDF files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-transkribus-htr-pipeline-job-form'
+        transkribus_htr_pipeline_models = kwargs.pop('transkribus_htr_pipeline_models', [])
+        service_manifest = SERVICES['transkribus-htr-pipeline']
+        version = kwargs.pop('version', service_manifest['latest_version'])
+        super().__init__(*args, **kwargs)
+        service_info = service_manifest['versions'][version]
+        if self.binarization.render_kw is None:
+            self.binarization.render_kw = {}
+        self.binarization.render_kw['disabled'] = True
+        if 'methods' in service_info:
+            if 'binarization' in service_info['methods']:
+                del self.binarization.render_kw['disabled']
+        self.model.choices = [('', 'Choose your option')]
+        self.model.choices += [(x['modelId'], x['name']) for x in transkribus_htr_pipeline_models]
+        self.model.default = ''
+        self.version.choices = [(x, x) for x in service_manifest['versions']]
+        self.version.data = version
+        self.version.default = service_manifest['latest_version']
+
+
+class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
+    encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
+    txt = FileField('File', validators=[FileRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
+    
+    def validate_encoding_detection(self, field):
+        service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]
+        if field.data:
+            if(
+                'methods' not in service_info
+                or 'encoding_detection' not in service_info['methods']
+            ):
+                raise ValidationError('Encoding detection is not available')
+
+    def validate_txt(form, field):
+        if field.data.mimetype != 'text/plain':
+            raise ValidationError('Plain text files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-job-form'
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        version = kwargs.pop('version', service_manifest['latest_version'])
+        super().__init__(*args, **kwargs)
+        service_info = service_manifest['versions'][version]
+        print(service_info)
+        if self.encoding_detection.render_kw is None:
+            self.encoding_detection.render_kw = {}
+        self.encoding_detection.render_kw['disabled'] = True
+        if 'methods' in service_info:
+            if 'encoding_detection' in service_info['methods']:
+                del self.encoding_detection.render_kw['disabled']
+        user_models = [
+            x for x in current_user.spacy_nlp_pipeline_models.order_by(SpaCyNLPPipelineModel.title).all()
+        ]
+        models = [
+            x for x in SpaCyNLPPipelineModel.query.filter(SpaCyNLPPipelineModel.user != current_user, SpaCyNLPPipelineModel.is_public == True).order_by(SpaCyNLPPipelineModel.title).all()
+            if version in x.compatible_service_versions
+        ]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
+        self.model.default = ''
+        self.version.choices = [(x, x) for x in service_manifest['versions']]
+        self.version.data = version
+        self.version.default = version

app/blueprints/services/routes.py

@@ -0,0 +1,224 @@
+from flask import abort, current_app, flash, redirect, render_template, request, url_for
+from flask_login import current_user
+import requests
+from app import db, hashids
+from app.models import (
+    Job,
+    JobInput,
+    JobStatus,
+    TesseractOCRPipelineModel,
+    SpaCyNLPPipelineModel
+)
+from . import bp, SERVICES
+from .forms import (
+    CreateFileSetupPipelineJobForm,
+    CreateTesseractOCRPipelineJobForm,
+    CreateTranskribusHTRPipelineJobForm,
+    CreateSpacyNLPPipelineJobForm
+)
+
+
+@bp.route('/services')
+def services():
+    return redirect(url_for('main.dashboard'))
+
+
+@bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
+def file_setup_pipeline():
+    service = 'file-setup-pipeline'
+    service_manifest = SERVICES[service]
+    version = request.args.get('version', service_manifest['latest_version'])
+    if version not in service_manifest['versions']:
+        abort(404)
+    form = CreateFileSetupPipelineJobForm(prefix='create-job-form', version=version)
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            job = Job.create(
+                title=form.title.data,
+                description=form.description.data,
+                service=service,
+                service_args={},
+                service_version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        for input_file in form.images.data:
+            try:
+                JobInput.create(input_file, job=job)
+            except (AttributeError, OSError):
+                abort(500)
+        job.status = JobStatus.SUBMITTED
+        db.session.commit()
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
+    return render_template(
+        'services/file_setup_pipeline.html.j2',
+        title=service_manifest['name'],
+        form=form
+    )
+
+
+@bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
+def tesseract_ocr_pipeline():
+    service_name = 'tesseract-ocr-pipeline'
+    service_manifest = SERVICES[service_name]
+    version = request.args.get('version', service_manifest['latest_version'])
+    if version not in service_manifest['versions']:
+        abort(404)
+    form = CreateTesseractOCRPipelineJobForm(prefix='create-job-form', version=version)
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            job = Job.create(
+                title=form.title.data,
+                description=form.description.data,
+                service=service_name,
+                service_args={
+                    'binarization': form.binarization.data,
+                    'model': hashids.decode(form.model.data),
+                    'ocropus_nlbin_threshold': float(form.ocropus_nlbin_threshold.data)
+                },
+                service_version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        try:
+            JobInput.create(form.pdf.data, job=job)
+        except (AttributeError, OSError):
+            abort(500)
+        job.status = JobStatus.SUBMITTED
+        db.session.commit()
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
+    tesseract_ocr_pipeline_models = [
+        x for x in TesseractOCRPipelineModel.query.all()
+        if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
+    ]
+    user_tesseract_ocr_pipeline_models_count = len(current_user.tesseract_ocr_pipeline_models.all())
+    return render_template(
+        'services/tesseract_ocr_pipeline.html.j2',
+        title=service_manifest['name'],
+        form=form,
+        tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models,
+        user_tesseract_ocr_pipeline_models_count=user_tesseract_ocr_pipeline_models_count
+    )
+
+
+@bp.route('/transkribus-htr-pipeline', methods=['GET', 'POST'])
+def transkribus_htr_pipeline():
+    if not current_app.config.get('NOPAQUE_TRANSKRIBUS_ENABLED'):
+        abort(404)
+    service = 'transkribus-htr-pipeline'
+    service_manifest = SERVICES[service]
+    version = request.args.get('version', service_manifest['latest_version'])
+    if version not in service_manifest['versions']:
+        abort(404)
+    r = requests.get(
+        'https://transkribus.eu/TrpServer/rest/models/text',
+        headers={'Accept': 'application/json'}
+    )
+    if r.status_code != 200:
+        abort(500)
+    transkribus_htr_pipeline_models = r.json()['trpModelMetadata']
+    transkribus_htr_pipeline_models.append({'modelId': 48513, 'name': 'Caroline Minuscle', 'language': 'lat', 'isoLanguages': ['lat']})
+    form = CreateTranskribusHTRPipelineJobForm(
+        prefix='create-job-form',
+        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models,
+        version=version
+    )
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            job = Job.create(
+                title=form.title.data,
+                description=form.description.data,
+                service=service,
+                service_args={
+                    'binarization': form.binarization.data,
+                    'model': form.model.data
+                },
+                service_version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        try:
+            JobInput.create(form.pdf.data, job=job)
+        except (AttributeError, OSError):
+            abort(500)
+        job.status = JobStatus.SUBMITTED
+        db.session.commit()
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
+    return render_template(
+        'services/transkribus_htr_pipeline.html.j2',
+        title=service_manifest['name'],
+        form=form,
+        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models
+    )
+
+
+@bp.route('/spacy-nlp-pipeline', methods=['GET', 'POST'])
+def spacy_nlp_pipeline():
+    service = 'spacy-nlp-pipeline'
+    service_manifest = SERVICES[service]
+    version = request.args.get('version', SERVICES[service]['latest_version'])
+    if version not in service_manifest['versions']:
+        abort(404)
+    form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
+    spacy_nlp_pipeline_models = SpaCyNLPPipelineModel.query.all()
+    user_spacy_nlp_pipeline_models_count = len(current_user.spacy_nlp_pipeline_models.all())
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            job = Job.create(
+                title=form.title.data,
+                description=form.description.data,
+                service=service,
+                service_args={
+                    'encoding_detection': form.encoding_detection.data,
+                    'model': form.model.data
+                },
+                service_version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        try:
+            JobInput.create(form.txt.data, job=job)
+        except (AttributeError, OSError):
+            abort(500)
+        job.status = JobStatus.SUBMITTED
+        db.session.commit()
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
+    return render_template(
+        'services/spacy_nlp_pipeline.html.j2',
+        title=service_manifest['name'],
+        form=form,
+        spacy_nlp_pipeline_models=spacy_nlp_pipeline_models,
+        user_spacy_nlp_pipeline_models_count=user_spacy_nlp_pipeline_models_count
+    )
+
+
+@bp.route('/corpus-analysis')
+def corpus_analysis():
+    return render_template(
+        'services/corpus_analysis.html.j2',
+        title='Corpus Analysis'
+    )

app/blueprints/services/services.yml

@@ -0,0 +1,66 @@
+# TODO: This could also be done via GitLab/GitHub APIs
+file-setup-pipeline:
+  name: 'File Setup Pipeline'
+  publisher: 'Bielefeld University - CRC 1288 - INF'
+  latest_version: '0.1.0'
+  versions:
+    0.1.0:
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup-pipeline/-/releases/v0.1.0'
+tesseract-ocr-pipeline:
+  name: 'Tesseract OCR Pipeline'
+  publisher: 'Bielefeld University - CRC 1288 - INF'
+  latest_version: '0.1.2'
+  versions:
+    0.1.0:
+      methods:
+        - 'binarization'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.0'
+    0.1.1:
+      methods:
+        - 'binarization'
+        - 'ocropus_nlbin_threshold'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
+    0.1.2:
+      methods:
+        - 'binarization'
+        - 'ocropus_nlbin_threshold'
+      publishing_year: 2023
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.2'
+transkribus-htr-pipeline:
+  name: 'Transkribus HTR Pipeline'
+  publisher: 'Bielefeld University - CRC 1288 - INF'
+  latest_version: '0.1.1'
+  versions:
+    0.1.0:
+      methods:
+        - 'binarization'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/transkribus-htr-pipeline/-/releases/v0.1.0'
+    0.1.1:
+      methods:
+        - 'binarization'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/transkribus-htr-pipeline/-/releases/v0.1.1'
+spacy-nlp-pipeline:
+  name: 'SpaCy NLP Pipeline'
+  publisher: 'Bielefeld University - CRC 1288 - INF'
+  latest_version: '0.1.1'
+  versions:
+    0.1.0:
+      methods:
+        - 'encoding_detection'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'
+    0.1.1:
+      methods:
+        - 'encoding_detection'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.1'
+    0.1.2:
+      methods:
+        - 'encoding_detection'
+      publishing_year: 2024
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2'

app/blueprints/settings/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('settings', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes

app/blueprints/settings/routes.py

@@ -0,0 +1,10 @@
+from flask import g, url_for
+from flask_login import current_user
+from app.blueprints.users.settings.routes import settings as settings_route
+from . import bp
+
+
+@bp.route('/settings', methods=['GET', 'POST'])
+def settings():
+    g._nopaque_redirect_location_on_post = url_for('.settings')
+    return settings_route(current_user.id)

app/blueprints/users/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('users', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import cli, events, json_routes, routes, settings

app/blueprints/users/cli.py

@@ -0,0 +1,12 @@
+from app.models import User
+from app import db
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset terms of use accept '''
+    for user in [x for x in User.query.all() if x.terms_of_use_accepted]:
+        print(f'Resetting user {user.username}')
+        user.terms_of_use_accepted = False
+    db.session.commit()

app/blueprints/users/events.py

@@ -0,0 +1,47 @@
+from flask_login import current_user
+from flask_socketio import join_room, leave_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import User
+
+
+@socketio.on('GET /users/<user_id>')
+@socketio_login_required
+def get_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.get(user_id)
+    if user is None:
+        return {'status': 404, 'statusText': 'Not found'}
+    if not (user == current_user or current_user.is_administrator):
+        return {'status': 403, 'statusText': 'Forbidden'}
+    return {
+        'body': user.to_json_serializeable(backrefs=True, relationships=True),
+        'status': 200,
+        'statusText': 'OK'
+    }
+
+
+@socketio.on('SUBSCRIBE /users/<user_id>')
+@socketio_login_required
+def subscribe_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.get(user_id)
+    if user is None:
+        return {'status': 404, 'statusText': 'Not found'}
+    if not (user == current_user or current_user.is_administrator):
+        return {'status': 403, 'statusText': 'Forbidden'}
+    join_room(f'/users/{user.hashid}')
+    return {'status': 200, 'statusText': 'OK'}
+
+
+@socketio.on('UNSUBSCRIBE /users/<user_id>')
+@socketio_login_required
+def unsubscribe_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.get(user_id)
+    if user is None:
+        return {'status': 404, 'statusText': 'Not found'}
+    if not (user == current_user or current_user.is_administrator):
+        return {'status': 403, 'statusText': 'Forbidden'}
+    leave_room(f'/users/{user.hashid}')
+    return {'status': 200, 'statusText': 'OK'}

app/blueprints/users/json_routes.py

@@ -0,0 +1,69 @@
+from flask import abort, current_app
+from flask_login import current_user, logout_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation
+from app.models import Avatar, User
+from . import bp
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_user(user_id):
+    def _delete_user(app, user_id):
+        with app.app_context():
+            user = User.query.get(user_id)
+            user.delete()
+            db.session.commit()
+
+    user = User.query.get_or_404(user_id)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_user,
+        args=(current_app._get_current_object(), user.id)
+    )
+    if user == current_user:
+        logout_user()
+    thread.start()
+    response_data = {
+        'message': f'User "{user.username}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/<hashid:user_id>/avatar', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_user_avatar(user_id):
+    def _delete_avatar(app, avatar_id):
+        with app.app_context():
+            avatar = Avatar.query.get(avatar_id)
+            avatar.delete()
+            db.session.commit()
+
+    user = User.query.get_or_404(user_id)
+    if user.avatar is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_avatar,
+        args=(current_app._get_current_object(), user.avatar.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Avatar marked for deletion'
+    }
+    return response_data, 202
+
+@bp.route('/accept-terms-of-use', methods=['POST'])
+@content_negotiation(produces='application/json')
+def accept_terms_of_use():
+    if not (current_user.is_authenticated or current_user.confirmed):
+        abort(403)
+    current_user.terms_of_use_accepted = True
+    db.session.commit()
+    response_data = {
+        'message': 'You accepted the terms of use',
+    }
+    return response_data, 202

app/blueprints/users/nevents.py

@@ -0,0 +1,114 @@
+from flask_login import current_user
+from flask_socketio import join_room
+from app import hashids, socketio
+from app.decorators import socketio_admin_required, socketio_login_required
+from app.models import User
+
+
+@socketio.on('GET /users')
+@socketio_admin_required
+def get_users():
+    users = User.query.filter_by().all()
+    return {
+        'body': [user.to_json_serializable() for user in users],
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /users')
+@socketio_admin_required
+def subscribe_users():
+    join_room('/users')
+    return {'options': {'status': 200, 'statusText': 'OK'}}
+
+
+@socketio.on('GET /users/<user_id>')
+@socketio_login_required
+def get_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.get(user_id)
+    if user is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (user == current_user or current_user.is_administrator):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    return {
+        'body': user.to_json_serializable(),
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /users/<user_id>')
+@socketio_login_required
+def subscribe_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.get(user_id)
+    if user is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (user == current_user or current_user.is_administrator):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    join_room(f'/users/{user.hashid}')
+    return {'options': {'status': 200, 'statusText': 'OK'}}
+
+
+@socketio.on('GET /public_users')
+@socketio_login_required
+def get_public_users():
+    users = User.query.filter_by(is_public=True).all()
+    return {
+        'body': [
+            user.to_json_serializable(filter_by_privacy_settings=True)
+            for user in users
+        ],
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /users')
+@socketio_admin_required
+def subscribe_users():
+    join_room('/public_users')
+    return {'options': {'status': 200, 'statusText': 'OK'}}
+
+
+@socketio.on('GET /public_users/<user_id>')
+@socketio_login_required
+def get_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.filter_by(id=user_id, is_public=True).first()
+    if user is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (user == current_user or current_user.is_administrator):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    return {
+        'body': user.to_json_serializable(filter_by_privacy_settings=True),
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /public_users/<user_id>')
+@socketio_login_required
+def subscribe_user(user_hashid):
+    user_id = hashids.decode(user_hashid)
+    user = User.query.filter_by(id=user_id, is_public=True).first()
+    if user is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (user == current_user or current_user.is_administrator):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    join_room(f'/public_users/{user.hashid}')
+    return {'options': {'status': 200, 'statusText': 'OK'}}

app/blueprints/users/routes.py

@@ -0,0 +1,43 @@
+from flask import (
+    abort,
+    redirect, 
+    render_template, 
+    send_from_directory,
+    url_for
+)
+from flask_login import current_user
+from app.models import User
+from . import bp
+
+
+@bp.route('')
+def users():
+    return redirect(url_for('main.social_area', _anchor='users'))
+
+
+@bp.route('/<hashid:user_id>')
+def user(user_id):
+    user = User.query.get_or_404(user_id)
+    if not (user.is_public or user == current_user or current_user.is_administrator):
+        abort(403)
+    return render_template(
+        'users/user.html.j2',
+        title=user.username,
+        user=user
+    )
+
+
+@bp.route('/<hashid:user_id>/avatar')
+def user_avatar(user_id):
+    user = User.query.get_or_404(user_id)
+    if not (user.is_public or user == current_user or current_user.is_administrator):
+        abort(403)
+    if user.avatar is None:
+        return redirect(url_for('static', filename='images/user_avatar.png'))
+    return send_from_directory(
+        user.avatar.path.parent,
+        user.avatar.path.name,
+        as_attachment=True,
+        download_name=user.avatar.filename,
+        mimetype=user.avatar.mimetype
+    )

app/blueprints/users/settings/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/users/settings/forms.py

@@ -0,0 +1,160 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired, FileSize
+from wtforms import (
+    PasswordField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    ValidationError
+)
+from wtforms.validators import (
+    DataRequired,
+    Email,
+    EqualTo,
+    Length,
+    Regexp
+)
+from app.models import User, UserSettingJobStatusMailNotificationLevel
+
+
+class UpdateAccountInformationForm(FlaskForm):
+    email = StringField(
+        'E-Mail',
+        validators=[DataRequired(), Length(max=254), Email()]
+    )
+    username = StringField(
+        'Username',
+        validators=[
+            DataRequired(),
+            Length(max=64),
+            Regexp(
+                User.username_pattern,
+                message=(
+                    'Usernames must have only letters, numbers, dots or '
+                    'underscores'
+                )
+            )
+        ]
+    )
+    submit = SubmitField()
+    
+    def __init__(self, user, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-account-information-form'
+        super().__init__(*args, **kwargs)
+        self.user = user
+
+    def validate_email(self, field):
+        if (field.data != self.user.email
+                and User.query.filter_by(email=field.data).first()):
+            raise ValidationError('Email already registered')
+
+    def validate_username(self, field):
+        if (field.data != self.user.username
+                and User.query.filter_by(username=field.data).first()):
+            raise ValidationError('Username already in use')
+
+
+class UpdateProfileInformationForm(FlaskForm):
+    full_name = StringField(
+        'Full name',
+        validators=[Length(max=128)]
+    )
+    about_me = TextAreaField(
+        'About me', 
+        validators=[
+            Length(max=254)
+        ]
+    )
+    website = StringField(
+        'Website',
+        validators=[
+            Length(max=254)
+        ]
+    )
+    organization = StringField(
+        'Organization',
+        validators=[
+            Length(max=128)
+        ]
+    )
+    location = StringField(
+        'Location',
+        validators=[
+            Length(max=128)
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-profile-information-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateAvatarForm(FlaskForm):
+    avatar = FileField('File', validators=[FileRequired(), FileSize(2_000_000)])
+    submit = SubmitField()
+
+    def validate_avatar(self, field):
+        valid_mimetypes = ['image/jpeg', 'image/png']
+        if field.data.mimetype not in valid_mimetypes:
+            raise ValidationError('JPEG and PNG files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-avatar-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdatePasswordForm(FlaskForm):
+    password = PasswordField('Old password', validators=[DataRequired()])
+    new_password = PasswordField(
+        'New password',
+        validators=[
+            DataRequired(),
+            EqualTo('new_password_2', message='Passwords must match')
+        ]
+    )
+    new_password_2 = PasswordField(
+        'New password confirmation',
+        validators=[
+            DataRequired(),
+            EqualTo('new_password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-password-form'
+        super().__init__(*args, **kwargs)
+        self.user = user
+
+    def validate_current_password(self, field):
+        if not self.user.verify_password(field.data):
+            raise ValidationError('Invalid password')
+
+
+class UpdateNotificationsForm(FlaskForm):
+    job_status_mail_notification_level = SelectField(
+        'Job status mail notification level',
+        choices=[
+            (x.name, x.name.capitalize())
+            for x in UserSettingJobStatusMailNotificationLevel
+        ],
+        validators=[DataRequired()]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-notifications-form'
+        super().__init__(*args, **kwargs)

app/blueprints/users/settings/json_routes.py

@@ -0,0 +1,49 @@
+from flask import abort, request
+from flask_login import current_user
+from app import db
+from app.decorators import content_negotiation
+from app.models import User, ProfilePrivacySettings
+from . import bp
+
+
+@bp.route('/<hashid:user_id>/settings/profile-privacy/is-public', methods=['PUT'])
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_user_profile_privacy_setting_is_public(user_id):
+    user = User.query.get_or_404(user_id)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    enabled = request.json
+    if not isinstance(enabled, bool):
+        abort(400)
+    user.is_public = enabled
+    db.session.commit()
+    response_data = {
+        'message': 'Profile privacy settings updated',
+        'category': 'settings'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:user_id>/settings/profile-privacy/<string:profile_privacy_setting_name>', methods=['PUT'])
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_user_profile_privacy_settings(user_id, profile_privacy_setting_name):
+    user = User.query.get_or_404(user_id)
+    try:
+        profile_privacy_setting = ProfilePrivacySettings[profile_privacy_setting_name]
+    except KeyError:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+    enabled = request.json
+    if not isinstance(enabled, bool):
+        abort(400)
+    if enabled:
+        user.add_profile_privacy_setting(profile_privacy_setting)
+    else:
+        user.remove_profile_privacy_setting(profile_privacy_setting)
+    db.session.commit()
+    response_data = {
+        'message': 'Profile privacy settings updated',
+        'category': 'settings'
+    }
+    return response_data, 200

app/blueprints/users/settings/routes.py

@@ -0,0 +1,93 @@
+from flask import abort, flash, g, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import Avatar, User
+from . import bp
+from .forms import (
+    UpdateAvatarForm,
+    UpdatePasswordForm,
+    UpdateNotificationsForm,
+    UpdateAccountInformationForm,
+    UpdateProfileInformationForm
+)
+
+
+@bp.route('/<hashid:user_id>/settings', methods=['GET', 'POST'])
+def settings(user_id):
+    user = User.query.get_or_404(user_id)
+    if not (user == current_user or current_user.is_administrator):
+        abort(403)
+
+    redirect_location_on_post = g.pop(
+        '_nopaque_redirect_location_on_post',
+        url_for('.settings', user_id=user_id)
+    )
+
+    update_account_information_form = UpdateAccountInformationForm(user)
+    update_profile_information_form = UpdateProfileInformationForm(user)
+    update_avatar_form = UpdateAvatarForm()
+    update_password_form = UpdatePasswordForm(user)
+    update_notifications_form = UpdateNotificationsForm(user)
+
+    # region handle update profile information form
+    if update_profile_information_form.submit.data and update_profile_information_form.validate():
+        user.about_me = update_profile_information_form.about_me.data
+        user.location = update_profile_information_form.location.data
+        user.organization = update_profile_information_form.organization.data
+        user.website = update_profile_information_form.website.data
+        user.full_name = update_profile_information_form.full_name.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(redirect_location_on_post)
+    # endregion handle update profile information form
+
+    # region handle update avatar form
+    if update_avatar_form.submit.data and update_avatar_form.validate():
+        try:
+            Avatar.create(
+                update_avatar_form.avatar.data,
+                user=user
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(redirect_location_on_post)
+    # endregion handle update avatar form
+
+    # region handle update account information form
+    if update_account_information_form.submit.data and update_account_information_form.validate():
+        user.email = update_account_information_form.email.data
+        user.username = update_account_information_form.username.data
+        db.session.commit()
+        flash('Profile settings updated')
+        return redirect(redirect_location_on_post)
+    # endregion handle update account information form
+
+    # region handle update password form
+    if update_password_form.submit.data and update_password_form.validate():
+        user.password = update_password_form.new_password.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(redirect_location_on_post)
+    # endregion handle update password form
+
+    # region handle update notifications form
+    if update_notifications_form.submit.data and update_notifications_form.validate():
+        user.setting_job_status_mail_notification_level = \
+            update_notifications_form.job_status_mail_notification_level.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(redirect_location_on_post)
+    # endregion handle update notifications form
+
+    return render_template(
+        'users/settings/settings.html.j2',
+        title='Settings',
+        update_account_information_form=update_account_information_form,
+        update_avatar_form=update_avatar_form,
+        update_notifications_form=update_notifications_form,
+        update_password_form=update_password_form,
+        update_profile_information_form=update_profile_information_form,
+        user=user
+    )

app/blueprints/workshops/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('workshops', __name__)
+from . import routes

app/blueprints/workshops/routes.py

@@ -0,0 +1,15 @@
+from flask import redirect, render_template, url_for
+from . import bp
+
+
+@bp.route('')
+def workshops():
+    return redirect(url_for('main.dashboard'))
+
+
+@bp.route('/fgho_sommerschule_2023')
+def fgho_sommerschule_2023():
+    return render_template(
+        'workshops/fgho_sommerschule_2023.html.j2',
+        title='FGHO Sommerschule 2023',
+    )