Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

.dockerignore

@@ -1 +1,29 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+README.md
+
+
 data

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.13-slim-buster
+FROM python:3.8.10-slim-buster
 
 
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
@@ -7,10 +7,12 @@ LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
 ARG DOCKER_GID
 ARG UID
 ARG GID
-ENV LANG=C.UTF-8
 
 
 ENV FLASK_APP nopaque.py
+ENV LANG=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
 
 
 RUN apt-get update \

README.md

@@ -40,7 +40,7 @@ username@hostname:~$ <YOUR EDITOR> db.env
 username@hostname:~$ <YOUR EDITOR> .env
 # Create docker-compose.override.yml file
 username@hostname:~$ touch docker-compose.override.yml
-# Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose.<example>.yml)
+# Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples inside the docker-compose directory)
 username@hostname:~$ <YOUR EDITOR> docker-compose.override.yml
 # Build docker images
 username@hostname:~$ docker-compose build

app/__init__.py

@@ -1,9 +1,12 @@
+from apifairy import APIFairy
 from config import Config
+from docker import DockerClient
 from flask import Flask
 from flask_apscheduler import APScheduler
 from flask_assets import Environment
 from flask_login import LoginManager
 from flask_mail import Mail
+from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_paranoid import Paranoid
 from flask_socketio import SocketIO
@@ -11,18 +14,21 @@ from flask_sqlalchemy import SQLAlchemy
 from flask_hashids import Hashids
 
 
-assets: Environment = Environment()
+apifairy = APIFairy()
-db: SQLAlchemy = SQLAlchemy()
+assets = Environment()
-hashids: Hashids = Hashids()
+db = SQLAlchemy()
-login: LoginManager = LoginManager()
+docker_client = DockerClient()
+hashids = Hashids()
+login = LoginManager()
 login.login_view = 'auth.login'
 login.login_message = 'Please log in to access this page.'
-mail: Mail = Mail()
+ma = Marshmallow()
-migrate: Migrate = Migrate()
+mail = Mail()
-paranoid: Paranoid = Paranoid()
+migrate = Migrate()
+paranoid = Paranoid()
 paranoid.redirect_view = '/'
-scheduler: APScheduler = APScheduler()  # TODO: Use this!
+scheduler = APScheduler()
-socketio: SocketIO = SocketIO()
+socketio = SocketIO()
 
 
 def create_app(config: Config = Config) -> Flask:
@@ -30,19 +36,24 @@ def create_app(config: Config = Config) -> Flask:
     app: Flask = Flask(__name__)
     app.config.from_object(config)
     config.init_app(app)
+    docker_client.login(
+        app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
+        password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
+        registry=app.config['NOPAQUE_DOCKER_REGISTRY']
+    )
 
+    apifairy.init_app(app)
     assets.init_app(app)
     db.init_app(app)
     hashids.init_app(app)
     login.init_app(app)
+    ma.init_app(app)
     mail.init_app(app)
     migrate.init_app(app, db)
     paranoid.init_app(app)
+    scheduler.init_app(app)
     socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
 
-    from app import socketio_event_listeners
-    from app import sqlalchemy_event_listeners
-
     from .admin import bp as admin_blueprint
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
 
@@ -52,8 +63,8 @@ def create_app(config: Config = Config) -> Flask:
     from .auth import bp as auth_blueprint
     app.register_blueprint(auth_blueprint, url_prefix='/auth')
 
-    from .contribute import bp as contribute_blueprint
+    from .contributions import bp as contributions_blueprint
-    app.register_blueprint(contribute_blueprint, url_prefix='/contribute')
+    app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
     from .corpora import bp as corpora_blueprint
     app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
@@ -65,7 +76,7 @@ def create_app(config: Config = Config) -> Flask:
     app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
 
     from .main import bp as main_blueprint
-    app.register_blueprint(main_blueprint)
+    app.register_blueprint(main_blueprint, url_prefix='/')
 
     from .services import bp as services_blueprint
     app.register_blueprint(services_blueprint, url_prefix='/services')
@@ -73,6 +84,9 @@ def create_app(config: Config = Config) -> Flask:
     from .settings import bp as settings_blueprint
     app.register_blueprint(settings_blueprint, url_prefix='/settings')
 
+    from .users import bp as users_blueprint
+    app.register_blueprint(users_blueprint, url_prefix='/users')
+
     from .test import bp as test_blueprint
     app.register_blueprint(test_blueprint, url_prefix='/test')
 

app/admin/forms.py

@@ -10,7 +10,9 @@ class AdminEditUserForm(FlaskForm):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.role.choices = [
+        self.role.choices = [(x.hashid, x.name) for x in Role.query.all()]
-            (role.hashid, role.name)
+
-            for role in Role.query.order_by(Role.name).all()
+    def prefill(self, user):
-        ]
+        ''' Pre-fill the form with data of an exististing user '''
+        self.confirmed.data = user.confirmed
+        self.role.data = user.role.hashid

app/admin/routes.py

@@ -1,14 +1,14 @@
+from flask import current_app, flash, redirect, render_template, url_for
+from flask_login import login_required
+from threading import Thread
 from app import db, hashids
 from app.decorators import admin_required
 from app.models import Role, User, UserSettingJobStatusMailNotificationLevel
-from app.settings import tasks as settings_tasks
 from app.settings.forms import (
     EditGeneralSettingsForm,
     EditInterfaceSettingsForm,
     EditNotificationSettingsForm
 )
-from flask import flash, redirect, render_template, url_for
-from flask_login import login_required
 from . import bp
 from .forms import AdminEditUserForm
 
@@ -24,20 +24,17 @@ def before_request():
     pass
 
 
-@bp.route('/')
+@bp.route('')
 def index():
     return redirect(url_for('.users'))
 
 
 @bp.route('/users')
 def users():
-    dict_users = {
+    json_users = [x.to_json(backrefs=True) for x in User.query.all()]
-        user.id: user.to_dict(backrefs=True, relationships=False)
-        for user in User.query.all()
-    }
     return render_template(
         'admin/users.html.j2',
-        dict_users=dict_users,
+        json_users=json_users,
         title='Users'
     )
 
@@ -48,59 +45,45 @@ def user(user_id):
     return render_template('admin/user.html.j2', title='User', user=user)
 
 
-@bp.route('/users/<hashid:user_id>/delete')
-def delete_user(user_id):
-    settings_tasks.delete_user(user_id)
-    flash('User has been marked for deletion')
-    return redirect(url_for('.users'))
-
-
 @bp.route('/users/<hashid:user_id>/edit', methods=['GET', 'POST'])
 def edit_user(user_id):
     user = User.query.get_or_404(user_id)
     admin_edit_user_form = AdminEditUserForm(
-        prefix='admin_edit_user_form'
+        prefix='admin-edit-user-form'
     )
     edit_general_settings_form = EditGeneralSettingsForm(
         user,
-        prefix='edit_general_settings_form'
+        prefix='edit-general-settings-form'
     )
     edit_interface_settings_form = EditInterfaceSettingsForm(
-        prefix='edit_interface_settings_form'
+        prefix='edit-interface-settings-form'
     )
     edit_notification_settings_form = EditNotificationSettingsForm(
-        prefix='edit_notification_settings_form'
+        prefix='edit-notification-settings-form'
     )
-    if (
+    if (admin_edit_user_form.submit.data
-        admin_edit_user_form.submit.data
+            and admin_edit_user_form.validate()):
-        and admin_edit_user_form.validate()
-    ):
         user.confirmed = admin_edit_user_form.confirmed.data
         role_id = hashids.decode(admin_edit_user_form.role.data)
         user.role = Role.query.get(role_id)
+        db.session.commit()
         flash('Your changes have been saved')
         return redirect(url_for('.edit_user', user_id=user.id))
-    if (
+    if (edit_general_settings_form.submit.data
-        edit_general_settings_form.submit.data
+            and edit_general_settings_form.validate()):
-        and edit_general_settings_form.validate()
-    ):
         user.email = edit_general_settings_form.email.data
         user.username = edit_general_settings_form.username.data
         db.session.commit()
         flash('Your changes have been saved')
         return redirect(url_for('.edit_user', user_id=user.id))
-    if (
+    if (edit_interface_settings_form.submit.data
-        edit_interface_settings_form.submit.data
+            and edit_interface_settings_form.validate()):
-        and edit_interface_settings_form.validate()
-    ):
         user.setting_dark_mode = edit_interface_settings_form.dark_mode.data
         db.session.commit()
         flash('Your changes have been saved')
         return redirect(url_for('.edit_user', user_id=user.id))
-    if (
+    if (edit_notification_settings_form.submit.data
-        edit_notification_settings_form.submit.data
+            and edit_notification_settings_form.validate()):
-        and edit_notification_settings_form.validate()
-    ):
         user.setting_job_status_mail_notification_level = \
             UserSettingJobStatusMailNotificationLevel[
                 edit_notification_settings_form.job_status_mail_notification_level.data  # noqa
@@ -108,13 +91,10 @@ def edit_user(user_id):
         db.session.commit()
         flash('Your changes have been saved')
         return redirect(url_for('.edit_user', user_id=user.id))
-    admin_edit_user_form.confirmed.data = user.confirmed
+    admin_edit_user_form.prefill(user)
-    admin_edit_user_form.role.data = user.role.hashid
+    edit_general_settings_form.prefill(user)
-    edit_general_settings_form.email.data = user.email
+    edit_interface_settings_form.prefill(user)
-    edit_general_settings_form.username.data = user.username
+    edit_notification_settings_form.prefill(user)
-    edit_interface_settings_form.dark_mode.data = user.setting_dark_mode
-    edit_notification_settings_form.job_status_mail_notification_level.data = \
-        user.setting_job_status_mail_notification_level.name
     return render_template(
         'admin/edit_user.html.j2',
         admin_edit_user_form=admin_edit_user_form,
@@ -124,3 +104,20 @@ def edit_user(user_id):
         title='Edit user',
         user=user
     )
+
+
+@bp.route('/users/<hashid:user_id>/delete', methods=['DELETE'])
+def delete_user(user_id):
+    def _delete_user(app, user_id):
+        with app.app_context():
+            user = User.query.get(user_id)
+            user.delete()
+            db.session.commit()
+
+    User.query.get_or_404(user_id)
+    thread = Thread(
+        target=_delete_user,
+        args=(current_app._get_current_object(), user_id)
+    )
+    thread.start()
+    return {}, 202

app/api/__init__.py

@@ -1,25 +1,14 @@
 from flask import Blueprint
-from flask_restx import Api
 
-from .tokens import ns as tokens_ns
 
 bp = Blueprint('api', __name__)
-authorizations = {
-    'basicAuth': {
-        'type': 'basic'
-    },
-    'apiKey': {
-        'type': 'apiKey',
-        'in': 'header',
-        'name': 'Authorization'
-    }
-}
-api = Api(
-    bp,
-    authorizations=authorizations,
-    description='An API to interact with nopaque',
-    title='nopaque API',
-    version='1.0'
-)
 
-api.add_namespace(tokens_ns)
+
+from .tokens import bp as tokens_blueprint
+bp.register_blueprint(tokens_blueprint, url_prefix='/tokens')
+
+from .users import bp as users_blueprint
+bp.register_blueprint(users_blueprint, url_prefix='/users')
+
+from .jobs import bp as jobs_blueprint
+bp.register_blueprint(jobs_blueprint, url_prefix='/jobs')

app/api/auth.py

@@ -1,34 +1,49 @@
-from app.models import User
+from flask import current_app
 from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth
-from sqlalchemy import or_
+from werkzeug.exceptions import Forbidden, Unauthorized
-from werkzeug.http import HTTP_STATUS_CODES
+from app.models import User
+
 
 basic_auth = HTTPBasicAuth()
 token_auth = HTTPTokenAuth()
-
+auth_error_responses = {
+    Unauthorized.code: Unauthorized.description,
+    Forbidden.code: Forbidden.description
+}
 
 @basic_auth.verify_password
 def verify_password(email_or_username, password):
-    user = User.query.filter(
+    user = User.query.filter((User.email == email_or_username.lower()) | (User.username == email_or_username)).first()
-        or_(
+    if user is not None and user.verify_password(password):
-            User.username == email_or_username,
-            User.email == email_or_username.lower()
-        )
-    ).first()
-    if user and user.verify_password(password):
         return user
 
 
 @basic_auth.error_handler
 def basic_auth_error(status):
-    return {'error': HTTP_STATUS_CODES.get(status, 'Unknown error')}, status
+    error = (Forbidden if status == 403 else Unauthorized)()
+    return {
+        'code': error.code,
+        'message': error.name,
+        'description': error.description,
+    }, error.code, {'WWW-Authenticate': 'Form'}
 
 
 @token_auth.verify_token
 def verify_token(token):
-    return User.check_token(token) if token else None
+    return User.verify_access_token(token) if token else None
 
 
 @token_auth.error_handler
 def token_auth_error(status):
-    return {'error': HTTP_STATUS_CODES.get(status, 'Unknown error')}, status
+    error = (Forbidden if status == 403 else Unauthorized)()
+    return {
+        'code': error.code,
+        'message': error.name,
+        'description': error.description,
+    }, error.code
+
+
+@basic_auth.get_user_roles
+@token_auth.get_user_roles
+def get_user_roles(user):
+    return [user.role.name]

app/api/jobs.py

@@ -0,0 +1,102 @@
+
+from apifairy import authenticate, response
+from apifairy.decorators import body, other_responses
+from flask import abort, Blueprint
+from werkzeug.exceptions import InternalServerError
+from app import db, hashids
+from app.models import Job, JobInput, JobStatus, TesseractOCRModel
+from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRModelSchema
+from .auth import auth_error_responses, token_auth
+
+
+bp = Blueprint('jobs', __name__)
+job_schema = JobSchema()
+jobs_schema = JobSchema(many=True)
+spacy_nlp_pipeline_job_schema = SpaCyNLPPipelineJobSchema()
+tesseract_ocr_pipeline_job_schema = TesseractOCRPipelineJobSchema()
+tesseract_ocr_model_schema = TesseractOCRModelSchema()
+tesseract_ocr_models_schema = TesseractOCRModelSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(jobs_schema)
+@other_responses(auth_error_responses)
+def get_jobs():
+    """Get all jobs"""
+    return Job.query.all()
+
+
+@bp.route('/tesseract-ocr-pipeline', methods=['POST'])
+@authenticate(token_auth)
+@body(tesseract_ocr_pipeline_job_schema, location='form')
+@response(job_schema)
+@other_responses({**auth_error_responses, InternalServerError.code: InternalServerError.description})
+def create_tesseract_ocr_pipeline_job(args):
+    """Create a new Tesseract OCR Pipeline job"""
+    current_user = token_auth.current_user()
+    try:
+        job = Job.create(
+            title=args['title'],
+            description=args['description'],
+            service='tesseract-ocr-pipeline',
+            service_args={
+                'model': hashids.decode(args['model_id']),
+                'binarization': args['binarization']
+            },
+            service_version=args['service_version'],
+            user=current_user
+        )
+    except OSError:
+        abort(500)
+    try:
+        JobInput.create(args['pdf'], job=job)
+    except OSError:
+        abort(500)
+    job.status = JobStatus.SUBMITTED
+    db.session.commit()
+    return job, 201
+
+
+@bp.route('/tesseract-ocr-pipeline/models', methods=['GET'])
+@authenticate(token_auth)
+@response(tesseract_ocr_models_schema)
+@other_responses(auth_error_responses)
+def get_tesseract_ocr_models():
+    """Get all Tesseract OCR Models"""
+    return TesseractOCRModel.query.all()
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_job(job_id):
+    """Delete a job by id"""
+    current_user = token_auth.current_user()
+    job = Job.query.get(job_id)
+    if job is None:
+        abort(404)
+    if not (job.user == current_user or current_user.is_administrator()):
+        abort(403)
+    try:
+        job.delete()
+    except OSError as e:
+        abort(500)
+    db.session.commit()
+    return {}, 204
+
+
+@bp.route('/<hashid:job_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(job_schema)
+@other_responses(auth_error_responses)
+def get_job(job_id):
+    """Get a job by id"""
+    current_user = token_auth.current_user()
+    job = Job.query.get(job_id)
+    if job is None:
+        abort(404)
+    if not (job.user == current_user or current_user.is_administrator()):
+        abort(403)
+    return job

app/api/schemas.py

@@ -0,0 +1,165 @@
+from apifairy.fields import FileField
+from marshmallow import validate, validates, ValidationError
+from marshmallow.decorators import post_dump
+from app import ma
+from app.auth import USERNAME_REGEX
+from app.models import Job, JobStatus, TesseractOCRModel, Token, User, UserSettingJobStatusMailNotificationLevel
+from app.services import SERVICES
+
+
+
+class EmptySchema(ma.Schema):
+    pass
+
+
+class TokenSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = Token
+        ordered = True
+
+    access_token = ma.String(required=True)
+    refresh_token = ma.String()
+
+
+class TesseractOCRModelSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = TesseractOCRModel
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    user_hashid = ma.String(data_key='user_id', dump_only=True)
+    title = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=64)
+    )
+    description = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    version = ma.String(
+        required=True,
+        validate=validate.Length(min=1, max=16)
+    )
+    compatible_service_versions = ma.List(
+        ma.String(required=True, validate=validate.Length(min=1, max=16)),
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    publisher = ma.String(
+        required=True,
+        validate=validate.Length(min=1, max=128)
+    )
+    publisher_url = ma.String(
+        validate=[validate.URL(), validate.Length(min=1, max=512)]
+    )
+    publishing_url = ma.String(
+        required=True,
+        validate=[validate.URL(), validate.Length(min=1, max=512)]
+    )
+    publishing_year = ma.Int(
+        required=True
+    )
+    shared = ma.Boolean(required=True)
+
+
+class JobSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = Job
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    user_hashid = ma.String(data_key='user_id', dump_only=True)
+    title = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=32)
+    )
+    description = ma.auto_field(
+        required=True,
+        validate=validate.Length(min=1, max=255)
+    )
+    creation_date = ma.auto_field(dump_only=True)
+    end_date = ma.auto_field(dump_only=True)
+    service = ma.String(
+        dump_only=True,
+        validate=validate.OneOf(SERVICES.keys())
+    )
+    service_args = ma.Dict(dump_only=True)
+    service_version = ma.String(dump_only=True)
+    status = ma.String(
+        dump_only=True,
+        validate=validate.OneOf(list(JobStatus.__members__.keys()))
+    )
+
+    @post_dump(pass_original=True)
+    def post_dump(self, serialized_job, job, **kwargs):
+        serialized_job['status'] = job.status.name
+        return serialized_job
+
+
+class TesseractOCRPipelineJobSchema(JobSchema):
+    binarization = ma.Boolean(load_only=True, missing=False)
+    model_id = ma.String(required=True, load_only=True)
+    service_version = ma.auto_field(
+        required=True,
+        validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+    )
+    pdf = FileField()
+
+    @validates('pdf')
+    def validate_pdf(self, value):
+        if value.mimetype != 'application/pdf':
+            raise ValidationError('PDF files only!')
+
+
+class SpaCyNLPPipelineJobSchema(JobSchema):
+    binarization = ma.Boolean(load_only=True, missing=False)
+    model_id = ma.String(required=True, load_only=True)
+    service_version = ma.auto_field(
+        required=True,
+        validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+    )
+    txt = FileField(required=True)
+
+    @validates('txt')
+    def validate_txt(self, value):
+        if value.mimetype != 'text/plain':
+            raise ValidationError('Plain text files only!')
+
+
+class UserSchema(ma.SQLAlchemySchema):
+    class Meta:
+        model = User
+        ordered = True
+
+    hashid = ma.String(data_key='id', dump_only=True)
+    username = ma.auto_field(
+        validate=[
+            validate.Length(min=1, max=64),
+            validate.Regexp(USERNAME_REGEX, error='Usernames must have only letters, numbers, dots or underscores')
+        ]
+    )
+    email = ma.auto_field(validate=validate.Email())
+    member_since = ma.auto_field(dump_only=True)
+    last_seen = ma.auto_field(dump_only=True)
+    password = ma.String(load_only=True)
+    last_seen = ma.auto_field(dump_only=True)
+    setting_dark_mode = ma.auto_field()
+    setting_job_status_mail_notification_level = ma.String(
+        validate=validate.OneOf(list(UserSettingJobStatusMailNotificationLevel.__members__.keys()))
+    )
+
+    @validates('email')
+    def validate_email(self, email):
+        if User.query.filter(User.email == email).first():
+            raise ValidationError('Email already registered')
+
+    @validates('username')
+    def validate_username(self, username):
+        if User.query.filter(User.username == username).first():
+            raise ValidationError('Username already in use')
+
+    @post_dump(pass_original=True)
+    def post_dump(self, serialized_user, user, **kwargs):
+        serialized_user['setting_job_status_mail_notification_level'] = \
+            user.setting_job_status_mail_notification_level.name
+        return serialized_user

app/api/tokens.py

@@ -1,27 +1,58 @@
+from apifairy import authenticate, body, response, other_responses
+from flask import Blueprint, request, abort
 from app import db
-from flask_restx import Namespace, Resource
+from app.models import Token, User
-from .auth import basic_auth, token_auth
+from .auth import basic_auth
+from .schemas import EmptySchema, TokenSchema
 
 
-ns = Namespace('tokens', description='Token operations')
+bp = Blueprint('tokens', __name__)
+token_schema = TokenSchema()
 
 
-@ns.route('')
+@bp.route('', methods=['DELETE'])
-class API_Tokens(Resource):
+@response(EmptySchema, status_code=204, description='Token revoked')
-    '''Get or revoke a user authentication token'''
+@other_responses({401: 'Invalid access token'})
+def delete_token():
+    """Revoke an access token"""
+    access_token = request.headers['Authorization'].split()[1]
+    token = Token.query.filter(Token.access_token == access_token).first()
+    if token is None:  # pragma: no cover
+        abort(401)
+    token.expire()
+    db.session.commit()
+    return {}
 
-    @ns.doc(security='basicAuth')
-    @basic_auth.login_required
-    def post(self):
-        '''Get a user token'''
-        token = basic_auth.current_user().get_token()
-        db.session.commit()
-        return {'token': 'Bearer ' + token}
 
-    @ns.doc(security='apiKey')
+@bp.route('', methods=['POST'])
-    @token_auth.login_required
+@authenticate(basic_auth)
-    def delete(self):
+@response(token_schema)
-        '''Revoke a user token'''
+@other_responses({401: 'Invalid username or password'})
-        token_auth.current_user().revoke_token()
+def create_token():
-        db.session.commit()
+    """Create new access and refresh tokens"""
-        return '', 204
+    user = basic_auth.current_user()
+    token = user.generate_auth_token()
+    db.session.add(token)
+    Token.clean()  # keep token table clean of old tokens
+    db.session.commit()
+    return token, 200
+
+
+@bp.route('', methods=['PUT'])
+@body(token_schema)
+@response(token_schema, description='Newly issued access and refresh tokens')
+@other_responses({401: 'Invalid access or refresh token'})
+def refresh_token(args):
+    """Refresh an access token"""
+    access_token = args.get('access_token')
+    refresh_token = args.get('refresh_token')
+    if access_token is None or refresh_token is None:
+        abort(401)
+    token = User.verify_refresh_token(refresh_token, access_token)
+    if token is None:
+        abort(401)
+    token.expire()
+    new_token = token.user.generate_auth_token()
+    db.session.add_all([token, new_token])
+    db.session.commit()
+    return new_token, 200

app/api/users.py

@@ -0,0 +1,99 @@
+
+from apifairy import authenticate, body, response
+from apifairy.decorators import other_responses
+from flask import abort, Blueprint, current_app
+from werkzeug.exceptions import InternalServerError
+from app import db
+from app.email import create_message, send
+from app.models import User
+from .schemas import EmptySchema, UserSchema
+from .auth import auth_error_responses, token_auth
+
+
+bp = Blueprint('users', __name__)
+user_schema = UserSchema()
+users_schema = UserSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(users_schema)
+@other_responses(auth_error_responses)
+def get_users():
+    """Get all users"""
+    return User.query.all()
+
+
+@bp.route('', methods=['POST'])
+@body(user_schema)
+@response(user_schema, 201)
+@other_responses({InternalServerError.code: InternalServerError.description})
+def create_user(args):
+    """Create a new user"""
+    try:
+        user = User.create(
+            email=args['email'].lower(),
+            password=args['password'],
+            username=args['username']
+        )
+    except OSError:
+        abort(500)
+    msg = create_message(
+        user.email,
+        'Confirm Your Account',
+        'auth/email/confirm',
+        token=user.generate_confirm_token(),
+        user=user
+    )
+    send(msg)
+    db.session.commit()
+    return user, 201
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_user(user_id):
+    """Delete a user by id"""
+    current_user = token_auth.current_user()
+    user = User.query.get(user_id)
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator()):
+        abort(403)
+    user.delete()
+    db.session.commit()
+    return {}, 204
+
+
+@bp.route('/<hashid:user_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user(user_id):
+    """Retrieve a user by id"""
+    current_user = token_auth.current_user()
+    user = User.query.get(user_id)
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator()):
+        abort(403)
+    return user
+
+
+@bp.route('/<username>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user_by_username(username):
+    """Retrieve a user by username"""
+    current_user = token_auth.current_user()
+    user = User.query.filter(User.username == username).first()
+    if user is None:
+        abort(404)
+    if not (user == current_user or current_user.is_administrator()):
+        abort(403)
+    return user

app/auth/forms.py

@@ -1,4 +1,3 @@
-from app.models import User
 from flask_wtf import FlaskForm
 from wtforms import (
     BooleanField,
@@ -7,32 +6,45 @@ from wtforms import (
     SubmitField,
     ValidationError
 )
-from wtforms.validators import DataRequired, InputRequired, Email, EqualTo, Length, Regexp
+from wtforms.validators import InputRequired, Email, EqualTo, Length, Regexp
+from app.models import User
 from . import USERNAME_REGEX
 
 
-class LoginForm(FlaskForm):
-    user = StringField('Email or username', validators=[DataRequired()])
-    password = PasswordField('Password', validators=[DataRequired()])
-    remember_me = BooleanField('Keep me logged in')
-    submit = SubmitField('Log In')
-
-
 class RegistrationForm(FlaskForm):
-    email = StringField('Email', validators=[DataRequired(), Email()])
+    email = StringField(
-    username = StringField('Username',
+        'Email',
+        validators=[InputRequired(), Email(), Length(max=254)]
+    )
+    username = StringField(
+        'Username',
         validators=[
             InputRequired(),
-            Length(1, 64),
+            Length(max=64),
             Regexp(
                 USERNAME_REGEX,
-                message='Usernames must have only letters, numbers, dots or underscores'  # noqa
+                message=(
-           )
+                    'Usernames must have only letters, numbers, dots or '
+                    'underscores'
+                )
+            )
        ]
     )
-    password = PasswordField('Password', validators=[DataRequired(), EqualTo('password_confirmation', message='Passwords must match')])
+    password = PasswordField(
-    password_confirmation = PasswordField('Password confirmation', validators=[DataRequired(), EqualTo('password', message='Passwords must match')])
+        'Password',
-    submit = SubmitField('Register')
+        validators=[
+            InputRequired(),
+            EqualTo('password_2', message='Passwords must match')
+        ]
+    )
+    password_2 = PasswordField(
+        'Password confirmation',
+        validators=[
+            InputRequired(),
+            EqualTo('password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()
 
     def validate_email(self, field):
         if User.query.filter_by(email=field.data.lower()).first():
@@ -43,12 +55,31 @@ class RegistrationForm(FlaskForm):
             raise ValidationError('Username already in use')
 
 
-class ResetPasswordForm(FlaskForm):
+class LoginForm(FlaskForm):
-    password = PasswordField('New password', validators=[DataRequired(), EqualTo('password_confirmation', message='Passwords must match')])
+    user = StringField('Email or username', validators=[InputRequired()])
-    password_confirmation = PasswordField('Password confirmation', validators=[DataRequired(), EqualTo('password', message='Passwords must match')])
+    password = PasswordField('Password', validators=[InputRequired()])
-    submit = SubmitField('Reset Password')
+    remember_me = BooleanField('Keep me logged in')
+    submit = SubmitField()
 
 
 class ResetPasswordRequestForm(FlaskForm):
-    email = StringField('Email', validators=[DataRequired(), Email()])
+    email = StringField('Email', validators=[InputRequired(), Email()])
-    submit = SubmitField('Reset Password')
+    submit = SubmitField()
+
+
+class ResetPasswordForm(FlaskForm):
+    password = PasswordField(
+        'New password',
+        validators=[
+            InputRequired(),
+            EqualTo('password_2', message='Passwords must match')
+        ]
+    )
+    password_2 = PasswordField(
+        'New password confirmation',
+        validators=[
+            InputRequired(),
+            EqualTo('password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()

app/auth/routes.py

@@ -1,10 +1,5 @@
-from app import db
-from app.email import create_message, send
-from app.models import User
-from datetime import datetime
 from flask import (
     abort,
-    current_app,
     flash,
     redirect,
     render_template,
@@ -12,7 +7,9 @@ from flask import (
     url_for
 )
 from flask_login import current_user, login_user, login_required, logout_user
-from sqlalchemy import or_
+from app import db
+from app.email import create_message, send
+from app.models import User
 from . import bp
 from .forms import (
     LoginForm,
@@ -29,69 +26,32 @@ def before_request():
     unconfirmed view if user is unconfirmed.
     """
     if current_user.is_authenticated:
-        current_user.last_seen = datetime.utcnow()
+        current_user.ping()
         db.session.commit()
-        if (
+        if (not current_user.confirmed
-            not current_user.confirmed
+                and request.endpoint
-            and request.endpoint
+                and request.blueprint != 'auth'
-            and request.blueprint != 'auth'
+                and request.endpoint != 'static'):
-            and request.endpoint != 'static'
-        ):
             return redirect(url_for('auth.unconfirmed'))
 
 
-@bp.route('/login', methods=['GET', 'POST'])
-def login():
-    if current_user.is_authenticated:
-        return redirect(url_for('main.dashboard'))
-    form = LoginForm(prefix='login-form')
-    if form.validate_on_submit():
-        user = User.query.filter(
-            or_(
-                User.username == form.user.data,
-                User.email == form.user.data.lower()
-            )
-        ).first()
-        if user and user.verify_password(form.password.data):
-            login_user(user, form.remember_me.data)
-            next = request.args.get('next')
-            if next is None or not next.startswith('/'):
-                next = url_for('main.dashboard')
-            return redirect(next)
-        flash('Invalid email/username or password', category='error')
-    return render_template('auth/login.html.j2', form=form, title='Log in')
-
-
-@bp.route('/logout')
-@login_required
-def logout():
-    logout_user()
-    flash('You have been logged out')
-    return redirect(url_for('main.index'))
-
-
 @bp.route('/register', methods=['GET', 'POST'])
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
     form = RegistrationForm(prefix='registration-form')
     if form.validate_on_submit():
-        user = User(
-            email=form.email.data.lower(),
-            password=form.password.data,
-            username=form.username.data
-        )
-        db.session.add(user)
-        db.session.flush(objects=[user])
-        db.session.refresh(user)
         try:
-            user.makedirs()
+            user = User.create(
-        except OSError as e:
+                email=form.email.data.lower(),
-            current_app.logger.error(e)
+                password=form.password.data,
-            db.session.rollback()
+                username=form.username.data
+            )
+        except OSError:
             flash('Internal Server Error', category='error')
             abort(500)
-        token = user.generate_confirmation_token()
+        flash(f'User "{user.username}" created')
+        token = user.generate_confirm_token()
         msg = create_message(
             user.email,
             'Confirm Your Account',
@@ -110,36 +70,46 @@ def register():
     )
 
 
-@bp.route('/confirm/<token>')
+@bp.route('/login', methods=['GET', 'POST'])
+def login():
+    if current_user.is_authenticated:
+        return redirect(url_for('main.dashboard'))
+    form = LoginForm(prefix='login-form')
+    if form.validate_on_submit():
+        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+        if user and user.verify_password(form.password.data):
+            login_user(user, form.remember_me.data)
+            next = request.args.get('next')
+            if next is None or not next.startswith('/'):
+                next = url_for('main.dashboard')
+            flash('You have been logged in')
+            return redirect(next)
+        flash('Invalid email/username or password', category='error')
+    return render_template('auth/login.html.j2', form=form, title='Log in')
+
+
+@bp.route('/logout')
 @login_required
-def confirm(token):
+def logout():
-    if current_user.confirmed:
+    logout_user()
-        return redirect(url_for('main.dashboard'))
+    flash('You have been logged out')
-    if current_user.confirm(token):
+    return redirect(url_for('main.index'))
-        db.session.commit()
-        flash('You have confirmed your account')
-        return redirect(url_for('main.dashboard'))
-    else:
-        flash(
-            'The confirmation link is invalid or has expired',
-            category='error'
-        )
-        return redirect(url_for('.unconfirmed'))
 
 
 @bp.route('/unconfirmed')
+@login_required
 def unconfirmed():
-    if current_user.is_anonymous:
+    if current_user.confirmed:
-        return redirect(url_for('main.index'))
-    elif current_user.confirmed:
         return redirect(url_for('main.dashboard'))
     return render_template('auth/unconfirmed.html.j2', title='Unconfirmed')
 
 
 @bp.route('/confirm')
 @login_required
-def resend_confirmation():
+def confirm_request():
-    token = current_user.generate_confirmation_token()
+    if current_user.confirmed:
+        return redirect(url_for('main.dashboard'))
+    token = current_user.generate_confirm_token()
     msg = create_message(
         current_user.email,
         'Confirm Your Account',
@@ -149,10 +119,23 @@ def resend_confirmation():
     )
     send(msg)
     flash('A new confirmation email has been sent to you by email')
-    return redirect(url_for('auth.unconfirmed'))
+    return redirect(url_for('.unconfirmed'))
 
 
-@bp.route('/reset', methods=['GET', 'POST'])
+@bp.route('/confirm/<token>')
+@login_required
+def confirm(token):
+    if current_user.confirmed:
+        return redirect(url_for('main.dashboard'))
+    if current_user.confirm(token):
+        db.session.commit()
+        flash('You have confirmed your account')
+        return redirect(url_for('main.dashboard'))
+    flash('The confirmation link is invalid or has expired', category='error')
+    return redirect(url_for('.unconfirmed'))
+
+
+@bp.route('/reset_password', methods=['GET', 'POST'])
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
@@ -160,7 +143,7 @@ def reset_password_request():
     if form.validate_on_submit():
         user = User.query.filter_by(email=form.email.data.lower()).first()
         if user is not None:
-            token = user.generate_reset_token()
+            token = user.generate_reset_password_token()
             msg = create_message(
                 user.email,
                 'Reset Your Password',
@@ -170,7 +153,8 @@ def reset_password_request():
             )
             send(msg)
         flash(
-            'An email with instructions to reset your password has been sent to you'  # noqa
+            'An email with instructions to reset your password has been sent '
+            'to you'
         )
         return redirect(url_for('.login'))
     return render_template(
@@ -180,7 +164,7 @@ def reset_password_request():
     )
 
 
-@bp.route('/reset/<token>', methods=['GET', 'POST'])
+@bp.route('/reset_password/<token>', methods=['GET', 'POST'])
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
@@ -190,8 +174,7 @@ def reset_password(token):
             db.session.commit()
             flash('Your password has been updated')
             return redirect(url_for('.login'))
-        else:
+        return redirect(url_for('main.index'))
-            return redirect(url_for('main.index'))
     return render_template(
         'auth/reset_password.html.j2',
         form=form,

app/cli.py

@@ -1,9 +1,8 @@
 from flask import current_app
 from flask_migrate import upgrade
-from . import db
-from .models import Corpus, Role, User, TesseractOCRModel, TranskribusHTRModel
 import click
 import os
+from app.models import Role, User, TesseractOCRModel, TranskribusHTRModel
 
 
 def _make_default_dirs():
@@ -41,22 +40,6 @@ def register(app):
         current_app.logger.info('Insert/Update default TranskribusHTRModels')
         TranskribusHTRModel.insert_defaults()
 
-    @app.cli.group()
-    def daemon():
-        ''' Daemon commands. '''
-        pass
-
-    @daemon.command('run')
-    def run_daemon():
-        ''' Run daemon '''
-        corpus: Corpus
-        for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0):
-            corpus.num_analysis_sessions = 0
-        db.session.commit()
-        from app.daemon import Daemon
-        daemon: Daemon = Daemon()
-        daemon.run()
-
     @app.cli.group()
     def converter():
         ''' Converter commands. '''

app/contributions/__init__.py

@@ -1,5 +1,5 @@
 from flask import Blueprint
 
 
-bp = Blueprint('contribute', __name__)
+bp = Blueprint('contributions', __name__)
 from . import routes

app/contributions/routes.py

@@ -1,9 +1,6 @@
-from app import db
-from app.decorators import permission_required
-from app.models import Permission, Role, User
-from app.settings import tasks as settings_tasks
-from flask import flash, redirect, render_template, url_for
 from flask_login import login_required
+from app.decorators import permission_required
+from app.models import Permission
 from . import bp
 
 
@@ -14,6 +11,6 @@ def before_request():
     pass
 
 
-@bp.route('/')
+@bp.route('')
-def index():
+def contributions():
     pass

app/corpora/cqi_over_socketio/__init__.py

@@ -1,11 +1,11 @@
-from app import db, hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus, CorpusStatus
 from flask import session
 from flask_login import current_user
 from flask_socketio import ConnectionRefusedError
 from threading import Lock
 import cqi
+from app import db, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus, CorpusStatus
 
 
 '''

app/corpora/cqi_over_socketio/cqi.py

@@ -1,9 +1,9 @@
+from socket import gaierror
+import cqi
 from app import socketio
 from app.decorators import socketio_login_required
-from socket import gaierror
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio
-import cqi
 
 
 @socketio.on('cqi.connect', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora.py

@@ -1,8 +1,8 @@
+import cqi
 from app import socketio
 from app.decorators import socketio_login_required
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio
-import cqi
 
 
 @socketio.on('cqi.corpora.get', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora_corpus.py

@@ -1,11 +1,11 @@
+from flask import session
+import cqi
+import math
 from app import db, socketio
 from app.decorators import socketio_login_required
 from app.models import Corpus
-from flask import session
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio, lookups_by_cpos
-import cqi
-import math
 
 
 @socketio.on('cqi.corpora.corpus.drop', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py

@@ -1,8 +1,8 @@
+import cqi
 from app import socketio
 from app.decorators import socketio_login_required
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio
-import cqi
 
 
 @socketio.on('cqi.corpora.corpus.alignment_attributes.get', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py

@@ -1,8 +1,8 @@
+import cqi
 from app import socketio
 from app.decorators import socketio_login_required
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio
-import cqi
 
 
 @socketio.on('cqi.corpora.corpus.positional_attributes.get', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py

@@ -1,8 +1,8 @@
+import cqi
 from app import socketio
 from app.decorators import socketio_login_required
 from . import NAMESPACE as ns
 from .utils import cqi_over_socketio
-import cqi
 
 
 @socketio.on('cqi.corpora.corpus.structural_attributes.get', namespace=ns)

app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py

@@ -1,13 +1,13 @@
-from app import socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus
 from flask import session
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, export_subcorpus
 import cqi
 import json
 import math
 import os
+from app import socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus
+from . import NAMESPACE as ns
+from .utils import cqi_over_socketio, export_subcorpus
 
 
 @socketio.on('cqi.corpora.corpus.subcorpora.get', namespace=ns)

app/corpora/forms.py

@@ -1,80 +1,67 @@
 from flask_wtf import FlaskForm
 from flask_wtf.file import FileField, FileRequired
-from werkzeug.utils import secure_filename
+from wtforms import StringField, SubmitField, ValidationError, IntegerField
-from wtforms import (
+from wtforms.validators import InputRequired, Length
-    StringField,
-    SubmitField,
-    ValidationError,
-    IntegerField
-)
-from wtforms.validators import DataRequired, InputRequired, Length
 
 
-class AddCorpusFileForm(FlaskForm):
+class CreateCorpusForm(FlaskForm):
-    '''
+    description = StringField(
-    Form to add a .vrt corpus file to the current corpus.
+        'Description',
-    '''
+        validators=[InputRequired(), Length(max=255)]
-    # Required fields
+    )
-    author = StringField('Author', validators=[InputRequired(), Length(1, 255)])
+    title = StringField('Title', validators=[InputRequired(), Length(max=32)])
-    publishing_year = IntegerField('Publishing year', validators=[InputRequired()])
-    title = StringField('Title', validators=[InputRequired(), Length(1, 255)])
-    vrt = FileField('File', validators=[FileRequired()])
-    # Optional fields
-    address = StringField('Adress', validators=[Length(0, 255)])
-    booktitle = StringField('Booktitle', validators=[Length(0, 255)])
-    chapter = StringField('Chapter', validators=[Length(0, 255)])
-    editor = StringField('Editor', validators=[Length(0, 255)])
-    institution = StringField('Institution', validators=[Length(0, 255)])
-    journal = StringField('Journal', validators=[Length(0, 255)])
-    pages = StringField('Pages', validators=[Length(0, 255)])
-    publisher = StringField('Publisher', validators=[Length(0, 255)])
-    school = StringField('School', validators=[Length(0, 255)])
     submit = SubmitField()
 
+
+class CorpusFileBaseForm(FlaskForm):
+    author = StringField(
+        'Author',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    address = StringField('Adress', validators=[Length(max=255)])
+    booktitle = StringField('Booktitle', validators=[Length(max=255)])
+    chapter = StringField('Chapter', validators=[Length(max=255)])
+    editor = StringField('Editor', validators=[Length(max=255)])
+    institution = StringField('Institution', validators=[Length(max=255)])
+    journal = StringField('Journal', validators=[Length(max=255)])
+    pages = StringField('Pages', validators=[Length(max=255)])
+    publisher = StringField('Publisher', validators=[Length(max=255)])
+    school = StringField('School', validators=[Length(max=255)])
+    submit = SubmitField()
+
+
+class CreateCorpusFileForm(CorpusFileBaseForm):
+    vrt = FileField('File', validators=[FileRequired()])
+
     def validate_vrt(self, field):
         if not field.data.filename.lower().endswith('.vrt'):
             raise ValidationError('VRT files only!')
 
-class EditCorpusFileForm(FlaskForm):
-    '''
-    Form to edit meta data of one corpus file.
-    '''
-    # Required fields
-    author = StringField('Author', validators=[InputRequired(), Length(1, 255)])
-    publishing_year = IntegerField('Publishing year', validators=[InputRequired()])
-    title = StringField('Title', validators=[InputRequired(), Length(1, 255)])
-    # Optional fields
-    address = StringField('Adress', validators=[Length(0, 255)])
-    booktitle = StringField('Booktitle', validators=[Length(0, 255)])
-    chapter = StringField('Chapter', validators=[Length(0, 255)])
-    editor = StringField('Editor', validators=[Length(0, 255)])
-    institution = StringField('Institution', validators=[Length(0, 255)])
-    journal = StringField('Journal', validators=[Length(0, 255)])
-    pages = StringField('Pages', validators=[Length(0, 255)])
-    publisher = StringField('Publisher', validators=[Length(0, 255)])
-    school = StringField('School', validators=[Length(0, 255)])
-    submit = SubmitField()
 
-
+class EditCorpusFileForm(CorpusFileBaseForm):
-class AddCorpusForm(FlaskForm):
+    def prefill(self, corpus_file):
-    '''
+        ''' Pre-fill the form with data of an exististing corpus file '''
-    Form to add a a new corpus.
+        self.address.data = corpus_file.address
-    '''
+        self.author.data = corpus_file.author
-    description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
+        self.booktitle.data = corpus_file.booktitle
-    title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
+        self.chapter.data = corpus_file.chapter
-    submit = SubmitField()
+        self.editor.data = corpus_file.editor
+        self.institution.data = corpus_file.institution
+        self.journal.data = corpus_file.journal
+        self.pages.data = corpus_file.pages
+        self.publisher.data = corpus_file.publisher
+        self.publishing_year.data = corpus_file.publishing_year
+        self.school.data = corpus_file.school
+        self.title.data = corpus_file.title
 
 
 class ImportCorpusForm(FlaskForm):
-    '''
+    pass
-    Form to import a corpus.
-    '''
-    description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
-    archive = FileField('File', validators=[FileRequired()])
-    title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
-    submit = SubmitField()
-
-    def validate_archive(self, field):
-        valid_mimetypes = ['application/zip', 'application/x-zip', 'application/x-zip-compressed']
-        if field.data.mimetype not in valid_mimetypes:
-            raise ValidationError('ZIP files only!')

app/corpora/routes.py

@@ -1,143 +1,44 @@
-from app import db
-from app.models import Corpus, CorpusFile, CorpusStatus
 from flask import (
     abort,
     current_app,
     flash,
-    make_response,
+    Markup,
     redirect,
     render_template,
-    url_for,
     send_from_directory
 )
 from flask_login import current_user, login_required
-from werkzeug.utils import secure_filename
+from threading import Thread
-from zipfile import ZipFile
-from . import bp
-from . import tasks
-from .forms import (
-    AddCorpusFileForm,
-    AddCorpusForm,
-    EditCorpusFileForm,
-    ImportCorpusForm
-)
 import os
-import shutil
+from app import db
-import tempfile
+from app.models import Corpus, CorpusFile, CorpusStatus
-import glob
+from . import bp
-import xml.etree.ElementTree as ET
+from .forms import CreateCorpusFileForm, CreateCorpusForm, EditCorpusFileForm
 
 
-@bp.route('/add', methods=['GET', 'POST'])
+@bp.route('/create', methods=['GET', 'POST'])
 @login_required
-def add_corpus():
+def create_corpus():
-    form = AddCorpusForm(prefix='add-corpus-form')
+    form = CreateCorpusForm(prefix='create-corpus-form')
     if form.validate_on_submit():
-        corpus = Corpus(
-            user=current_user,
-            description=form.description.data,
-            title=form.title.data
-        )
-        db.session.add(corpus)
-        db.session.flush()
-        db.session.refresh(corpus)
         try:
-            corpus.makedirs()
+            corpus = Corpus.create(
-        except OSError as e:
+                title=form.title.data,
-            current_app.logger.error(e)
+                description=form.description.data,
-            db.session.rollback()
+                user=current_user
-            flash('Internal Server Error', category='error')
+            )
+        except OSError:
             abort(500)
         db.session.commit()
-        flash(f'Corpus "{corpus.title}" added', category='corpus')
+        message = Markup(
-        return redirect(url_for('.corpus', corpus_id=corpus.id))
+            f'Corpus "<a href="{corpus.url}">{corpus.title}</a>" created'
-    return render_template(
-        'corpora/add_corpus.html.j2',
-        form=form,
-        title='Add corpus'
-    )
-
-
-@bp.route('/import', methods=['GET', 'POST'])
-@login_required
-def import_corpus():
-    form = ImportCorpusForm(prefix='import-corpus-form')
-    if form.is_submitted():
-        if not form.validate():
-            return make_response(form.errors, 400)
-        corpus = Corpus(
-            user=current_user,
-            description=form.description.data,
-            title=form.title.data
         )
-        db.session.add(corpus)
+        flash(message, 'corpus')
-        db.session.flush(objects=[corpus])
+        return redirect(corpus.url)
-        db.session.refresh(corpus)
-        try:
-            corpus.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
-            flash('Internal Server Error', category='error')
-            return make_response({'redirect_url': url_for('.import_corpus')}, 500)  # noqa
-        # Save the uploaded zip file in a temporary directory
-        tmp_dir_base = os.path.join(current_app.config['NOPAQUE_DATA_DIR'], 'tmp')  # noqa
-        with tempfile.TemporaryDirectory(dir=tmp_dir_base) as tmp_dir:
-            archive_file = os.path.join(tmp_dir, 'corpus.zip')
-            try:
-                form.archive.data.save(archive_file)
-            except OSError as e:
-                current_app.logger.error(e)
-                db.session.rollback()
-                flash('Internal Server Error1', category='error')
-                return make_response({'redirect_url': url_for('.import_corpus')}, 500)  # noqa
-            shutil.unpack_archive(archive_file, extract_dir=tmp_dir)
-            for vrt_filename in [x for x in os.listdir(tmp_dir) if x.endswith('.vrt')]:
-                vrt_file = os.path.join(tmp_dir, vrt_filename)
-                element_tree = ET.parse(vrt_file)
-                text_node = element_tree.find('text')
-                corpus_file = CorpusFile(
-                    author=text_node.get('author'),
-                    corpus=corpus,
-                    filename=vrt_filename,
-                    mimetype='application/vrt+xml',
-                    publishing_year=int(text_node.get('publishing_year')),
-                    title=text_node.get('title')
-                )
-                if 'address' not in text_node.attrib:
-                    corpus_file.address = text_node.get('address')
-                if 'booktitle' not in text_node.attrib:
-                    corpus_file.booktitle = text_node.get('booktitle')
-                if 'chapter' not in text_node.attrib:
-                    corpus_file.chapter = text_node.get('chapter')
-                if 'editor' not in text_node.attrib:
-                    corpus_file.editor = text_node.get('editor')
-                if 'institution' not in text_node.attrib:
-                    corpus_file.institution = text_node.get('institution')
-                if 'journal' not in text_node.attrib:
-                    corpus_file.journal = text_node.get('journal')
-                if 'pages' not in text_node.attrib:
-                    corpus_file.pages = text_node.get('pages')
-                if 'publisher' not in text_node.attrib:
-                    corpus_file.publisher = text_node.get('publisher')
-                if 'school' not in text_node.attrib:
-                    corpus_file.school = text_node.get('school')
-                db.session.add(corpus_file)
-                db.session.flush(objects=[corpus_file])
-                db.session.refresh(corpus)
-                try:
-                    shutil.copy2(vrt_file, corpus_file.path)
-                except Exception as e:
-                    db.session.rollback()
-                    flash('Internal Server Error2', category='error')
-                    return make_response({'redirect_url': url_for('.import_corpus')}, 500)  # noqa
-        db.session.commit()
-        flash(f'Corpus "{corpus.title}" imported', 'corpus')
-        return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
     return render_template(
-        'corpora/import_corpus.html.j2',
+        'corpora/create_corpus.html.j2',
         form=form,
-        title='Import Corpus'
+        title='Create corpus'
     )
 
 
@@ -154,6 +55,26 @@ def corpus(corpus_id):
     )
 
 
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+@login_required
+def delete_corpus(corpus_id):
+    def _delete_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.delete()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if not (corpus.user == current_user or current_user.is_administrator()):
+        abort(403)
+    thread = Thread(
+        target=_delete_corpus,
+        args=(current_app._get_current_object(), corpus_id)
+    )
+    thread.start()
+    return {}, 202
+
+
 @bp.route('/<hashid:corpus_id>/analyse')
 @login_required
 def analyse_corpus(corpus_id):
@@ -165,95 +86,132 @@ def analyse_corpus(corpus_id):
     )
 
 
-@bp.route('/<hashid:corpus_id>/build')
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
 @login_required
 def build_corpus(corpus_id):
+    def _build_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.build()
+            db.session.commit()
+
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.user == current_user or current_user.is_administrator()):
         abort(403)
-    if corpus.files.all():
+    # Check if the corpus has corpus files
-        tasks.build_corpus(corpus_id)
+    if not corpus.files.all():
-        flash(
+        response = {'errors': {'message': 'Corpus file(s) required'}}
-            f'Corpus "{corpus.title}" marked for building',
+        return response, 409
-            category='corpus'
+    thread = Thread(
-        )
+        target=_build_corpus,
-    else:
+        args=(current_app._get_current_object(), corpus_id)
-        flash(
+    )
-            f'Can\'t build corpus "{corpus.title}": No corpus file(s)',
+    thread.start()
-            category='error'
+    return {}, 202
-        )
-    return redirect(url_for('.corpus', corpus_id=corpus_id))
 
 
-@bp.route('/<hashid:corpus_id>/delete')
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
 @login_required
-def delete_corpus(corpus_id):
+def create_corpus_file(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.user == current_user or current_user.is_administrator()):
         abort(403)
-    flash(f'Corpus "{corpus.title}" marked for deletion', 'corpus')
+    form = CreateCorpusFileForm(prefix='create-corpus-file-form')
-    tasks.delete_corpus(corpus_id)
+    if form.is_submitted():
-    return redirect(url_for('main.dashboard'))
+        if not form.validate():
-
+            response = {'errors': form.errors}
-
+            return response, 400
-@bp.route('/<hashid:corpus_id>/export')
+        try:
-@login_required
+            corpus_file = CorpusFile.create(
-def export_corpus(corpus_id):
+                form.vrt.data,
-    abort(503)
+                address=form.address.data,
-    corpus = Corpus.query.get_or_404(corpus_id)
+                author=form.author.data,
-    if not (corpus.user == current_user or current_user.is_administrator()):
+                booktitle=form.booktitle.data,
-        abort(403)
+                chapter=form.chapter.data,
-    return send_from_directory(
+                editor=form.editor.data,
-        as_attachment=True,
+                institution=form.institution.data,
-        directory=os.path.join(corpus.user.path, 'corpora'),
+                journal=form.journal.data,
-        filename=corpus.archive_file,
+                pages=form.pages.data,
-        mimetype='zip'
+                publisher=form.publisher.data,
+                publishing_year=form.publishing_year.data,
+                school=form.school.data,
+                title=form.title.data,
+                mimetype='application/vrt+xml',
+                corpus=corpus
+            )
+        except OSError:
+            abort(500)
+        corpus.status = CorpusStatus.UNPREPARED
+        db.session.commit()
+        message = Markup(
+            'Corpus file'
+            f'"<a href="{corpus_file.url}">{corpus_file.filename}</a>" added'
+        )
+        flash(message, category='corpus')
+        return {}, 201, {'Location': corpus.url}
+    return render_template(
+        'corpora/create_corpus_file.html.j2',
+        corpus=corpus,
+        form=form,
+        title='Add corpus file'
     )
 
 
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])  # noqa
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>',
+          methods=['GET', 'POST'])
 @login_required
 def corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter(
+    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-        CorpusFile.corpus_id == corpus_id,
+    if corpus_file.corpus.id != corpus_id:
-        CorpusFile.id == corpus_file_id
+        abort(404)
-    ).first_or_404()
+    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-    if not (
-        corpus_file.corpus.user == current_user
-        or current_user.is_administrator()
-    ):
         abort(403)
     form = EditCorpusFileForm(prefix='edit-corpus-file-form')
     if form.validate_on_submit():
-        corpus_file.address = form.address.data
+        has_changes = False
-        corpus_file.author = form.author.data
+        if corpus_file.address != form.address.data:
-        corpus_file.booktitle = form.booktitle.data
+            corpus_file.address = form.address.data
-        corpus_file.chapter = form.chapter.data
+            has_changes = True
-        corpus_file.editor = form.editor.data
+        if corpus_file.author != form.author.data:
-        corpus_file.institution = form.institution.data
+            corpus_file.author = form.author.data
-        corpus_file.journal = form.journal.data
+            has_changes = True
-        corpus_file.pages = form.pages.data
+        if corpus_file.booktitle != form.booktitle.data:
-        corpus_file.publisher = form.publisher.data
+            corpus_file.booktitle = form.booktitle.data
-        corpus_file.publishing_year = form.publishing_year.data
+            has_changes = True
-        corpus_file.school = form.school.data
+        if corpus_file.chapter != form.chapter.data:
-        corpus_file.title = form.title.data
+            corpus_file.chapter = form.chapter.data
-        corpus_file.corpus.status = CorpusStatus.UNPREPARED
+            has_changes = True
+        if corpus_file.editor != form.editor.data:
+            corpus_file.editor = form.editor.data
+            has_changes = True
+        if corpus_file.institution != form.institution.data:
+            corpus_file.institution = form.institution.data
+            has_changes = True
+        if corpus_file.journal != form.journal.data:
+            corpus_file.journal = form.journal.data
+            has_changes = True
+        if corpus_file.pages != form.pages.data:
+            corpus_file.pages = form.pages.data
+            has_changes = True
+        if corpus_file.publisher != form.publisher.data:
+            corpus_file.publisher = form.publisher.data
+            has_changes = True
+        if corpus_file.publishing_year != form.publishing_year.data:
+            corpus_file.publishing_year = form.publishing_year.data
+            has_changes = True
+        if corpus_file.school != form.school.data:
+            corpus_file.school = form.school.data
+            has_changes = True
+        if corpus_file.title != form.title.data:
+            corpus_file.title = form.title.data
+            has_changes = True
+        if has_changes:
+            corpus_file.corpus.status = CorpusStatus.UNPREPARED
         db.session.commit()
-        flash(f'Corpus file "{corpus_file.filename}" edited', category='corpus')  # noqa
+        message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
-        return redirect(url_for('.corpus', corpus_id=corpus_id))
+        flash(message, category='corpus')
-    # If no form is submitted or valid, fill out fields with current values
+        return redirect(corpus_file.corpus.url)
-    form.address.data = corpus_file.address
+    form.prefill(corpus_file)
-    form.author.data = corpus_file.author
-    form.booktitle.data = corpus_file.booktitle
-    form.chapter.data = corpus_file.chapter
-    form.editor.data = corpus_file.editor
-    form.institution.data = corpus_file.institution
-    form.journal.data = corpus_file.journal
-    form.pages.data = corpus_file.pages
-    form.publisher.data = corpus_file.publisher
-    form.publishing_year.data = corpus_file.publishing_year
-    form.school.data = corpus_file.school
-    form.title.data = corpus_file.title
     return render_template(
         'corpora/corpus_file.html.j2',
         corpus=corpus_file.corpus,
@@ -263,91 +221,52 @@ def corpus_file(corpus_id, corpus_file_id):
     )
 
 
-@bp.route('/<hashid:corpus_id>/files/add', methods=['GET', 'POST'])
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
-@login_required
-def add_corpus_file(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    form = AddCorpusFileForm(prefix='add-corpus-file-form')
-    if form.is_submitted():
-        if not form.validate():
-            return make_response(form.errors, 400)
-        # Save the file
-        corpus_file = CorpusFile(
-            address=form.address.data,
-            author=form.author.data,
-            booktitle=form.booktitle.data,
-            chapter=form.chapter.data,
-            corpus=corpus,
-            editor=form.editor.data,
-            filename=form.vrt.data.filename,
-            institution=form.institution.data,
-            journal=form.journal.data,
-            mimetype='application/vrt+xml',
-            pages=form.pages.data,
-            publisher=form.publisher.data,
-            publishing_year=form.publishing_year.data,
-            school=form.school.data,
-            title=form.title.data
-        )
-        db.session.add(corpus_file)
-        db.session.flush(objects=[corpus_file])
-        db.session.refresh(corpus_file)
-        try:
-            form.vrt.data.save(corpus_file.path)
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
-            flash('Internal Server Error', category='error')
-            return make_response({'redirect_url': url_for('.add_corpus_file', corpus_id=corpus.id)}, 500)  # noqa
-        corpus.status = CorpusStatus.UNPREPARED
-        db.session.commit()
-        flash(f'Corpus file "{corpus_file.filename}" added', category='corpus')
-        return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)  # noqa
-    return render_template(
-        'corpora/add_corpus_file.html.j2',
-        corpus=corpus,
-        form=form,
-        title='Add corpus file'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/delete')
 @login_required
 def delete_corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter(
+    def _delete_corpus_file(app, corpus_file_id):
-        CorpusFile.corpus_id == corpus_id,
+        with app.app_context():
-        CorpusFile.id == corpus_file_id
+            corpus_file = CorpusFile.query.get(corpus_file_id)
-    ).first_or_404()
+            corpus_file.delete()
-    if not (
+            db.session.commit()
-        corpus_file.corpus.user == current_user
+
-        or current_user.is_administrator()
+    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-    ):
+    if corpus_file.corpus.id != corpus_id:
+        abort(404)
+    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
         abort(403)
-    flash(
+    thread = Thread(
-        f'Corpus file "{corpus_file.filename}" marked for deletion',
+        target=_delete_corpus_file,
-        category='corpus'
+        args=(current_app._get_current_object(), corpus_file_id)
     )
-    tasks.delete_corpus_file(corpus_file_id)
+    thread.start()
-    return redirect(url_for('.corpus', corpus_id=corpus_id))
+    return {}, 202
 
 
 @bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
 @login_required
 def download_corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter(
+    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-        CorpusFile.corpus_id == corpus_id,
+    if corpus_file.corpus.id != corpus_id:
-        CorpusFile.id == corpus_file_id
+        abort(404)
-    ).first_or_404()
+    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-    if not (
-        corpus_file.corpus.user == current_user
-        or current_user.is_administrator()
-    ):
         abort(403)
     return send_from_directory(
+        os.path.dirname(corpus_file.path),
+        os.path.basename(corpus_file.path),
         as_attachment=True,
         attachment_filename=corpus_file.filename,
-        directory=os.path.dirname(corpus_file.path),
+        mimetype=corpus_file.mimetype
-        filename=os.path.basename(corpus_file.path)
+    )
-    )
+
+
+@bp.route('/import', methods=['GET', 'POST'])
+@login_required
+def import_corpus():
+    abort(503)
+
+
+@bp.route('/<hashid:corpus_id>/export')
+@login_required
+def export_corpus(corpus_id):
+    abort(503)

app/corpora/tasks.py

@@ -1,34 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import Corpus, CorpusFile
-
-
-@background
-def build_corpus(corpus_id, *args, **kwargs):
-    app = kwargs['app']
-    with app.app_context():
-        corpus = Corpus.query.get(corpus_id)
-        if corpus is None:
-            raise Exception(f'Corpus {corpus_id} not found')
-        corpus.build()
-        db.session.commit()
-
-
-@background
-def delete_corpus(corpus_id, *args, **kwargs):
-    with kwargs['app'].app_context():
-        corpus = Corpus.query.get(corpus_id)
-        if corpus is None:
-            raise Exception(f'Corpus {corpus_id} not found')
-        corpus.delete()
-        db.session.commit()
-
-
-@background
-def delete_corpus_file(corpus_file_id, *args, **kwargs):
-    with kwargs['app'].app_context():
-        corpus_file = CorpusFile.query.get(corpus_file_id)
-        if corpus_file is None:
-            raise Exception(f'Corpus file {corpus_file_id} not found')
-        corpus_file.delete()
-        db.session.commit()

app/daemon/__init__.py

@@ -1,23 +1,11 @@
 from app import db
-from flask import current_app
+from flask import Flask
-from time import sleep
+from .corpus_utils import check_corpora
-from .corpus_utils import CheckCorporaMixin
+from .job_utils import check_jobs
-from .job_utils import CheckJobsMixin
-import docker
 
 
-class Daemon(CheckCorporaMixin, CheckJobsMixin):
+def daemon(app: Flask):
-    def __init__(self):
+    with app.app_context():
-        self.docker = docker.from_env()
+        check_corpora()
-        self.docker.login(
+        check_jobs()
-            username=current_app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
+        db.session.commit()
-            password=current_app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
-            registry=current_app.config['NOPAQUE_DOCKER_REGISTRY']
-        )
-
-    def run(self):
-        while True:
-            self.check_corpora()
-            self.check_jobs()
-            db.session.commit()
-            sleep(1.5)

app/daemon/corpus_utils.py

@@ -1,3 +1,4 @@
+from app import docker_client
 from app.models import Corpus, CorpusStatus
 from flask import current_app
 import docker
@@ -5,250 +6,216 @@ import os
 import shutil
 
 
-class CheckCorporaMixin:
+def check_corpora():
-    def check_corpora(self):
+    corpora = Corpus.query.all()
-        corpora = Corpus.query.all()
+    for corpus in [x for x in corpora if x.status == CorpusStatus.SUBMITTED]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.SUBMITTED):  # noqa
+        _create_build_corpus_service(corpus)
-            self.create_build_corpus_service(corpus)
+    for corpus in [x for x in corpora if x.status in [CorpusStatus.QUEUED, CorpusStatus.BUILDING]]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.QUEUED or x.status == CorpusStatus.BUILDING):  # noqa
+        _checkout_build_corpus_service(corpus)
-            self.checkout_build_corpus_service(corpus)
+    for corpus in [x for x in corpora if x.status == CorpusStatus.BUILT and x.num_analysis_sessions > 0]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.BUILT and x.num_analysis_sessions > 0):  # noqa
+        corpus.status = CorpusStatus.STARTING_ANALYSIS_SESSION
-            corpus.status = CorpusStatus.STARTING_ANALYSIS_SESSION
+    for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0):  # noqa
+        corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
-            corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
+    for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION):  # noqa
+        _checkout_analysing_corpus_container(corpus)
-            self.checkout_analysing_corpus_container(corpus)
+    for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION):  # noqa
+        _create_cqpserver_container(corpus)
-            self.create_cqpserver_container(corpus)
+    for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
-        for corpus in (x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION):  # noqa
+        _remove_cqpserver_container(corpus)
-            self.remove_cqpserver_container(corpus)
 
-    def create_build_corpus_service(self, corpus):
+def _create_build_corpus_service(corpus):
-        ''' # Docker service settings # '''
+    ''' # Docker service settings # '''
-        ''' ## Command ## '''
+    ''' ## Command ## '''
-        command = ['bash', '-c']
+    command = ['bash', '-c']
-        command.append(
+    command.append(
-            f'mkdir /corpora/data/nopaque_{corpus.id}'
+        f'mkdir /corpora/data/nopaque_{corpus.id}'
-            ' && '
+        ' && '
-            'cwb-encode'
+        'cwb-encode'
-            ' -c utf8'
+        ' -c utf8'
-            f' -d /corpora/data/nopaque_{corpus.id}'
+        f' -d /corpora/data/nopaque_{corpus.id}'
-            ' -f /root/files/corpus.vrt'
+        ' -f /root/files/corpus.vrt'
-            f' -R /usr/local/share/cwb/registry/nopaque_{corpus.id}'
+        f' -R /usr/local/share/cwb/registry/nopaque_{corpus.id}'
-            ' -P pos -P lemma -P simple_pos'
+        ' -P pos -P lemma -P simple_pos'
-            ' -S ent:0+type -S s:0'
+        ' -S ent:0+type -S s:0'
-            ' -S text:0+address+author+booktitle+chapter+editor+institution+journal+pages+publisher+publishing_year+school+title'  # noqa
+        ' -S text:0+address+author+booktitle+chapter+editor+institution+journal+pages+publisher+publishing_year+school+title'
-            ' -xsB -9'
+        ' -xsB -9'
-            ' && '
+        ' && '
-            f'cwb-make -V NOPAQUE_{corpus.id}'
+        f'cwb-make -V NOPAQUE_{corpus.id}'
+    )
+    ''' ## Constraints ## '''
+    constraints = ['node.role==worker']
+    ''' ## Image ## '''
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+    ''' ## Labels ## '''
+    labels = {
+        'origin': current_app.config['SERVER_NAME'],
+        'type': 'corpus.build',
+        'corpus_id': str(corpus.id)
+    }
+    ''' ## Mounts ## '''
+    mounts = []
+    ''' ### Data mount ### '''
+    data_mount_source = os.path.join(corpus.path, 'cwb', 'data')
+    data_mount_target = '/corpora/data'
+    data_mount = f'{data_mount_source}:{data_mount_target}:rw'
+    # Make sure that their is no data in the data directory
+    shutil.rmtree(data_mount_source, ignore_errors=True)
+    os.makedirs(data_mount_source)
+    mounts.append(data_mount)
+    ''' ### File mount ### '''
+    file_mount_source = os.path.join(corpus.path, 'cwb', 'corpus.vrt')
+    file_mount_target = '/root/files/corpus.vrt'
+    file_mount = f'{file_mount_source}:{file_mount_target}:ro'
+    mounts.append(file_mount)
+    ''' ### Registry mount ### '''
+    registry_mount_source = os.path.join(corpus.path, 'cwb', 'registry')
+    registry_mount_target = '/usr/local/share/cwb/registry'
+    registry_mount = f'{registry_mount_source}:{registry_mount_target}:rw'
+    # Make sure that their is no data in the registry directory
+    shutil.rmtree(registry_mount_source, ignore_errors=True)
+    os.makedirs(registry_mount_source)
+    mounts.append(registry_mount)
+    ''' ## Name ## '''
+    name = f'build-corpus_{corpus.id}'
+    ''' ## Restart policy ## '''
+    restart_policy = docker.types.RestartPolicy()
+    try:
+        docker_client.services.create(
+            image,
+            command=command,
+            constraints=constraints,
+            labels=labels,
+            mounts=mounts,
+            name=name,
+            restart_policy=restart_policy,
+            user='0:0'
         )
-        ''' ## Constraints ## '''
+    except docker.errors.DockerException as e:
-        constraints = ['node.role==worker']
+        current_app.logger.error(f'Create service "{name}" failed: {e}')
-        ''' ## Image ## '''
+        return
-        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+    corpus.status = CorpusStatus.QUEUED
-        ''' ## Labels ## '''
-        labels = {
-            'origin': current_app.config['SERVER_NAME'],
-            'type': 'corpus.build',
-            'corpus_id': str(corpus.id)
-        }
-        ''' ## Mounts ## '''
-        mounts = []
-        ''' ### Data mount ### '''
-        data_mount_source = os.path.join(corpus.path, 'cwb', 'data')
-        data_mount_target = '/corpora/data'
-        data_mount = f'{data_mount_source}:{data_mount_target}:rw'
-        # Make sure that their is no data in the data directory
-        shutil.rmtree(data_mount_source, ignore_errors=True)
-        os.makedirs(data_mount_source)
-        mounts.append(data_mount)
-        ''' ### File mount ### '''
-        file_mount_source = os.path.join(corpus.path, 'cwb', 'corpus.vrt')
-        file_mount_target = '/root/files/corpus.vrt'
-        file_mount = f'{file_mount_source}:{file_mount_target}:ro'
-        mounts.append(file_mount)
-        ''' ### Registry mount ### '''
-        registry_mount_source = os.path.join(corpus.path, 'cwb', 'registry')
-        registry_mount_target = '/usr/local/share/cwb/registry'
-        registry_mount = f'{registry_mount_source}:{registry_mount_target}:rw'
-        # Make sure that their is no data in the registry directory
-        shutil.rmtree(registry_mount_source, ignore_errors=True)
-        os.makedirs(registry_mount_source)
-        mounts.append(registry_mount)
-        ''' ## Name ## '''
-        name = f'build-corpus_{corpus.id}'
-        ''' ## Restart policy ## '''
-        restart_policy = docker.types.RestartPolicy()
-        try:
-            self.docker.services.create(
-                image,
-                command=command,
-                constraints=constraints,
-                labels=labels,
-                mounts=mounts,
-                name=name,
-                restart_policy=restart_policy,
-                user='0:0'
-            )
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Create service "{name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
-            return
-        corpus.status = CorpusStatus.QUEUED
 
-    def checkout_build_corpus_service(self, corpus):
+def _checkout_build_corpus_service(corpus):
-        service_name = f'build-corpus_{corpus.id}'
+    service_name = f'build-corpus_{corpus.id}'
-        try:
+    try:
-            service = self.docker.services.get(service_name)
+        service = docker_client.services.get(service_name)
-        except docker.errors.NotFound as e:
+    except docker.errors.NotFound as e:
-            current_app.logger.error(
+        current_app.logger.error(f'Get service "{service_name}" failed: {e}')
-                f'Get service "{service_name}" failed '
+        corpus.status = CorpusStatus.FAILED
-                f'due to "docker.errors.NotFound": {e}'
+        return
-            )
+    except docker.errors.DockerException as e:
-            corpus.status = CorpusStatus.FAILED
+        current_app.logger.error(f'Get service "{service_name}" failed: {e}')
-            return
+    service_tasks = service.tasks()
-        except docker.errors.APIError as e:
+    if not service_tasks:
-            current_app.logger.error(
+        return
-                f'Get service "{service_name}" failed '
+    task_state = service_tasks[0].get('Status').get('State')
-                f'due to "docker.errors.APIError": {e}'
+    if corpus.status == CorpusStatus.QUEUED and task_state != 'pending':
-            )
+        corpus.status = CorpusStatus.BUILDING
-        service_tasks = service.tasks()
+        return
-        if not service_tasks:
+    elif corpus.status == CorpusStatus.BUILDING and task_state == 'complete':
-            return
+        corpus.status = CorpusStatus.BUILT
-        task_state = service_tasks[0].get('Status').get('State')
+    elif corpus.status == CorpusStatus.BUILDING and task_state == 'failed':
-        if corpus.status == CorpusStatus.QUEUED and task_state != 'pending':  # noqa
+        corpus.status = CorpusStatus.FAILED
-            corpus.status = CorpusStatus.BUILDING
+    else:
-            return
+        return
-        elif corpus.status == CorpusStatus.BUILDING and task_state == 'complete':  # noqa
+    try:
-            corpus.status = CorpusStatus.BUILT
+        service.remove()
-        elif corpus.status == CorpusStatus.BUILDING and task_state == 'failed':  # noqa
+    except docker.errors.DockerException as e:
-            corpus.status = CorpusStatus.FAILED
+        current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
-        else:
-            return
-        try:
-            service.remove()
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Remove service "{service_name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
 
-    def create_cqpserver_container(self, corpus):
+def _create_cqpserver_container(corpus):
-        ''' # Docker container settings # '''
+    ''' # Docker container settings # '''
-        ''' ## Command ## '''
+    ''' ## Command ## '''
-        command = []
+    command = []
-        command.append(
+    command.append(
-            'echo "host *;" > cqpserver.init'
+        'echo "host *;" > cqpserver.init'
-            ' && '
+        ' && '
-            'echo "user anonymous \\"\\";" >> cqpserver.init'
+        'echo "user anonymous \\"\\";" >> cqpserver.init'
-            ' && '
+        ' && '
-            'cqpserver -I cqpserver.init'
+        'cqpserver -I cqpserver.init'
-        )
+    )
-        ''' ## Detach ## '''
+    ''' ## Detach ## '''
-        detach = True
+    detach = True
-        ''' ## Entrypoint ## '''
+    ''' ## Entrypoint ## '''
-        entrypoint = ['bash', '-c']
+    entrypoint = ['bash', '-c']
-        ''' ## Image ## '''
+    ''' ## Image ## '''
-        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
-        ''' ## Name ## '''
+    ''' ## Name ## '''
-        name = f'cqpserver_{corpus.id}'
+    name = f'cqpserver_{corpus.id}'
-        ''' ## Network ## '''
+    ''' ## Network ## '''
-        network = 'nopaque_default'
+    network = 'nopaque_default'
-        ''' ## Volumes ## '''
+    ''' ## Volumes ## '''
-        volumes = []
+    volumes = []
-        ''' ### Corpus data volume ### '''
+    ''' ### Corpus data volume ### '''
-        data_volume_source = os.path.join(corpus.path, 'cwb', 'data')
+    data_volume_source = os.path.join(corpus.path, 'cwb', 'data')
-        data_volume_target = '/corpora/data'
+    data_volume_target = '/corpora/data'
-        data_volume = f'{data_volume_source}:{data_volume_target}:rw'
+    data_volume = f'{data_volume_source}:{data_volume_target}:rw'
-        volumes.append(data_volume)
+    volumes.append(data_volume)
-        ''' ### Corpus registry volume ### '''
+    ''' ### Corpus registry volume ### '''
-        registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry')
+    registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry')
-        registry_volume_target = '/usr/local/share/cwb/registry'
+    registry_volume_target = '/usr/local/share/cwb/registry'
-        registry_volume = f'{registry_volume_source}:{registry_volume_target}:rw'  # noqa
+    registry_volume = f'{registry_volume_source}:{registry_volume_target}:rw'
-        volumes.append(registry_volume)
+    volumes.append(registry_volume)
-        # Check if a cqpserver container already exists. If this is the case,
+    # Check if a cqpserver container already exists. If this is the case,
-        # remove it and create a new one
+    # remove it and create a new one
-        try:
+    try:
-            container = self.docker.containers.get(name)
+        container = docker_client.containers.get(name)
-        except docker.errors.NotFound:
+    except docker.errors.NotFound:
-            pass
+        pass
-        except docker.errors.APIError as e:
+    except docker.errors.DockerException as e:
-            current_app.logger.error(
+        current_app.logger.error(f'Get container "{name}" failed: {e}')
-                f'Get container "{name}" failed '
+        return
-                f'due to "docker.errors.APIError": {e}'
+    else:
-            )
-            return
-        else:
-            try:
-                container.remove(force=True)
-            except docker.errors.APIError as e:
-                current_app.logger.error(
-                    f'Remove container "{name}" failed '
-                    f'due to "docker.errors.APIError": {e}'
-                )
-                return
-        try:
-            self.docker.containers.run(
-                image,
-                command=command,
-                detach=detach,
-                entrypoint=entrypoint,
-                name=name,
-                network=network,
-                user='0:0',
-                volumes=volumes
-            )
-        except docker.errors.ImageNotFound as e:
-            current_app.logger.error(
-                f'Run container "{name}" failed '
-                f'due to "docker.errors.ImageNotFound" error: {e}'
-            )
-            corpus.status = CorpusStatus.FAILED
-            return
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Run container "{name}" failed '
-                f'due to "docker.errors.APIError" error: {e}'
-            )
-            return
-        corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
-
-    def checkout_analysing_corpus_container(self, corpus):
-        container_name = f'cqpserver_{corpus.id}'
-        try:
-            self.docker.containers.get(container_name)
-        except docker.errors.NotFound as e:
-            current_app.logger.error(
-                f'Get container "{container_name}" failed '
-                f'due to "docker.errors.NotFound": {e}'
-            )
-            corpus.num_analysis_sessions = 0
-            corpus.status = CorpusStatus.BUILT
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Get container "{container_name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
-
-    def remove_cqpserver_container(self, corpus):
-        container_name = f'cqpserver_{corpus.id}'
-        try:
-            container = self.docker.containers.get(container_name)
-        except docker.errors.NotFound:
-            corpus.status = CorpusStatus.BUILT
-            return
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Get container "{container_name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
-            return
         try:
             container.remove(force=True)
-        except docker.errors.APIError as e:
+        except docker.errors.DockerException as e:
-            current_app.logger.error(
+            current_app.logger.error(f'Remove container "{name}" failed: {e}')
-                f'Remove container "{container_name}" failed '
+            return
-                f'due to "docker.errors.APIError": {e}'
+    try:
-            )
+        docker_client.containers.run(
+            image,
+            command=command,
+            detach=detach,
+            entrypoint=entrypoint,
+            name=name,
+            network=network,
+            user='0:0',
+            volumes=volumes
+        )
+    except docker.errors.ImageNotFound as e:
+        current_app.logger.error(
+            f'Run container "{name}" failed '
+            f'due to "docker.errors.ImageNotFound" error: {e}'
+        )
+        corpus.status = CorpusStatus.FAILED
+        return
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Run container "{name}" failed: {e}')
+        return
+    corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
+
+def _checkout_analysing_corpus_container(corpus):
+    container_name = f'cqpserver_{corpus.id}'
+    try:
+        docker_client.containers.get(container_name)
+    except docker.errors.NotFound as e:
+        current_app.logger.error(f'Get container "{container_name}" failed: {e}')
+        corpus.num_analysis_sessions = 0
+        corpus.status = CorpusStatus.BUILT
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Get container "{container_name}" failed: {e}')
+
+def _remove_cqpserver_container(corpus):
+    container_name = f'cqpserver_{corpus.id}'
+    try:
+        container = docker_client.containers.get(container_name)
+    except docker.errors.NotFound:
+        corpus.status = CorpusStatus.BUILT
+        return
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Get container "{container_name}" failed: {e}')
+        return
+    try:
+        container.remove(force=True)
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Remove container "{container_name}" failed: {e}')

app/daemon/job_utils.py

@@ -1,4 +1,4 @@
-from app import db
+from app import db, docker_client, hashids
 from app.models import (
     Job,
     JobResult,
@@ -15,217 +15,202 @@ import os
 import shutil
 
 
-class CheckJobsMixin:
+def check_jobs():
-    def check_jobs(self):
+    jobs = Job.query.all()
-        jobs = Job.query.all()
+    for job in [x for x in jobs if x.status == JobStatus.SUBMITTED]:
-        for job in (x for x in jobs if x.status == JobStatus.SUBMITTED):
+        _create_job_service(job)
-            self.create_job_service(job)
+    for job in [x for x in jobs if x.status in [JobStatus.QUEUED, JobStatus.RUNNING]]:
-        for job in (x for x in jobs if x.status in [JobStatus.QUEUED, JobStatus.RUNNING]):  # noqa
+        _checkout_job_service(job)
-            self.checkout_job_service(job)
+    for job in [x for x in jobs if x.status == JobStatus.CANCELING]:
-        for job in (x for x in jobs if x.status == JobStatus.CANCELING):
+        _remove_job_service(job)
-            self.remove_job_service(job)
 
-    def create_job_service(self, job):
+def _create_job_service(job):
-        ''' # Docker service settings # '''
+    ''' # Docker service settings # '''
-        ''' ## Service specific settings ## '''
+    ''' ## Service specific settings ## '''
-        if job.service == 'file-setup-pipeline':
+    if job.service == 'file-setup-pipeline':
-            mem_mb = 512
+        mem_mb = 512
-            n_cores = 2
+        n_cores = 2
-            executable = 'file-setup-pipeline'
+        executable = 'file-setup-pipeline'
-            image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup-pipeline:v{job.service_version}'  # noqa
+        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup-pipeline:v{job.service_version}'
-        elif job.service == 'tesseract-ocr-pipeline':
+    elif job.service == 'tesseract-ocr-pipeline':
-            mem_mb = 1024
+        mem_mb = 1024
-            n_cores = 4
+        n_cores = 4
-            executable = 'tesseract-ocr-pipeline'
+        executable = 'tesseract-ocr-pipeline'
-            image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}tesseract-ocr-pipeline:v{job.service_version}'  # noqa
+        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}tesseract-ocr-pipeline:v{job.service_version}'
-        elif job.service == 'transkribus-htr-pipeline':
+    elif job.service == 'transkribus-htr-pipeline':
-            mem_mb = 1024
+        mem_mb = 1024
-            n_cores = 4
+        n_cores = 4
-            executable = 'transkribus-htr-pipeline'
+        executable = 'transkribus-htr-pipeline'
-            image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}transkribus-htr-pipeline:v{job.service_version}'  # noqa
+        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}transkribus-htr-pipeline:v{job.service_version}'
-        elif job.service == 'spacy-nlp-pipeline':
+    elif job.service == 'spacy-nlp-pipeline':
-            mem_mb = 1024
+        mem_mb = 1024
-            n_cores = 1
+        n_cores = 1
-            executable = 'spacy-nlp-pipeline'
+        executable = 'spacy-nlp-pipeline'
-            image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}spacy-nlp-pipeline:v{job.service_version}'  # noqa
+        image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}spacy-nlp-pipeline:v{job.service_version}'
-        ''' ## Command ## '''
+    ''' ## Command ## '''
-        command = f'{executable} -i /input -o /output'
+    command = f'{executable} -i /input -o /output'
-        command += ' --log-dir /logs'
+    command += ' --log-dir /logs'
-        command += f' --mem-mb {mem_mb}'
+    command += f' --mem-mb {mem_mb}'
-        command += f' --n-cores {n_cores}'
+    command += f' --n-cores {n_cores}'
-        if job.service == 'spacy-nlp-pipeline':
+    if job.service == 'spacy-nlp-pipeline':
-            command += f' -m {job.service_args["model"]}'
+        command += f' -m {job.service_args["model"]}'
-            if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']:  # noqa
+        if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']:
-                command += ' --check-encoding'
+            command += ' --check-encoding'
-        elif job.service == 'tesseract-ocr-pipeline':
+    elif job.service == 'tesseract-ocr-pipeline':
-            command += f' -m {job.service_args["model"]}'
+        command += f' -m {job.service_args["model"]}'
-            if 'binarization' in job.service_args and job.service_args['binarization']:
+        if 'binarization' in job.service_args and job.service_args['binarization']:
-                command += ' --binarize'
+            command += ' --binarize'
-        elif job.service == 'transkribus-htr-pipeline':
+    elif job.service == 'transkribus-htr-pipeline':
-            transkribus_htr_model = TranskribusHTRModel.query.get(job.service_args['model'])
+        transkribus_htr_model = TranskribusHTRModel.query.get(job.service_args['model'])
-            command += f' -m {transkribus_htr_model.transkribus_model_id}'
+        command += f' -m {transkribus_htr_model.transkribus_model_id}'
-            readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME')
+        readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME')
-            command += f' --readcoop-username "{readcoop_username}"'
+        command += f' --readcoop-username "{readcoop_username}"'
-            readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD')
+        readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD')
-            command += f' --readcoop-password "{readcoop_password}"'
+        command += f' --readcoop-password "{readcoop_password}"'
-            if 'binarization' in job.service_args and job.service_args['binarization']:
+        if 'binarization' in job.service_args and job.service_args['binarization']:
-                command += ' --binarize'
+            command += ' --binarize'
-        ''' ## Constraints ## '''
+    ''' ## Constraints ## '''
-        constraints = ['node.role==worker']
+    constraints = ['node.role==worker']
-        ''' ## Labels ## '''
+    ''' ## Labels ## '''
-        labels = {
+    labels = {
-            'origin': current_app.config['SERVER_NAME'],
+        'origin': current_app.config['SERVER_NAME'],
-            'type': 'job',
+        'type': 'job',
-            'job_id': str(job.id)
+        'job_id': str(job.id)
-        }
+    }
-        ''' ## Mounts ## '''
+    ''' ## Mounts ## '''
-        mounts = []
+    mounts = []
-        ''' ### Input mount(s) ### '''
+    ''' ### Input mount(s) ### '''
-        input_mount_target_base = '/input'
+    input_mount_target_base = '/input'
-        if job.service == 'file-setup-pipeline':
+    if job.service == 'file-setup-pipeline':
-            input_mount_target_base += f'/{secure_filename(job.title)}'
+        input_mount_target_base += f'/{secure_filename(job.title)}'
-        for job_input in job.inputs:
+    for job_input in job.inputs:
-            input_mount_source = job_input.path
+        input_mount_source = job_input.path
-            input_mount_target = f'{input_mount_target_base}/{job_input.filename}'  # noqa
+        input_mount_target = f'{input_mount_target_base}/{job_input.filename}'
-            input_mount = f'{input_mount_source}:{input_mount_target}:ro'
+        input_mount = f'{input_mount_source}:{input_mount_target}:ro'
-            mounts.append(input_mount)
+        mounts.append(input_mount)
-        if job.service == 'tesseract-ocr-pipeline':
+    if job.service == 'tesseract-ocr-pipeline':
-            model = TesseractOCRModel.query.get(job.service_args['model'])
+        if isinstance(job.service_args['model'], str):
-            if model is None:
+            model_id = hashids.decode(job.service_args['model'])
-                job.status = JobStatus.FAILED
+        elif isinstance(job.service_args['model'], int):
-                return
+            model_id = job.service_args['model']
-            models_mount_source = model.path
-            models_mount_target = f'/usr/local/share/tessdata/{model.filename}'
-            models_mount = f'{models_mount_source}:{models_mount_target}:ro'
-            mounts.append(models_mount)
-        ''' ### Output mount ### '''
-        output_mount_source = os.path.join(job.path, 'results')
-        output_mount_target = '/output'
-        output_mount = f'{output_mount_source}:{output_mount_target}:rw'
-        # Make sure that their is no data in the output directory
-        shutil.rmtree(output_mount_source, ignore_errors=True)
-        os.makedirs(output_mount_source)
-        mounts.append(output_mount)
-        ''' ### Pipeline data mount ### '''
-        pyflow_data_mount_source = os.path.join(job.path, 'pipeline_data')
-        pyflow_data_mount_target = '/logs/pyflow.data'
-        pyflow_data_mount = f'{pyflow_data_mount_source}:{pyflow_data_mount_target}:rw'  # noqa
-        # Make sure that their is no data in the output directory
-        shutil.rmtree(pyflow_data_mount_source, ignore_errors=True)
-        os.makedirs(pyflow_data_mount_source)
-        mounts.append(pyflow_data_mount)
-        ''' ## Name ## '''
-        name = f'job_{job.id}'
-        ''' ## Resources ## '''
-        resources = docker.types.Resources(
-            cpu_reservation=n_cores * (10 ** 9),
-            mem_reservation=mem_mb * (10 ** 6)
-        )
-        ''' ## Restart policy ## '''
-        restart_policy = docker.types.RestartPolicy()
-        try:
-            self.docker.services.create(
-                image,
-                command=command,
-                constraints=constraints,
-                labels=labels,
-                mounts=mounts,
-                name=name,
-                resources=resources,
-                restart_policy=restart_policy,
-                user='0:0'
-            )
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Create service "{name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
-            return
-        job.status = JobStatus.QUEUED
-
-    def checkout_job_service(self, job):
-        service_name = f'job_{job.id}'
-        try:
-            service = self.docker.services.get(service_name)
-        except docker.errors.NotFound as e:
-            current_app.logger.error(
-                f'Get service "{service_name}" failed '
-                f'due to "docker.errors.NotFound": {e}'
-            )
-            job.status = JobStatus.FAILED
-            return
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Get service "{service_name}" failed '
-                f'due to "docker.errors.APIError": {e}'
-            )
-            return
-        service_tasks = service.tasks()
-        if not service_tasks:
-            return
-        task_state = service_tasks[0].get('Status').get('State')
-        if job.status == JobStatus.QUEUED and task_state != 'pending':
-            job.status = JobStatus.RUNNING
-            return
-        elif job.status == JobStatus.RUNNING and task_state == 'complete':  # noqa
-            job.status = JobStatus.COMPLETED
-            results_dir = os.path.join(job.path, 'results')
-            with open(os.path.join(results_dir, 'outputs.json')) as f:
-                outputs = json.load(f)
-            for output in outputs:
-                filename = os.path.basename(output['file'])
-                job_result = JobResult(
-                    filename=filename,
-                    job=job,
-                    mimetype=output['mimetype']
-                )
-                if 'description' in output:
-                    job_result.description = output['description']
-                db.session.add(job_result)
-                db.session.flush(objects=[job_result])
-                db.session.refresh(job_result)
-                os.rename(
-                    os.path.join(results_dir, output['file']),
-                    job_result.path
-                )
-        elif job.status == JobStatus.RUNNING and task_state == 'failed':
-            job.status = JobStatus.FAILED
         else:
+            job.status = JobStatus.FAILED
             return
-        job.end_date = datetime.utcnow()
+        model = TesseractOCRModel.query.get(model_id)
-        try:
+        if model is None:
-            service.remove()
+            job.status = JobStatus.FAILED
-        except docker.errors.APIError as e:
+            return
-            current_app.logger.error(
+        models_mount_source = model.path
-                f'Remove service "{service_name}" failed '
+        models_mount_target = f'/usr/local/share/tessdata/{model.filename}'
-                f'due to "docker.errors.APIError": {e}'
+        models_mount = f'{models_mount_source}:{models_mount_target}:ro'
-            )
+        mounts.append(models_mount)
+    ''' ### Output mount ### '''
+    output_mount_source = os.path.join(job.path, 'results')
+    output_mount_target = '/output'
+    output_mount = f'{output_mount_source}:{output_mount_target}:rw'
+    # Make sure that their is no data in the output directory
+    shutil.rmtree(output_mount_source, ignore_errors=True)
+    os.makedirs(output_mount_source)
+    mounts.append(output_mount)
+    ''' ### Pipeline data mount ### '''
+    pyflow_data_mount_source = os.path.join(job.path, 'pipeline_data')
+    pyflow_data_mount_target = '/logs/pyflow.data'
+    pyflow_data_mount = f'{pyflow_data_mount_source}:{pyflow_data_mount_target}:rw'
+    # Make sure that their is no data in the output directory
+    shutil.rmtree(pyflow_data_mount_source, ignore_errors=True)
+    os.makedirs(pyflow_data_mount_source)
+    mounts.append(pyflow_data_mount)
+    ''' ## Name ## '''
+    name = f'job_{job.id}'
+    ''' ## Resources ## '''
+    resources = docker.types.Resources(
+        cpu_reservation=n_cores * (10 ** 9),
+        mem_reservation=mem_mb * (10 ** 6)
+    )
+    ''' ## Restart policy ## '''
+    restart_policy = docker.types.RestartPolicy()
+    try:
+        docker_client.services.create(
+            image,
+            command=command,
+            constraints=constraints,
+            labels=labels,
+            mounts=mounts,
+            name=name,
+            resources=resources,
+            restart_policy=restart_policy,
+            user='0:0'
+        )
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Create service "{name}" failed: {e}')
+        return
+    job.status = JobStatus.QUEUED
 
-    def remove_job_service(self, job):
+def _checkout_job_service(job):
-        service_name = f'job_{job.id}'
+    service_name = f'job_{job.id}'
-        try:
+    try:
-            service = self.docker.services.get(service_name)
+        service = docker_client.services.get(service_name)
-        except docker.errors.NotFound:
+    except docker.errors.NotFound as e:
-            job.status = JobStatus.CANCELED
+        current_app.logger.error(f'Get service "{service_name}" failed: {e}')
-            return
+        job.status = JobStatus.FAILED
-        except docker.errors.APIError as e:
+        return
-            current_app.logger.error(
+    except docker.errors.DockerException as e:
-                f'Get service "{service_name}" failed '
+        current_app.logger.error(f'Get service "{service_name}" failed: {e}')
-                f'due to "docker.errors.APIError": {e}'
+        return
+    service_tasks = service.tasks()
+    if not service_tasks:
+        return
+    task_state = service_tasks[0].get('Status').get('State')
+    if job.status == JobStatus.QUEUED and task_state != 'pending':
+        job.status = JobStatus.RUNNING
+        return
+    elif job.status == JobStatus.RUNNING and task_state == 'complete':
+        job.status = JobStatus.COMPLETED
+        results_dir = os.path.join(job.path, 'results')
+        with open(os.path.join(results_dir, 'outputs.json')) as f:
+            outputs = json.load(f)
+        for output in outputs:
+            filename = os.path.basename(output['file'])
+            job_result = JobResult(
+                filename=filename,
+                job=job,
+                mimetype=output['mimetype']
             )
-            return
+            if 'description' in output:
-        try:
+                job_result.description = output['description']
-            service.update(mounts=None)
+            db.session.add(job_result)
-        except docker.errors.APIError as e:
+            db.session.flush(objects=[job_result])
-            current_app.logger.error(
+            db.session.refresh(job_result)
-                f'Update service "{service_name}" failed '
+            os.rename(
-                f'due to "docker.errors.APIError": {e}'
+                os.path.join(results_dir, output['file']),
-            )
+                job_result.path
-            return
-        try:
-            service.remove()
-        except docker.errors.APIError as e:
-            current_app.logger.error(
-                f'Remove "{service_name}" service failed '
-                f'due to "docker.errors.APIError": {e}'
             )
+    elif job.status == JobStatus.RUNNING and task_state == 'failed':
+        job.status = JobStatus.FAILED
+    else:
+        return
+    job.end_date = datetime.utcnow()
+    try:
+        service.remove()
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
+
+def _remove_job_service(job):
+    service_name = f'job_{job.id}'
+    try:
+        service = docker_client.services.get(service_name)
+    except docker.errors.NotFound:
+        job.status = JobStatus.CANCELED
+        return
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+        return
+    try:
+        service.update(mounts=None)
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Update service "{service_name}" failed: {e}')
+        return
+    try:
+        service.remove()
+    except docker.errors.DockerException as e:
+        current_app.logger.error(f'Remove "{service_name}" service failed: {e}')

app/decorators.py

@@ -2,7 +2,7 @@ from flask import abort, current_app
 from flask_login import current_user
 from functools import wraps
 from threading import Thread
-from .models import Permission
+from app.models import Permission
 
 
 def permission_required(permission):

app/email.py

@@ -1,27 +1,25 @@
 from flask import current_app, render_template
 from flask_mail import Message
-from typing import Any, Text
+from threading import Thread
-from . import mail
+from app import mail
-from .decorators import background
 
 
-def create_message(
+def create_message(recipient, subject, template, **kwargs):
-    recipient: str,
-    subject: str,
-    template: str,
-    **kwargs: Any
-) -> Message:
     subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
     msg: Message = Message(
-        f'{subject_prefix} {subject}',
+        body=render_template(f'{template}.txt.j2', **kwargs),
-        recipients=[recipient]
+        html=render_template(f'{template}.html.j2', **kwargs),
+        recipients=[recipient],
+        subject=f'{subject_prefix} {subject}'
     )
-    msg.body = render_template(f'{template}.txt.j2', **kwargs)
-    msg.html = render_template(f'{template}.html.j2', **kwargs)
     return msg
 
 
-@background
+def send(msg, *args, **kwargs):
-def send(msg: Message, *args, **kwargs):
+    def _send(app, msg):
-    with kwargs['app'].app_context():
+        with app.app_context():
-        mail.send(msg)
+            mail.send(msg)
+
+    thread = Thread(target=_send, args=[current_app._get_current_object(), msg])
+    thread.start()
+    return thread

app/errors/handlers.py

@@ -1,52 +1,11 @@
-from flask import render_template, request, jsonify
+from flask import render_template, request
+from werkzeug.exceptions import HTTPException
 from . import bp
 
 
-@bp.app_errorhandler(403)
+@bp.errorhandler(HTTPException)
-def forbidden(e):
+def generic_error_handler(e):
     if (request.accept_mimetypes.accept_json
             and not request.accept_mimetypes.accept_html):
-        response = jsonify({'error': 'forbidden'})
+        return {'errors': {'message': e.description}}, e.code
-        response.status_code = 403
+    return render_template('errors/error.html.j2', error=e), e.code
-        return response
-    return render_template('errors/403.html.j2', title='Forbidden'), 403
-
-
-@bp.app_errorhandler(404)
-def not_found(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        response = jsonify({'error': 'not found'})
-        response.status_code = 404
-        return response
-    return render_template('errors/404.html.j2', title='Not Found'), 404
-
-
-@bp.app_errorhandler(413)
-def payload_too_large(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        response = jsonify({'error': 'payload too large'})
-        response.status_code = 413
-        return response
-    return render_template('errors/413.html.j2', title='Payload Too Large'), 413
-
-
-@bp.app_errorhandler(500)
-def internal_server_error(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        response = jsonify({'error': 'internal server error'})
-        response.status_code = 500
-        return response
-    return render_template('errors/500.html.j2', title='Internal Server Error'), 500
-
-
-@bp.app_errorhandler(503)
-def service_unavailable_error(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        response = jsonify({'error': 'service unavailable'})
-        response.status_code = 503
-        return response
-    return render_template('errors/503.html.j2', title='Service Unavailable'), 503

app/jobs/routes.py

@@ -1,17 +1,16 @@
-from app.decorators import admin_required
-from app.models import Job, JobInput, JobResult, JobStatus
 from flask import (
     abort,
-    flash,
+    current_app,
-    redirect,
     render_template,
-    send_from_directory,
+    send_from_directory
-    url_for
 )
 from flask_login import current_user, login_required
-from . import bp
+from threading import Thread
-from . import tasks
 import os
+from app import db
+from app.decorators import admin_required
+from app.models import Job, JobInput, JobResult, JobStatus
+from . import bp
 
 
 @bp.route('/<hashid:job_id>')
@@ -27,68 +26,91 @@ def job(job_id):
     )
 
 
-@bp.route('/<hashid:job_id>/delete')
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
 @login_required
 def delete_job(job_id):
+    def _delete_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.delete()
+            db.session.commit()
+
     job = Job.query.get_or_404(job_id)
     if not (job.user == current_user or current_user.is_administrator()):
         abort(403)
-    tasks.delete_job(job_id)
+    thread = Thread(
-    flash(f'Job "{job.title}" marked for deletion', 'job')
+        target=_delete_job,
-    return redirect(url_for('main.dashboard'))
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    return {}, 202
+
+
+@bp.route('/<hashid:job_id>/log')
+@login_required
+@admin_required
+def job_log(job_id):
+    job = Job.query.get_or_404(job_id)
+    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+        response = {'errors': {'message': 'Job status is not completed or failed'}}
+        return response, 409
+    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+        log = log_file.read()
+    return log, 200, {'Content-Type': 'text/plain; charset=utf-8'}
+
+
+@bp.route('/<hashid:job_id>/restart', methods=['POST'])
+@login_required
+def restart_job(job_id):
+    def _restart_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.restart()
+            db.session.commit()
+
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator()):
+        abort(403)
+    if job.status == JobStatus.FAILED:
+        response = {'errors': {'message': 'Job status is not "failed"'}}
+        return response, 409
+    thread = Thread(
+        target=_restart_job,
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    return {}, 202
 
 
 @bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
 @login_required
 def download_job_input(job_id, job_input_id):
-    job_input = JobInput.query.filter(
+    job_input = JobInput.query.get_or_404(job_input_id)
-        JobInput.job_id == job_id,
+    if job_input.job.id != job_id:
-        JobInput.id == job_input_id
+        abort(404)
-    ).first_or_404()
+    if not (job_input.job.user == current_user or current_user.is_administrator()):
-    if not (
-        job_input.job.user == current_user
-        or current_user.is_administrator()
-    ):
         abort(403)
     return send_from_directory(
+        os.path.dirname(job_input.path),
+        os.path.basename(job_input.path),
         as_attachment=True,
         attachment_filename=job_input.filename,
-        directory=os.path.dirname(job_input.path),
+        mimetype=job_input.mimetype
-        filename=os.path.basename(job_input.path)
     )
 
 
-@bp.route('/<hashid:job_id>/restart')
-@login_required
-@admin_required
-def restart(job_id):
-    job = Job.query.get_or_404(job_id)
-    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-        flash(
-            f'Can\'t restart job "{job.title}": Status is not "Completed/Failed"',  # noqa
-            category='error'
-        )
-    else:
-        tasks.restart_job(job_id)
-        flash(f'Job "{job.title}" marked to get restarted', category='job')
-    return redirect(url_for('.job', job_id=job_id))
-
-
 @bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
 @login_required
 def download_job_result(job_id, job_result_id):
-    job_result = JobResult.query.filter(
+    job_result = JobResult.query.get_or_404(job_result_id)
-        JobResult.job_id == job_id,
+    if job_result.job.id != job_id:
-        JobResult.id == job_result_id
+        abort(404)
-    ).first_or_404()
+    if not (job_result.job.user == current_user or current_user.is_administrator()):
-    if not (
-        job_result.job.user == current_user
-        or current_user.is_administrator()
-    ):
         abort(403)
     return send_from_directory(
+        os.path.dirname(job_result.path),
+        os.path.basename(job_result.path),
         as_attachment=True,
         attachment_filename=job_result.filename,
-        directory=os.path.dirname(job_result.path),
+        mimetype=job_result.mimetype
-        filename=os.path.basename(job_result.path)
     )

app/jobs/tasks.py

@@ -1,27 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import Job
-
-
-@background
-def delete_job(job_id, *args, **kwargs):
-    with kwargs['app'].app_context():
-        job = Job.query.get(job_id)
-        if job is None:
-            raise Exception(f'Job {job_id} not found')
-        job.delete()
-        db.session.commit()
-
-
-@background
-def restart_job(job_id, *args, **kwargs):
-    with kwargs['app'].app_context():
-        job = Job.query.get(job_id)
-        if job is None:
-            raise Exception(f'Job {job_id} not found')
-        try:
-            job.restart()
-        except Exception:
-            pass
-        else:
-            db.session.commit()

app/main/routes.py

@@ -1,30 +1,27 @@
-from app.auth.forms import LoginForm
-from app.models import User
 from flask import flash, redirect, render_template, url_for
 from flask_login import login_required, login_user
+from app.auth.forms import LoginForm
+from app.models import User
 from . import bp
 
 
-@bp.route('/', methods=['GET', 'POST'])
+@bp.route('', methods=['GET', 'POST'])
 def index():
     form = LoginForm(prefix='login-form')
     if form.validate_on_submit():
-        user = User.query.filter_by(username=form.user.data).first()
+        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
-        if user is None:
+        if user and user.verify_password(form.password.data):
-            user = User.query.filter_by(email=form.user.data.lower()).first()
-        if user is not None and user.verify_password(form.password.data):
             login_user(user, form.remember_me.data)
+            flash('You have been logged in')
             return redirect(url_for('.dashboard'))
-        flash('Invalid email/username or password.')
+        flash('Invalid email/username or password', category='error')
+        redirect(url_for('.index'))
     return render_template('main/index.html.j2', form=form, title='nopaque')
 
 
 @bp.route('/faq')
 def faq():
-    return render_template(
+    return render_template('main/faq.html.j2', title='Frequently Asked Questions')
-        'main/faq.html.j2',
-        title='Frequently Asked Questions'
-    )
 
 
 @bp.route('/dashboard')
@@ -45,10 +42,7 @@ def news():
 
 @bp.route('/privacy_policy')
 def privacy_policy():
-    return render_template(
+    return render_template('main/privacy_policy.html.j2', title='Privacy statement (GDPR)')
-        'main/privacy_policy.html.j2',
-        title='Privacy statement (GDPR)'
-    )
 
 
 @bp.route('/terms_of_use')

app/query_results_models.py

@@ -42,21 +42,17 @@ class QueryResult(FileMixin, HashidMixin, db.Model):
         shutil.rmtree(self.path, ignore_errors=True)
         db.session.delete(self)
 
-    def to_dict(self, backrefs=False, relationships=False):
+    def to_json(self, backrefs=False, relationships=False):
-        dict_query_result = {
+        _json = {
             'id': self.hashid,
-            'user_id': self.user.hashid,
-            'download_url': self.download_url,
-            'url': self.url,
             'corpus_title': self.query_metadata['corpus_name'],
             'description': self.description,
             'filename': self.filename,
             'query': self.query_metadata['query'],
             'query_metadata': self.query_metadata,
             'title': self.title,
-            **self.file_mixin_to_dict(
+            **self.file_mixin_to_json(
                 backrefs=backrefs, relationships=relationships)
         }
         if backrefs:
-            dict_query_result['user'] = self.user.to_dict(
+            _json['user'] = self.user.to_json(backrefs=True, relationships=False)
-                backrefs=True, relationships=False)

app/services/forms.py

@@ -1,4 +1,3 @@
-from app.models import TesseractOCRModel, TranskribusHTRModel
 from flask_login import current_user
 from flask_wtf import FlaskForm
 from flask_wtf.file import FileField, FileRequired
@@ -10,19 +9,26 @@ from wtforms import (
     SubmitField,
     ValidationError
 )
-from wtforms.validators import DataRequired, InputRequired, Length
+from wtforms.validators import InputRequired, Length
+from app.models import TesseractOCRModel, TranskribusHTRModel
 from . import SERVICES
 
 
-class AddJobForm(FlaskForm):
+class CreateJobBaseForm(FlaskForm):
-    description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
+    description = StringField(
-    title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
+        'Description',
-    version = SelectField('Version', validators=[DataRequired()])
+        validators=[InputRequired(), Length(max=255)]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=32)]
+    )
+    version = SelectField('Version', validators=[InputRequired()])
     submit = SubmitField()
 
 
-class AddFileSetupPipelineJobForm(AddJobForm):
+class CreateFileSetupPipelineJobForm(CreateJobBaseForm):
-    images = MultipleFileField('File(s)', validators=[DataRequired()])
+    images = MultipleFileField('File(s)', validators=[InputRequired()])
 
     def validate_images(form, field):
         valid_mimetypes = ['image/jpeg', 'image/png', 'image/tiff']
@@ -39,18 +45,15 @@ class AddFileSetupPipelineJobForm(AddJobForm):
         self.version.default = service_manifest['latest_version']
 
 
-class AddTesseractOCRPipelineJobForm(AddJobForm):
+class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
     binarization = BooleanField('Binarization')
     pdf = FileField('File', validators=[FileRequired()])
-    model = SelectField('Model', validators=[DataRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
 
     def validate_binarization(self, field):
         service_info = SERVICES['tesseract-ocr-pipeline']['versions'][self.version.data]
         if field.data:
-            if(
+            if not('methods' in service_info and 'binarization' in service_info['methods']):
-                'methods' not in service_info
-                or 'binarization' not in service_info['methods']
-            ):
                 raise ValidationError('Binarization is not available')
 
     def validate_pdf(self, field):
@@ -81,10 +84,10 @@ class AddTesseractOCRPipelineJobForm(AddJobForm):
         self.version.default = service_manifest['latest_version']
 
 
-class AddTranskribusHTRPipelineJobForm(AddJobForm):
+class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
     binarization = BooleanField('Binarization')
     pdf = FileField('File', validators=[FileRequired()])
-    model = SelectField('Model', validators=[DataRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
 
     def validate_binarization(self, field):
         service_info = SERVICES['transkribus-htr-pipeline']['versions'][self.version.data]
@@ -123,10 +126,10 @@ class AddTranskribusHTRPipelineJobForm(AddJobForm):
         self.version.default = service_manifest['latest_version']
 
 
-class AddSpacyNLPPipelineJobForm(AddJobForm):
+class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
     encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
     txt = FileField('File', validators=[FileRequired()])
-    model = SelectField('Model', validators=[DataRequired()])
+    model = SelectField('Model', validators=[InputRequired()])
 
     def validate_encoding_detection(self, field):
         service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]

app/services/routes.py

@@ -1,3 +1,5 @@
+from flask import abort, current_app, flash, Markup, render_template, request
+from flask_login import current_user, login_required
 from app import db, hashids
 from app.models import (
     Job,
@@ -7,26 +9,13 @@ from app.models import (
     TRANSKRIBUS_HTR_MODELS,
     TranskribusHTRModel
 )
-from flask import (
+from . import bp, SERVICES
-    abort,
-    current_app,
-    flash,
-    make_response,
-    render_template,
-    request,
-    url_for
-)
-from flask_login import current_user, login_required
-from werkzeug.utils import secure_filename
-from . import bp
-from . import SERVICES
 from .forms import (
-    AddFileSetupPipelineJobForm,
+    CreateFileSetupPipelineJobForm,
-    AddTesseractOCRPipelineJobForm,
+    CreateTesseractOCRPipelineJobForm,
-    AddTranskribusHTRPipelineJobForm,
+    CreateTranskribusHTRPipelineJobForm,
-    AddSpacyNLPPipelineJobForm
+    CreateSpacyNLPPipelineJobForm
 )
-import json
 
 
 @bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
@@ -37,49 +26,32 @@ def file_setup_pipeline():
     version = request.args.get('version', service_manifest['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
-    form = AddFileSetupPipelineJobForm(prefix='add-job-form', version=version)
+    form = CreateFileSetupPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
-            return make_response(form.errors, 400)
+            response = {'errors': form.errors}
-        service_args = {}
+            return response, 400
-        job = Job(
-            user=current_user,
-            description=form.description.data,
-            service=service,
-            service_args=service_args,
-            service_version=form.version.data,
-            title=form.title.data
-        )
-        db.session.add(job)
-        db.session.flush(objects=[job])
-        db.session.refresh(job)
         try:
-            job.makedirs()
+            job = Job.create(
-        except OSError as e:
+                title=form.title.data,
-            current_app.logger.error(e)
+                description=form.description.data,
-            db.session.rollback()
+                service=service,
-            flash('Internal Server Error', 'error')
+                service_args={},
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
+                service_version=form.version.data,
-        for image_file in form.images.data:
+                user=current_user
-            job_input = JobInput(
-                filename=secure_filename(image_file.filename),
-                job=job,
-                mimetype=image_file.mimetype
             )
-            db.session.add(job_input)
+        except OSError:
-            db.session.flush(objects=[job_input])
+            abort(500)
-            db.session.refresh(job_input)
+        for input_file in form.images.data:
             try:
-                image_file.save(job_input.path)
+                JobInput.create(input_file, job=job)
-            except OSError as e:
+            except OSError:
-                current_app.logger.error(e)
+                abort(500)
-                db.session.rollback()
-                flash('Internal Server Error', 'error')
-                return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        flash(f'Job "{job.title}" added', 'job')
+        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
-        return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)  # noqa
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
     return render_template(
         'services/file_setup_pipeline.html.j2',
         form=form,
@@ -90,61 +62,43 @@ def file_setup_pipeline():
 @bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
 @login_required
 def tesseract_ocr_pipeline():
-    service = 'tesseract-ocr-pipeline'
+    service_name = 'tesseract-ocr-pipeline'
-    service_manifest = SERVICES[service]
+    service_manifest = SERVICES[service_name]
     version = request.args.get('version', service_manifest['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
-    form = AddTesseractOCRPipelineJobForm(prefix='add-job-form', version=version)
+    form = CreateTesseractOCRPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
-            return make_response(form.errors, 400)
+            response = {'errors': form.errors}
-        service_args = {}
+            return response, 400
-        service_args['model'] = hashids.decode(form.model.data)
-        if form.binarization.data:
-            service_args['binarization'] = True
-        job = Job(
-            user=current_user,
-            description=form.description.data,
-            service=service,
-            service_args=service_args,
-            service_version=form.version.data,
-            title=form.title.data
-        )
-        db.session.add(job)
-        db.session.flush(objects=[job])
-        db.session.refresh(job)
         try:
-            job.makedirs()
+            job = Job.create(
-        except OSError as e:
+                title=form.title.data,
-            current_app.logger.error(e)
+                description=form.description.data,
-            db.session.rollback()
+                service=service_name,
-            flash('Internal Server Error', 'error')
+                service_args={
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
+                    'binarization': form.binarization.data,
-        job_input = JobInput(
+                    'model': hashids.decode(form.model.data)
-            filename=secure_filename(form.pdf.data.filename),
+                },
-            job=job,
+                service_version=form.version.data,
-            mimetype=form.pdf.data.mimetype
+                user=current_user
-        )
+            )
-        db.session.add(job_input)
+        except OSError:
-        db.session.flush(objects=[job_input])
+            abort(500)
-        db.session.refresh(job_input)
         try:
-            form.pdf.data.save(job_input.path)
+            JobInput.create(form.pdf.data, job=job)
-        except OSError as e:
+        except OSError:
-            current_app.logger.error(e)
+            abort(500)
-            db.session.rollback()
-            flash('Internal Server Error', 'error')
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        flash(f'Job "{job.title}" added', 'job')
+        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
-        return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)  # noqa
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
     tesseract_ocr_models = [
-        x for x in TesseractOCRModel.query.filter().all()
+        x for x in TesseractOCRModel.query.all()
         if version in x.compatible_service_versions and (x.shared == True or x.user == current_user)
     ]
-    current_app.logger.warning(tesseract_ocr_models)
     return render_template(
         'services/tesseract_ocr_pipeline.html.j2',
         form=form,
@@ -163,57 +117,40 @@ def transkribus_htr_pipeline():
     version = request.args.get('version', service_manifest['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
-    form = AddTranskribusHTRPipelineJobForm(prefix='add-job-form', version=version)
+    form = CreateTranskribusHTRPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
-            return make_response(form.errors, 400)
+            response = {'errors': form.errors}
-        service_args = {}
+            return response, 400
-        service_args['model'] = hashids.decode(form.model.data)
-        if form.binarization.data:
-            service_args['binarization'] = True
-        job = Job(
-            user=current_user,
-            description=form.description.data,
-            service=service,
-            service_args=service_args,
-            service_version=form.version.data,
-            title=form.title.data
-        )
-        db.session.add(job)
-        db.session.flush(objects=[job])
-        db.session.refresh(job)
         try:
-            job.makedirs()
+            job = Job.create(
-        except OSError as e:
+                title=form.title.data,
-            current_app.logger.error(e)
+                description=form.description.data,
-            db.session.rollback()
+                service=service,
-            flash('Internal Server Error', 'error')
+                service_args={
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
+                    'binarization': form.binarization.data,
-        job_input = JobInput(
+                    'model': hashids.decode(form.model.data)
-            filename=secure_filename(form.pdf.data.filename),
+                },
-            job=job,
+                service_version=form.version.data,
-            mimetype=form.pdf.data.mimetype
+                user=current_user
-        )
+            )
-        db.session.add(job_input)
+        except OSError:
-        db.session.flush(objects=[job_input])
+            abort(500)
-        db.session.refresh(job_input)
         try:
-            form.pdf.data.save(job_input.path)
+            JobInput.create(form.pdf.data, job=job)
-        except OSError as e:
+        except OSError:
-            current_app.logger.error(e)
+            abort(500)
-            db.session.rollback()
-            flash('Internal Server Error', 'error')
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        flash(f'Job "{job.title}" added', 'job')
+        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
-        return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)  # noqa
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
     transkribus_htr_models = [
-        x for x in TranskribusHTRModel.query.filter().all()
+        x for x in TranskribusHTRModel.query.all()
         if x.shared == True or x.user == current_user
     ]
     return render_template(
-        f'services/transkribus_htr_pipeline.html.j2',
+        'services/transkribus_htr_pipeline.html.j2',
         form=form,
         title=service_manifest['name'],
         TRANSKRIBUS_HTR_MODELS=TRANSKRIBUS_HTR_MODELS,
@@ -229,51 +166,34 @@ def spacy_nlp_pipeline():
     version = request.args.get('version', SERVICES[service]['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
-    form = AddSpacyNLPPipelineJobForm(prefix='add-job-form', version=version)
+    form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
-            return make_response(form.errors, 400)
+            response = {'errors': form.errors}
-        service_args = {}
+            return response, 400
-        service_args['model'] = form.model.data
-        if form.encoding_detection.data:
-            service_args['encoding_detection'] = True
-        job = Job(
-            user=current_user,
-            description=form.description.data,
-            service=service,
-            service_args=service_args,
-            service_version=form.version.data,
-            title=form.title.data
-        )
-        db.session.add(job)
-        db.session.flush(objects=[job])
-        db.session.refresh(job)
         try:
-            job.makedirs()
+            job = Job.create(
-        except OSError as e:
+                title=form.title.data,
-            current_app.logger.error(e)
+                description=form.description.data,
-            db.session.rollback()
+                service=service,
-            flash('Internal Server Error', 'error')
+                service_args={
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
+                    'encoding_detection': form.encoding_detection.data,
-        job_input = JobInput(
+                    'model': form.model.data
-            filename=secure_filename(form.txt.data.filename),
+                },
-            job=job,
+                service_version=form.version.data,
-            mimetype=form.txt.data.mimetype
+                user=current_user
-        )
+            )
-        db.session.add(job_input)
+        except OSError:
-        db.session.flush(objects=[job_input])
+            abort(500)
-        db.session.refresh(job_input)
         try:
-            form.txt.data.save(job_input.path)
+            JobInput.create(form.txt.data, job=job)
-        except OSError as e:
+        except OSError:
-            current_app.logger.error(e)
+            abort(500)
-            db.session.rollback()
-            flash('Internal Server Error', 'error')
-            return make_response({'redirect_url': url_for('.service', service=service)}, 500)  # noqa
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        flash(f'Job "{job.title}" added', 'job')
+        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
-        return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)  # noqa
+        flash(message, 'job')
+        return {}, 201, {'Location': job.url}
     return render_template(
         'services/spacy_nlp_pipeline.html.j2',
         form=form,

app/services/services.yml

@@ -17,6 +17,11 @@ tesseract-ocr-pipeline:
         - 'binarization'
       publishing_year: 2022
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.0'
+    0.1.1:
+      methods:
+        - 'binarization'
+      publishing_year: 2022
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
 transkribus-htr-pipeline:
   name: 'Transkribus HTR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
@@ -47,4 +52,4 @@ spacy-nlp-pipeline:
         ru: 'Russian'
         zh: 'Chinese'
       publishing_year: 2022
-      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'

app/settings/forms.py

@@ -1,5 +1,3 @@
-from app.auth import USERNAME_REGEX
-from app.models import User, UserSettingJobStatusMailNotificationLevel
 from flask_wtf import FlaskForm
 from wtforms import (
     BooleanField,
@@ -9,14 +7,35 @@ from wtforms import (
     SubmitField,
     ValidationError
 )
-from wtforms.validators import DataRequired, InputRequired, Email, EqualTo, Length, Regexp
+from wtforms.validators import (
+    DataRequired,
+    InputRequired,
+    Email,
+    EqualTo,
+    Length,
+    Regexp
+)
+from app.models import User, UserSettingJobStatusMailNotificationLevel
+from app.auth import USERNAME_REGEX
 
 
 class ChangePasswordForm(FlaskForm):
     password = PasswordField('Old password', validators=[DataRequired()])
-    new_password = PasswordField('New password', validators=[DataRequired(), EqualTo('new_password_confirmation', message='Passwords must match')])
+    new_password = PasswordField(
-    new_password_confirmation = PasswordField('Confirm new password', validators=[DataRequired(), EqualTo('new_password', message='Passwords must match')])
+        'New password',
-    submit = SubmitField('Submit')
+        validators=[
+            DataRequired(),
+            EqualTo('new_password_2', message='Passwords must match')
+        ]
+    )
+    new_password_2 = PasswordField(
+        'New password confirmation',
+        validators=[
+            DataRequired(),
+            EqualTo('new_password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()
 
     def __init__(self, user, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -28,43 +47,51 @@ class ChangePasswordForm(FlaskForm):
 
 
 class EditGeneralSettingsForm(FlaskForm):
-    email = StringField('E-Mail', validators=[DataRequired(), Length(1, 254), Email()])
+    email = StringField(
+        'E-Mail',
+        validators=[InputRequired(), Length(max=254), Email()]
+    )
     username = StringField(
         'Username',
         validators=[
             InputRequired(),
-            Length(1, 64),
+            Length(max=64),
             Regexp(
                 USERNAME_REGEX,
-                message='Usernames must have only letters, numbers, dots or underscores'  # noqa
+                message=(
+                    'Usernames must have only letters, numbers, dots or '
+                    'underscores'
+                )
             )
         ]
     )
-    submit = SubmitField('Submit')
+    submit = SubmitField()
 
     def __init__(self, user, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.user = user
 
+    def prefill(self, user):
+        self.email.data = user.email
+        self.username.data = user.username
+
     def validate_email(self, field):
-        if (
+        if (field.data != self.user.email
-            field.data != self.user.email
+                and User.query.filter_by(email=field.data).first()):
-            and User.query.filter_by(email=field.data).first()
-        ):
             raise ValidationError('Email already registered')
 
     def validate_username(self, field):
-        if (
+        if (field.data != self.user.username
-            field.data != self.user.username
+                and User.query.filter_by(username=field.data).first()):
-            and User.query.filter_by(username=field.data).first()
-        ):
             raise ValidationError('Username already in use')
 
 
 class EditInterfaceSettingsForm(FlaskForm):
     dark_mode = BooleanField('Dark mode')
-    submit = SubmitField('Submit')
+    submit = SubmitField()
 
+    def prefill(self, user):
+        self.dark_mode.data = user.setting_dark_mode
 
 class EditNotificationSettingsForm(FlaskForm):
     job_status_mail_notification_level = SelectField(
@@ -72,11 +99,15 @@ class EditNotificationSettingsForm(FlaskForm):
         choices=[('', 'Choose your option')],
         validators=[DataRequired()]
     )
-    submit = SubmitField('Submit')
+    submit = SubmitField()
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.job_status_mail_notification_level.choices += [
-            (enum_member.name, enum_member.name.capitalize())
+            (x.name, x.name.capitalize())
-            for enum_member in UserSettingJobStatusMailNotificationLevel
+            for x in UserSettingJobStatusMailNotificationLevel
         ]
+
+    def prefill(self, user):
+        self.job_status_mail_notification_level.data = \
+            user.setting_job_status_mail_notification_level.name

app/settings/routes.py

@@ -1,32 +1,32 @@
 from flask import flash, redirect, render_template, url_for
-from flask_login import current_user, login_required, logout_user
+from flask_login import current_user, login_required
-from . import bp, tasks
+from app import db
+from app.models import UserSettingJobStatusMailNotificationLevel
+from . import bp
 from .forms import (
     ChangePasswordForm,
     EditGeneralSettingsForm,
     EditInterfaceSettingsForm,
     EditNotificationSettingsForm
 )
-from .. import db
-from ..models import UserSettingJobStatusMailNotificationLevel
 
 
 @bp.route('', methods=['GET', 'POST'])
 @login_required
-def index():
+def settings():
     change_password_form = ChangePasswordForm(
-        current_user._get_current_object(),
+        current_user,
-        prefix='change_password_form'
+        prefix='change-password-form'
     )
     edit_general_settings_form = EditGeneralSettingsForm(
-        current_user._get_current_object(),
+        current_user,
-        prefix='edit_general_settings_form'
+        prefix='edit-general-settings-form'
     )
     edit_interface_settings_form = EditInterfaceSettingsForm(
-        prefix='edit_interface_settings_form'
+        prefix='edit-interface-settings-form'
     )
     edit_notification_settings_form = EditNotificationSettingsForm(
-        prefix='edit_notification_settings_form'
+        prefix='edit-notification-settings-form'
     )
 
     if change_password_form.submit.data and change_password_form.validate():
@@ -34,58 +34,38 @@ def index():
         db.session.commit()
         flash('Your changes have been saved')
         return redirect(url_for('.index'))
-    if (
+    if (edit_general_settings_form.submit.data
-        edit_general_settings_form.submit.data
+            and edit_general_settings_form.validate()):
-        and edit_general_settings_form.validate()
-    ):
         current_user.email = edit_general_settings_form.email.data
         current_user.username = edit_general_settings_form.username.data
         db.session.commit()
         flash('Your changes have been saved')
-        return redirect(url_for('.index'))
+        return redirect(url_for('.settings'))
-    if (
+    if (edit_interface_settings_form.submit.data
-        edit_interface_settings_form.submit.data
+            and edit_interface_settings_form.validate()):
-        and edit_interface_settings_form.validate()
+        current_user.setting_dark_mode = (
-    ):
+            edit_interface_settings_form.dark_mode.data)
-        current_user.setting_dark_mode = \
-            edit_interface_settings_form.dark_mode.data
         db.session.commit()
         flash('Your changes have been saved')
-        return redirect(url_for('.index'))
+        return redirect(url_for('.settings'))
-    if (
+    if (edit_notification_settings_form.submit.data
-        edit_notification_settings_form.submit.data
+            and edit_notification_settings_form.validate()):
-        and edit_notification_settings_form.validate()
+        current_user.setting_job_status_mail_notification_level = (
-    ):
-        current_user.setting_job_status_mail_notification_level = \
             UserSettingJobStatusMailNotificationLevel[
                 edit_notification_settings_form.job_status_mail_notification_level.data  # noqa
             ]
+        )
         db.session.commit()
         flash('Your changes have been saved')
-        return redirect(url_for('.index'))
+        return redirect(url_for('.settings'))
-    edit_general_settings_form.email.data = current_user.email
+    edit_general_settings_form.prefill(current_user)
-    edit_general_settings_form.username.data = current_user.username
+    edit_interface_settings_form.prefill(current_user)
-    edit_interface_settings_form.dark_mode.data = \
+    edit_notification_settings_form.prefill(current_user)
-        current_user.setting_dark_mode
-    edit_notification_settings_form.job_status_mail_notification_level.data = \
-        current_user.setting_job_status_mail_notification_level.name
     return render_template(
-        'settings/index.html.j2',
+        'settings/settings.html.j2',
         change_password_form=change_password_form,
         edit_general_settings_form=edit_general_settings_form,
         edit_interface_settings_form=edit_interface_settings_form,
         edit_notification_settings_form=edit_notification_settings_form,
         title='Settings'
     )
-
-
-@bp.route('/delete')
-@login_required
-def delete():
-    """
-    View to delete current_user and all associated data.
-    """
-    tasks.delete_user(current_user.id)
-    logout_user()
-    flash('Your account has been marked for deletion')
-    return redirect(url_for('main.index'))

app/settings/tasks.py

@@ -1,13 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import User
-
-
-@background
-def delete_user(user_id, *args, **kwargs):
-    with kwargs['app'].app_context():
-        user = User.query.get(user_id)
-        if user is None:
-            raise Exception(f'User {user_id} not found')
-        user.delete()
-        db.session.commit()

app/socketio_event_listeners.py

@@ -1,41 +0,0 @@
-from app import hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import TesseractOCRModel, TranskribusHTRModel, User
-from flask_login import current_user
-from flask_socketio import join_room
-
-
-@socketio.on('users.user.get')
-@socketio_login_required
-def users_user_get(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'code': 404, 'msg': 'Not found'}
-    if not (user == current_user or current_user.is_administrator):
-        return {'code': 403, 'msg': 'Forbidden'}
-    # corpora = [x.to_dict() for x in user.corpora]
-    # jobs = [x.to_dict() for x in user.jobs]
-    # transkribus_htr_models = TranskribusHTRModel.query.filter(
-    #     (TranskribusHTRModel.shared == True) | (TranskribusHTRModel.user == user)
-    # ).all()
-    # tesseract_ocr_models = TesseractOCRModel.query.filter(
-    #     (TesseractOCRModel.shared == True) | (TesseractOCRModel.user == user)
-    # ).all()
-    # response = {
-    #     'code': 200,
-    #     'msg': 'OK',
-    #     'payload': {
-    #         'user': user.to_dict(),
-    #         'corpora': corpora,
-    #         'jobs': jobs,
-    #         'transkribus_htr_models': transkribus_htr_models,
-    #         'tesseract_ocr_models': tesseract_ocr_models
-    #     }
-    # }
-    join_room(f'users.{user.hashid}')
-    return {
-        'code': 200,
-        'msg': 'OK',
-        'payload': user.to_dict(backrefs=True, relationships=True)
-    }

app/sqlalchemy_event_listeners.py

@@ -1,87 +0,0 @@
-from app import db, mail, socketio
-from app.email import create_message
-from app.models import (
-    Corpus,
-    CorpusFile,
-    Job,
-    JobInput,
-    JobResult,
-    JobStatus,
-    UserSettingJobStatusMailNotificationLevel
-)
-from datetime import datetime
-from enum import Enum
-
-
-@db.event.listens_for(Corpus, 'after_delete')
-@db.event.listens_for(CorpusFile, 'after_delete')
-@db.event.listens_for(Job, 'after_delete')
-@db.event.listens_for(JobInput, 'after_delete')
-@db.event.listens_for(JobResult, 'after_delete')
-def ressource_after_delete(mapper, connection, ressource):
-    jsonpatch = [{'op': 'remove', 'path': ressource.jsonpatch_path}]
-    room = f'users.{ressource.user_hashid}'
-    socketio.emit('users.patch', jsonpatch, room=room)
-
-
-@db.event.listens_for(Corpus, 'after_insert')
-@db.event.listens_for(CorpusFile, 'after_insert')
-@db.event.listens_for(Job, 'after_insert')
-@db.event.listens_for(JobInput, 'after_insert')
-@db.event.listens_for(JobResult, 'after_insert')
-def ressource_after_insert_handler(mapper, connection, ressource):
-    value = ressource.to_dict(backrefs=False, relationships=False)
-    for attr in mapper.relationships:
-        value[attr.key] = {}
-    jsonpatch = [
-        {'op': 'add', 'path': ressource.jsonpatch_path, 'value': value}
-    ]
-    room = f'users.{ressource.user_hashid}'
-    socketio.emit('users.patch', jsonpatch, room=room)
-
-
-@db.event.listens_for(Corpus, 'after_update')
-@db.event.listens_for(CorpusFile, 'after_update')
-@db.event.listens_for(Job, 'after_update')
-@db.event.listens_for(JobInput, 'after_update')
-@db.event.listens_for(JobResult, 'after_update')
-def ressource_after_update_handler(mapper, connection, ressource):
-    jsonpatch = []
-    for attr in db.inspect(ressource).attrs:
-        if attr.key in mapper.relationships:
-            continue
-        if not attr.load_history().has_changes():
-            continue
-        if isinstance(attr.value, datetime):
-            value = attr.value.isoformat() + 'Z'
-        elif isinstance(attr.value, Enum):
-            value = attr.value.name
-        else:
-            value = attr.value
-        jsonpatch.append(
-            {
-                'op': 'replace',
-                'path': f'{ressource.jsonpatch_path}/{attr.key}',
-                'value': value
-            }
-        )
-        if isinstance(ressource, Job) and attr.key == 'status':
-            _job_status_email_handler(ressource)
-    if jsonpatch:
-        room = f'users.{ressource.user_hashid}'
-        socketio.emit('users.patch', jsonpatch, room=room)
-
-
-def _job_status_email_handler(job):
-    if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:  # noqa
-        return
-    if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:  # noqa
-        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-            return
-    msg = create_message(
-        job.user.email,
-        f'Status update for your Job "{job.title}"',
-        'tasks/email/notification',
-        job=job
-    )
-    mail.send(msg)

app/sqlalchemy_type_decorators.py

@@ -1,43 +0,0 @@
-from app import db
-import json
-
-
-class IntEnumColumn(db.TypeDecorator):
-    impl = db.Integer
-
-    def __init__(self, enum_type, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.enum_type = enum_type
-
-    def process_bind_param(self, value, dialect):
-        if isinstance(value, self.enum_type) and isinstance(value.value, int):
-            return value.value
-        elif isinstance(value, int):
-            return self.enum_type(value).value
-        else:
-            return TypeError()
-
-    def process_result_value(self, value, dialect):
-        return self.enum_type(value)
-
-
-class ContainerColumn(db.TypeDecorator):
-    impl = db.String
-
-    def __init__(self, container_type, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.container_type = container_type
-
-    def process_bind_param(self, value, dialect):
-        if isinstance(value, self.container_type):
-            return json.dumps(value)
-        elif (
-            isinstance(value, str)
-            and isinstance(json.loads(value), self.container_type)
-        ):
-            return value
-        else:
-            return TypeError()
-
-    def process_result_value(self, value, dialect):
-        return json.loads(value)

app/static/css/helpers.scss

@@ -0,0 +1,31 @@
+/*
+ * Spacing
+ */
+$spacing-shortcuts: ("margin": "mg", "padding": "pd");
+$spacing-directions: ("top": "t", "right": "r", "bottom": "b", "left": "l");
+$spacing-values: ("0": 0, "1": 0.25rem, "2": 0.5rem, "3": 0.75rem, "4": 1rem, "5": 1.5rem, "6": 3rem, "auto": auto);
+
+@each $spacing-shortcut-name, $spacing-shortcut-value in $spacing-shortcuts {
+  @each $spacing-name, $spacing-value in $spacing-values {
+    // All directions
+    .#{$spacing-shortcut-value}-#{$spacing-name} {
+      #{$spacing-shortcut-name}: $spacing-value !important;
+    }
+    // Horizontal axis
+    .#{$spacing-shortcut-value}x-#{$spacing-name} {
+      #{$spacing-shortcut-name}-left: $spacing-value !important;
+      #{$spacing-shortcut-name}-right: $spacing-value !important;
+    }
+    // Vertical axis
+    .#{$spacing-shortcut-value}y-#{$spacing-name} {
+      #{$spacing-shortcut-name}-top: $spacing-value !important;
+      #{$spacing-shortcut-name}-bottom: $spacing-value !important;
+    }
+    // Cardinal directions
+    @each $spacing-direction-name, $spacing-direction-value in $spacing-directions {
+      .#{$spacing-shortcut-value}#{$spacing-direction-value}-#{$spacing-name} {
+        #{$spacing-shortcut-name}-#{$spacing-direction-name}: $spacing-value !important;
+      }
+    }
+  }
+}

app/static/css/style.css

@@ -49,5 +49,14 @@ h1 .nopaque-icons, h2 .nopaque-icons, h3 .nopaque-icons, h4 .nopaque-icons, .tab
 .nopaque-icons.service-icon[data-service="spacy-nlp-pipeline"]:empty:before {content: "G";}
 .nopaque-icons.service-icon[data-service="corpus-analysis"]:empty:before {content: "H";}
 
-.hoverable {cursor: pointer;}
+.clickable {
+  cursor: pointer !important;
+  pointer-events: all !important;
+}
 .chip.s-attr .chip.p-attr {background-color: inherit;}
+
+
+.width-25 {width: 25%;}
+.width-50 {width: 50%;}
+.width-75 {width: 75%;}
+.width-100 {width: 100%;}

app/static/js/App.js

@@ -1,78 +1,111 @@
 class App {
   constructor() {
-    this.data = {users: {}};
+    this.data = {
-    this.eventListeners = {'users.patch': []};
+      promises: {getUser: {}, subscribeUser: {}},
-    this.promises = {users: {}};
+      users: {},
+    };
     this.socket = io({transports: ['websocket'], upgrade: false});
-    this.socket.on('users.patch', patch => this.usersPatchHandler(patch));
+    this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
   }
 
-  get users() {
+  getUser(userId) {
-    return this.data.users;
+    if (userId in this.data.promises.getUser) {
-  }
+      return this.data.promises.getUser[userId];
-
-  addEventListener(type, listener) {
-    if (!(type in this.eventListeners)) {
-      throw `Unknown event type: ${type}`;
     }
-    this.eventListeners[type].push(listener);
+
+    this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
+      fetch(`/users/${userId}?backrefs=true&relationships=true`, {headers: {Accept: 'application/json'}})
+        .then(
+          (response) => {return response.json();},
+          (response) => {
+            if (response.status === 403) {this.flash('Forbidden', 'error');}
+            if (response.status === 404) {this.flash('Not Found', 'error');}
+            reject(response);
+          }
+        )
+        .then(
+          (user) => {
+            this.data.users[userId] = user;
+            resolve(this.data.users[userId]);
+          },
+          (error) => {
+            console.error(error, 'error');
+            reject(error);
+          }
+        );
+    });
+
+    return this.data.promises.getUser[userId];
   }
 
-  flash(message, category) {
+  subscribeUser(userId) {
-    let iconPrefix;
+    if (userId in this.data.promises.subscribeUser) {
-    let toast;
+      return this.data.promises.subscribeUser[userId];
-    let toastCloseActionElement;
-
-    switch (category) {
-      case 'corpus':
-        iconPrefix = '<i class="left material-icons">book</i>';
-        break;
-      case 'error':
-        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
-        break;
-      case 'job':
-        iconPrefix = '<i class="left nopaque-icons">J</i>';
-        break;
-      default:
-        iconPrefix = '<i class="left material-icons">notifications</i>';
-        break;
     }
-    toast = M.toast(
-      {
-        html: `
-          <span>${iconPrefix}${message}</span>
-          <button class="btn-flat toast-action white-text" data-action="close">
-            <i class="material-icons">close</i>
-          </button>
-        `.trim()
-      }
-    );
-    toastCloseActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
-    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
-  }
 
-  getUserById(userId) {
+    this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
-    if (userId in this.promises.users) {
+      this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
-      return this.promises.users[userId];
-    }
-    this.promises.users[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('users.user.get', userId, response => {
         if (response.code === 200) {
-          this.data.users[userId] = response.payload;
+          resolve(response);
-          resolve(this.data.users[userId]);
         } else {
           reject(response);
         }
       });
     });
-    return this.promises.users[userId];
+
+    return this.data.promises.subscribeUser[userId];
   }
 
-  usersPatchHandler(patch) {
+  flash(message, category) {
-    let listener;
+    let iconPrefix = '';
+    switch (category) {
+      case 'corpus': {
+        iconPrefix = '<i class="left material-icons">book</i>';
+        break;
+      }
+      case 'error': {
+        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
+        break;
+      }
+      case 'job': {
+        iconPrefix = '<i class="left nopaque-icons">J</i>';
+        break;
+      }
+      default: {
+        iconPrefix = '<i class="left material-icons">notifications</i>';
+        break;
+      }
+    }
+    let toast = M.toast(
+      {
+        html: `
+          <span>${iconPrefix}${message}</span>
+          <button class="action-button btn-flat toast-action white-text" data-action="close">
+            <i class="material-icons">close</i>
+          </button>
+        `.trim()
+      }
+    );
+    let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
+    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
+  }
+  
+  onPatch(patch) {
+    // Filter Patch to only include operations on users that are initialized
+    let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
+    let filteredPatch = patch.filter(operation => regExp.test(operation.path));
 
-    this.data = jsonpatch.applyPatch(this.data, patch).newDocument;
+    // Handle job status updates
-    //this.data = jsonpatch.apply_patch(this.data, patch);
+    let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
-    for (listener of this.eventListeners['users.patch']) {listener(patch);}
+    let subFilteredPatch = filteredPatch
+      .filter((operation) => {return operation.op === 'replace';})
+      .filter((operation) => {return subRegExp.test(operation.path);});
+    for (let operation of subFilteredPatch) {
+      let [match, userId, jobId] = operation.path.match(subRegExp);
+      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
+    }
+
+    // Apply Patch
+    jsonpatch.applyPatch(this.data, filteredPatch);
   }
 }

app/static/js/Forms/CreateCorpusFileForm.js

@@ -0,0 +1,18 @@
+class CreateCorpusFileForm extends Form {
+  static autoInit() {
+    let createCorpusFileFormElements = document.querySelectorAll('.create-corpus-file-form');
+    for (let createCorpusFileFormElement of createCorpusFileFormElements) {
+      new CreateCorpusFileForm(createCorpusFileFormElement);
+    }
+  }
+
+  constructor(formElement) {
+    super(formElement);
+
+    this.addEventListener('requestLoad', (event) => {
+      if (event.target.status === 201) {
+        window.location.href = event.target.getResponseHeader('Location');
+      }
+    });
+  }
+}

app/static/js/Forms/CreateJobForm.js

@@ -0,0 +1,25 @@
+class CreateJobForm extends Form {
+  static autoInit() {
+    let createJobFormElements = document.querySelectorAll('.create-job-form');
+    for (let createJobFormElement of createJobFormElements) {
+      new CreateJobForm(createJobFormElement);
+    }
+  }
+
+  constructor(formElement) {
+    super(formElement);
+
+    let versionField = this.formElement.querySelector('#create-job-form-version');
+    versionField.addEventListener('change', (event) => {
+      let url = new URL(window.location.href);
+      url.search = `?version=${event.target.value}`;
+      window.location.href = url.toString();
+    });
+
+    this.addEventListener('requestLoad', (event) => {
+      if (event.target.status === 201) {
+        window.location.href = event.target.getResponseHeader('Location');
+      }
+    });
+  }
+}

app/static/js/Forms/Form.js

@@ -0,0 +1,141 @@
+class Form {
+  static autoInit() {
+    CreateCorpusFileForm.autoInit();
+    CreateJobForm.autoInit();
+  }
+
+  constructor(formElement) {
+    this.formElement = formElement;
+    this.eventListeners = {
+      'requestLoad': []
+    };
+    this.afterRequestListeners = [];
+
+    for (let selectElement of this.formElement.querySelectorAll('select')) {
+      selectElement.removeAttribute('required');
+    }
+
+    this.formElement.addEventListener('submit', (event) => {
+      event.preventDefault();
+      this.submit(event);
+    });
+  }
+
+  addEventListener(eventType, listener) {
+    if (eventType in this.eventListeners) {
+      this.eventListeners[eventType].push(listener);
+    } else {
+      throw `Unknown event type ${eventType}`;
+    }
+  }
+
+  submit(event) {
+    let request = new XMLHttpRequest();
+    let modalElement = Utils.elementFromString(
+      `
+        <div class="modal">
+          <div class="modal-content">
+            <h4><i class="material-icons left">file_upload</i>Submitting...</h4>
+            <div class="progress">
+              <div class="determinate" style="width: 0%"></div>
+            </div>
+          </div>
+          <div class="modal-footer">
+            <a class="action-button btn red waves-effect waves-light modal-close" data-action="cancel">Cancel</a>
+          </div>
+        </div>
+      `
+    );
+    document.querySelector('#modals').appendChild(modalElement);
+    let modal = M.Modal.init(
+      modalElement,
+      {
+        dismissible: false,
+        onCloseEnd: () => {
+          modal.destroy();
+          modalElement.remove();
+        }
+      }
+    );
+    modal.open();
+
+    // Remove all previous helper text elements that indicate errors
+    let errorHelperTextElements = this.formElement
+      .querySelectorAll('.helper-text[data-helper-text-type="error"]');
+    for (let errorHelperTextElement of errorHelperTextElements) {
+      errorHelperTextElement.remove();
+    }
+
+    // Check if select elements are filled out properly
+    for (let selectElement of this.formElement.querySelectorAll('select')) {
+      if (selectElement.value === '') {
+        let inputFieldElement = selectElement.closest('.input-field');
+        let errorHelperTextElement = Utils.elementFromString(
+          '<span class="helper-text error-color-text" data-helper-text-type="error">Please select an option.</span>'
+        );
+        inputFieldElement.appendChild(errorHelperTextElement);
+        inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
+        modal.close();
+        return;
+      }
+    }
+
+    // Setup abort handling
+    let cancelElement = modalElement.querySelector('.action-button[data-action="cancel"]');
+    cancelElement.addEventListener('click', (event) => {request.abort();});
+
+    // Setup load handling (after the request completed)
+    request.addEventListener('load', (event) => {
+      for (let listener of this.eventListeners['requestLoad']) {
+        listener(event);
+      }
+      if (request.status === 400) {
+        let responseJson = JSON.parse(request.responseText);
+        console.log(responseJson);
+        for (let [inputName, inputErrors] of Object.entries(responseJson.errors)) {
+          let inputFieldElement = this.formElement
+            .querySelector(`input[name$="${inputName}"], select[name$="${inputName}"]`)
+            .closest('.input-field');
+          for (let inputError of inputErrors) {
+            let errorHelperTextElement = Utils.elementFromString(
+              `<span class="helper-text error-color-text" data-helper-type="error">${inputError}</span>`
+            );
+            inputFieldElement.appendChild(errorHelperTextElement);
+          }
+        }
+      }
+      if (request.status === 500) {
+        app.flash('Internal Server Error', 'error');
+      }
+      modal.close();
+    });
+
+    // Setup progress handling
+    let progressBarElement = modalElement.querySelector('.progress > .determinate');
+    request.upload.addEventListener('progress', (event) => {
+      let progress = Math.floor(100 * event.loaded / event.total);
+      progressBarElement.style.width = `${progress}%`;
+    });
+
+    request.open(this.formElement.method, this.formElement.action);
+    request.setRequestHeader('Accept', 'application/json');
+    let formData = new FormData(this.formElement);
+    switch (this.formElement.enctype) {
+      case 'application/x-www-form-urlencoded':
+        let urlSearchParams = new URLSearchParams(formData);
+        request.send(urlSearchParams);
+        break;
+      case 'multipart/form-data': {
+        request.send(formData);
+        break;
+      }
+      case 'text/plain': {
+        throw 'enctype "text/plain" is not supported';
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+}

app/static/js/JobStatusNotifier.js

@@ -1,22 +0,0 @@
-class JobStatusNotifier {
-  constructor(userId) {
-    this.userId = userId;
-  }
-
-  usersPatchHandler(patch) {
-    let filteredPatch;
-    let jobId;
-    let match;
-    let operation;
-    let re;
-
-    re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/status$`)
-    filteredPatch = patch
-      .filter(operation => operation.op === 'replace')
-      .filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
-      [match, jobId] = operation.path.match(re);
-      app.flash(`[<a href="/jobs/${jobId}">${app.users[this.userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
-    }
-  }
-}

app/static/js/RessourceDisplays/CorpusDisplay.js

@@ -2,12 +2,20 @@ class CorpusDisplay extends RessourceDisplay {
   constructor(displayElement) {
     super(displayElement);
     this.corpusId = displayElement.dataset.corpusId;
+    this.displayElement
+      .querySelector('.action-button[data-action="build-request"]')
+      .addEventListener('click', (event) => {
+        Utils.buildCorpusRequest(this.userId, this.corpusId);
+      });
+    this.displayElement
+      .querySelector('.action-button[data-action="delete-request"]')
+      .addEventListener('click', (event) => {
+        Utils.deleteCorpusRequest(this.userId, this.corpusId);
+      });
   }
 
   init(user) {
-    let corpus;
+    let corpus = user.corpora[this.corpusId];
-
-    corpus = user.corpora[this.corpusId];
     this.setCreationDate(corpus.creation_date);
     this.setDescription(corpus.description);
     this.setLastEditedDate(corpus.last_edited_date);
@@ -16,17 +24,20 @@ class CorpusDisplay extends RessourceDisplay {
     this.setNumTokens(corpus.num_tokens);
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let filteredPatch;
+    let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}`);
-    let operation;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let re;
+    for (let operation of filteredPatch) {
-
-    re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'replace':
+        case 'remove': {
-          re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/last_edited_date$`);
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}$`);
+          if (re.test(operation.path)) {
+            window.location.href = '/dashboard#corpora';
+          }
+          break;
+        }
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/last_edited_date$`);
           if (re.test(operation.path)) {
             this.setLastEditedDate(operation.value);
             break;
@@ -42,8 +53,10 @@ class CorpusDisplay extends RessourceDisplay {
             break;
           }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }
@@ -55,7 +68,7 @@ class CorpusDisplay extends RessourceDisplay {
   setNumTokens(numTokens) {
     this.setElements(
       this.displayElement.querySelectorAll('.corpus-token-ratio'),
-      `${numTokens}/${app.users[this.userId].corpora[this.corpusId].max_num_tokens}`
+      `${numTokens}/${app.data.users[this.userId].corpora[this.corpusId].max_num_tokens}`
     );
   }
 
@@ -64,31 +77,28 @@ class CorpusDisplay extends RessourceDisplay {
   }
 
   setStatus(status) {
-    let element;
+    let elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger')
-    let elements;
+    for (let element of elements) {
-
-    elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger')
-    for (element of elements) {
       if (['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
         element.classList.remove('disabled');
       } else {
         element.classList.add('disabled');
       }
     }
-    elements = this.displayElement.querySelectorAll('.corpus-build-trigger');
+    elements = this.displayElement.querySelectorAll('.action-button[data-action="build-request"]');
-    for (element of elements) {
+    for (let element of elements) {
-      if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(app.users[this.userId].corpora[this.corpusId].files).length > 0) {
+      if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(app.data.users[this.userId].corpora[this.corpusId].files).length > 0) {
         element.classList.remove('disabled');
       } else {
         element.classList.add('disabled');
       }
     }
     elements = this.displayElement.querySelectorAll('.corpus-status');
-    for (element of elements) {
+    for (let element of elements) {
       element.dataset.corpusStatus = status;
     }
     elements = this.displayElement.querySelectorAll('.corpus-status-spinner');
-    for (element of elements) {
+    for (let element of elements) {
       if (['SUBMITTED', 'QUEUED', 'BUILDING', 'STARTING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
         element.classList.remove('hide');
       } else {

app/static/js/RessourceDisplays/JobDisplay.js

@@ -2,12 +2,25 @@ class JobDisplay extends RessourceDisplay {
   constructor(displayElement) {
     super(displayElement);
     this.jobId = this.displayElement.dataset.jobId;
+    this.displayElement
+      .querySelector('.action-button[data-action="delete-request"]')
+      .addEventListener('click', (event) => {
+        Utils.deleteJobRequest(this.userId, this.jobId);
+      });
+    this.displayElement
+      .querySelector('.action-button[data-action="get-log-request"]')
+      .addEventListener('click', (event) => {
+        Utils.getJobLogRequest(this.userId, this.jobId);
+      });
+    this.displayElement
+      .querySelector('.action-button[data-action="restart-request"]')
+      .addEventListener('click', (event) => {
+        Utils.restartJobRequest(this.userId, this.jobId);
+      });
   }
 
   init(user) {
-    let job;
+    let job = user.jobs[this.jobId];
-
-    job = user.jobs[this.jobId];
     this.setCreationDate(job.creation_date);
     this.setEndDate(job.creation_date);
     this.setDescription(job.description);
@@ -18,17 +31,20 @@ class JobDisplay extends RessourceDisplay {
     this.setTitle(job.title);
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let filteredPatch;
+    let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}`);
-    let operation;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let re;
+    for (let operation of filteredPatch) {
-
-    re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'replace':
+        case 'remove': {
-          re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/end_date$`);
+          let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}$`);
+          if (re.test(operation.path)) {
+            window.location.href = '/dashboard#jobs';
+          }
+          break;
+        }
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/end_date$`);
           if (re.test(operation.path)) {
             this.setEndDate(operation.value);
             break;
@@ -39,8 +55,10 @@ class JobDisplay extends RessourceDisplay {
             break;
           }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }
@@ -54,29 +72,42 @@ class JobDisplay extends RessourceDisplay {
   }
 
   setStatus(status) {
-    let element;
+    let elements = this.displayElement.querySelectorAll('.job-status');
-    let elements;
+    for (let element of elements) {
-
-    elements = this.displayElement.querySelectorAll('.job-status');
-    for (element of elements) {
       element.dataset.jobStatus = status;
     }
     elements = this.displayElement.querySelectorAll('.job-status-spinner');
-    for (element of elements) {
+    for (let element of elements) {
       if (['COMPLETED', 'FAILED'].includes(status)) {
         element.classList.add('hide');
       } else {
         element.classList.remove('hide');
       }
     }
-    elements = this.displayElement.querySelectorAll('.job-restart-trigger');
+    elements = this.displayElement.querySelectorAll('.job-log-trigger');
-    for (element of elements) {
+    for (let element of elements) {
       if (['COMPLETED', 'FAILED'].includes(status)) {
         element.classList.remove('hide');
       } else {
         element.classList.add('hide');
       }
     }
+    elements = this.displayElement.querySelectorAll('.action-button[data-action="get-log-request"]');
+    for (let element of elements) {
+      if (['COMPLETED', 'FAILED'].includes(status)) {
+        element.classList.remove('disabled');
+      } else {
+        element.classList.add('disabled');
+      }
+    }
+    elements = this.displayElement.querySelectorAll('.action-button[data-action="restart-request"]');
+    for (let element of elements) {
+      if (status === 'FAILED') {
+        element.classList.remove('disabled');
+      } else {
+        element.classList.add('disabled');
+      }
+    }
   }
 
   setCreationDate(creationDate) {

app/static/js/RessourceDisplays/RessourceDisplay.js

@@ -2,30 +2,42 @@ class RessourceDisplay {
   constructor(displayElement) {
     this.displayElement = displayElement;
     this.userId = this.displayElement.dataset.userId;
-    app.addEventListener('users.patch', patch => this.usersPatchHandler(patch));
+    this.isInitialized = false;
-    app.getUserById(this.userId).then(user => this.init(user));
+    if (this.userId) {
+      app.subscribeUser(this.userId)
+        .then((response) => {
+          app.socket.on('PATCH', (patch) => {
+            if (this.isInitialized) {this.onPatch(patch);}
+          });
+        });
+      app.getUser(this.userId)
+        .then((user) => {
+          this.init(user);
+          this.isInitialized = true;
+        });
+    }
   }
 
   init(user) {throw 'Not implemented';}
 
-  usersPatchHandler(patch) {throw 'Not implemented';}
+  onPatch(patch) {throw 'Not implemented';}
 
   setElement(element, value) {
     switch (element.tagName) {
-      case 'INPUT':
+      case 'INPUT': {
         element.value = value;
         M.updateTextFields();
         break;
-      default:
+      }
+      default: {
         element.innerText = value;
         break;
+      }
     }
   }
 
   setElements(elements, value) {
-    let element;
+    for (let element of elements) {
-
-    for (element of elements) {
       this.setElement(element, value);
     }
   }

app/static/js/RessourceLists/CorpusFileList.js

@@ -1,19 +1,47 @@
 class CorpusFileList extends RessourceList {
+  static autoInit() {
+    for (let corpusFileListElement of document.querySelectorAll('.corpus-file-list:not(.no-autoinit)')) {
+      new CorpusFileList(corpusFileListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search corpus file</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th>Filename</th>
+              <th>Author</th>
+              <th>Title</th>
+              <th>Publishing year</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable">
+      <tr class="clickable hoverable">
         <td><span class="filename"></span></td>
         <td><span class="author"></span></td>
         <td><span class="title"></span></td>
         <td><span class="publishing-year"></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+          <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
-          <a class="action-button btn-floating tooltipped nopaque-service-color darken waves-effect waves-light" data-action="download" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">file_download</i></a>
+          <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="download" data-service="corpus-analysis"><i class="material-icons">file_download</i></a>
-          <a class="action-button btn-floating tooltipped nopaque-service-color darken waves-effect waves-light" data-action="view" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">send</i></a>
+          <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: corpusFile => {
+    ressourceMapper: (corpusFile) => {
       return {
         'id': corpusFile.id,
         'author': corpusFile.author,
@@ -23,7 +51,7 @@ class CorpusFileList extends RessourceList {
         'title': corpusFile.title
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['creation-date', {order: 'desc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -34,7 +62,6 @@ class CorpusFileList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...CorpusFileList.options, ...options});
     this.corpusId = listElement.dataset.corpusId;
@@ -44,92 +71,59 @@ class CorpusFileList extends RessourceList {
     this._init(user.corpora[this.corpusId].files);
   }
 
-  onclick(event) {
+  onClick(event) {
-    let action;
+    let actionButtonElement = event.target.closest('.action-button');
-    let actionButtonElement;
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
-    let corpusFileElement;
+    let corpusFileElement = event.target.closest('tr');
-    let corpusFileId;
+    let corpusFileId = corpusFileElement.dataset.id;
-    let deleteModal;
-    let deleteModalElement;
-    let tmp;
-
-    corpusFileElement = event.target.closest('tr[data-id]');
-    if (corpusFileElement === null) {return;}
-    corpusFileId = corpusFileElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
     switch (action) {
-      case 'delete':
+      case 'delete': {
-        tmp = document.createElement('div');
+        Utils.deleteCorpusFileRequest(this.userId, this.corpusId, corpusFileId);
-        tmp.innerHTML = `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm corpus deletion</h4>
-              <p>Do you really want to delete the corpus file <b>${app.users[this.userId].corpora[this.corpusId].files[corpusFileId].filename}</b>? It will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-              <a class="btn modal-close red waves-effect waves-light" href="/corpora/${this.corpusId}/files/${corpusFileId}/delete"><i class="material-icons left">delete</i>Delete</a>
-            </div>
-          </div>
-        `.trim();
-        deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
-        deleteModal = M.Modal.init(
-          deleteModalElement,
-          {
-            onCloseEnd: () => {
-              deleteModal.destroy();
-              deleteModalElement.remove();
-            }
-          }
-        );
-        deleteModal.open();
         break;
-      case 'download':
+      }
+      case 'download': {
         window.location.href = `/corpora/${this.corpusId}/files/${corpusFileId}/download`;
         break;
-      case 'view':
+      }
+      case 'view': {
         window.location.href = `/corpora/${this.corpusId}/files/${corpusFileId}`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let corpusFileId;
+    let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)`);
-    let filteredPatch;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let match;
+    for (let operation of filteredPatch) {
-    let operation;
-    let re;
-    let valueName;
-
-    re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'add':
+        case 'add': {
-          re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
-          if (re.test(operation.path)) {
+          if (re.test(operation.path)) {this.add(operation.value);}
-            this.add(operation.value);
-          }
           break;
-        case 'remove':
+        }
-          re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
+        case 'remove': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
           if (re.test(operation.path)) {
-            [match, corpusFileId] = operation.path.match(re);
+            let [match, corpusFileId] = operation.path.match(re);
             this.remove(corpusFileId);
           }
           break;
-        case 'replace':
+        }
-          re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)/(author|filename|publishing_year|title)$`);
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)/(author|filename|publishing_year|title)$`);
           if (re.test(operation.path)) {
-            [match, corpusFileId, valueName] = operation.path.match(re);
+            let [match, corpusFileId, valueName] = operation.path.match(re);
             this.replace(corpusFileId, valueName.replace('_', '-'), operation.value);
           }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }

app/static/js/RessourceLists/CorpusList.js

@@ -1,17 +1,44 @@
 class CorpusList extends RessourceList {
+  static autoInit() {
+    for (let corpusListElement of document.querySelectorAll('.corpus-list:not(.no-autoinit)')) {
+      new CorpusList(corpusListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search corpus</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th></th>
+              <th>Title and Description</th>
+              <th>Status</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable">
+      <tr class="clickable hoverable">
         <td><a class="btn-floating disabled"><i class="material-icons service-color darken" data-service="corpus-analysis">book</i></a></td>
         <td><b class="title"></b><br><i class="description"></i></td>
         <td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+          <a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
-          <a class="action-button btn-floating nopaque-service-color darken tooltipped waves-effect waves-light" data-action="view" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">send</i></a>
+          <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: corpus => {
+    ressourceMapper: (corpus) => {
       return {
         'id': corpus.id,
         'creation-date': corpus.creation_date,
@@ -20,7 +47,7 @@ class CorpusList extends RessourceList {
         'title': corpus.title
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['creation-date', {order: 'desc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -30,96 +57,63 @@ class CorpusList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...CorpusList.options, ...options});
   }
 
   init(user) {
-    super._init(user.corpora);
+    this._init(user.corpora);
   }
 
-  onclick(event) {
+  onClick(event) {
-    let action;
+    let actionButtonElement = event.target.closest('.action-button');
-    let actionButtonElement;
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
-    let corpusElement;
+    let corpusElement = event.target.closest('tr');
-    let corpusId;
+    let corpusId = corpusElement.dataset.id;
-    let deleteModal;
-    let deleteModalElement;
-    let tmp;
-
-    corpusElement = event.target.closest('tr[data-id]');
-    if (corpusElement === null) {return;}
-    corpusId = corpusElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
     switch (action) {
-      case 'delete':
+      case 'delete-request': {
-        tmp = document.createElement('div');
+        Utils.deleteCorpusRequest(this.userId, corpusId);
-        tmp.innerHTML = `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm corpus deletion</h4>
-              <p>Do you really want to delete the corpus <b>${app.users[this.userId].corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-              <a class="btn modal-close red waves-effect waves-light" href="/corpora/${corpusId}/delete"><i class="material-icons left">delete</i>Delete</a>
-            </div>
-          </div>
-        `.trim();
-        deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
-        deleteModal = M.Modal.init(
-          deleteModalElement,
-          {
-            onCloseEnd: () => {
-              deleteModal.destroy();
-              deleteModalElement.remove();
-            }
-          }
-        );
-        deleteModal.open();
         break;
-      case 'view':
+      }
+      case 'view': {
         window.location.href = `/corpora/${corpusId}`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let corpusId;
+    let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)`);
-    let filteredPatch;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let match;
+    for (let operation of filteredPatch) {
-    let operation;
-    let re;
-    let valueName;
-
-    re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'add':
+        case 'add': {
-          re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
+          let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
           if (re.test(operation.path)) {this.add(operation.value);}
           break;
-        case 'remove':
+        }
-          re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
+        case 'remove': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
           if (re.test(operation.path)) {
-            [match, corpusId] = operation.path.match(re);
+            let [match, corpusId] = operation.path.match(re);
             this.remove(corpusId);
           }
           break;
-        case 'replace':
+        }
-          re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)/(status|description|title)$`);
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)/(status|description|title)$`);
           if (re.test(operation.path)) {
-            [match, corpusId, valueName] = operation.path.match(re);
+            let [match, corpusId, valueName] = operation.path.match(re);
             this.replace(corpusId, valueName, operation.value);
           }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }

app/static/js/RessourceLists/JobInputList.js

@@ -1,21 +1,46 @@
 class JobInputList extends RessourceList {
+  static autoInit() {
+    for (let jobInputListElement of document.querySelectorAll('.job-input-list:not(.no-autoinit)')) {
+      new JobInputList(jobInputListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search job input</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th>Filename</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable">
+      <tr class="clickable hoverable">
         <td><span class="filename"></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+          <a class="action-button btn-floating waves-effect waves-light" data-action="download"><i class="material-icons">file_download</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: jobInput => {
+    ressourceMapper: (jobInput) => {
       return {
         'id': jobInput.id,
         'creation-date': jobInput.creation_date,
         'filename': jobInput.filename
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['filename', {order: 'asc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -23,7 +48,6 @@ class JobInputList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...JobInputList.options, ...options});
     this.jobId = listElement.dataset.jobId;
@@ -33,26 +57,21 @@ class JobInputList extends RessourceList {
     this._init(user.jobs[this.jobId].inputs);
   }
 
-  onclick(event) {
+  onClick(event) {
-    let jobInputElement;
+    let actionButtonElement = event.target.closest('.action-button');
-    let jobInputId;
+    let action = actionButtonElement === null ? 'download' : actionButtonElement.dataset.action;
-    let action;
+    let jobInputElement = event.target.closest('tr');
-    let actionButtonElement;
+    let jobInputId = jobInputElement.dataset.id;
-
-    jobInputElement = event.target.closest('tr[data-id]');
-    if (jobInputElement === null) {return;}
-    jobInputId = jobInputElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    if (actionButtonElement === null) {return;}
-    action = actionButtonElement.dataset.action;
     switch (action) {
-      case 'download':
+      case 'download': {
         window.location.href = `/jobs/${this.jobId}/inputs/${jobInputId}/download`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 
-  usersPatchHandler(patch) {return;}
+  onPatch(patch) {return;}
 }

app/static/js/RessourceLists/JobList.js

@@ -1,17 +1,44 @@
 class JobList extends RessourceList {
+  static autoInit() {
+    for (let jobListElement of document.querySelectorAll('.job-list:not(.no-autoinit)')) {
+      new JobList(jobListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search job</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th>Service</th>
+              <th>Title and Description</th>
+              <th>Status</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable service-color lighten">
+      <tr class="clickable hoverable service-color lighten">
         <td><a class="btn-floating disabled"><i class="service-1 nopaque-icons service-color darken service-icon"></i></a></td>
         <td><b class="title"></b><br><i class="description"></i></td>
         <td><span class="status badge new job-status-color job-status-text" data-badge-caption=""></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+          <a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
-          <a class="service-2 action-button btn-floating nopaque-service-color darken tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+          <a class="action-button btn-floating service-color darken waves-effect waves-light service-2" data-action="view"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: job => {
+    ressourceMapper: (job) => {
       return {
         'id': job.id,
         'creation-date': job.creation_date,
@@ -23,7 +50,7 @@ class JobList extends RessourceList {
         'title': job.title
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['creation-date', {order: 'desc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -36,7 +63,6 @@ class JobList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...JobList.options, ...options});
   }
@@ -45,89 +71,55 @@ class JobList extends RessourceList {
     this._init(user.jobs);
   }
 
-  onclick(event) {
+  onClick(event) {
-    let action;
+    let actionButtonElement = event.target.closest('.action-button');
-    let actionButtonElement;
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
-    let deleteModal;
+    let jobElement = event.target.closest('tr');
-    let deleteModalElement;
+    let jobId = jobElement.dataset.id;
-    let jobElement;
-    let jobId;
-    let tmp;
-
-    jobElement = event.target.closest('tr[data-id]');
-    if (jobElement === null) {return;}
-    jobId = jobElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
     switch (action) {
-      case 'delete':
+      case 'delete-request': {
-        tmp = document.createElement('div');
+        Utils.deleteJobRequest(this.userId, jobId);
-        tmp.innerHTML = `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm job deletion</h4>
-              <p>Do you really want to delete the job <b>${app.users[this.userId].jobs[jobId].title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-              <a class="btn modal-close red waves-effect waves-light" href="/jobs/${jobId}/delete"><i class="material-icons left">delete</i>Delete</a>
-            </div>
-          </div>
-        `.trim();
-        deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
-        deleteModal = M.Modal.init(
-          deleteModalElement,
-          {
-            onCloseEnd: () => {
-              deleteModal.destroy();
-              deleteModalElement.remove();
-            }
-          }
-        );
-        deleteModal.open();
         break;
-      case 'view':
+      }
+      case 'view': {
         window.location.href = `/jobs/${jobId}`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let filteredPatch;
+    let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)`);
-    let jobId;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let match;
+    for (let operation of filteredPatch) {
-    let operation;
-    let re;
-    let valueName;
-
-    re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'add':
+        case 'add': {
-          re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
+          let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
-          if (re.test(operation.path)) {
+          if (re.test(operation.path)) {this.add(operation.value);}
-            this.add(operation.value);
-          }
           break;
-        case 'remove':
+        }
-          re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
+        case 'remove': {
+          let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
           if (re.test(operation.path)) {
-            [match, jobId] = operation.path.match(re);
+            let [match, jobId] = operation.path.match(re);
             this.remove(jobId);
           }
           break;
-        case 'replace':
+        }
-          re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/(service|status|description|title)$`);
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/(service|status|description|title)$`);
           if (re.test(operation.path)) {
-            [match, jobId, valueName] = operation.path.match(re);
+            let [match, jobId, valueName] = operation.path.match(re);
             this.replace(jobId, valueName, operation.value);
           }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }

app/static/js/RessourceLists/JobResultList.js

@@ -1,15 +1,41 @@
 class JobResultList extends RessourceList {
+  static autoInit() {
+    for (let jobResultListElement of document.querySelectorAll('.job-result-list:not(.no-autoinit)')) {
+      new JobResultList(jobResultListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search job result</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th>Description</th>
+              <th>Filename</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable">
+      <tr class="clickable hoverable">
         <td><span class="description"></span></td>
         <td><span class="filename"></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+          <a class="action-button btn-floating waves-effect waves-light" data-action="download"><i class="material-icons">file_download</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: jobResult => {
+    ressourceMapper: (jobResult) => {
       return {
         'id': jobResult.id,
         'creation-date': jobResult.creation_date,
@@ -17,7 +43,7 @@ class JobResultList extends RessourceList {
         'filename': jobResult.filename
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['filename', {order: 'asc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -26,7 +52,6 @@ class JobResultList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...JobResultList.options, ...options});
     this.jobId = listElement.dataset.jobId;
@@ -36,44 +61,35 @@ class JobResultList extends RessourceList {
     super._init(user.jobs[this.jobId].results);
   }
 
-  onclick(event) {
+  onClick(event) {
-    let action;
+    let actionButtonElement = event.target.closest('.action-button');
-    let actionButtonElement;
+    let action = actionButtonElement === null ? 'download' : actionButtonElement.dataset.action;
-    let jobResultElement;
+    let jobResultElement = event.target.closest('tr');
-    let jobResultId;
+    let jobResultId = jobResultElement.dataset.id;
-
-    jobResultElement = event.target.closest('tr[data-id]');
-    if (jobResultElement === null) {return;}
-    jobResultId = jobResultElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    if (actionButtonElement === null) {return;}
-    action = actionButtonElement.dataset.action;
     switch (action) {
-      case 'download':
+      case 'download': {
         window.location.href = `/jobs/${this.jobId}/results/${jobResultId}/download`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 
-  usersPatchHandler(patch) {
+  onPatch(patch) {
-    let filteredPatch;
+    let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)`);
-    let operation;
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
-    let re;
+    for (let operation of filteredPatch) {
-
-    re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)`);
-    filteredPatch = patch.filter(operation => re.test(operation.path));
-    for (operation of filteredPatch) {
       switch(operation.op) {
-        case 'add':
+        case 'add': {
-          re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)$`);
+          let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)$`);
-          if (re.test(operation.path)) {
+          if (re.test(operation.path)) {this.add(operation.value);}
-            this.add(operation.value);
-          }
           break;
-        default:
+        }
+        default: {
           break;
+        }
       }
     }
   }

app/static/js/RessourceLists/QueryResultList.js

@@ -1,12 +1,18 @@
 class QueryResultList extends RessourceList {
+  static autoInit() {
+    for (let queryResultListElement of document.querySelectorAll('.query-result-list:not(.no-autoinit)')) {
+      new QueryResultList(queryResultListElement);
+    }
+  }
+
   static options = {
     item: `
       <tr class="hoverable">
         <td><b class="title"></b><br><i class="description"></i><br></td>
         <td><span class="corpus-title"></span><br><span class="query"></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+          <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
-          <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+          <a class="action-button btn-floating waves-effect waves-light" data-action="view"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim(),
@@ -20,7 +26,7 @@ class QueryResultList extends RessourceList {
         'title': queryResult.title
       };
     },
-    sortValueName: 'creation-date',
+    sortArgs: ['creation-date', {order: 'desc'}],
     valueNames: [
       {data: ['id']},
       {data: ['creation-date']},
@@ -31,7 +37,6 @@ class QueryResultList extends RessourceList {
     ]
   };
 
-
   constructor(listElement, options = {}) {
     super(listElement, {...QueryResultList.options, ...options});
   }
@@ -89,7 +94,7 @@ class QueryResultList extends RessourceList {
     }
   }
 
-  usersPatchHandler(patch) {
+  onPATCH(patch) {
     let filteredPatch;
     let match;
     let operation;

app/static/js/RessourceLists/RessourceList.js

@@ -3,45 +3,22 @@ class RessourceList {
    * This class is not meant to be used directly, instead it should be used as
    * a base class for concrete ressource list implementations.
    */
-  static autoInit() {
-    const nopaqueRessourceListElements = document.querySelectorAll('.nopaque-ressource-list[data-ressource-type]:not(.no-autoinit)');
-    let nopaqueRessourceListElement;
 
-    for (nopaqueRessourceListElement of nopaqueRessourceListElements) {
+  static autoInit() {
-      switch (nopaqueRessourceListElement.dataset.ressourceType) {
+    CorpusList.autoInit();
-        case 'Corpus':
+    CorpusFileList.autoInit();
-          new CorpusList(nopaqueRessourceListElement);
+    JobList.autoInit();
-          break;
+    JobInputList.autoInit();
-        case 'CorpusFile':
+    JobResultList.autoInit();
-          new CorpusFileList(nopaqueRessourceListElement);
+    QueryResultList.autoInit();
-          break;
+    UserList.autoInit();
-        case 'Job':
-          new JobList(nopaqueRessourceListElement);
-          break;
-        case 'JobInput':
-          new JobInputList(nopaqueRessourceListElement);
-          break;
-        case 'JobResult':
-          new JobResultList(nopaqueRessourceListElement);
-          break;
-        case 'QueryResult':
-          new QueryResultList(nopaqueRessourceListElement);
-          break;
-        case 'User':
-          new UserList(nopaqueRessourceListElement);
-          break;
-        default:
-          break;
-      }
-    }
   }
+
   static options = {page: 5, pagination: {innerWindow: 4, outerWindow: 1}};
 
-
   constructor(listElement, options = {}) {
-    let i;
-
     if (!(listElement.hasAttribute('id'))) {
+      let i;
       for (i = 0; true; i++) {
         if (document.querySelector(`#ressource-list-${i}`)) {continue;}
         listElement.id = `ressource-list-${i}`;
@@ -56,9 +33,14 @@ class RessourceList {
       this.ressourceMapper = options.ressourceMapper;
       delete options.ressourceMapper;
     }
-    if ('sortValueName' in options) {
+    if ('initialHtmlGenerator' in options) {
-      this.sortValueName = options.sortValueName;
+      this.initialHtmlGenerator = options.initialHtmlGenerator;
-      delete options.sortValueName;
+      listElement.innerHTML = this.initialHtmlGenerator(listElement.id);
+      delete options.initialHtmlGenerator;
+    }
+    if ('sortArgs' in options) {
+      this.sortArgs = options.sortArgs;
+      delete options.sortArgs;
     }
     this.listjs = new List(listElement, {...RessourceList.options, ...options});
     this.listjs.list.innerHTML = `
@@ -87,47 +69,54 @@ class RessourceList {
         </td>
       </tr>
     `.trim();
-    this.listjs.list.style.cursor = 'pointer';
     this.userId = this.listjs.listContainer.dataset.userId;
-    this.listjs.list.addEventListener('click', event => this.onclick(event));
+    this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    this.isInitialized = false;
     if (this.userId) {
-      app.addEventListener('users.patch', patch => this.usersPatchHandler(patch));
+      app.subscribeUser(this.userId)
-      app.getUserById(this.userId).then(
+        .then((response) => {
-        user => this.init(user),
+          app.socket.on('PATCH', (patch) => {
-        error => {throw JSON.stringify(error);}
+            if (this.isInitialized) {this.onPatch(patch);}
-      );
+          });
+        });
+      app.getUser(this.userId)
+        .then((user) => {
+          this.init(user);
+          this.isInitialized = true;
+        });
     }
   }
 
   _init(ressources) {
     this.listjs.clear();
     this.add(Object.values(ressources));
-    let emptyListElementHTML = `
+    this.listjs.list.insertAdjacentHTML(
-      <tr class="show-if-only-child">
+      'afterbegin',
-        <td colspan="100%">
+      `
-          <span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
+        <tr class="show-if-only-child">
-          <p>No ressource available.</p>
+          <td colspan="100%">
-        </td>
+            <span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
-      </tr>
+            <p>No ressource available.</p>
-    `.trim();
+          </td>
-    this.listjs.list.insertAdjacentHTML('afterbegin', emptyListElementHTML);
+        </tr>
+      `.trim()
+    );
   }
 
   init(user) {throw 'Not implemented';}
 
-  onclick(event) {throw 'Not implemented';}
+  onClick(event) {throw 'Not implemented';}
 
-  usersPatchHandler(patch) {throw 'Not implemented';}
+  onPatch(patch) {throw 'Not implemented';}
 
   add(ressources) {
     let values = Array.isArray(ressources) ? ressources : [ressources];
-
     if ('ressourceMapper' in this) {
-      values = values.map(value => this.ressourceMapper(value));
+      values = values.map((value) => {return this.ressourceMapper(value);});
     }
     this.listjs.add(values, () => {
-      if ('sortValueName' in this) {
+      if ('sortArgs' in this) {
-        this.listjs.sort(this.sortValueName, {order: 'desc'});
+        this.listjs.sort(...this.sortArgs);
       }
     });
   }
@@ -137,6 +126,6 @@ class RessourceList {
   }
 
   replace(id, valueName, newValue) {
-      this.listjs.get('id', id)[0].values({[valueName]: newValue});
+    this.listjs.get('id', id)[0].values({[valueName]: newValue});
   }
 }

app/static/js/RessourceLists/UserList.js

@@ -1,31 +1,60 @@
 class UserList extends RessourceList {
+  static autoInit() {
+    for (let userListElement of document.querySelectorAll('.user-list:not(.no-autoinit)')) {
+      new UserList(userListElement);
+    }
+  }
+
   static options = {
+    initialHtmlGenerator: (id) => {
+      return `
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="${id}-search" class="search" type="search"></input>
+          <label for="${id}-search">Search user</label>
+        </div>
+        <table>
+          <thead>
+            <tr>
+              <th>Id</th>
+              <th>Username</th>
+              <th>Email</th>
+              <th>Last seen</th>
+              <th>Role</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody class="list"></tbody>
+        </table>
+        <ul class="pagination"></ul>
+      `.trim();
+    },
     item: `
-      <tr class="hoverable">
+      <tr class="clickable hoverable">
         <td><span class="id-1"></span></td>
         <td><span class="username"></span></td>
         <td><span class="email"></span></td>
         <td><span class="last-seen"></span></td>
         <td><span class="role"></span></td>
         <td class="right-align">
-          <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+          <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
-          <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="edit" data-position="top" data-tooltip="Edit"><i class="material-icons">edit</i></a>
+          <a class="action-button btn-floating waves-effect waves-light" data-action="edit"><i class="material-icons">edit</i></a>
-          <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+          <a class="action-button btn-floating waves-effect waves-light" data-action="view"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim(),
-    ressourceMapper: user => {
+    ressourceMapper: (user) => {
       return {
         'id': user.id,
         'id-1': user.id,
         'username': user.username,
         'email': user.email,
-        'last-seen': new Date(user.last_seen).toLocaleString("en-US"),
+        'last-seen': new Date(user.last_seen).toLocaleString('en-US'),
         'member-since': user.member_since,
         'role': user.role.name
       };
     },
-    sortValueName: 'member-since',
+    sortArgs: ['member-since', {order: 'desc'}],
     valueNames: [
       {data: ['id']},
       {data: ['member-since']},
@@ -37,8 +66,6 @@ class UserList extends RessourceList {
     ]
   };
 
-
-
   constructor(listElement, options = {}) {
     super(listElement, {...UserList.options, ...options});
   }
@@ -47,55 +74,28 @@ class UserList extends RessourceList {
     super._init(Object.values(users));
   }
 
-  onclick(event) {
+  onClick(event) {
-    let action;
+    let actionButtonElement = event.target.closest('.action-button');
-    let actionButtonElement;
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
-    let deleteModal;
+    let userElement = event.target.closest('tr');
-    let deleteModalElement;
+    let userId = userElement.dataset.id;
-    let tmp;
-    let userElement;
-    let userId;
-
-    userElement = event.target.closest('tr[data-id]');
-    if (userElement === null) {return;}
-    userId = userElement.dataset.id;
-    actionButtonElement = event.target.closest('.action-button[data-action]');
-    action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
     switch (action) {
-      case 'delete':
+      case 'delete': {
-        tmp = document.createElement('div');
+        Utils.deleteUserRequest(userId);
-        tmp.innerHTML = `
+        if (userId === currentUserId) {window.location.href = '/';}
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm user deletion</h4>
-              <p>Do you really want to delete user <b>${userId}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-              <a class="btn modal-close red waves-effect waves-light" href="/admin/users/${userId}/delete"><i class="material-icons left">delete</i>Delete</a>
-            </div>
-          </div>
-        `.trim();
-        deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
-        deleteModal = M.Modal.init(
-          deleteModalElement,
-          {
-            onCloseEnd: () => {
-              deleteModal.destroy();
-              deleteModalElement.remove();
-            }
-          }
-        );
-        deleteModal.open();
         break;
-      case 'edit':
+      }
+      case 'edit': {
         window.location.href = `/admin/users/${userId}/edit`;
         break;
-      case 'view':
+      }
+      case 'view': {
         window.location.href = `/admin/users/${userId}`;
         break;
-      default:
+      }
+      default: {
         break;
+      }
     }
   }
 }

app/static/js/UploadForm.js

@@ -1,125 +0,0 @@
-class UploadForm {
-  static autoInit() {
-    const nopaqueSubmitForms = document.querySelectorAll('.nopaque-upload-form');
-    let nopaqueSubmitForm;
-
-    for (nopaqueSubmitForm of nopaqueSubmitForms) {
-      new UploadForm(nopaqueSubmitForm);
-    }
-  }
-
-
-  constructor(formElement) {
-    this.formElement = formElement;
-    this.request = new XMLHttpRequest();
-
-    this.formElement.addEventListener('submit', (event) => {
-      event.preventDefault();
-      this.submit();
-    });
-  }
-
-  submit() {
-    const selectElements = this.formElement.querySelectorAll('select');
-    let abortElement;
-    let helperTextElement;
-    let helperTextElements;
-    let inputFieldElement;
-    let modal;
-    let modalElement;
-    let progressElement;
-    let selectElement;
-    let tmp;
-
-    // Check if select elements are filled out properly
-    for (selectElement of selectElements) {
-      if (selectElement.value === '') {
-        inputFieldElement = selectElement.closest('.input-field');
-        inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
-        helperTextElements = inputFieldElement.querySelectorAll('.helper-text');
-        for (helperTextElement of helperTextElements) {
-          helperTextElement.remove();
-        }
-        inputFieldElement.insertAdjacentHTML(
-          'beforeend',
-          '<span class="helper-text error-color-text">Please select an option.</span>'
-        );
-        return;
-      }
-    }
-
-    // Setup modal
-    tmp = document.createElement('div');
-    tmp.innerHTML = `
-      <div class="modal">
-        <div class="modal-content">
-          <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
-          <div class="progress">
-            <div class="determinate" style="width: 0%"></div>
-          </div>
-        </div>
-        <div class="modal-footer">
-          <a href="#!" class="btn red waves-effect waves-light abort">Cancel</a>
-        </div>
-      </div>
-    `.trim();
-    modalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
-    modal = M.Modal.init(
-      modalElement,
-      {
-        dismissible: false,
-        onCloseEnd: () => {
-          modal.destroy();
-          modalElement.remove();
-        }
-      }
-    );
-    modal.open();
-
-    // Setup abort handling
-    abortElement = modalElement.querySelector('.abort');
-    abortElement.addEventListener('click', event => {this.request.abort();});
-    this.request.addEventListener('abort', event => {
-      this.request.abort();
-      modal.close();
-    });
-
-    // Setup load handling (after the request completed)
-    this.request.addEventListener('load', event => {
-      const response = JSON.parse(this.request.responseText);
-      let inputError;
-      let inputErrors;
-      let inputFieldElement;
-      let inputName;
-
-      if (this.request.status === 201) {
-        window.location.href = response.redirect_url;
-      }
-      if (this.request.status === 400) {
-        for ([inputName, inputErrors] of Object.entries(response)) {
-          inputFieldElement = this.formElement.querySelector(`input[name="${inputName}"], select[name="${inputName}"]`).closest('.input-field');
-          for (inputError of inputErrors) {
-            inputFieldElement.insertAdjacentHTML(
-              'beforeend',
-              `<span class="helper-text red-text">${inputError}</span>`
-            );
-          }
-        }
-      }
-      if (this.request.status === 500) {
-        location.reload();
-      }
-      modal.close();
-    });
-
-    // Setup progress handling
-    progressElement = modalElement.querySelector('.progress > .determinate');
-    this.request.upload.addEventListener('progress', event => {
-      const progress = Math.floor(100 * event.loaded / event.total);
-      progressElement.style.width = `${progress}%`;
-    });
-
-    this.request.open('POST', window.location.href);
-    this.request.send(new FormData(this.formElement));
-  }
-}

app/static/js/Utils.js

@@ -0,0 +1,326 @@
+class Utils {
+  static elementFromString(string) {
+    let tmpElement = document.createElement('div');
+    tmpElement.innerHTML = string.trim();
+    return tmpElement.firstChild;
+  }
+
+  static buildCorpusRequest(userId, corpusId) {
+    return new Promise((resolve, reject) => {
+      let corpus = app.data.users[userId].corpora[corpusId];
+
+      fetch(`/corpora/${corpus.id}/build`, {method: 'POST', headers: {Accept: 'application/json'}})
+        .then(
+          (response) => {
+            app.flash(`Corpus "${corpus.title}" marked for building`, 'corpus');
+            resolve(response);
+          },
+          (response) => {
+            if (response.status === 403) {app.flash('Forbidden', 'error');}
+            if (response.status === 404) {app.flash('Not Found', 'error');}
+            if (response.status === 409) {app.flash('Conflict', 'error');}
+            reject(response);
+          }
+        );
+    });
+  }
+
+  static deleteCorpusRequest(userId, corpusId) {
+    return new Promise((resolve, reject) => {
+      let corpus = app.data.users[userId].corpora[corpusId];
+
+      let modalElement = Utils.elementFromString(
+        `
+          <div class="modal">
+            <div class="modal-content">
+              <h4>Confirm job deletion</h4>
+              <p>Do you really want to delete the job <b>${corpus.title}</b>? All files will be permanently deleted!</p>
+            </div>
+            <div class="modal-footer">
+              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+            </div>
+          </div>
+        `
+      );
+      document.querySelector('#modals').appendChild(modalElement);
+      let modal = M.Modal.init(
+        modalElement,
+        {
+          dismissible: false,
+          onCloseEnd: () => {
+            modal.destroy();
+            modalElement.remove();
+          }
+        }
+      );
+      
+      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+      confirmElement.addEventListener('click', (event) => {
+        let corpusTitle = corpus.title;
+        fetch(`/corpora/${corpus.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+          .then(
+            (response) => {
+              app.flash(`Corpus "${corpusTitle}" marked for deletion`, 'corpus');
+              resolve(response);
+            },
+            (response) => {
+              if (response.status === 403) {app.flash('Forbidden', 'error');}
+              if (response.status === 404) {app.flash('Not Found', 'error');}
+              reject(response);
+            }
+          );
+      });
+      modal.open();
+    });
+  }
+
+  static deleteCorpusFileRequest(userId, corpusId, corpusFileId) {
+    return new Promise((resolve, reject) => {
+      let corpus = app.data.users[userId].corpora[corpusId];
+      let corpusFile = corpus.files[corpusFileId];
+
+      let modalElement = Utils.elementFromString(
+        `
+          <div class="modal">
+            <div class="modal-content">
+              <h4>Confirm job deletion</h4>
+              <p>Do you really want to delete the job <b>${corpusFile.title}</b>? All files will be permanently deleted!</p>
+            </div>
+            <div class="modal-footer">
+              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+            </div>
+          </div>
+        `
+      );
+      document.querySelector('#modals').appendChild(modalElement);
+      let modal = M.Modal.init(
+        modalElement,
+        {
+          dismissible: false,
+          onCloseEnd: () => {
+            modal.destroy();
+            modalElement.remove();
+          }
+        }
+      );
+
+      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+      confirmElement.addEventListener('click', (event) => {
+        let corpusFileTitle = corpusFile.title;
+        fetch(`/corpora/${corpusId}/files/${corpusFileId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+          .then(
+            (response) => {
+              app.flash(`Corpus file "${corpusFileTitle}" marked for deletion`, 'corpus');
+              resolve(response);
+            },
+            (response) => {
+              if (response.status === 403) {app.flash('Forbidden', 'error');}
+              if (response.status === 404) {app.flash('Not Found', 'error');}
+              reject(response);
+            }
+          );
+      });
+      modal.open();
+    });
+  }
+
+  static deleteJobRequest(userId, jobId) {
+    return new Promise((resolve, reject) => {
+      let job = app.data.users[userId].jobs[jobId];
+
+      let modalElement = Utils.elementFromString(
+        `
+          <div class="modal">
+            <div class="modal-content">
+              <h4>Confirm job deletion</h4>
+              <p>Do you really want to delete the job <b>${job.title}</b>? All files will be permanently deleted!</p>
+            </div>
+            <div class="modal-footer">
+              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+            </div>
+          </div>
+        `
+      );
+      document.querySelector('#modals').appendChild(modalElement);
+      let modal = M.Modal.init(
+        modalElement,
+        {
+          dismissible: false,
+          onCloseEnd: () => {
+            modal.destroy();
+            modalElement.remove();
+          }
+        }
+      );
+
+      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+      confirmElement.addEventListener('click', (event) => {
+        let jobTitle = job.title;
+        fetch(`/jobs/${job.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+          .then(
+            (response) => {
+              app.flash(`Job "${jobTitle}" marked for deletion`, 'job');
+              resolve(response);
+            },
+            (response) => {
+              if (response.status === 403) {app.flash('Forbidden', 'error');}
+              if (response.status === 404) {app.flash('Not Found', 'error');}
+              reject(response);
+            }
+          );
+      });
+      modal.open();
+    });
+  }
+
+  static getJobLogRequest(userId, jobId) {
+    return new Promise((resolve, reject) => {
+      let job = app.data.users[userId].jobs[jobId];
+
+      fetch(`/jobs/${job.id}/log`, {method: 'GET', headers: {Accept: 'application/json, text/plain'}})
+        .then(
+          (response) => {
+            resolve(response);
+            return response.text();
+          },
+          (response) => {
+            if (response.status === 403) {app.flash('Forbidden', 'error');}
+            if (response.status === 404) {app.flash('Not Found', 'error');}
+            reject(response);
+          }
+        )
+        .then(
+          (text) => {
+            let modalElement = Utils.elementFromString(
+              `
+                <div class="modal">
+                  <div class="modal-content">
+                    <h4>Job logs</h4>
+                    <pre><code>${text}</code></pre>
+                  </div>
+                  <div class="modal-footer">
+                    <a class="btn modal-close waves-effect waves-light">Close</a>
+                  </div>
+                </div>
+              `
+            );
+            document.querySelector('#modals').appendChild(modalElement);
+            let modal = M.Modal.init(
+              modalElement,
+              {
+                onCloseEnd: () => {
+                  modal.destroy();
+                  modalElement.remove();
+                }
+              }
+            );
+            modal.open();
+          }
+        );
+    });
+  }
+
+  static restartJobRequest(userId, jobId) {
+    return new Promise((resolve, reject) => {
+      let job = app.data.users[userId].jobs[jobId];
+
+      let modalElement = Utils.elementFromString(
+        `
+          <div class="modal">
+            <div class="modal-content">
+              <h4>Confirm job restart</h4>
+              <p>Do you really want to restart the job <b>${job.title}</b>? All log and result files will be permanently deleted.</p>
+            </div>
+            <div class="modal-footer">
+              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Restart</a>
+            </div>
+          </div>
+        `
+      );
+      document.querySelector('#modals').appendChild(modalElement);
+      let modal = M.Modal.init(
+        modalElement,
+        {
+          dismissible: false,
+          onCloseEnd: () => {
+            modal.destroy();
+            modalElement.remove();
+          }
+        }
+      );
+
+      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+      confirmElement.addEventListener('click', (event) => {
+        let jobTitle = job.title;
+        fetch(`/jobs/${job.id}/restart`, {method: 'POST', headers: {Accept: 'application/json'}})
+          .then(
+            (response) => {
+              app.flash(`Job "${jobTitle}" restarted.`, 'job');
+              resolve(response);
+            },
+            (response) => {
+              if (response.status === 403) {app.flash('Forbidden', 'error');}
+              if (response.status === 404) {app.flash('Not Found', 'error');}
+              if (response.status === 409) {app.flash('Conflict', 'error');}
+              reject(response);
+            }
+          );
+      });
+      modal.open();
+    });
+  }
+
+  static deleteUserRequest(userId) {
+    return new Promise((resolve, reject) => {
+      let user = app.data.users[userId];
+
+      let modalElement = Utils.elementFromString(
+        `
+          <div class="modal">
+            <div class="modal-content">
+              <h4>Confirm job deletion</h4>
+              <p>Do you really want to delete the user <b>${user.username}</b>? All files will be permanently deleted!</p>
+            </div>
+            <div class="modal-footer">
+              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+            </div>
+          </div>
+        `
+      );
+      document.querySelector('#modals').appendChild(modalElement);
+      let modal = M.Modal.init(
+        modalElement,
+        {
+          dismissible: false,
+          onCloseEnd: () => {
+            modal.destroy();
+            modalElement.remove();
+          }
+        }
+      );
+
+      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+      confirmElement.addEventListener('click', (event) => {
+        let userName = user.username;
+        fetch(`/users/${user.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+          .then(
+            (response) => {
+              app.flash(`User "${userName}" marked for deletion`);
+              resolve(response);
+            },
+            (response) => {
+              if (response.status === 403) {app.flash('Forbidden', 'error');}
+              if (response.status === 404) {app.flash('Not Found', 'error');}
+              reject(response);
+            }
+          );
+      });
+      modal.open();
+    });
+  }
+}

app/templates/_navbar.html.j2

@@ -29,7 +29,7 @@
 <ul class="dropdown-content" id="nav-more-dropdown">
   <li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons left">help</i>Manual</a></li>
   {% if current_user.is_authenticated %}
-  <li><a href="{{ url_for('settings.index') }}"><i class="material-icons left">settings</i>Settings</a></li>
+  <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>Settings</a></li>
   <li class="divider" tabindex="-1"></li>
   <li><a href="{{ url_for('auth.logout') }}">Log out</a></li>
   {% else %}

app/templates/_roadmap.html.j2

@@ -11,19 +11,19 @@
       <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
       <li class="tab"><a{%if request.path == url_for('services.spacy_nlp_pipeline') %} class="active"{% endif %} href="{{ url_for('services.spacy_nlp_pipeline') }}" target="_self">NLP</a></li>
       <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-      <li class="tab"><a{%if request.path == url_for('corpora.add_corpus') %} class="active"{% endif %} href="{{ url_for('corpora.add_corpus') }}" target="_self">Add corpus</a></li>
+      <li class="tab"><a{%if request.path == url_for('corpora.create_corpus') %} class="active"{% endif %} href="{{ url_for('corpora.create_corpus') }}" target="_self">Create corpus</a></li>
       <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
       {% if corpus %}
-      <li class="tab"><a{%if request.path == url_for('corpora.add_corpus_file', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" target="_self">Add corpus file(s)</a></li>
+      <li class="tab"><a{%if request.path == url_for('corpora.create_corpus_file', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.create_corpus_file', corpus_id=corpus.id) }}" target="_self">Create corpus file(s)</a></li>
       {% else %}
-      <li class="tab disabled tooltipped" data-tooltip="Select a corpus first" target="_self"><a>Add corpus file(s)</a></li>
+      <li class="tab disabled tooltipped" data-tooltip="Select a corpus first" target="_self"><a>Create corpus file(s)</a></li>
       {% endif %}
       <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
       {% if corpus %}
       {% if corpus.files.all() %}
       <li class="tab"><a{%if request.path == url_for('corpora.analyse_corpus', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" target="_self">Corpus analysis</a></li>
       {% else %}
-      <li class="tab disabled tooltipped" data-tooltip="Add at least one corpus file first"><a>Corpus analysis</a></li>
+      <li class="tab disabled tooltipped" data-tooltip="Create at least one corpus file first"><a>Corpus analysis</a></li>
       {% endif %}
       {% else %}
       <li class="tab disabled tooltipped" data-tooltip="Select a corpus first"><a>Corpus analysis</a></li>

app/templates/_scripts.html.j2

@@ -1,16 +1,18 @@
-<script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.0/fast-json-patch.min.js" integrity="sha512-KrvLlmKBiDoTa0Fke92aFoEv4xS0+cuYGP27nt39w0yLZWvVOhArmZ29uuOe3uOOBcbnkpvnLhkvYcYjahSOwg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.1/fast-json-patch.min.js" integrity="sha512-5uDdefwnzyq4N+SkmMBmekZLZNmc6dLixvVxCdlHBfqpyz0N3bzLdrJ55OLm7QrZmgZuhLGgHLDtJwU6RZoFCA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/2.3.1/list.min.js" integrity="sha512-93wYgwrIFL+b+P3RvYxi/WUFRXXUDSLCT2JQk9zhVGXuS2mHl2axj6d+R6pP+gcU5isMHRj1u0oYE/mWyt/RjA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.0/socket.io.min.js" integrity="sha512-pxLMWs4E33rW9tdIhovcCp2dCo9k4Q8eHw7CETjyjdXf4aX6wvsEBq+KdOJJRFALr6FxNoXx+jksgbE74TZjEw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.min.js" integrity="sha512-mHO4BJ0ELk7Pb1AzhTi3zvUeRgq3RXVOu9tTRfnA6qOxGK4pG2u57DJYolI4KrEnnLTcH9/J5wNOozRTDaybXg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 {%- assets
   filters='rjsmin',
   output='gen/app.%(version)s.js',
   'js/App.js',
+  'js/Utils.js',
+  'js/Forms/Form.js',
+  'js/Forms/CreateCorpusFileForm.js',
+  'js/Forms/CreateJobForm.js',
   'js/CorpusAnalysis/CQiClient.js',
   'js/CorpusAnalysis/CorpusAnalysisApp.js',
   'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
   'js/CorpusAnalysis/CorpusAnalysisReader.js',
-  'js/CorpusAnalysis/QueryBuilder.js',
-  'js/JobStatusNotifier.js',
   'js/RessourceDisplays/RessourceDisplay.js',
   'js/RessourceDisplays/CorpusDisplay.js',
   'js/RessourceDisplays/JobDisplay.js',
@@ -21,8 +23,7 @@
   'js/RessourceLists/JobInputList.js',
   'js/RessourceLists/JobResultList.js',
   'js/RessourceLists/QueryResultList.js',
-  'js/RessourceLists/UserList.js',
+  'js/RessourceLists/UserList.js'
-  'js/UploadForm.js'
 %}
 <script src="{{ ASSET_URL }}"></script>
 {%- endassets %}
@@ -33,12 +34,8 @@
   const jobStatusNotifier = new JobStatusNotifier(currentUserId);
 
   // Initialize components for current user
-  app.addEventListener('users.patch', patch => jobStatusNotifier.usersPatchHandler(patch));
+  app.subscribeUser(currentUserId).catch((error) => {throw JSON.stringify(error);});
-  app.getUserById(currentUserId)
+  app.getUser(currentUserId, true, true);
-    .then(
-      user => {return;},
-      error => {throw JSON.stringify(error);}
-    );
   {%- endif %}
 
   // Disable all option elements with no value
@@ -59,7 +56,7 @@
     {alignment: 'right', constrainWidth: false, coverTrigger: false}
   );
   RessourceList.autoInit();
-  UploadForm.autoInit();
+  Form.autoInit();
 
   // Display flashed messages
   for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {

app/templates/_sidenav.html.j2

@@ -23,7 +23,7 @@
   <li class="service-color service-color-border border-darken" data-service="corpus-analysis" style="border-left: 10px solid; margin-top: 5px;"><a href="{{ url_for('services.corpus_analysis') }}"><i class="nopaque-icons service-icon" data-service="corpus-analysis"></i>Corpus analysis</a></li>
   <li><div class="divider"></div></li>
   <li><a class="subheader">Account</a></li>
-  <li><a href="{{ url_for('settings.index') }}"><i class="material-icons">settings</i>Settings</a></li>
+  <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons">settings</i>Settings</a></li>
   <li><a href="{{ url_for('auth.logout') }}">Log out</a></li>
   {% if current_user.can(Permission.ADMINISTRATE) or current_user.can(Permission.USE_API) %}
   <li><div class="divider"></div></li>
@@ -31,10 +31,10 @@
   {% if current_user.can(Permission.ADMINISTRATE) %}
   <li><a href="{{ url_for('admin.index') }}"><i class="material-icons">admin_panel_settings</i>Administration</a></li>
   {% endif %}
-  {% if current_user.can(Permission.CONTRIBUTE) %}
-  <li><a href="{{ url_for('contribute.index') }}"><i class="material-icons">new_label</i>Contribute</a></li>
-  {% endif %}
   {% if current_user.can(Permission.USE_API) %}
-  <li><a href="{{ url_for('api.doc') }}"><i class="material-icons">api</i>API</a></li>
+  <li><a href="{{ url_for('apifairy.docs') }}"><i class="material-icons">api</i>API</a></li>
+  {% endif %}
+  {% if current_user.can(Permission.CONTRIBUTE) %}
+  <li><a href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>Contribute</a></li>
   {% endif %}
 </ul>

app/templates/_styles.html.j2

@@ -8,6 +8,7 @@
   filters='pyscss',
   output='gen/app.%(version)s.css',
   'css/colors.scss',
+  'css/helpers.scss',
   'css/style.css'
 %}
 <link href="{{ ASSET_URL }}" media="screen,projection" rel="stylesheet">

app/templates/admin/edit_user.html.j2

@@ -15,8 +15,8 @@
         <div class="card">
           <div class="card-content">
             <span class="card-title">General settings</span>
-            {{ wtf.render_field(edit_general_settings_form.username, data_length='64', material_icon='person') }}
+            {{ wtf.render_field(edit_general_settings_form.username, material_icon='person') }}
-            {{ wtf.render_field(edit_general_settings_form.email, data_length='254', material_icon='email') }}
+            {{ wtf.render_field(edit_general_settings_form.email, material_icon='email') }}
           </div>
           <div class="card-action">
             <div class="right-align">

app/templates/admin/user.html.j2

@@ -37,52 +37,20 @@
       </div>
     </div>
 
-    <div class="col s12 l6 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ user.hashid }}">
+    <div class="col s12 l6">
       <h3>Corpora</h3>
       <div class="card">
         <div class="card-content">
-          <div class="input-field">
+          <div class="corpus-list" data-user-id="{{ user.hashid }}"></div>
-            <i class="material-icons prefix">search</i>
-            <input id="search-corpus" class="search" type="search"></input>
-            <label for="search-corpus">Search corpus</label>
-          </div>
-          <table>
-            <thead>
-              <tr>
-                <th></th>
-                <th>Title and Description</th>
-                <th>Status</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
       </div>
     </div>
 
-    <div class="col s12 l6 nopaque-ressource-list" data-ressource-type="Job" data-user-id="{{ user.hashid }}">
+    <div class="col s12 l6">
       <h3>Jobs</h3>
       <div class="card">
         <div class="card-content">
-          <div class="input-field">
+          <div class="job-list" data-user-id="{{ user.hashid }}"></div>
-            <i class="material-icons prefix">search</i>
-            <input id="search-job" class="search" type="search"></input>
-            <label for="search-job">Search job</label>
-          </div>
-          <table>
-            <thead>
-              <tr>
-                <th>Service</th>
-                <th>Title and Description</th>
-                <th>Status</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
       </div>
     </div>

app/templates/admin/users.html.j2

@@ -8,28 +8,10 @@
       <h1 id="title">{{ title }}</h1>
     </div>
 
-    <div class="col s12 nopaque-ressource-list no-autoinit" data-ressource-type="User" id="users">
+    <div class="col s12">
       <div class="card">
         <div class="card-content">
-          <div class="input-field">
+          <div class="user-list no-autoinit"></div>
-            <i class="material-icons prefix">search</i>
-            <input id="search-user" class="search" type="text"></input>
-            <label for="search-user">Search user</label>
-          </div>
-          <table>
-            <thead>
-              <tr>
-                <th>Id</th>
-                <th>Username</th>
-                <th>Email</th>
-                <th>Last seen</th>
-                <th>Role</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
       </div>
     </div>
@@ -40,7 +22,11 @@
 {% block scripts %}
 {{ super() }}
 <script>
-  let userList = new UserList(document.querySelector('#users'));
+  for (let user of {{ json_users|tojson }}) {
-  userList.init({{ dict_users|tojson }});
+    if (user.id in app.data.users) {continue;}
+    app.data.users[user.id] = user;
+  }
+  let userList = new UserList(document.querySelector('.user-list'));
+  userList.init(app.data.users);
 </script>
 {% endblock scripts %}

app/templates/auth/login.html.j2

@@ -2,53 +2,30 @@
 {% from "auth/_breadcrumbs.html.j2" import breadcrumbs with context %}
 {% import "materialize/wtf.html.j2" as wtf %}
 
-{% block styles %}
-{{ super() }}
-<style>
-  main {
-    background-image: url("{{ url_for('static', filename='images/parallax_lq/04_german_text_book_paper.jpg') }}");
-    background-repeat: no-repeat;
-    background-size: cover;
-  }
-</style>
-{% endblock styles %}
 
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12 m4">
+    <div class="col s12 m8 offset-m2">
-      <div class="card medium">
+      <h1 id="title">{{ title }}</h1>
-        <div class="card-content">
+      <p>Want to boost your research and get going? Nopaque is free and no download is needed. <a href="{{ url_for('.register') }}">Register now</a>!</p>
-          <h1 id="title">{{ title }}</h1>
-          <p>Want to boost your research and get going? nopaque is free and no download is needed. Register now!</p>
-        </div>
-        <div class="card-action right-align">
-          <a class="btn" href="{{ url_for('.register') }}"><i class="material-icons left">person_add</i>Register</a>
-        </div>
-      </div>
-    </div>
 
-    <div class="col s12 m8">
+      <form method="POST">
-      <div class="card medium">
+        <div class="card-panel">
-        <form method="POST">
+          {{ form.hidden_tag() }}
-          <div class="card-content">
+          {{ wtf.render_field(form.user, material_icon='person') }}
-            {{ form.hidden_tag() }}
+          {{ wtf.render_field(form.password, material_icon='vpn_key') }}
-            {{ wtf.render_field(form.user, material_icon='person') }}
+          <div class="row">
-            {{ wtf.render_field(form.password, material_icon='vpn_key') }}
+            <div class="col s6 left-align">
-            <div class="row" style="margin-bottom: 0;">
+              <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
-              <div class="col s6 left-align">
+            </div>
-                <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
+            <div class="col s6 right-align">
-              </div>
+              {{ wtf.render_field(form.remember_me) }}
-              <div class="col s6 right-align">
-                {{ wtf.render_field(form.remember_me) }}
-              </div>
             </div>
           </div>
-          <div class="card-action right-align">
+          {{ wtf.render_field(form.submit, material_icon='send', class_='width-100') }}
-            {{ wtf.render_field(form.submit, material_icon='send') }}
+        </div>
-          </div>
+      </form>
-        </form>
-      </div>
     </div>
   </div>
 </div>

app/templates/auth/register.html.j2

@@ -2,47 +2,31 @@
 {% from "auth/_breadcrumbs.html.j2" import breadcrumbs with context %}
 {% import "materialize/wtf.html.j2" as wtf %}
 
-{% block styles %}
-{{ super() }}
-<style>
-  main {
-    background-image: url("{{ url_for('static', filename='images/parallax_lq/02_concept_document_focus_letter.jpg') }}");
-    background-repeat: no-repeat;
-    background-size: cover;
-  }
-</style>
-{% endblock styles %}
 
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12 m4">
+    <div class="col s12 m8 offset-m2">
-      <div class="card medium">
+      <h1 id="title">{{ title }}</h1>
-        <div class="card-content">
+      <p>
-          <h1 id="title">{{ title }}</h1>
+        Simply enter a username and password to receive your registration email.
-          <p>Simply enter a username and password to receive your registration email. After that you can start right away.</p>
+        After that you can start right away. It goes without saying that the
-          <p>It goes without saying that the <a href="{{ url_for('main.privacy_policy') }}">General Data Protection Regulation</a> applies, only necessary data is stored.</p>
+        <a href="{{ url_for('main.privacy_policy') }}">General Data Protection Regulation</a>
-          <p>Please also read our <a href="{{ url_for('main.terms_of_use') }}">terms of use</a> before signing up for nopaque!</p>
+        applies, only necessary data is stored. Please also read our
-        </div>
+        <a href="{{ url_for('main.terms_of_use') }}">terms of use</a> before
-      </div>
+        signing up for nopaque!
-    </div>
+      </p>
 
-    <div class="col s12 m8">
+      <form method="POST">
-      <div class="card medium">
+        <div class="card-panel">
-        <form method="POST">
+          {{ form.hidden_tag() }}
-          <div class="card-content">
+          {{ wtf.render_field(form.username, material_icon='person') }}
-            {{ form.hidden_tag() }}
+          {{ wtf.render_field(form.password, material_icon='vpn_key') }}
-            {{ wtf.render_field(form.username, material_icon='person') }}
+          {{ wtf.render_field(form.password_2, material_icon='vpn_key') }}
-            {{ wtf.render_field(form.password, material_icon='vpn_key') }}
+          {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
-            {{ wtf.render_field(form.password_confirmation, material_icon='vpn_key') }}
+          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
-            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
+        </div>
-          </div>
+      </form>
-          <div class="card-action right-align">
-            {{ wtf.render_field(form.submit, material_icon='send') }}
-          </div>
-        </form>
-      </div>
-    </div>
   </div>
 </div>
 {% endblock page_content %}

app/templates/auth/reset_password.html.j2

@@ -5,27 +5,18 @@
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12">
+    <div class="col s12 m8 offset-m2">
       <h1 id="title">{{ title }}</h1>
-    </div>
-
-    <div class="col s12 m4">
       <p>Enter a new password and confirm it! After that, the entered password is your new one!</p>
-    </div>
 
-    <div class="col s12 m8">
+      <form method="POST">
-      <div class="card">
+        <div class="card-panel">
-        <form method="POST">
+          {{ form.hidden_tag() }}
-          <div class="card-content">
+          {{ wtf.render_field(form.password) }}
-            {{ form.hidden_tag() }}
+          {{ wtf.render_field(form.password_2) }}
-            {{ wtf.render_field(form.password) }}
+          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
-            {{ wtf.render_field(form.password_confirmation) }}
+        </div>
-          </div>
+      </form>
-          <div class="card-action right-align">
-            {{ wtf.render_field(form.submit, material_icon='send') }}
-          </div>
-        </form>
-      </div>
     </div>
   </div>
 </div>

app/templates/auth/reset_password_request.html.j2

@@ -5,26 +5,17 @@
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12">
+    <div class="col s12 m8 offset-m2">
       <h1 id="title">{{ title }}</h1>
-    </div>
-
-    <div class="col s12 m4">
       <p>After entering your email address you will receive instructions on how to reset your password.</p>
-    </div>
 
-    <div class="col s12 m8">
+      <form method="POST">
-      <div class="card">
+        <div class="card-panel">
-        <form method="POST">
+          {{ form.hidden_tag() }}
-          <div class="card-content">
+          {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
-            {{ form.hidden_tag() }}
+          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
-            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
+        </div>
-          </div>
+      </form>
-          <div class="card-action right-align">
-            {{ wtf.render_field(form.submit, material_icon='send') }}
-          </div>
-        </form>
-      </div>
     </div>
   </div>
 </div>

app/templates/auth/unconfirmed.html.j2

@@ -6,20 +6,13 @@
   <div class="row">
     <div class="col s12">
       <h1 id="title">{{ title }}</h1>
-    </div>
+      <p>Hello, <b>{{ current_user.username }}</b>.</p>
-
+      <p>
-    <div class="col s12">
+        You have not confirmed your account yet. Before you can access this
-      <div class="card">
+        site you need to confirm your account. Check your inbox, you should
-        <div class="card-content">
+        have received an email with a confirmation link.
-          <span class="card-title">Hello, {{ current_user.username }}!</span>
+      </p>
-          <p><b>You have not confirmed your account yet.</b></p>
+      <p>Need another confirmation email? <a href="{{ url_for('.confirm_request') }}">Get a new one</a>.</p>
-          <p>Before you can access this site you need to confirm your account. Check your inbox, you should have received an email with a confirmation link.</p>
-          <p>Need another confirmation email? Click the button below!</p>
-        </div>
-        <div class="card-action right-align">
-          <a class="btn" href="{{ url_for('.resend_confirmation') }}">Resend confirmation mail</a>
-        </div>
-      </div>
     </div>
   </div>
 </div>

app/templates/corpora/_breadcrumbs.html.j2

@@ -2,8 +2,8 @@
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 <li class="tab"><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" target="_self">My corpora</a></li>
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-{% if request.path == url_for('.add_corpus') %}
+{% if request.path == url_for('.create_corpus') %}
-<li class="tab"><a class="active" href="{{ url_for('.add_corpus') }}" target="_self">{{ title }}</a></li>
+<li class="tab"><a class="active" href="{{ url_for('.create_corpus') }}" target="_self">{{ title }}</a></li>
 {% elif request.path == url_for('.import_corpus') %}
 <li class="tab"><a class="active" href="{{ url_for('.import_corpus') }}" target="_self">{{ title }}</a></li>
 {% elif request.path == url_for('.corpus', corpus_id=corpus.id) %}
@@ -12,12 +12,12 @@
 <li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 <li class="tab"><a class="active" href="{{ url_for('.analyse_corpus', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
-{% elif request.path == url_for('.add_corpus_file', corpus_id=corpus.id) %}
+{% elif request.path == url_for('.create_corpus_file', corpus_id=corpus.id) %}
 <li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 <li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id, _anchor='files') }}" target="_self">Corpus files</a></li>
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-<li class="tab"><a class="active" href="{{ url_for('.add_corpus_file', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
+<li class="tab"><a class="active" href="{{ url_for('.create_corpus_file', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
 {% elif request.path == url_for('.corpus_file', corpus_file_id=corpus_file.id, corpus_id=corpus.id) %}
 <li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>

app/templates/corpora/corpus.html.j2

@@ -65,38 +65,21 @@
         </div>
         <div class="card-action right-align">
           <a class="btn corpus-analyse-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
-          <a class="btn corpus-build-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.build_corpus', corpus_id=corpus.id) }}"><i class="nopaque-icons left">K</i>Build</a>
+          <a class="action-button btn disabled waves-effect waves-light" data-action="build-request"><i class="nopaque-icons left">K</i>Build</a>
           <a class="btn disabled export-corpus-trigger waves-effect waves-light" href="{{ url_for('corpora.export_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">import_export</i>Export</a>
-          <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-corpus-modal"><i class="material-icons left">delete</i>Delete</a>
+          <a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
         </div>
       </div>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-corpus-id="{{ corpus.hashid }}" data-ressource-type="CorpusFile" data-user-id="{{ corpus.user.hashid }}">
+    <div class="col s12">
       <div class="card">
         <div class="card-content">
           <span class="card-title" id="files">Corpus files</span>
-          <div class="input-field">
+          <div class="corpus-file-list" data-user-id="{{ corpus.user.hashid }}" data-corpus-id="{{ corpus.hashid }}"></div>
-            <i class="material-icons prefix">search</i>
-            <input class="search" id="search-corpus-files" type="search"></input>
-            <label for="search-corpus-files">Search corpus files</label>
-          </div>
-          <table>
-            <thead>
-              <tr>
-                <th>Filename</th>
-                <th>Author</th>
-                <th>Title</th>
-                <th>Publishing year</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
-          <a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
+          <a href="{{ url_for('corpora.create_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
         </div>
       </div>
     </div>
@@ -104,20 +87,6 @@
 </div>
 {% endblock page_content %}
 
-{% block modals %}
-{{ super() }}
-<div id="delete-corpus-modal" class="modal">
-  <div class="modal-content">
-    <h4>Confirm corpus deletion</h4>
-    <p>Do you really want to delete the corpus <span class="corpus-title"></span>? All files will be permanently deleted!</p>
-  </div>
-  <div class="modal-footer">
-    <a class="btn modal-close waves-effect waves-light" href="#!">Cancel</a>
-    <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">delete</i>Delete</a>
-  </div>
-</div>
-{% endblock modals %}
-
 {% block scripts %}
 {{ super() }}
 <script>

app/templates/corpora/create_corpus_file.html.j2

@@ -13,11 +13,10 @@
 
     <div class="col s12 m4">
       <p>Fill out the following form to add a corpus file in verticalized text format (.vrt).</p>
-      <p><b>Do not use the .stand-off.vrt file!</b></p>
     </div>
 
     <div class="col s12 m8">
-      <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+      <form class="create-corpus-file-form" enctype="multipart/form-data" method="POST">
         <div class="card">
           <div class="card-content">
             {{ form.hidden_tag() }}
@@ -52,39 +51,8 @@
             </div>
           </li>
         </ul>
-        <br>
-        <ul class="collapsible hoverable">
-          <li>
-            <div class="collapsible-header"><i class="material-icons">add</i>Add metadata with BibTex</div>
-            <div class="collapsible-body">
-              <span>
-                <div class="row">
-                  <div class="col s12">
-
-                  </div>
-                </div>
-              </span>
-            </div>
-          </li>
-        </ul>
       </form>
     </div>
   </div>
 </div>
 {% endblock page_content %}
-
-
-{% block modals %}
-{{ super() }}
-<div id="progress-modal" class="modal">
-  <div class="modal-content">
-    <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
-    <div class="progress">
-      <div class="determinate" style="width: 0%"></div>
-    </div>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="modal-close waves-effect waves-light btn red abort-request">Cancel</a>
-  </div>
-</div>
-{% endblock modals %}

app/templates/errors/403.html.j2

@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
-  <h1 id="title">{{ title }}</h1>
-  <p class="light">{{ request.path }}</p>
-  <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
-  <div class="modal-content">
-    <h2>About the "{{ title }}" error</h2>
-    <p>The request contained valid data and was understood by the server, but the server is refusing action. This may be due to the user not having the necessary permissions for a resource or needing an account of some sort, or attempting a prohibited action (e.g. creating a duplicate record where only one is allowed). This code is also typically used if the request provided authentication by answering the WWW-Authenticate header field challenge, but the server did not accept that authentication. The request should not be repeated.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/errors/404.html.j2

@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
-  <h1 id="title">{{ title }}</h1>
-  <p class="light">{{ request.path }}</p>
-  <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
-  <div class="modal-content">
-    <h2>About the "{{ title }}" error</h2>
-    <p>The requested resource could not be found but may be available in the future. Subsequent requests by the client are permissible.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/errors/413.html.j2

@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
-  <h1 id="title">{{ title }}</h1>
-  <p class="light">{{ request.path }}</p>
-  <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
-  <div class="modal-content">
-    <h2>About the "{{ title }}" error</h2>
-    <p>The request is larger than the server is willing or able to process. Previously called "Request Entity Too Large".</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/errors/500.html.j2

@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
-  <h1 id="title">{{ title }}</h1>
-  <p class="light">{{ request.path }}</p>
-  <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
-  <div class="modal-content">
-    <h2>About the "{{ title }}" error</h2>
-    <p>A generic error message, given when an unexpected condition was encountered and no more specific message is suitable.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/errors/503.html.j2

@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
-  <h1 id="title">{{ title }}</h1>
-  <p class="light">{{ request.path }}</p>
-  <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
-  <div class="modal-content">
-    <h2>About the "{{ title }}" error</h2>
-    <p>The server cannot handle the request (because it is overloaded or down for maintenance). Generally, this is a temporary state.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/errors/error.html.j2

@@ -0,0 +1,10 @@
+{% extends "base.html.j2" %}
+
+{% set title = error.name %}
+
+{% block page_content %}
+<div class="container">
+  <h1 id="title">{{ error.name }}</h1>
+  <p>{{ error.description }}</p>
+</div>
+{% endblock page_content %}

app/templates/jobs/_breadcrumbs.html.j2

@@ -1,8 +1,8 @@
 {% set breadcrumbs %}
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 <li class="tab"><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" target="_self">My jobs</a></li>
-<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 {% if request.path == url_for('.job', job_id=job.id) %}
+<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
 <li class="tab"><a class="active" href="{{ url_for('.job', job_id=job.id) }}" target="_self">{{ job.title }}</a></li>
 {% endif %}
 {% endset %}

app/templates/jobs/job.html.j2

@@ -79,60 +79,31 @@
         </div>
         <div class="card-action right-align">
           {% if current_user.is_administrator()  %}
-          <a class="btn hide modal-trigger restart-job-trigger waves-effect waves-light" data-target="restart-job-modal"><i class="material-icons left">repeat</i>Restart</a>
+          <a class="action-button btn disabled waves-effect waves-light" data-action="get-log-request"><i class="material-icons left">text_snippet</i>Log</a>
           {% endif %}
-          <!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> -->
+          <a class="action-button btn disabled waves-effect waves-light" data-action="restart-request"><i class="material-icons left">repeat</i>Restart</a>
-          <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-job-modal"><i class="material-icons left">delete</i>Delete</a>
+          <a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
         </div>
       </div>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-job-id="{{ job.hashid }}" data-ressource-type="JobInput" data-user-id="{{ job.user.hashid }}">
+    <div class="col s12">
       <div class="card">
         <div class="card-content">
           <div class="row">
-            <div class="col s12 m2">
+            <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
-              <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
+            <div class="job-input-list" data-user-id="{{ job.user.hashid }}" data-job-id="{{ job.hashid }}"></div>
-              <p>Original input files.</p>
-            </div>
-            <div class="col s12 m10">
-              <table>
-                <thead>
-                  <tr>
-                    <th>Filename</th>
-                    <th></th>
-                  </tr>
-                </thead>
-                <tbody class="list"></tbody>
-              </table>
-              <ul class="pagination"></ul>
-            </div>
           </div>
         </div>
       </div>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-job-id="{{ job.hashid }}" data-ressource-type="JobResult" data-user-id="{{ job.user.hashid }}">
+    <div class="col s12">
       <div class="card">
         <div class="card-content">
           <div class="row">
-            <div class="col s12 m2">
+            <span class="card-title"><i class="left material-icons" style="font-size: inherit;">done</i>Results</span>
-              <span class="card-title"><i class="left material-icons" style="font-size: inherit;">done</i>Results</span>
+            <div class="job-result-list" data-user-id="{{ job.user.hashid }}" data-job-id="{{ job.hashid }}"></div>
-              <p>Processed result files.</p>
-            </div>
-            <div class="col s12 m10">
-              <table>
-                <thead>
-                  <tr>
-                    <th>Description</th>
-                    <th>Filename</th>
-                    <th></th>
-                  </tr>
-                </thead>
-                <tbody class="list"></tbody>
-              </table>
-              <ul class="pagination"></ul>
-            </div>
           </div>
         </div>
       </div>
@@ -141,32 +112,6 @@
 </div>
 {% endblock page_content %}
 
-{% block modals %}
-{{ super() }}
-<div id="delete-job-modal" class="modal">
-  <div class="modal-content">
-    <h4>Confirm deletion</h4>
-    <p>Do you really want to delete the job <span class="job-title"></span>? All associated files will be permanently deleted.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-    <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
-  </div>
-</div>
-
-{% if current_user.is_administrator() %}
-<div id="restart-job-modal" class="modal">
-  <div class="modal-content">
-    <h4>Confirm restart</h4>
-    <p>Do you really want to restart the job <span class="job-title"></span>? All log and result files will be permanently deleted.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-    <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.restart', job_id=job.id) }}"><i class="material-icons left">restart</i>Restart</a>
-  </div>
-</div>
-{% endif %}
-{% endblock modals %}
 
 {% block scripts %}
 {{ super() }}

app/templates/main/dashboard.html.j2

@@ -6,100 +6,48 @@
   <div class="row">
     <div class="col s12">
       <h1 id="title">{{ title }}</h1>
-    </div>
 
-    <div class="col s12">
       <h3>My Corpora and Query results</h3>
       <p>Create a corpus to interactively perform linguistic analysis or import query results to save interesting passages.</p>
+
       <div class="row">
         <div class="col s12">
           <ul class="tabs">
             <li class="tab col s6"><a class="active" href="#corpora">Corpora</a></li>
-            <li class="tab col s6"><a href="#query-results">Query results</a></li>
+            <li class="tab col s6 disabled"><a href="#query-results">Query results</a></li>
           </ul>
         </div>
-        <div class="col s12 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ current_user.hashid }}" id="corpora">
+        <div class="col s12" id="corpora">
           <div class="card">
             <div class="card-content">
-              <div class="input-field">
+              <div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
-                <i class="material-icons prefix">search</i>
-                <input id="search-corpus" class="search" type="search"></input>
-                <label for="search-corpus">Search corpus</label>
-              </div>
-              <table>
-                <thead>
-                  <tr>
-                    <th></th>
-                    <th>Title and Description</th>
-                    <th>Status</th>
-                    <th></th>
-                  </tr>
-                </thead>
-                <tbody class="list"></tbody>
-              </table>
-              <ul class="pagination"></ul>
             </div>
             <div class="card-action right-align">
-              <a class="btn waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
+              <a class="btn disabled waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
-              <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
+              <a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
-            </div>
-          </div>
-        </div>
-        {# <div class="col s12 nopaque-ressource-list" data-ressource-type="QueryResult" data-user-id="{{ current_user.hashid }}" id="query-results"> #}
-        <div class="col s12" id="query-results">
-          <div class="card">
-            <div class="card-content">
-              <div class="input-field">
-                <i class="material-icons prefix">search</i>
-                <input id="search-query-results" class="search" type="search"></input>
-                <label for="search-query-results">Search query result</label>
-              </div>
-              <table>
-                <thead>
-                  <tr>
-                    <th>Title and Description</th>
-                    <th>Corpus and Query</th>
-                    <th></th>
-                  </tr>
-                </thead>
-                <tbody class="list"></tbody>
-              </table>
-              <ul class="pagination"></ul>
-            </div>
-            <div class="card-action right-align">
-              <a class="waves-effect waves-light btn disabled">Add query result<i class="material-icons right">file_upload</i></a>
             </div>
           </div>
         </div>
       </div>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-ressource-type="Job" data-user-id="{{ current_user.hashid }}" id="jobs">
+    <div class="col s12" id="jobs">
       <h3>My Jobs</h3>
-      <p>A job is the execution of a service provided by nopaque. You can create any number of jobs and let them be processed simultaneously.</p>
+      <p>
+        A job is the execution of a service provided by nopaque. You can
+        create any number of jobs and let them be processed simultaneously. We
+        <b>strongly recommend</b> that you create a folder on your computer where you
+        save the various files that nopaque provides you with after each
+        pre-processing step. You will need the result of each step for the
+        next step.
+      </p>
+      <p><b>Where is my Job data?</b> Don't worry, please read <a href="{{ url_for('main.news', _anchor='april-2022-update') }}">this news</a> entry</p>
       <div class="card">
         <div class="card-content">
-          <p><b>Where is my Job data?</b> Don't worry, please read <a href="{{ url_for('main.news', _anchor='april-2022-update') }}">this news</a> entry</p>
+          <div class="job-list" data-user-id="{{ current_user.hashid }}"></div>
-          <div class="input-field">
-            <i class="material-icons prefix">search</i>
-            <input id="search-job" class="search" type="search"></input>
-            <label for="search-job">Search job</label>
-          </div>
-          <table>
-            <thead>
-              <tr>
-                <th>Service</th>
-                <th>Title and Description</th>
-                <th>Status</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
-          <p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p>
+          <p><a class="btn modal-trigger waves-effect waves-light" data-target="create-job-modal"><i class="material-icons left">add</i>Create job</a></p>
         </div>
       </div>
     </div>
@@ -109,7 +57,7 @@
 
 {% block modals %}
 {{ super() }}
-<div id="new-job-modal" class="modal">
+<div id="create-job-modal" class="modal">
   <div class="modal-content">
     <h4>Select a service</h4>
     <p>&nbsp;</p>
@@ -153,7 +101,7 @@
     </div>
   </div>
   <div class="modal-footer">
-    <a href="#!" class="modal-close waves-effect waves-light btn-flat">Close</a>
+    <a class="btn-flat modal-close waves-effect waves-light">Close</a>
   </div>
 </div>
 {% endblock modals %}

app/templates/services/corpus_analysis.html.j2

@@ -22,36 +22,20 @@
       <p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ current_user.hashid }}" id="corpora">
+    <div class="col s12" id="corpora">
       <h2>My Corpora</h2>
       <div class="card">
         <div class="card-content">
-          <div class="input-field">
+          <div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
-            <i class="material-icons prefix">search</i>
-            <input id="search-corpus" class="search" type="search"></input>
-            <label for="search-corpus">Search corpus</label>
-          </div>
-          <table class="highlight">
-            <thead>
-              <tr>
-                <th></th>
-                <th>Title and Description</th>
-                <th>Status</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody class="list"></tbody>
-          </table>
-          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
-          <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
+          <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
-          <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
+          <a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
         </div>
       </div>
     </div>
 
-    <div class="col s12 nopaque-ressource-list" data-ressource-type="QueryResult" data-user-id="{{ current_user.hashid }}" id="query-results">
+    <div class="col s12 query-result-list" data-user-id="{{ current_user.hashid }}" id="query-results">
       <h2>My query results</h2>
       <div class="card">
         <div class="card-content">

app/templates/services/file_setup_pipeline.html.j2

@@ -39,7 +39,7 @@
     <div class="col s12">
       <h2>Submit a job</h2>
       <div class="card">
-        <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+        <form class="create-job-form" enctype="multipart/form-data" method="POST">
           <div class="card-content">
             {{ form.hidden_tag() }}
             <div class="row">

app/templates/services/spacy_nlp_pipeline.html.j2

@@ -57,7 +57,7 @@
     <div class="col s12">
       <h2>Submit a job</h2>
       <div class="card">
-        <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+        <form class="create-job-form" enctype="multipart/form-data" method="POST">
           <div class="card-content">
             {{ form.hidden_tag() }}
             <div class="row">

app/templates/services/tesseract_ocr_pipeline.html.j2

@@ -39,7 +39,7 @@
     <div class="col s12">
       <h2>Submit a job</h2>
       <div class="card">
-        <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+        <form class="create-job-form" enctype="multipart/form-data" method="POST">
           <div class="card-content">
             {{ form.hidden_tag() }}
             <div class="row">
@@ -178,28 +178,4 @@
     <a href="#!" class="modal-close waves-effect waves-light btn">Close</a>
   </div>
 </div>
-
-<div id="progress-modal" class="modal">
-  <div class="modal-content">
-    <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
-    <div class="progress">
-      <div class="determinate" style="width: 0%"></div>
-    </div>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="modal-close waves-effect waves-light btn red abort-request">Cancel</a>
-  </div>
-</div>
 {% endblock modals %}
-
-{% block scripts %}
-{{ super() }}
-<script>
-  let versionField = document.querySelector('#add-job-form-version');
-  versionField.addEventListener('change', (event) => {
-    let url = new URL(window.location.href);
-    url.search = `?version=${event.target.value}`;
-    window.location.href = url.toString();
-  });
-</script>
-{% endblock scripts %}

app/templates/services/transkribus_htr_pipeline.html.j2

@@ -44,7 +44,7 @@
     <div class="col s12">
       <h2>Submit a job</h2>
       <div class="card">
-        <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+        <form class="create-job-form" enctype="multipart/form-data" method="POST">
           <div class="card-content">
             {{ form.hidden_tag() }}
             <div class="row">

app/templates/settings/_breadcrumbs.html.j2

@@ -1,6 +1,6 @@
 {% set breadcrumbs %}
 <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-{% if request.path == url_for('settings.index') %}
+{% if request.path == url_for('settings.settings') %}
-<li class="tab"><a{%if request.path == url_for('settings.index') %} class="active"{% endif %} href="{{ url_for('settings.index') }}" target="_self">Settings</a></li>
+<li class="tab"><a{%if request.path == url_for('settings.settings') %} class="active"{% endif %} href="{{ url_for('settings.settings') }}" target="_self">Settings</a></li>
 {% endif %}
 {% endset %}