Merge branch 'development'

.gitignore

@@ -6,6 +6,8 @@ logs/
 !logs/dummy
 *.env
 
+*.pjentsch-testing
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.vscode/settings.json

@@ -1 +1,23 @@
-{}
+{
+    "editor.rulers": [79],
+    "files.insertFinalNewline": true,
+    "python.terminal.activateEnvironment": false,
+    "[css]": {
+        "editor.tabSize": 2
+    },
+    "[scss]": {
+        "editor.tabSize": 2
+    },
+    "[html]": {
+        "editor.tabSize": 2
+    },
+    "[javascript]": {
+        "editor.tabSize": 2
+    },
+    "[jinja-html]": {
+        "editor.tabSize": 2
+    },
+    "[jinja-js]": {
+        "editor.tabSize": 2
+    }
+}

app/__init__.py

@@ -4,6 +4,7 @@ from docker import DockerClient
 from flask import Flask
 from flask_apscheduler import APScheduler
 from flask_assets import Environment
+from flask_breadcrumbs import Breadcrumbs, default_breadcrumb_root
 from flask_login import LoginManager
 from flask_mail import Mail
 from flask_marshmallow import Marshmallow
@@ -12,10 +13,12 @@ from flask_paranoid import Paranoid
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 from flask_hashids import Hashids
+from werkzeug.exceptions import HTTPException
 
 
 apifairy = APIFairy()
 assets = Environment()
+breadcrumbs = Breadcrumbs()
 db = SQLAlchemy()
 docker_client = DockerClient()
 hashids = Hashids()
@@ -33,7 +36,7 @@ socketio = SocketIO()
 
 def create_app(config: Config = Config) -> Flask:
     ''' Creates an initialized Flask (WSGI Application) object. '''
-    app: Flask = Flask(__name__)
+    app = Flask(__name__)
     app.config.from_object(config)
     config.init_app(app)
     docker_client.login(
@@ -44,6 +47,7 @@ def create_app(config: Config = Config) -> Flask:
 
     apifairy.init_app(app)
     assets.init_app(app)
+    breadcrumbs.init_app(app)
     db.init_app(app)
     hashids.init_app(app)
     login.init_app(app)
@@ -55,36 +59,45 @@ def create_app(config: Config = Config) -> Flask:
     socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
 
     from .admin import bp as admin_blueprint
+    default_breadcrumb_root(admin_blueprint, '.admin')
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
 
     from .api import bp as api_blueprint
     app.register_blueprint(api_blueprint, url_prefix='/api')
 
     from .auth import bp as auth_blueprint
-    app.register_blueprint(auth_blueprint, url_prefix='/auth')
+    default_breadcrumb_root(auth_blueprint, '.')
+    app.register_blueprint(auth_blueprint)
 
     from .contributions import bp as contributions_blueprint
+    default_breadcrumb_root(contributions_blueprint, '.contributions')
     app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
     from .corpora import bp as corpora_blueprint
-    app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
+    default_breadcrumb_root(corpora_blueprint, '.corpora')
+    app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
 
-    from .errors import bp as errors_blueprint
-    app.register_blueprint(errors_blueprint)
+    from .errors import bp as errors_bp
+    app.register_blueprint(errors_bp)
 
     from .jobs import bp as jobs_blueprint
+    default_breadcrumb_root(jobs_blueprint, '.jobs')
     app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
 
     from .main import bp as main_blueprint
-    app.register_blueprint(main_blueprint, url_prefix='/')
+    default_breadcrumb_root(main_blueprint, '.')
+    app.register_blueprint(main_blueprint, cli_group=None)
 
     from .services import bp as services_blueprint
+    default_breadcrumb_root(services_blueprint, '.services')
     app.register_blueprint(services_blueprint, url_prefix='/services')
 
     from .settings import bp as settings_blueprint
+    default_breadcrumb_root(settings_blueprint, '.settings')
     app.register_blueprint(settings_blueprint, url_prefix='/settings')
 
     from .users import bp as users_blueprint
+    default_breadcrumb_root(users_blueprint, '.users')
     app.register_blueprint(users_blueprint, url_prefix='/users')
 
     return app

app/admin/__init__.py

@@ -1,5 +1,20 @@
 from flask import Blueprint
+from flask_login import login_required
+from app.decorators import admin_required
 
 
 bp = Blueprint('admin', __name__)
-from . import routes
+
+
+@bp.before_request
+@login_required
+@admin_required
+def before_request():
+    '''
+    Ensures that the routes in this package can be visited only by users with
+    administrator privileges (login_required and admin_required).
+    '''
+    pass
+
+
+from . import json_routes, routes

app/admin/forms.py

@@ -1,13 +1,16 @@
-from app.models import Role
 from flask_wtf import FlaskForm
-from wtforms import BooleanField, SelectField, SubmitField
+from wtforms import SelectField, SubmitField
+from app.models import Role
 
 
-class AdminEditUserForm(FlaskForm):
-    confirmed = BooleanField('Confirmed')
+class UpdateUserForm(FlaskForm):
     role = SelectField('Role')
-    submit = SubmitField('Submit')
+    submit = SubmitField()
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, user, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = {'role': user.role.hashid}
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-user-form'
         super().__init__(*args, **kwargs)
         self.role.choices = [(x.hashid, x.name) for x in Role.query.all()]

app/admin/json_routes.py

@@ -0,0 +1,23 @@
+from flask import abort, request
+from app import db
+from app.decorators import content_negotiation
+from app.models import User
+from . import bp
+
+
+@bp.route('/users/<hashid:user_id>/confirmed', methods=['PUT'])
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_user_role(user_id):
+    confirmed = request.json
+    if not isinstance(confirmed, bool):
+        abort(400)
+    user = User.query.get_or_404(user_id)
+    user.confirmed = confirmed
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'User "{user.username}" is now '
+            f'{"confirmed" if confirmed else "unconfirmed"}'
+        )
+    }
+    return response_data, 200

app/admin/routes.py

@@ -1,111 +1,146 @@
-from flask import current_app, flash, redirect, render_template, url_for
-from flask_login import login_required
-from threading import Thread
+from flask import abort, flash, redirect, render_template, url_for
+from flask_breadcrumbs import register_breadcrumb
 from app import db, hashids
-from app.decorators import admin_required
-from app.models import Role, User, UserSettingJobStatusMailNotificationLevel
-from app.settings.forms import (
-    EditNotificationSettingsForm
+from app.models import Avatar, Corpus, Role, User
+from app.users.settings.forms import (
+    UpdateAvatarForm,
+    UpdatePasswordForm,
+    UpdateNotificationsForm,
+    UpdateAccountInformationForm,
+    UpdateProfileInformationForm
 )
-from app.users.forms import EditProfileSettingsForm
 from . import bp
-from .forms import AdminEditUserForm
-
-
-@bp.before_request
-@login_required
-@admin_required
-def before_request():
-    '''
-    Ensures that the routes in this package can be visited only by users with
-    administrator privileges (login_required and admin_required).
-    '''
-    pass
+from .forms import UpdateUserForm
+from app.users.utils import (
+    user_endpoint_arguments_constructor as user_eac,
+    user_dynamic_list_constructor as user_dlc
+)
 
 
 @bp.route('')
-def index():
-    return redirect(url_for('.users'))
+@register_breadcrumb(bp, '.', '<i class="material-icons left">admin_panel_settings</i>Administration')
+def admin():
+    return render_template(
+        'admin/admin.html.j2',
+        title='Administration'
+    )
+
+
+@bp.route('/corpora')
+@register_breadcrumb(bp, '.corpora', 'Corpora')
+def corpora():
+    corpora = Corpus.query.all()
+    return render_template(
+        'admin/corpora.html.j2',
+        title='Corpora',
+        corpora=corpora
+    )
 
 
 @bp.route('/users')
+@register_breadcrumb(bp, '.users', '<i class="material-icons left">group</i>Users')
 def users():
-    users = [x.to_json_serializeable(backrefs=True) for x in User.query.all()]
+    users = User.query.all()
     return render_template(
         'admin/users.html.j2',
-        users=users,
-        title='Users'
+        title='Users',
+        users=users
     )
 
 
 @bp.route('/users/<hashid:user_id>')
+@register_breadcrumb(bp, '.users.entity', '', dynamic_list_constructor=user_dlc)
 def user(user_id):
     user = User.query.get_or_404(user_id)
-    return render_template('admin/user.html.j2', title='User', user=user)
+    corpora = Corpus.query.filter(Corpus.user == user).all()
+    return render_template(
+        'admin/user.html.j2',
+        title=user.username,
+        user=user,
+        corpora=corpora
+    )
 
 
-@bp.route('/users/<hashid:user_id>/edit', methods=['GET', 'POST'])
-def edit_user(user_id):
+@bp.route('/users/<hashid:user_id>/settings', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.users.entity.settings', '<i class="material-icons left">settings</i>Settings')
+def user_settings(user_id):
     user = User.query.get_or_404(user_id)
-    admin_edit_user_form = AdminEditUserForm(
-        data={'confirmed': user.confirmed, 'role': user.role.hashid},
-        prefix='admin-edit-user-form'
+    update_account_information_form = UpdateAccountInformationForm(user)
+    update_profile_information_form = UpdateProfileInformationForm(user)
+    update_avatar_form = UpdateAvatarForm()
+    update_password_form = UpdatePasswordForm(user)
+    update_notifications_form = UpdateNotificationsForm(user)
+    update_user_form = UpdateUserForm(user)
+
+    # region handle update profile information form
+    if update_profile_information_form.submit.data and update_profile_information_form.validate():
+        user.about_me = update_profile_information_form.about_me.data
+        user.location = update_profile_information_form.location.data
+        user.organization = update_profile_information_form.organization.data
+        user.website = update_profile_information_form.website.data
+        user.full_name = update_profile_information_form.full_name.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update profile information form
+
+    # region handle update avatar form
+    if update_avatar_form.submit.data and update_avatar_form.validate():
+        try:
+            Avatar.create(
+                update_avatar_form.avatar.data,
+                user=user
             )
-    edit_profile_settings_form = EditProfileSettingsForm(
-        user,
-        data=user.to_json_serializeable(),
-        prefix='edit-profile-settings-form'
-    )
-    edit_notification_settings_form = EditNotificationSettingsForm(
-        data=user.to_json_serializeable(),
-        prefix='edit-notification-settings-form'
-    )
-    if (admin_edit_user_form.submit.data
-            and admin_edit_user_form.validate()):
-        user.confirmed = admin_edit_user_form.confirmed.data
-        role_id = hashids.decode(admin_edit_user_form.role.data)
+        except (AttributeError, OSError):
+            abort(500)
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update avatar form
+
+    # region handle update account information form
+    if update_account_information_form.submit.data and update_account_information_form.validate():
+        user.email = update_account_information_form.email.data
+        user.username = update_account_information_form.username.data
+        db.session.commit()
+        flash('Profile settings updated')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update account information form
+
+    # region handle update password form
+    if update_password_form.submit.data and update_password_form.validate():
+        user.password = update_password_form.new_password.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update password form
+
+    # region handle update notifications form
+    if update_notifications_form.submit.data and update_notifications_form.validate():
+        user.setting_job_status_mail_notification_level = \
+            update_notifications_form.job_status_mail_notification_level.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update notifications form
+
+    # region handle update user form
+    if update_user_form.submit.data and update_user_form.validate():
+        role_id = hashids.decode(update_user_form.role.data)
         user.role = Role.query.get(role_id)
         db.session.commit()
         flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
-    if (edit_profile_settings_form.submit.data
-            and edit_profile_settings_form.validate()):
-        user.email = edit_profile_settings_form.email.data
-        user.username = edit_profile_settings_form.username.data
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
-    if (edit_notification_settings_form.submit.data
-            and edit_notification_settings_form.validate()):
-        user.setting_job_status_mail_notification_level = \
-            UserSettingJobStatusMailNotificationLevel[
-                edit_notification_settings_form.job_status_mail_notification_level.data  # noqa
-            ]
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
+        return redirect(url_for('.user_settings', user_id=user.id))
+    # endregion handle update user form
+
     return render_template(
-        'admin/edit_user.html.j2',
-        admin_edit_user_form=admin_edit_user_form,
-        edit_profile_settings_form=edit_profile_settings_form,
-        edit_notification_settings_form=edit_notification_settings_form,
-        title='Edit user',
+        'admin/user_settings.html.j2',
+        title='Settings',
+        update_account_information_form=update_account_information_form,
+        update_avatar_form=update_avatar_form,
+        update_notifications_form=update_notifications_form,
+        update_password_form=update_password_form,
+        update_profile_information_form=update_profile_information_form,
+        update_user_form=update_user_form,
         user=user
     )
-
-
-@bp.route('/users/<hashid:user_id>/delete', methods=['DELETE'])
-def delete_user(user_id):
-    def _delete_user(app, user_id):
-        with app.app_context():
-            user = User.query.get(user_id)
-            user.delete()
-            db.session.commit()
-
-    User.query.get_or_404(user_id)
-    thread = Thread(
-        target=_delete_user,
-        args=(current_app._get_current_object(), user_id)
-    )
-    thread.start()
-    return {}, 202

app/api/schemas.py

@@ -2,7 +2,6 @@ from apifairy.fields import FileField
 from marshmallow import validate, validates, ValidationError
 from marshmallow.decorators import post_dump
 from app import ma
-from app.auth import USERNAME_REGEX
 from app.models import (
     Job,
     JobStatus,
@@ -142,7 +141,10 @@ class UserSchema(ma.SQLAlchemySchema):
     username = ma.auto_field(
         validate=[
             validate.Length(min=1, max=64),
-            validate.Regexp(USERNAME_REGEX, error='Usernames must have only letters, numbers, dots or underscores')
+            validate.Regexp(
+                User.username_pattern,
+                error='Usernames must have only letters, numbers, dots or underscores'
+            )
         ]
     )
     email = ma.auto_field(validate=validate.Email())

app/auth/__init__.py

@@ -1,8 +1,5 @@
 from flask import Blueprint
 
 
-USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
-
-
 bp = Blueprint('auth', __name__)
 from . import routes

app/auth/forms.py

@@ -8,7 +8,6 @@ from wtforms import (
 )
 from wtforms.validators import InputRequired, Email, EqualTo, Length, Regexp
 from app.models import User
-from . import USERNAME_REGEX
 
 
 class RegistrationForm(FlaskForm):
@@ -22,7 +21,7 @@ class RegistrationForm(FlaskForm):
             InputRequired(),
             Length(max=64),
             Regexp(
-                USERNAME_REGEX,
+                User.username_pattern,
                 message=(
                     'Usernames must have only letters, numbers, dots or '
                     'underscores'
@@ -44,8 +43,17 @@ class RegistrationForm(FlaskForm):
             EqualTo('password', message='Passwords must match')
         ]
     )
+    terms_of_use_accepted = BooleanField(
+        'I have read and accept the terms of use',
+        validators=[InputRequired()]
+    )
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'registration-form'
+        super().__init__(*args, **kwargs)
+
     def validate_email(self, field):
         if User.query.filter_by(email=field.data.lower()).first():
             raise ValidationError('Email already registered')
@@ -61,11 +69,21 @@ class LoginForm(FlaskForm):
     remember_me = BooleanField('Keep me logged in')
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'login-form'
+        super().__init__(*args, **kwargs)
+
 
 class ResetPasswordRequestForm(FlaskForm):
     email = StringField('Email', validators=[InputRequired(), Email()])
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-request-form'
+        super().__init__(*args, **kwargs)
+
 
 class ResetPasswordForm(FlaskForm):
     password = PasswordField(
@@ -83,3 +101,8 @@ class ResetPasswordForm(FlaskForm):
         ]
     )
     submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-form'
+        super().__init__(*args, **kwargs)

app/auth/routes.py

@@ -1,11 +1,5 @@
-from flask import (
-    abort,
-    flash,
-    redirect,
-    render_template,
-    request,
-    url_for
-)
+from flask import abort, flash, redirect, render_template, request, url_for
+from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user, login_user, login_required, logout_user
 from app import db
 from app.email import create_message, send
@@ -36,16 +30,18 @@ def before_request():
 
 
 @bp.route('/register', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.register', 'Register')
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = RegistrationForm(prefix='registration-form')
+    form = RegistrationForm()
     if form.validate_on_submit():
         try:
             user = User.create(
                 email=form.email.data.lower(),
                 password=form.password.data,
-                username=form.username.data
+                username=form.username.data,
+                terms_of_use_accepted=form.terms_of_use_accepted.data
             )
         except OSError:
             flash('Internal Server Error', category='error')
@@ -65,16 +61,17 @@ def register():
         return redirect(url_for('.login'))
     return render_template(
         'auth/register.html.j2',
-        form=form,
-        title='Register'
+        title='Register',
+        form=form
     )
 
 
 @bp.route('/login', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.login', 'Login')
 def login():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = LoginForm(prefix='login-form')
+    form = LoginForm()
     if form.validate_on_submit():
         user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
         if user and user.verify_password(form.password.data):
@@ -85,7 +82,11 @@ def login():
             flash('You have been logged in')
             return redirect(next)
         flash('Invalid email/username or password', category='error')
-    return render_template('auth/login.html.j2', form=form, title='Log in')
+    return render_template(
+        'auth/login.html.j2',
+        title='Log in',
+        form=form
+    )
 
 
 @bp.route('/logout')
@@ -97,14 +98,18 @@ def logout():
 
 
 @bp.route('/unconfirmed')
+@register_breadcrumb(bp, '.unconfirmed', 'Unconfirmed')
 @login_required
 def unconfirmed():
     if current_user.confirmed:
         return redirect(url_for('main.dashboard'))
-    return render_template('auth/unconfirmed.html.j2', title='Unconfirmed')
+    return render_template(
+        'auth/unconfirmed.html.j2',
+        title='Unconfirmed'
+    )
 
 
-@bp.route('/confirm')
+@bp.route('/confirm-request')
 @login_required
 def confirm_request():
     if current_user.confirmed:
@@ -135,11 +140,12 @@ def confirm(token):
     return redirect(url_for('.unconfirmed'))
 
 
-@bp.route('/reset_password', methods=['GET', 'POST'])
+@bp.route('/reset-password-request', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.reset_password_request', 'Password Reset')
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = ResetPasswordRequestForm(prefix='reset-password-request-form')
+    form = ResetPasswordRequestForm()
     if form.validate_on_submit():
         user = User.query.filter_by(email=form.email.data.lower()).first()
         if user is not None:
@@ -159,16 +165,17 @@ def reset_password_request():
         return redirect(url_for('.login'))
     return render_template(
         'auth/reset_password_request.html.j2',
-        form=form,
-        title='Password Reset'
+        title='Password Reset',
+        form=form
     )
 
 
-@bp.route('/reset_password/<token>', methods=['GET', 'POST'])
+@bp.route('/reset-password/<token>', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.reset_password', 'Password Reset')
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = ResetPasswordForm(prefix='reset-password-form')
+    form = ResetPasswordForm()
     if form.validate_on_submit():
         if User.reset_password(token, form.password.data):
             db.session.commit()
@@ -177,7 +184,7 @@ def reset_password(token):
         return redirect(url_for('main.index'))
     return render_template(
         'auth/reset_password.html.j2',
-        form=form,
         title='Password Reset',
+        form=form,
         token=token
     )

app/cli.py

@@ -1,72 +0,0 @@
-from flask import current_app
-from flask_migrate import upgrade
-import click
-import os
-from app.models import (
-    Role,
-    User,
-    TesseractOCRPipelineModel,
-    SpaCyNLPPipelineModel
-)
-
-
-def _make_default_dirs():
-    base_dir = current_app.config['NOPAQUE_DATA_DIR']
-
-    default_directories = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
-    ]
-    for directory in default_directories:
-        if os.path.exists(directory):
-            if not os.path.isdir(directory):
-                raise NotADirectoryError(f'{directory} is not a directory')
-        else:
-            os.mkdir(directory)
-
-
-def register(app):
-    @app.cli.command()
-    def deploy():
-        ''' Run deployment tasks. '''
-        # Make default directories
-        _make_default_dirs()
-
-        # migrate database to latest revision
-        upgrade()
-
-        # Insert/Update default database values
-        current_app.logger.info('Insert/Update default roles')
-        Role.insert_defaults()
-        current_app.logger.info('Insert/Update default users')
-        User.insert_defaults()
-        current_app.logger.info('Insert/Update default SpaCyNLPPipelineModels')
-        SpaCyNLPPipelineModel.insert_defaults()
-        current_app.logger.info('Insert/Update default TesseractOCRPipelineModels')
-        TesseractOCRPipelineModel.insert_defaults()
-
-    @app.cli.group()
-    def converter():
-        ''' Converter commands. '''
-        pass
-
-    @converter.command()
-    @click.argument('json_db')
-    @click.argument('data_dir')
-    def sandpaper(json_db, data_dir):
-        ''' Sandpaper converter '''
-        from app.converters.sandpaper import convert
-        convert(json_db, data_dir)
-
-    @app.cli.group()
-    def test():
-        ''' Test commands. '''
-        pass
-
-    @test.command('run')
-    def run_test():
-        ''' Run unit tests. '''
-        from unittest import TestLoader, TextTestRunner
-        from unittest.suite import TestSuite
-        tests: TestSuite = TestLoader().discover('tests')
-        TextTestRunner(verbosity=2).run(tests)

app/contributions/__init__.py

@@ -1,5 +1,23 @@
 from flask import Blueprint
+from flask_login import login_required
 
 
 bp = Blueprint('contributions', __name__)
-from . import routes
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import (
+    routes,
+    spacy_nlp_pipeline_models,
+    tesseract_ocr_pipeline_models,
+    transkribus_htr_pipeline_models
+)

app/contributions/forms.py

@@ -1,16 +1,11 @@
-from flask import current_app
 from flask_wtf import FlaskForm
-from flask_wtf.file import FileField, FileRequired
 from wtforms import (
-    BooleanField,
     StringField,
     SubmitField,
     SelectMultipleField,
-    IntegerField,
-    ValidationError
+    IntegerField
 )
 from wtforms.validators import InputRequired, Length
-from app.services import SERVICES
 
 
 class ContributionBaseForm(FlaskForm):
@@ -48,74 +43,5 @@ class ContributionBaseForm(FlaskForm):
     submit = SubmitField()
 
 
-class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
-    tesseract_model_file = FileField(
-        'File',
-        validators=[FileRequired()]
-    )
-    
-    def validate_tesseract_model_file(self, field):
-        if not field.data.filename.lower().endswith('.traineddata'):
-            raise ValidationError('traineddata files only!')
-
-    def __init__(self, *args, **kwargs):
-        service_manifest = SERVICES['tesseract-ocr-pipeline']
-        super().__init__(*args, **kwargs)
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
-    spacy_model_file = FileField(
-        'File',
-        validators=[FileRequired()]
-    )
-    pipeline_name = StringField(
-        'Pipeline name',
-        validators=[InputRequired(), Length(max=64)]
-    )
-
-    def validate_spacy_model_file(self, field):
-        if not field.data.filename.lower().endswith('.tar.gz'):
-            raise ValidationError('.tar.gz files only!')
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        service_manifest = SERVICES['spacy-nlp-pipeline']
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class EditContributionBaseForm(ContributionBaseForm):
+class UpdateContributionBaseForm(ContributionBaseForm):
     pass
-
-class EditTesseractOCRPipelineModelForm(EditContributionBaseForm):
-    def __init__(self, *args, **kwargs):
-        service_manifest = SERVICES['tesseract-ocr-pipeline']
-        super().__init__(*args, **kwargs)
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class EditSpaCyNLPPipelineModelForm(EditContributionBaseForm):
-    pipeline_name = StringField(
-        'Pipeline name',
-        validators=[InputRequired(), Length(max=64)]
-    )
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        service_manifest = SERVICES['spacy-nlp-pipeline']
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''

app/contributions/routes.py

@@ -1,233 +1,9 @@
-from flask import (
-    abort,
-    current_app,
-    flash,
-    Markup,
-    redirect,
-    render_template,
-    url_for
-)
-from flask_login import login_required, current_user
-from threading import Thread
-from app import db
-from app.decorators import permission_required 
-from app.models import (
-    Permission,
-    SpaCyNLPPipelineModel,
-    TesseractOCRPipelineModel
-)
+from flask import redirect, url_for
+from flask_breadcrumbs import register_breadcrumb
 from . import bp
-from .forms import (
-    CreateSpaCyNLPPipelineModelForm,
-    CreateTesseractOCRPipelineModelForm,
-    EditSpaCyNLPPipelineModelForm,
-    EditTesseractOCRPipelineModelForm
-)
 
 
-@bp.before_request
-@login_required
-def before_request():
-    pass
-
-
-@bp.route('/')
+@bp.route('')
+@register_breadcrumb(bp, '.', '<i class="material-icons left">new_label</i>My Contributions')
 def contributions():
-    return render_template(
-        'contributions/contributions.html.j2',
-        title='Contributions'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models')
-def tesseract_ocr_pipeline_models():
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_models.html.j2',
-        title='Tesseract OCR Pipeline Models'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
-def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    form = EditTesseractOCRPipelineModelForm(
-        data=tesseract_ocr_pipeline_model.to_json_serializeable(),
-        prefix='edit-tesseract-ocr-pipeline-model-form'
-    )
-    if form.validate_on_submit():
-        form.populate_obj(tesseract_ocr_pipeline_model)
-        if db.session.is_modified(tesseract_ocr_pipeline_model):
-            message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model.url}">{tesseract_ocr_pipeline_model.title}</a>" updated')
-            flash(message)
-            db.session.commit()
-        return redirect(url_for('.tesseract_ocr_pipeline_models'))
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_model.html.j2',
-        form=form,
-        tesseract_ocr_pipeline_model=tesseract_ocr_pipeline_model,
-        title=f'{tesseract_ocr_pipeline_model.title} {tesseract_ocr_pipeline_model.version}'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
-def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
-    def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
-        with app.app_context():
-            tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
-            tesseract_ocr_pipeline_model.delete()
-            db.session.commit()
-
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_tesseract_ocr_pipeline_model,
-        args=(current_app._get_current_object(), tesseract_ocr_pipeline_model_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
-def create_tesseract_ocr_pipeline_model():
-    form = CreateTesseractOCRPipelineModelForm(prefix='create-tesseract-ocr-pipeline-model-form')
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.create(
-                form.tesseract_model_file.data,
-                compatible_service_versions=form.compatible_service_versions.data,
-                description=form.description.data,
-                publisher=form.publisher.data,
-                publisher_url=form.publisher_url.data,
-                publishing_url=form.publishing_url.data,
-                publishing_year=form.publishing_year.data,
-                is_public=False,
-                title=form.title.data,
-                version=form.version.data,
-                user=current_user
-            )
-        except OSError:
-            abort(500)
-        db.session.commit()
-        tesseract_ocr_pipeline_model_url = url_for(
-            '.tesseract_ocr_pipeline_model',
-            tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id
-        )
-        message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model_url}">{tesseract_ocr_pipeline_model.title}</a>" created')
-        flash(message)
-        return {}, 201, {'Location': tesseract_ocr_pipeline_model_url}
-    return render_template(
-        'contributions/create_tesseract_ocr_pipeline_model.html.j2',
-        form=form,
-        title='Create Tesseract OCR Pipeline Model'
-    )
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/toggle-public-status', methods=['POST'])
-@permission_required(Permission.CONTRIBUTE)
-def toggle_tesseract_ocr_pipeline_model_public_status(tesseract_ocr_pipeline_model_id):
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    tesseract_ocr_pipeline_model.is_public = not tesseract_ocr_pipeline_model.is_public
-    db.session.commit()
-    return {}, 201
-
-
-@bp.route('/spacy-nlp-pipeline-models')
-def spacy_nlp_pipeline_models():
-    return render_template(
-        'contributions/spacy_nlp_pipeline_models.html.j2',
-        title='SpaCy NLP Pipeline Models'
-    )
-
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
-def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    form = EditSpaCyNLPPipelineModelForm(
-        data=spacy_nlp_pipeline_model.to_json_serializeable(),
-        prefix='edit-spacy-nlp-pipeline-model-form'
-    )
-    if form.validate_on_submit():
-        form.populate_obj(spacy_nlp_pipeline_model)
-        if db.session.is_modified(spacy_nlp_pipeline_model):
-            message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model.url}">{spacy_nlp_pipeline_model.title}</a>" updated')
-            flash(message)
-            db.session.commit()
-        return redirect(url_for('.spacy_nlp_pipeline_models'))
-    return render_template(
-        'contributions/spacy_nlp_pipeline_model.html.j2',
-        form=form,
-        spacy_nlp_pipeline_model=spacy_nlp_pipeline_model,
-        title=f'{spacy_nlp_pipeline_model.title} {spacy_nlp_pipeline_model.version}'
-    )
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
-def delete_spacy_model(spacy_nlp_pipeline_model_id):
-    def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
-        with app.app_context():
-            spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
-            spacy_nlp_pipeline_model.delete()
-            db.session.commit()
-    
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_spacy_model,
-        args=(current_app._get_current_object(), spacy_nlp_pipeline_model_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
-def create_spacy_nlp_pipeline_model():
-    form = CreateSpaCyNLPPipelineModelForm(prefix='create-spacy-nlp-pipeline-model-form')
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.create(
-                form.spacy_model_file.data,
-                compatible_service_versions=form.compatible_service_versions.data,
-                description=form.description.data,
-                pipeline_name=form.pipeline_name.data,
-                publisher=form.publisher.data,
-                publisher_url=form.publisher_url.data,
-                publishing_url=form.publishing_url.data,
-                publishing_year=form.publishing_year.data,
-                is_public=False,
-                title=form.title.data,
-                version=form.version.data,
-                user=current_user
-            )
-        except OSError:
-            abort(500)
-        db.session.commit()
-        spacy_nlp_pipeline_model_url = url_for(
-            '.spacy_nlp_pipeline_model',
-            spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id
-        )
-        message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model_url}">{spacy_nlp_pipeline_model.title}</a>" created')
-        flash(message)
-        return {}, 201, {'Location': spacy_nlp_pipeline_model_url}
-    return render_template(
-        'contributions/create_spacy_nlp_pipeline_model.html.j2',
-        form=form,
-        title='Create SpaCy NLP Pipeline Model'
-    )
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/toggle-public-status', methods=['POST'])
-@permission_required(Permission.CONTRIBUTE)
-def toggle_spacy_nlp_pipeline_model_public_status(spacy_nlp_pipeline_model_id):
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    spacy_nlp_pipeline_model.is_public = not spacy_nlp_pipeline_model.is_public
-    db.session.commit()
-    return {}, 201
+    return redirect(url_for('main.dashboard', _anchor='contributions'))

app/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/contributions/spacy_nlp_pipeline_models/forms.py

@@ -0,0 +1,48 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import StringField, ValidationError
+from wtforms.validators import InputRequired, Length
+from app.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
+    spacy_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+
+    def validate_spacy_model_file(self, field):
+        if not field.data.filename.lower().endswith('.tar.gz'):
+            raise ValidationError('.tar.gz files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateSpaCyNLPPipelineModelForm(UpdateContributionBaseForm):
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -0,0 +1,52 @@
+from flask import abort, current_app, request
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import SpaCyNLPPipelineModel
+from .. import bp
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_spacy_model(spacy_nlp_pipeline_model_id):
+    def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
+        with app.app_context():
+            snpm = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
+            snpm.delete()
+            db.session.commit()
+    
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    thread = Thread(
+        target=_delete_spacy_model,
+        args=(current_app._get_current_object(), snpm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'SpaCy NLP Pipeline Model "{snpm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    snpm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'SpaCy NLP Pipeline Model "{snpm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/contributions/spacy_nlp_pipeline_models/routes.py

@@ -0,0 +1,77 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
+from app import db
+from app.models import SpaCyNLPPipelineModel
+from . import bp
+from .forms import (
+    CreateSpaCyNLPPipelineModelForm,
+    UpdateSpaCyNLPPipelineModelForm
+)
+from .utils import (
+    spacy_nlp_pipeline_model_dlc as spacy_nlp_pipeline_model_dlc
+)
+
+
+@bp.route('/spacy-nlp-pipeline-models')
+@register_breadcrumb(bp, '.spacy_nlp_pipeline_models', 'SpaCy NLP Pipeline Models')
+def spacy_nlp_pipeline_models():
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
+        title='SpaCy NLP Pipeline Models'
+    )
+
+
+@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.create', 'Create')
+def create_spacy_nlp_pipeline_model():
+    form = CreateSpaCyNLPPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            snpm = SpaCyNLPPipelineModel.create(
+                form.spacy_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                pipeline_name=form.pipeline_name.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
+        return {}, 201, {'Location': url_for('.spacy_nlp_pipeline_models')}
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/create.html.j2',
+        title='Create SpaCy NLP Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.entity', '', dynamic_list_constructor=spacy_nlp_pipeline_model_dlc)
+def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    form = UpdateSpaCyNLPPipelineModelForm(data=snpm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(snpm)
+        if db.session.is_modified(snpm):
+            flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.spacy_nlp_pipeline_models'))
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_model.html.j2',
+        title=f'{snpm.title} {snpm.version}',
+        form=form,
+        spacy_nlp_pipeline_model=snpm
+    )

app/contributions/spacy_nlp_pipeline_models/utils.py

@@ -0,0 +1,13 @@
+from flask import request, url_for
+from app.models import SpaCyNLPPipelineModel
+
+
+def spacy_nlp_pipeline_model_dlc():
+    snpm_id = request.view_args['spacy_nlp_pipeline_model_id']
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(snpm_id)
+    return [
+        {
+            'text': f'{snpm.title} {snpm.version}',
+            'url': url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=snpm_id)
+        }
+    ]

app/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/contributions/tesseract_ocr_pipeline_models/forms.py

@@ -0,0 +1,39 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import ValidationError
+from app.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
+    tesseract_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+    
+    def validate_tesseract_model_file(self, field):
+        if not field.data.filename.lower().endswith('.traineddata'):
+            raise ValidationError('traineddata files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateTesseractOCRPipelineModelForm(UpdateContributionBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -0,0 +1,52 @@
+from flask import abort, current_app, request
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import TesseractOCRPipelineModel
+from . import bp
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
+    def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
+        with app.app_context():
+            topm = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
+            topm.delete()
+            db.session.commit()
+
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    thread = Thread(
+        target=_delete_tesseract_ocr_pipeline_model,
+        args=(current_app._get_current_object(), topm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'Tesseract OCR Pipeline Model "{topm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    topm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Tesseract OCR Pipeline Model "{topm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -0,0 +1,76 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
+from app import db
+from app.models import TesseractOCRPipelineModel
+from . import bp
+from .forms import (
+    CreateTesseractOCRPipelineModelForm,
+    UpdateTesseractOCRPipelineModelForm
+)
+from .utils import (
+    tesseract_ocr_pipeline_model_dlc as tesseract_ocr_pipeline_model_dlc
+)
+
+
+@bp.route('/tesseract-ocr-pipeline-models')
+@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models', 'Tesseract OCR Pipeline Models')
+def tesseract_ocr_pipeline_models():
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
+        title='Tesseract OCR Pipeline Models'
+    )
+
+
+@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.create', 'Create')
+def create_tesseract_ocr_pipeline_model():
+    form = CreateTesseractOCRPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            topm = TesseractOCRPipelineModel.create(
+                form.tesseract_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
+        return {}, 201, {'Location': url_for('.tesseract_ocr_pipeline_models')}
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/create.html.j2',
+        title='Create Tesseract OCR Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.entity', '', dynamic_list_constructor=tesseract_ocr_pipeline_model_dlc)
+def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator()):
+        abort(403)
+    form = UpdateTesseractOCRPipelineModelForm(data=topm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(topm)
+        if db.session.is_modified(topm):
+            flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.tesseract_ocr_pipeline_models'))
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_model.html.j2',
+        title=f'{topm.title} {topm.version}',
+        form=form,
+        tesseract_ocr_pipeline_model=topm
+    )

app/contributions/tesseract_ocr_pipeline_models/utils.py

@@ -0,0 +1,13 @@
+from flask import request, url_for
+from app.models import TesseractOCRPipelineModel
+
+
+def tesseract_ocr_pipeline_model_dlc():
+    topm_id = request.view_args['tesseract_ocr_pipeline_model_id']
+    topm = TesseractOCRPipelineModel.query.get_or_404(topm_id)
+    return [
+        {
+            'text': f'{topm.title} {topm.version}',
+            'url': url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=topm_id)
+        }
+    ]

app/contributions/transkribus_htr_pipeline_models/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import routes

app/contributions/transkribus_htr_pipeline_models/routes.py

@@ -0,0 +1,7 @@
+from flask import abort
+from . import bp
+
+
+@bp.route('/transkribus_htr_pipeline_models')
+def transkribus_htr_pipeline_models():
+    return abort(503)

app/converters/cli.py

@@ -0,0 +1,22 @@
+import click
+from . import bp
+from .sandpaper import SandpaperConverter
+
+
+@bp.cli.group('converter')
+def converter():
+    ''' Converter commands. '''
+    pass
+
+@converter.group('sandpaper')
+def sandpaper_converter():
+    ''' Sandpaper converter commands. '''
+    pass
+
+@sandpaper_converter.command('run')
+@click.argument('json_db_file')
+@click.argument('data_dir')
+def run_sandpaper_converter(json_db_file, data_dir):
+    ''' Run the sandpaper converter. '''
+    sandpaper_converter = SandpaperConverter(json_db_file, data_dir)
+    sandpaper_converter.run()

app/converters/sandpaper.py

@@ -7,20 +7,25 @@ import os
 import shutil
 
 
-def convert(json_db_file, data_dir):
-    with open(json_db_file, 'r') as f:
+class SandpaperConverter:
+    def __init__(self, json_db_file, data_dir):
+        self.json_db_file = json_db_file
+        self.data_dir = data_dir
+
+    def run(self):
+        with open(self.json_db_file, 'r') as f:
             json_db = json.loads(f.read())
 
         for json_user in json_db:
             if not json_user['confirmed']:
                 current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
                 continue
-        user_dir = os.path.join(data_dir, str(json_user['id']))
-        convert_user(json_user, user_dir)
+            user_dir = os.path.join(self.data_dir, str(json_user['id']))
+            self.convert_user(json_user, user_dir)
             db.session.commit()
 
 
-def convert_user(json_user, user_dir):
+    def convert_user(self, json_user, user_dir):
         current_app.logger.info(f'Create User {json_user["username"]}...')
         user = User(
             confirmed=json_user['confirmed'],
@@ -44,11 +49,11 @@ def convert_user(json_user, user_dir):
                 current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
                 continue
             corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
-        convert_corpus(json_corpus, user, corpus_dir)
+            self.convert_corpus(json_corpus, user, corpus_dir)
         current_app.logger.info('Done')
 
 
-def convert_corpus(json_corpus, user, corpus_dir):
+    def convert_corpus(self, json_corpus, user, corpus_dir):
         current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
         corpus = Corpus(
             user=user,
@@ -66,11 +71,11 @@ def convert_corpus(json_corpus, user, corpus_dir):
             db.session.rollback()
             raise Exception('Internal Server Error')
         for json_corpus_file in json_corpus['files'].values():
-        convert_corpus_file(json_corpus_file, corpus, corpus_dir)
+            self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
         current_app.logger.info('Done')
 
 
-def convert_corpus_file(json_corpus_file, corpus, corpus_dir):
+    def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
         current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
         corpus_file = CorpusFile(
             corpus=corpus,

app/corpora/__init__.py

@@ -1,5 +1,19 @@
 from flask import Blueprint
+from flask_login import login_required
 
 
 bp = Blueprint('corpora', __name__)
-from . import cqi_over_socketio, routes  # noqa
+bp.cli.short_help = 'Corpus commands.'
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import cli, cqi_over_socketio, files, followers, routes, json_routes

app/corpora/cli.py

@@ -0,0 +1,24 @@
+from app.models import Corpus, CorpusStatus
+import os
+import shutil
+from app import db
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset built corpora. '''
+    status = [
+        CorpusStatus.QUEUED,
+        CorpusStatus.BUILDING,
+        CorpusStatus.BUILT,
+        CorpusStatus.STARTING_ANALYSIS_SESSION,
+        CorpusStatus.RUNNING_ANALYSIS_SESSION,
+        CorpusStatus.CANCELING_ANALYSIS_SESSION
+    ]
+    for corpus in [x for x in Corpus.query.all() if x.status in status]:
+        print(f'Resetting corpus {corpus}')
+        shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
+        corpus.status = CorpusStatus.UNPREPARED
+        corpus.num_analysis_sessions = 0
+    db.session.commit()

app/corpora/cqi_over_socketio/cqi_corpora_corpus.py

@@ -38,7 +38,7 @@ def cqi_corpora_corpus_query(cqi_client: cqi.CQiClient, corpus_name: str, subcor
 @cqi_over_socketio
 def cqi_corpora_corpus_update_db(cqi_client: cqi.CQiClient, corpus_name: str):
     corpus = Corpus.query.get(session['d']['corpus_id'])
-    corpus.num_tokens = cqi_client.corpora.get('CORPUS').attrs['size']
+    corpus.num_tokens = cqi_client.corpora.get(corpus_name).attrs['size']
     db.session.commit()
 
 

app/corpora/decorators.py

@@ -0,0 +1,33 @@
+from flask import abort
+from flask_login import current_user
+from functools import wraps
+from app.models import Corpus, CorpusFollowerAssociation
+
+
+def corpus_follower_permission_required(*permissions):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            corpus_id = kwargs.get('corpus_id')
+            corpus = Corpus.query.get_or_404(corpus_id)
+            if not (corpus.user == current_user or current_user.is_administrator()):
+                cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+                if cfa is None:
+                    abort(403)
+                if not all([cfa.role.has_permission(p) for p in permissions]):
+                    abort(403)
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator
+
+
+def corpus_owner_or_admin_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        corpus_id = kwargs.get('corpus_id')
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if not (corpus.user == current_user or current_user.is_administrator()):
+            abort(403)
+        return f(*args, **kwargs)
+    return decorated_function
+

app/corpora/events.py

@@ -0,0 +1,45 @@
+from flask_login import current_user
+from flask_socketio import join_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus
+
+
+@socketio.on('GET /corpora/<corpus_id>')
+@socketio_login_required
+def get_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator()
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    return {
+        'body': corpus.to_json_serializable(),
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /corpora/<corpus_id>')
+@socketio_login_required
+def subscribe_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator()
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    join_room(f'/corpora/{corpus.hashid}')
+    return {'options': {'status': 200, 'statusText': 'OK'}}

app/corpora/files/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/corpora/files/forms.py

@@ -0,0 +1,54 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired
+from wtforms import (
+    StringField,
+    SubmitField,
+    ValidationError,
+    IntegerField
+)
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusFileBaseForm(FlaskForm):
+    author = StringField(
+        'Author',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    address = StringField('Adress', validators=[Length(max=255)])
+    booktitle = StringField('Booktitle', validators=[Length(max=255)])
+    chapter = StringField('Chapter', validators=[Length(max=255)])
+    editor = StringField('Editor', validators=[Length(max=255)])
+    institution = StringField('Institution', validators=[Length(max=255)])
+    journal = StringField('Journal', validators=[Length(max=255)])
+    pages = StringField('Pages', validators=[Length(max=255)])
+    publisher = StringField('Publisher', validators=[Length(max=255)])
+    school = StringField('School', validators=[Length(max=255)])
+    submit = SubmitField()
+
+
+class CreateCorpusFileForm(CorpusFileBaseForm):
+    vrt = FileField('File', validators=[FileRequired()])
+
+    def validate_vrt(self, field):
+        if not field.data.filename.lower().endswith('.vrt'):
+            raise ValidationError('VRT files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-file-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusFileForm(CorpusFileBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-file-form'
+        super().__init__(*args, **kwargs)

app/corpora/files/json_routes.py

@@ -0,0 +1,30 @@
+from flask import abort, current_app
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation
+from app.models import CorpusFile
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def delete_corpus_file(corpus_id, corpus_file_id):
+    def _delete_corpus_file(app, corpus_file_id):
+        with app.app_context():
+            corpus_file = CorpusFile.query.get(corpus_file_id)
+            corpus_file.delete()
+            db.session.commit()
+
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    thread = Thread(
+        target=_delete_corpus_file,
+        args=(current_app._get_current_object(), corpus_file.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus File "{corpus_file.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 202

app/corpora/files/routes.py

@@ -0,0 +1,100 @@
+from flask import (
+    abort,
+    flash,
+    redirect,
+    render_template,
+    send_from_directory,
+    url_for
+)
+from flask_breadcrumbs import register_breadcrumb
+import os
+from app import db
+from app.models import Corpus, CorpusFile, CorpusStatus
+from ..decorators import corpus_follower_permission_required
+from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
+from . import bp
+from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
+from .utils import corpus_file_dynamic_list_constructor as corpus_file_dlc
+
+
+@bp.route('/<hashid:corpus_id>/files')
+@register_breadcrumb(bp, '.entity.files', 'Files', endpoint_arguments_constructor=corpus_eac)
+def corpus_files(corpus_id):
+    return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
+
+
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.entity.files.create', 'Create', endpoint_arguments_constructor=corpus_eac)
+@corpus_follower_permission_required('MANAGE_FILES')
+def create_corpus_file(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    form = CreateCorpusFileForm()
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            corpus_file = CorpusFile.create(
+                form.vrt.data,
+                address=form.address.data,
+                author=form.author.data,
+                booktitle=form.booktitle.data,
+                chapter=form.chapter.data,
+                editor=form.editor.data,
+                institution=form.institution.data,
+                journal=form.journal.data,
+                pages=form.pages.data,
+                publisher=form.publisher.data,
+                publishing_year=form.publishing_year.data,
+                school=form.school.data,
+                title=form.title.data,
+                mimetype='application/vrt+xml',
+                corpus=corpus
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        corpus.status = CorpusStatus.UNPREPARED
+        db.session.commit()
+        flash(f'Corpus File "{corpus_file.filename}" added', category='corpus')
+        return '', 201, {'Location': corpus.url}
+    return render_template(
+        'corpora/files/create.html.j2',
+        title='Add corpus file',
+        form=form,
+        corpus=corpus
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.entity.files.entity', '', dynamic_list_constructor=corpus_file_dlc)
+@corpus_follower_permission_required('MANAGE_FILES')
+def corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(corpus_file)
+        if db.session.is_modified(corpus_file):
+            corpus_file.corpus.status = CorpusStatus.UNPREPARED
+            db.session.commit()
+            flash(f'Corpus file "{corpus_file.filename}" updated', category='corpus')
+        return redirect(corpus_file.corpus.url)
+    return render_template(
+        'corpora/files/corpus_file.html.j2',
+        title='Edit corpus file',
+        form=form,
+        corpus=corpus_file.corpus,
+        corpus_file=corpus_file
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
+@corpus_follower_permission_required('VIEW')
+def download_corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    return send_from_directory(
+        os.path.dirname(corpus_file.path),
+        os.path.basename(corpus_file.path),
+        as_attachment=True,
+        attachment_filename=corpus_file.filename,
+        mimetype=corpus_file.mimetype
+    )

app/corpora/files/utils.py

@@ -0,0 +1,15 @@
+from flask import request, url_for
+from app.models import CorpusFile
+from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
+
+
+def corpus_file_dynamic_list_constructor():
+    corpus_id = request.view_args['corpus_id']
+    corpus_file_id = request.view_args['corpus_file_id']
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    return [
+        {
+            'text': f'{corpus_file.author}: {corpus_file.title} ({corpus_file.publishing_year})',
+            'url': url_for('.corpus_file', corpus_id=corpus_id, corpus_file_id=corpus_file_id)
+        }
+    ]

app/corpora/followers/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes

app/corpora/followers/json_routes.py

@@ -0,0 +1,76 @@
+from flask import abort, flash, jsonify, make_response, request
+from flask_login import current_user
+from app import db
+from app.decorators import content_negotiation
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+# @bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
+# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
+# @content_negotiation(consumes='application/json', produces='application/json')
+# def create_corpus_followers(corpus_id):
+#     usernames = request.json
+#     if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
+#         abort(400)
+#     corpus = Corpus.query.get_or_404(corpus_id)
+#     for username in usernames:
+#         user = User.query.filter_by(username=username, is_public=True).first_or_404()
+#         user.follow_corpus(corpus)
+#     db.session.commit()
+#     response_data = {
+#         'message': f'Users are now following "{corpus.title}"',
+#         'category': 'corpus'
+#     }
+#     return response_data, 200
+
+
+# @bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
+# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
+# @content_negotiation(consumes='application/json', produces='application/json')
+# def update_corpus_follower_role(corpus_id, follower_id):
+#     role_name = request.json
+#     if not isinstance(role_name, str):
+#         abort(400)
+#     cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+#     if cfr is None:
+#         abort(400)
+#     cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+#     cfa.role = cfr
+#     db.session.commit()
+#     response_data = {
+#         'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
+#         'category': 'corpus'
+#     }
+#     return response_data, 200
+
+
+# @bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
+# def delete_corpus_follower(corpus_id, follower_id):
+#     cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+#     if not (
+#         current_user.id == follower_id
+#         or current_user == cfa.corpus.user 
+#         or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
+#         or current_user.is_administrator()):
+#         abort(403)
+#     if current_user.id == follower_id:
+#         flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
+#         response = make_response()
+#         response.status_code = 204
+#     else:
+#         response_data = {
+#             'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
+#             'category': 'corpus'
+#         }
+#         response = jsonify(response_data)
+#         response.status_code = 200
+#     cfa.follower.unfollow_corpus(cfa.corpus)
+#     db.session.commit()
+#     return response

app/corpora/forms.py

@@ -1,13 +1,5 @@
 from flask_wtf import FlaskForm
-from flask_wtf.file import FileField, FileRequired
-from wtforms import (
-    BooleanField,
-    StringField,
-    SubmitField,
-    TextAreaField,
-    ValidationError,
-    IntegerField
-)
+from wtforms import StringField, SubmitField, TextAreaField
 from wtforms.validators import InputRequired, Length
 
 
@@ -34,50 +26,8 @@ class UpdateCorpusForm(CorpusBaseForm):
         super().__init__(*args, **kwargs)
 
 
-class CorpusFileBaseForm(FlaskForm):
-    author = StringField(
-        'Author',
-        validators=[InputRequired(), Length(max=255)]
-    )
-    publishing_year = IntegerField(
-        'Publishing year',
-        validators=[InputRequired()]
-    )
-    title = StringField(
-        'Title',
-        validators=[InputRequired(), Length(max=255)]
-    )
-    address = StringField('Adress', validators=[Length(max=255)])
-    booktitle = StringField('Booktitle', validators=[Length(max=255)])
-    chapter = StringField('Chapter', validators=[Length(max=255)])
-    editor = StringField('Editor', validators=[Length(max=255)])
-    institution = StringField('Institution', validators=[Length(max=255)])
-    journal = StringField('Journal', validators=[Length(max=255)])
-    pages = StringField('Pages', validators=[Length(max=255)])
-    publisher = StringField('Publisher', validators=[Length(max=255)])
-    school = StringField('School', validators=[Length(max=255)])
-    submit = SubmitField()
-
-
-class CreateCorpusFileForm(CorpusFileBaseForm):
-    vrt = FileField('File', validators=[FileRequired()])
-
-    def __init__(self, *args, **kwargs):
-        if 'prefix' not in kwargs:
-            kwargs['prefix'] = 'create-corpus-file-form'
-        super().__init__(*args, **kwargs)
-
-    def validate_vrt(self, field):
-        if not field.data.filename.lower().endswith('.vrt'):
-            raise ValidationError('VRT files only!')
-
-
-class UpdateCorpusFileForm(CorpusFileBaseForm):
-    def __init__(self, *args, **kwargs):
-        if 'prefix' not in kwargs:
-            kwargs['prefix'] = 'update-corpus-file-form'
-        super().__init__(*args, **kwargs)
-
-
 class ImportCorpusForm(FlaskForm):
-    pass
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'import-corpus-form'
+        super().__init__(*args, **kwargs)

app/corpora/json_routes.py

@@ -0,0 +1,111 @@
+from datetime import datetime
+from flask import abort, current_app, request, url_for
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation
+from app.models import Corpus, CorpusFollowerRole
+from . import bp
+from .decorators import corpus_follower_permission_required, corpus_owner_or_admin_required
+
+
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+@corpus_owner_or_admin_required
+@content_negotiation(produces='application/json')
+def delete_corpus(corpus_id):
+    def _delete_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.delete()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    thread = Thread(
+        target=_delete_corpus,
+        args=(current_app._get_current_object(), corpus.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def build_corpus(corpus_id):
+    def _build_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.build()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if len(corpus.files.all()) == 0:
+        abort(409)
+    thread = Thread(
+        target=_build_corpus,
+        args=(current_app._get_current_object(), corpus_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for building',
+        'category': 'corpus'
+    }
+    return response_data, 202
+
+
+# @bp.route('/<hashid:corpus_id>/generate-share-link', methods=['POST'])
+# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
+# @content_negotiation(consumes='application/json', produces='application/json')
+# def generate_corpus_share_link(corpus_id):
+#     data = request.json
+#     if not isinstance(data, dict):
+#         abort(400)
+#     expiration = data.get('expiration')
+#     if not isinstance(expiration, str):
+#         abort(400)
+#     role_name = data.get('role')
+#     if not isinstance(role_name, str):
+#         abort(400)
+#     expiration_date = datetime.strptime(expiration, '%b %d, %Y')
+#     cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+#     if cfr is None:
+#         abort(400)
+#     corpus = Corpus.query.get_or_404(corpus_id)
+#     token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date)
+#     corpus_share_link = url_for(
+#         'corpora.follow_corpus',
+#         corpus_id=corpus_id,
+#         token=token,
+#         _external=True
+#     )
+#     response_data = {
+#         'message': 'Corpus share link generated',
+#         'category': 'corpus',
+#         'corpusShareLink': corpus_share_link
+#     }
+#     return response_data, 200
+    
+
+
+# @bp.route('/<hashid:corpus_id>/is_public', methods=['PUT'])
+# @corpus_owner_or_admin_required
+# @content_negotiation(consumes='application/json', produces='application/json')
+# def update_corpus_is_public(corpus_id):
+#     is_public = request.json
+#     if not isinstance(is_public, bool):
+#         abort(400)
+#     corpus = Corpus.query.get_or_404(corpus_id)
+#     corpus.is_public = is_public
+#     db.session.commit()
+#     response_data = {
+#         'message': (
+#             f'Corpus "{corpus.title}" is now'
+#             f' {"public" if is_public else "private"}'
+#         ),
+#         'category': 'corpus'
+#     }
+#     return response_data, 200

app/corpora/routes.py

@@ -1,139 +1,30 @@
-from datetime import datetime
-from flask import (
-    abort,
-    current_app,
-    flash,
-    Markup,
-    redirect,
-    render_template,
-    request,
-    send_from_directory,
-    url_for
+from flask import abort, flash, redirect, render_template, url_for
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
 )
-from flask_login import current_user, login_required
-from threading import Thread
-import jwt
-import os
-from app import db, hashids
-from app.models import Corpus, CorpusFile, CorpusStatus, User
 from . import bp
-from .forms import (
-    CreateCorpusFileForm,
-    CreateCorpusForm,
-    UpdateCorpusFileForm
+from .decorators import corpus_follower_permission_required
+from .forms import CreateCorpusForm
+from .utils import (
+    corpus_endpoint_arguments_constructor as corpus_eac,
+    corpus_dynamic_list_constructor as corpus_dlc
 )
 
 
-# @bp.route('/share/<token>', methods=['GET', 'POST'])
-# def share_corpus(token):
-#     try:
-#         payload = jwt.decode(
-#             token,
-#             current_app.config['SECRET_KEY'],
-#             algorithms=['HS256'],
-#             issuer=current_app.config['SERVER_NAME'],
-#             options={'require': ['iat', 'iss', 'sub']}
-#         )
-#     except jwt.PyJWTError:
-#         return False
-#     corpus_hashid = payload.get('sub')
-#     corpus_id = hashids.decode(corpus_hashid)
-#     return redirect(url_for('.corpus', corpus_id=corpus_id))
-
-
-@bp.route('/<hashid:corpus_id>/enable_is_public', methods=['POST'])
-@login_required
-def enable_corpus_is_public(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    corpus.is_public = True
-    db.session.commit()
-    return '', 204
-
-
-@bp.route('/<hashid:corpus_id>/disable_is_public', methods=['POST'])
-@login_required
-def disable_corpus_is_public(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    corpus.is_public = False
-    db.session.commit()
-    return '', 204
-
-
-# @bp.route('/<hashid:corpus_id>/follow', methods=['GET', 'POST'])
-# @login_required
-# def follow_corpus(corpus_id):
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     user_hashid = request.args.get('user_id')
-#     if user_hashid is None:
-#         user = current_user
-#     else:
-#         if not current_user.is_administrator():
-#             abort(403)
-#         else:
-#             user_id = hashids.decode(user_hashid)
-#             user = User.query.get_or_404(user_id)
-#     if not user.is_following_corpus(corpus):
-#         user.follow_corpus(corpus)
-#     db.session.commit()
-#     flash(f'You are following {corpus.title} now', category='corpus')
-#     return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/unfollow', methods=['GET', 'POST'])
-@login_required
-def unfollow_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    user_hashid = request.args.get('user_id')
-    if user_hashid is None:
-        user = current_user
-    elif current_user.is_administrator():
-        user_id = hashids.decode(user_hashid)
-        user = User.query.get_or_404(user_id)
-    else:
-        abort(403)
-    if user.is_following_corpus(corpus):
-        user.unfollow_corpus(corpus)
-    db.session.commit()
-    flash(f'You are not following {corpus.title} anymore', category='corpus')
-    return {}, 202
-
-
-# @bp.route('/add_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
-# def add_permission(corpus_id, user_id, permission):
-#     a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
-#     a.add_permission(permission)
-#     db.session.commit()
-#     return 'ok'
-
-
-# @bp.route('/remove_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
-# def remove_permission(corpus_id, user_id, permission):
-#     a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
-#     a.remove_permission(permission)
-#     db.session.commit()
-#     return 'ok'
-
-
-@bp.route('/public')
-@login_required
-def public_corpora():
-    corpora = [
-        c.to_json_serializeable()
-        for c in Corpus.query.filter(Corpus.is_public == True).all()
-    ]
-    return render_template(
-        'corpora/public_corpora.html.j2',
-        corpora=corpora,
-        title='Corpora'
-    )
+@bp.route('')
+@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">I</i>My Corpora')
+def corpora():
+    return redirect(url_for('main.dashboard', _anchor='corpora'))
 
 
 @bp.route('/create', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.create', 'Create')
 def create_corpus():
     form = CreateCorpusForm()
     if form.validate_on_submit():
@@ -146,224 +37,85 @@ def create_corpus():
         except OSError:
             abort(500)
         db.session.commit()
-        message = Markup(
-            f'Corpus "<a href="{corpus.url}">{corpus.title}</a>" created'
-        )
-        flash(message, 'corpus')
+        flash(f'Corpus "{corpus.title}" created', 'corpus')
         return redirect(corpus.url)
     return render_template(
-        'corpora/create_corpus.html.j2',
-        form=form,
-        title='Create corpus'
+        'corpora/create.html.j2',
+        title='Create corpus',
+        form=form
     )
 
 
-@bp.route('/<hashid:corpus_id>', methods=['GET', 'POST'])
-@login_required
+@bp.route('/<hashid:corpus_id>')
+@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=corpus_dlc)
 def corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
+    cfrs = CorpusFollowerRole.query.all()
+    # TODO: Better solution for filtering admin
+    users = User.query.filter(User.is_public == True, User.id != current_user.id, User.id != corpus.user.id, User.role_id < 4).all()
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+    if cfa is None:
+        if corpus.user == current_user or current_user.is_administrator():
+            cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
+        else:
+            cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
+    else:
+        cfr = cfa.role
     if corpus.user == current_user or current_user.is_administrator():
-        # now = datetime.utcnow()
-        # payload = {
-        #         'exp': now + timedelta(weeks=1),
-        #         'iat': now,
-        #         'iss': current_app.config['SERVER_NAME'],
-        #         'sub': corpus.hashid
-        #     }
-        # token = jwt.encode(
-        #         payload,
-        #         current_app.config['SECRET_KEY'],
-        #         algorithm='HS256'
-        #     )
         return render_template(
             'corpora/corpus.html.j2',
+            title=corpus.title,
             corpus=corpus,
-            # token=token,
-            title='Corpus'
-        )
-    if current_user.is_following_corpus(corpus) or corpus.is_public:
-        corpus_files = [x.to_json_serializeable() for x in corpus.files]
-        return render_template(
-            'corpora/public_corpus.html.j2',
-            corpus=corpus,
-            corpus_files=corpus_files,
-            title='Corpus'
+            cfr=cfr,
+            cfrs=cfrs,
+            users = users
         )
+    if (current_user.is_following_corpus(corpus) or corpus.is_public):
+        abort(404)
+        # cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all()
+        # return render_template(
+        #     'corpora/public_corpus.html.j2',
+        #     title=corpus.title,
+        #     corpus=corpus,
+        #     cfrs=cfrs,
+        #     cfr=cfr,
+        #     cfas=cfas,
+        #     users = users
+        # )
     abort(403)
 
 
-@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
-@login_required
-def delete_corpus(corpus_id):
-    def _delete_corpus(app, corpus_id):
-        with app.app_context():
-            corpus = Corpus.query.get(corpus_id)
-            corpus.delete()
-            db.session.commit()
 
+@bp.route('/<hashid:corpus_id>/analysis')
+@corpus_follower_permission_required('VIEW')
+@register_breadcrumb(bp, '.entity.analysis', 'Analysis', endpoint_arguments_constructor=corpus_eac)
+def analysis(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_corpus,
-        args=(current_app._get_current_object(), corpus_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/analyse')
-@login_required
-def analyse_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user
-            or current_user.is_administrator()
-            or current_user.is_following_corpus(corpus)):
-        abort(403)
     return render_template(
-        'corpora/analyse_corpus.html.j2',
+        'corpora/analysis.html.j2',
         corpus=corpus,
         title=f'Analyse Corpus {corpus.title}'
     )
 
 
-@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
-@login_required
-def build_corpus(corpus_id):
-    def _build_corpus(app, corpus_id):
-        with app.app_context():
-            corpus = Corpus.query.get(corpus_id)
-            corpus.build()
-            db.session.commit()
-
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    # Check if the corpus has corpus files
-    if not corpus.files.all():
-        response = {'errors': {'message': 'Corpus file(s) required'}}
-        return response, 409
-    thread = Thread(
-        target=_build_corpus,
-        args=(current_app._get_current_object(), corpus_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
-@login_required
-def create_corpus_file(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    form = CreateCorpusFileForm()
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            corpus_file = CorpusFile.create(
-                form.vrt.data,
-                address=form.address.data,
-                author=form.author.data,
-                booktitle=form.booktitle.data,
-                chapter=form.chapter.data,
-                editor=form.editor.data,
-                institution=form.institution.data,
-                journal=form.journal.data,
-                pages=form.pages.data,
-                publisher=form.publisher.data,
-                publishing_year=form.publishing_year.data,
-                school=form.school.data,
-                title=form.title.data,
-                mimetype='application/vrt+xml',
-                corpus=corpus
-            )
-        except (AttributeError, OSError):
-            abort(500)
-        corpus.status = CorpusStatus.UNPREPARED
-        db.session.commit()
-        message = Markup(
-            'Corpus file'
-            f'"<a href="{corpus_file.url}">{corpus_file.filename}</a>" added'
-        )
-        flash(message, category='corpus')
-        return {}, 201, {'Location': corpus.url}
-    return render_template(
-        'corpora/create_corpus_file.html.j2',
-        corpus=corpus,
-        form=form,
-        title='Add corpus file'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
-@login_required
-def corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
-    if form.validate_on_submit():
-        form.populate_obj(corpus_file)
-        if db.session.is_modified(corpus_file):
-            corpus_file.corpus.status = CorpusStatus.UNPREPARED
-            db.session.commit()
-            message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
-            flash(message, category='corpus')
-        return redirect(corpus_file.corpus.url)
-    return render_template(
-        'corpora/corpus_file.html.j2',
-        corpus=corpus_file.corpus,
-        corpus_file=corpus_file,
-        form=form,
-        title='Edit corpus file'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
-@login_required
-def delete_corpus_file(corpus_id, corpus_file_id):
-    def _delete_corpus_file(app, corpus_file_id):
-        with app.app_context():
-            corpus_file = CorpusFile.query.get(corpus_file_id)
-            corpus_file.delete()
-            db.session.commit()
-
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_corpus_file,
-        args=(current_app._get_current_object(), corpus_file_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
-@login_required
-def download_corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    return send_from_directory(
-        os.path.dirname(corpus_file.path),
-        os.path.basename(corpus_file.path),
-        as_attachment=True,
-        attachment_filename=corpus_file.filename,
-        mimetype=corpus_file.mimetype
-    )
+# @bp.route('/<hashid:corpus_id>/follow/<token>')
+# def follow_corpus(corpus_id, token):
+#     corpus = Corpus.query.get_or_404(corpus_id)
+#     if current_user.follow_corpus_by_token(token):
+#         db.session.commit()
+#         flash(f'You are following "{corpus.title}" now', category='corpus')
+#         return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+#     abort(403)
 
 
 @bp.route('/import', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.import', 'Import')
 def import_corpus():
     abort(503)
 
 
 @bp.route('/<hashid:corpus_id>/export')
-@login_required
+@corpus_follower_permission_required('VIEW')
+@register_breadcrumb(bp, '.entity.export', 'Export', endpoint_arguments_constructor=corpus_eac)
 def export_corpus(corpus_id):
     abort(503)

app/corpora/utils.py

@@ -0,0 +1,17 @@
+from flask import request, url_for
+from app.models import Corpus
+
+
+def corpus_endpoint_arguments_constructor():
+    return {'corpus_id': request.view_args['corpus_id']}
+
+
+def corpus_dynamic_list_constructor():
+    corpus_id = request.view_args['corpus_id']
+    corpus = Corpus.query.get_or_404(corpus_id)
+    return [
+        {
+            'text': f'<i class="material-icons left">book</i>{corpus.title}',
+            'url': url_for('.corpus', corpus_id=corpus_id)
+        }
+    ]

app/decorators.py

@@ -1,7 +1,9 @@
-from flask import abort, current_app
+from flask import abort, current_app, request
 from flask_login import current_user
 from functools import wraps
 from threading import Thread
+from typing import List, Union
+from werkzeug.exceptions import NotAcceptable
 from app.models import Permission
 
 
@@ -61,3 +63,37 @@ def background(f):
         thread.start()
         return thread
     return wrapped
+
+
+def content_negotiation(
+    produces: Union[str, List[str], None] = None,
+    consumes: Union[str, List[str], None] = None
+):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            provided = request.mimetype
+            if consumes is None:
+                consumeables = None
+            elif isinstance(consumes, str):
+                consumeables = {consumes}
+            elif isinstance(consumes, list) and all(isinstance(x, str) for x in consumes):
+                consumeables = {*consumes}
+            else:
+                raise TypeError()
+            accepted = {*request.accept_mimetypes.values()}
+            if produces is None:
+                produceables = None
+            elif isinstance(produces, str):
+                produceables = {produces}
+            elif isinstance(produces, list) and all(isinstance(x, str) for x in produces):
+                produceables = {*produces}
+            else:
+                raise TypeError()
+            if produceables is not None and len(produceables & accepted) == 0:
+                raise NotAcceptable()
+            if consumeables is not None and provided not in consumeables:
+                raise NotAcceptable()
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator

app/errors/handlers.py

@@ -1,11 +1,14 @@
-from flask import render_template, request
+from flask import jsonify, render_template, request
 from werkzeug.exceptions import HTTPException
 from . import bp
 
 
-@bp.errorhandler(HTTPException)
-def generic_error_handler(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        return {'errors': {'message': e.description}}, e.code
-    return render_template('errors/error.html.j2', error=e), e.code
+@bp.app_errorhandler(HTTPException)
+def handle_http_exception(error):
+    ''' Generic HTTP exception handler '''
+    accept_json = request.accept_mimetypes.accept_json
+    accept_html = request.accept_mimetypes.accept_html
+    if accept_json and not accept_html:
+        response = jsonify(str(error))
+        return response, error.code
+    return render_template('errors/error.html.j2', error=error), error.code

app/jobs/__init__.py

@@ -1,5 +1,18 @@
 from flask import Blueprint
+from flask_login import login_required
 
 
 bp = Blueprint('jobs', __name__)
-from . import routes
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/jobs/json_routes.py

@@ -0,0 +1,74 @@
+from flask import abort, current_app
+from flask_login import current_user
+from threading import Thread
+import os
+from app import db
+from app.decorators import admin_required, content_negotiation
+from app.models import Job, JobStatus
+from . import bp
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_job(job_id):
+    def _delete_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.delete()
+            db.session.commit()
+
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator()):
+        abort(403)
+    thread = Thread(
+        target=_delete_job,
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Job "{job.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/<hashid:job_id>/log')
+@admin_required
+@content_negotiation(produces='application/json')
+def job_log(job_id):
+    job = Job.query.get_or_404(job_id)
+    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+        response = {'errors': {'message': 'Job status is not completed or failed'}}
+        return response, 409
+    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+        log = log_file.read()
+    response_data = {
+        'message': '',
+        'jobLog': log
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:job_id>/restart', methods=['POST'])
+@content_negotiation(produces='application/json')
+def restart_job(job_id):
+    def _restart_job(app, job_id):
+        with app.app_context():
+            job = Job.query.get(job_id)
+            job.restart()
+            db.session.commit()
+
+    job = Job.query.get_or_404(job_id)
+    if not (job.user == current_user or current_user.is_administrator()):
+        abort(403)
+    if job.status == JobStatus.FAILED:
+        response = {'errors': {'message': 'Job status is not "failed"'}}
+        return response, 409
+    thread = Thread(
+        target=_restart_job,
+        args=(current_app._get_current_object(), job_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Job "{job.title}" marked for restarting'
+    }
+    return response_data, 202

app/jobs/routes.py

@@ -1,93 +1,40 @@
 from flask import (
     abort,
-    current_app,
+    redirect,
     render_template,
-    send_from_directory
+    send_from_directory,
+    url_for
 )
-from flask_login import current_user, login_required
-from threading import Thread
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
 import os
-from app import db
-from app.decorators import admin_required
-from app.models import Job, JobInput, JobResult, JobStatus
+from app.models import Job, JobInput, JobResult
 from . import bp
+from .utils import job_dynamic_list_constructor as job_dlc
+
+
+@bp.route('')
+@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">J</i>My Jobs')
+def corpora():
+    return redirect(url_for('main.dashboard', _anchor='jobs'))
 
 
 @bp.route('/<hashid:job_id>')
-@login_required
+@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=job_dlc)
 def job(job_id):
     job = Job.query.get_or_404(job_id)
     if not (job.user == current_user or current_user.is_administrator()):
         abort(403)
     return render_template(
         'jobs/job.html.j2',
-        job=job,
-        title='Job'
+        title='Job',
+        job=job
     )
 
 
-@bp.route('/<hashid:job_id>', methods=['DELETE'])
-@login_required
-def delete_job(job_id):
-    def _delete_job(app, job_id):
-        with app.app_context():
-            job = Job.query.get(job_id)
-            job.delete()
-            db.session.commit()
-
-    job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_job,
-        args=(current_app._get_current_object(), job_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:job_id>/log')
-@login_required
-@admin_required
-def job_log(job_id):
-    job = Job.query.get_or_404(job_id)
-    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-        response = {'errors': {'message': 'Job status is not completed or failed'}}
-        return response, 409
-    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
-        log = log_file.read()
-    return log, 200, {'Content-Type': 'text/plain; charset=utf-8'}
-
-
-@bp.route('/<hashid:job_id>/restart', methods=['POST'])
-@login_required
-def restart_job(job_id):
-    def _restart_job(app, job_id):
-        with app.app_context():
-            job = Job.query.get(job_id)
-            job.restart()
-            db.session.commit()
-
-    job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    if job.status == JobStatus.FAILED:
-        response = {'errors': {'message': 'Job status is not "failed"'}}
-        return response, 409
-    thread = Thread(
-        target=_restart_job,
-        args=(current_app._get_current_object(), job_id)
-    )
-    thread.start()
-    return {}, 202
-
-
 @bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
-@login_required
 def download_job_input(job_id, job_input_id):
-    job_input = JobInput.query.get_or_404(job_input_id)
-    if job_input.job.id != job_id:
-        abort(404)
+    job_input = JobInput.query.filter_by(job_id=job_id, id=job_input_id).first_or_404()
     if not (job_input.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(
@@ -100,11 +47,8 @@ def download_job_input(job_id, job_input_id):
 
 
 @bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
-@login_required
 def download_job_result(job_id, job_result_id):
-    job_result = JobResult.query.get_or_404(job_result_id)
-    if job_result.job.id != job_id:
-        abort(404)
+    job_result = JobResult.query.filter_by(job_id=job_id, id=job_result_id).first_or_404()
     if not (job_result.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(

app/jobs/utils.py

@@ -0,0 +1,13 @@
+from flask import request, url_for
+from app.models import Job
+
+
+def job_dynamic_list_constructor():
+    job_id = request.view_args['job_id']
+    job = Job.query.get_or_404(job_id)
+    return [
+        {
+            'text': f'<i class="nopaque-icons left service-icons" data-service="{job.service}"></i>{job.title}',
+            'url': url_for('.job', job_id=job_id)
+        }
+    ]

app/main/__init__.py

@@ -1,5 +1,5 @@
 from flask import Blueprint
 
 
-bp = Blueprint('main', __name__)
-from . import routes
+bp = Blueprint('main', __name__, cli_group=None)
+from . import cli, routes

app/main/cli.py

@@ -0,0 +1,45 @@
+from flask import current_app
+from flask_migrate import upgrade
+import os
+from app.models import (
+    CorpusFollowerRole,
+    Role,
+    SpaCyNLPPipelineModel,
+    TesseractOCRPipelineModel,
+    User
+)
+from . import bp
+
+
+@bp.cli.command('deploy')
+def deploy():
+    ''' Run deployment tasks. '''
+    # Make default directories
+    print('Make default directories')
+    base_dir = current_app.config['NOPAQUE_DATA_DIR']
+    default_dirs = [
+        os.path.join(base_dir, 'tmp'),
+        os.path.join(base_dir, 'users')
+    ]
+    for dir in default_dirs:
+        if os.path.exists(dir):
+            if not os.path.isdir(dir):
+                raise NotADirectoryError(f'{dir} is not a directory')
+        else:
+            os.mkdir(dir)
+
+    # migrate database to latest revision
+    print('Migrate database to latest revision')
+    upgrade()
+
+    # Insert/Update default database values
+    print('Insert/Update default Roles')
+    Role.insert_defaults()
+    print('Insert/Update default Users')
+    User.insert_defaults()
+    print('Insert/Update default CorpusFollowerRoles')
+    CorpusFollowerRole.insert_defaults()
+    print('Insert/Update default SpaCyNLPPipelineModels')
+    SpaCyNLPPipelineModel.insert_defaults()
+    print('Insert/Update default TesseractOCRPipelineModels')
+    TesseractOCRPipelineModel.insert_defaults()

app/main/routes.py

@@ -1,13 +1,16 @@
 from flask import flash, redirect, render_template, url_for
+from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user, login_required, login_user
 from app.auth.forms import LoginForm
 from app.models import Corpus, User
+from sqlalchemy import or_
 from . import bp
 
 
-@bp.route('', methods=['GET', 'POST'])
+@bp.route('/', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.', '<i class="material-icons">home</i>')
 def index():
-    form = LoginForm(prefix='login-form')
+    form = LoginForm()
     if form.validate_on_submit():
         user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
         if user and user.verify_password(form.password.data):
@@ -16,54 +19,74 @@ def index():
             return redirect(url_for('.dashboard'))
         flash('Invalid email/username or password', category='error')
         redirect(url_for('.index'))
-    return render_template('main/index.html.j2', form=form, title='nopaque')
-
-
-@bp.route('/faq')
-def faq():
-    return render_template('main/faq.html.j2', title='Frequently Asked Questions')
-
-
-@bp.route('/dashboard')
-@login_required
-def dashboard():
-    # users = [
-    #     u.to_json_serializeable(filter_by_privacy_settings=True) for u
-    #     in User.query.filter(User.is_public == True, User.id != current_user.id).all()
-    # ]
-    # corpora = [
-    #     c.to_json_serializeable() for c
-    #     in Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
-    # ]
     return render_template(
-        'main/dashboard.html.j2',
-        title='Dashboard',
-        # users=users,
-        # corpora=corpora
+        'main/index.html.j2',
+        title='nopaque',
+        form=form
     )
 
 
-@bp.route('/dashboard2')
+@bp.route('/faq')
+@register_breadcrumb(bp, '.faq', 'Frequently Asked Questions')
+def faq():
+    return render_template(
+        'main/faq.html.j2',
+        title='Frequently Asked Questions'
+    )
+
+
+@bp.route('/dashboard')
+@register_breadcrumb(bp, '.dashboard', '<i class="material-icons left">dashboard</i>Dashboard')
 @login_required
-def dashboard2():
-    return render_template('main/dashboard2.html.j2', title='Dashboard')
+def dashboard():
+    return render_template(
+        'main/dashboard.html.j2',
+        title='Dashboard'
+    )
 
 
-@bp.route('/user_manual')
-def user_manual():
-    return render_template('main/user_manual.html.j2', title='User manual')
+# @bp.route('/user_manual')
+# @register_breadcrumb(bp, '.user_manual', '<i class="material-icons left">help</i>User manual')
+# def user_manual():
+#     return render_template('main/user_manual.html.j2', title='User manual')
 
 
 @bp.route('/news')
+@register_breadcrumb(bp, '.news', '<i class="material-icons left">email</i>News')
 def news():
-    return render_template('main/news.html.j2', title='News')
+    return render_template(
+        'main/news.html.j2',
+        title='News'
+    )
 
 
 @bp.route('/privacy_policy')
+@register_breadcrumb(bp, '.privacy_policy', 'Private statement (GDPR)')
 def privacy_policy():
-    return render_template('main/privacy_policy.html.j2', title='Privacy statement (GDPR)')
+    return render_template(
+        'main/privacy_policy.html.j2',
+        title='Privacy statement (GDPR)'
+    )
 
 
 @bp.route('/terms_of_use')
+@register_breadcrumb(bp, '.terms_of_use', 'Terms of Use')
 def terms_of_use():
-    return render_template('main/terms_of_use.html.j2', title='Terms of Use')
+    return render_template(
+        'main/terms_of_use.html.j2',
+        title='Terms of Use'
+    )
+
+
+# @bp.route('/social-area')
+# @register_breadcrumb(bp, '.social_area', '<i class="material-icons left">group</i>Social Area')
+# @login_required
+# def social_area():
+#     corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
+#     users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
+#     return render_template(
+#         'main/social_area.html.j2',
+#         title='Social Area',
+#         corpora=corpora,
+#         users=users
+#     )

app/models.py

@@ -1,16 +1,18 @@
 from datetime import datetime, timedelta
 from enum import Enum, IntEnum
-from flask import current_app, url_for
+from flask import abort, current_app, url_for
 from flask_hashids import HashidMixin
 from flask_login import UserMixin
 from sqlalchemy.ext.associationproxy import association_proxy
 from time import sleep
 from tqdm import tqdm
+from typing import Union
 from werkzeug.security import generate_password_hash, check_password_hash
 from werkzeug.utils import secure_filename
 import json
 import jwt
 import os
+import re
 import requests
 import secrets
 import shutil
@@ -36,6 +38,16 @@ class CorpusStatus(IntEnum):
     RUNNING_ANALYSIS_SESSION = 8
     CANCELING_ANALYSIS_SESSION = 9
 
+    @staticmethod
+    def get(corpus_status: Union['CorpusStatus', int, str]) -> 'CorpusStatus':
+        if isinstance(corpus_status, CorpusStatus):
+            return corpus_status
+        if isinstance(corpus_status, int):
+            return CorpusStatus(corpus_status)
+        if isinstance(corpus_status, str):
+            return CorpusStatus[corpus_status]
+        raise TypeError('corpus_status must be CorpusStatus, int, or str')
+
 
 class JobStatus(IntEnum):
     INITIALIZING = 1
@@ -47,6 +59,16 @@ class JobStatus(IntEnum):
     COMPLETED = 7
     FAILED = 8
 
+    @staticmethod
+    def get(job_status: Union['JobStatus', int, str]) -> 'JobStatus':
+        if isinstance(job_status, JobStatus):
+            return job_status
+        if isinstance(job_status, int):
+            return JobStatus(job_status)
+        if isinstance(job_status, str):
+            return JobStatus[job_status]
+        raise TypeError('job_status must be JobStatus, int, or str')
+
 
 class Permission(IntEnum):
     '''
@@ -57,6 +79,16 @@ class Permission(IntEnum):
     CONTRIBUTE = 2
     USE_API = 4
 
+    @staticmethod
+    def get(permission: Union['Permission', int, str]) -> 'Permission':
+        if isinstance(permission, Permission):
+            return permission
+        if isinstance(permission, int):
+            return Permission(permission)
+        if isinstance(permission, str):
+            return Permission[permission]
+        raise TypeError('permission must be Permission, int, or str')
+
 
 class UserSettingJobStatusMailNotificationLevel(IntEnum):
     NONE = 1
@@ -69,10 +101,31 @@ class ProfilePrivacySettings(IntEnum):
     SHOW_LAST_SEEN = 2
     SHOW_MEMBER_SINCE = 4
 
-class CorpusFollowPermission(IntEnum):
+    @staticmethod
+    def get(profile_privacy_setting: Union['ProfilePrivacySettings', int, str]) -> 'ProfilePrivacySettings':
+        if isinstance(profile_privacy_setting, ProfilePrivacySettings):
+            return profile_privacy_setting
+        if isinstance(profile_privacy_setting, int):
+            return ProfilePrivacySettings(profile_privacy_setting)
+        if isinstance(profile_privacy_setting, str):
+            return ProfilePrivacySettings[profile_privacy_setting]
+        raise TypeError('profile_privacy_setting must be ProfilePrivacySettings, int, or str')
+
+class CorpusFollowerPermission(IntEnum):
     VIEW = 1
-    CONTRIBUTE = 2
-    ADMINISTRATE = 4
+    MANAGE_FILES = 2
+    MANAGE_FOLLOWERS = 4
+    MANAGE_CORPUS = 8
+
+    @staticmethod
+    def get(corpus_follower_permission: Union['CorpusFollowerPermission', int, str]) -> 'CorpusFollowerPermission':
+        if isinstance(corpus_follower_permission, CorpusFollowerPermission):
+            return corpus_follower_permission
+        if isinstance(corpus_follower_permission, int):
+            return CorpusFollowerPermission(corpus_follower_permission)
+        if isinstance(corpus_follower_permission, str):
+            return CorpusFollowerPermission[corpus_follower_permission]
+        raise TypeError('corpus_follower_permission must be CorpusFollowerPermission, int, or str')
 # endregion enums
 
 
@@ -180,16 +233,19 @@ class Role(HashidMixin, db.Model):
     def __repr__(self):
         return f'<Role {self.name}>'
 
-    def add_permission(self, permission):
-        if not self.has_permission(permission):
-            self.permissions += permission
+    def has_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        return self.permissions & p.value == p.value
     
-    def has_permission(self, permission):
-        return self.permissions & permission == permission
+    def add_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if not self.has_permission(p):
+            self.permissions += p.value
     
-    def remove_permission(self, permission):
-        if self.has_permission(permission):
-            self.permissions -= permission
+    def remove_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if self.has_permission(p):
+            self.permissions -= p.value
 
     def reset_permissions(self):
         self.permissions = 0
@@ -199,8 +255,13 @@ class Role(HashidMixin, db.Model):
             'id': self.hashid,
             'default': self.default,
             'name': self.name,
-            'permissions': self.permissions
+            'permissions': [
+                x.name for x in Permission
+                if self.has_permission(x.value)
+            ]
         }
+        if backrefs:
+            pass
         if relationships:
             json_serializeable['users'] = {
                 x.hashid: x.to_json_serializeable(relationships=True)
@@ -252,6 +313,27 @@ class Token(db.Model):
         self.access_expiration = datetime.utcnow()
         self.refresh_expiration = datetime.utcnow()
 
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'access_token': self.access_token,
+            'access_expiration': (
+                None if self.access_expiration is None
+                else f'{self.access_expiration.isoformat()}Z'
+            ),
+            'refresh_token': self.refresh_token,
+            'refresh_expiration': (
+                None if self.refresh_expiration is None
+                else f'{self.refresh_expiration.isoformat()}Z'
+            )
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable
+
     @staticmethod
     def clean():
         """Remove any tokens that have been expired for more than a day."""
@@ -284,35 +366,143 @@ class Avatar(HashidMixin, FileMixin, db.Model):
             'id': self.hashid,
             **self.file_mixin_to_json_serializeable()
         }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
-class CorpusFollowerAssociation(db.Model):
+class CorpusFollowerRole(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='role'
+    )
+
+    def __repr__(self):
+        return f'<CorpusFollowerRole {self.name}>'
+
+    def has_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        return self.permissions & perm.value == perm.value
+    
+    def add_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if not self.has_permission(perm):
+            self.permissions += perm.value
+    
+    def remove_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if self.has_permission(perm):
+            self.permissions -= perm.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name
+                for x in CorpusFollowerPermission
+                if self.has_permission(x)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['corpus_follower_association'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpus_follower_association
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'Anonymous': [],
+            'Viewer': [
+                CorpusFollowerPermission.VIEW
+            ],
+            'Contributor': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES
+            ],
+            'Administrator': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES,
+                CorpusFollowerPermission.MANAGE_FOLLOWERS,
+                CorpusFollowerPermission.MANAGE_CORPUS
+
+            ]
+        }
+        default_role_name = 'Viewer'
+        for role_name, permissions in roles.items():
+            role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+            if role is None:
+                role = CorpusFollowerRole(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()
+
+
+class CorpusFollowerAssociation(HashidMixin, db.Model):
     __tablename__ = 'corpus_follower_associations'
     # Primary key
     id = db.Column(db.Integer, primary_key=True)
     # Foreign keys
-    following_user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
-    followed_corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
-    # Fields
-    permissions = db.Column(db.Integer, default=0, nullable=False)
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    follower_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    role_id = db.Column(db.Integer, db.ForeignKey('corpus_follower_roles.id'))
     # Relationships
-    followed_corpus = db.relationship('Corpus', back_populates='following_user_associations')
-    following_user = db.relationship('User', back_populates='followed_corpus_associations')
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='corpus_follower_associations'
+    )
+    follower = db.relationship(
+        'User',
+        back_populates='corpus_follower_associations'
+    )
+    role = db.relationship(
+        'CorpusFollowerRole',
+        back_populates='corpus_follower_associations'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = CorpusFollowerRole.query.filter_by(default=True).first()
+        super().__init__(**kwargs)
 
     def __repr__(self):
-        return f'<CorpusFollowerAssociation {self.following_user.__repr__()} ~ {self.followed_corpus.__repr__()}>'
+        return f'<CorpusFollowerAssociation {self.follower.__repr__()} ~ {self.role.__repr__()} ~ {self.corpus.__repr__()}>'
 
-    def has_permission(self, permission):
-        return self.permissions & permission == permission
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'corpus': self.corpus.to_json_serializeable(backrefs=True),
+            'follower': self.follower.to_json_serializeable(),
+            'role': self.role.to_json_serializeable()
+        }
+        if backrefs:
+            pass
+        if relationships:
+            pass
+        return json_serializeable
 
-    def add_permission(self, permission):
-        if not self.has_permission(permission):
-            self.permissions += permission
-    
-    def remove_permission(self, permission):
-        if self.has_permission(permission):
-            self.permissions -= permission
 
 class User(HashidMixin, UserMixin, db.Model):
     __tablename__ = 'users'
@@ -323,8 +513,10 @@ class User(HashidMixin, UserMixin, db.Model):
     # Fields
     email = db.Column(db.String(254), index=True, unique=True)
     username = db.Column(db.String(64), index=True, unique=True)
+    username_pattern = re.compile(r'^[A-Za-zÄÖÜäöüß0-9_.]*$')
     password_hash = db.Column(db.String(128))
     confirmed = db.Column(db.Boolean, default=False)
+    terms_of_use_accepted = db.Column(db.Boolean, default=False)
     member_since = db.Column(db.DateTime(), default=datetime.utcnow)
     setting_job_status_mail_notification_level = db.Column(
         IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
@@ -351,14 +543,15 @@ class User(HashidMixin, UserMixin, db.Model):
         cascade='all, delete-orphan',
         lazy='dynamic'
     )
-    followed_corpus_associations = db.relationship(
+    corpus_follower_associations = db.relationship(
         'CorpusFollowerAssociation',
-        back_populates='following_user'
+        back_populates='follower',
+        cascade='all, delete-orphan'
     )
     followed_corpora = association_proxy(
-        'followed_corpus_associations',
-        'followed_corpus',
-        creator=lambda c: CorpusFollowerAssociation(followed_corpus=c)
+        'corpus_follower_associations',
+        'corpus',
+        creator=lambda c: CorpusFollowerAssociation(corpus=c)
     )
     jobs = db.relationship(
         'Job',
@@ -390,13 +583,13 @@ class User(HashidMixin, UserMixin, db.Model):
     )
 
     def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = (
+                Role.query.filter_by(name='Administrator').first()
+                if kwargs['email'] == current_app.config['NOPAQUE_ADMIN']
+                else Role.query.filter_by(default=True).first()
+            )
         super().__init__(**kwargs)
-        if self.role is not None:
-            return
-        if self.email == current_app.config['NOPAQUE_ADMIN']:
-            self.role = Role.query.filter_by(name='Administrator').first()
-        else:
-            self.role = Role.query.filter_by(default=True).first()
 
     def __repr__(self):
         return f'<User {self.username}>'
@@ -495,7 +688,7 @@ class User(HashidMixin, UserMixin, db.Model):
             db.session.commit()
 
     def can(self, permission):
-        return self.role.has_permission(permission)
+        return self.role is not None and self.role.has_permission(permission)
 
     def confirm(self, confirmation_token):
         try:
@@ -506,7 +699,6 @@ class User(HashidMixin, UserMixin, db.Model):
                 issuer=current_app.config['SERVER_NAME'],
                 options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
             )
-            current_app.logger.warning(payload)
         except jwt.PyJWTError:
             return False
         if payload.get('purpose') != 'user.confirm':
@@ -577,42 +769,97 @@ class User(HashidMixin, UserMixin, db.Model):
 
     #region Profile Privacy settings
     def has_profile_privacy_setting(self, setting):
-        return self.profile_privacy_settings & setting == setting
+        s = ProfilePrivacySettings.get(setting)
+        return self.profile_privacy_settings & s.value == s.value
     
     def add_profile_privacy_setting(self, setting):
-        if not self.has_profile_privacy_setting(setting):
-            self.profile_privacy_settings += setting
+        s = ProfilePrivacySettings.get(setting)
+        if not self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings += s.value
 
     def remove_profile_privacy_setting(self, setting):
-        if self.has_profile_privacy_setting(setting):
-            self.profile_privacy_settings -= setting
+        s = ProfilePrivacySettings.get(setting)
+        if self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings -= s.value
 
     def reset_profile_privacy_settings(self):
         self.profile_privacy_settings = 0
     #endregion Profile Privacy settings
 
-    def follow_corpus(self, corpus):
-        if not self.is_following_corpus(corpus):
-            self.followed_corpora.append(corpus)
+    def follow_corpus(self, corpus, role=None):
+        if role is None:
+            cfr = CorpusFollowerRole.query.filter_by(default=True).first()
+        else:
+            cfr = role
+        if self.is_following_corpus(corpus):
+            cfa = CorpusFollowerAssociation.query.filter_by(corpus=corpus, follower=self).first()
+            if cfa.role != cfr:
+                cfa.role = cfr
+        else:
+            cfa = CorpusFollowerAssociation(corpus=corpus, role=cfr, follower=self)
+            db.session.add(cfa)
 
     def unfollow_corpus(self, corpus):
-        if self.is_following_corpus(corpus):
+        if not self.is_following_corpus(corpus):
+            return
         self.followed_corpora.remove(corpus)
 
     def is_following_corpus(self, corpus):
         return corpus in self.followed_corpora
     
+    def generate_follow_corpus_token(self, corpus_hashid, role_name, expiration=7):
+        now = datetime.utcnow()
+        payload = {
+            'exp': expiration,
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'User.follow_corpus',
+            'role_name': role_name,
+            'sub': corpus_hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+    
+    def follow_corpus_by_token(self, token):
+        try:
+            payload = jwt.decode(
+                token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'role_name', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'User.follow_corpus':
+            return False
+        corpus_hashid = payload.get('sub')
+        corpus_id = hashids.decode(corpus_hashid)
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if corpus is None:
+            return False
+        role_name = payload.get('role_name')
+        role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+        if role is None:
+            return False
+        self.follow_corpus(corpus, role)
+        # db.session.add(self)
+        return True
 
     def to_json_serializeable(self, backrefs=False, relationships=False, filter_by_privacy_settings=False):
         json_serializeable = {
             'id': self.hashid,
             'confirmed': self.confirmed,
+            # 'avatar': url_for('users.user_avatar', user_id=self.id),
             'email': self.email,
             'last_seen': (
                 None if self.last_seen is None
-                else self.last_seen.strftime('%Y-%m-%d %H:%M')
+                else f'{self.last_seen.isoformat()}Z'
             ),
-            'member_since': self.member_since.strftime('%Y-%m-%d'),
+            'member_since': f'{self.member_since.isoformat()}Z',
             'username': self.username,
             'full_name': self.full_name,
             'about_me': self.about_me,
@@ -621,19 +868,21 @@ class User(HashidMixin, UserMixin, db.Model):
             'organization': self.organization,
             'job_status_mail_notification_level': \
                     self.setting_job_status_mail_notification_level.name,
+            'profile_privacy_settings': {
                 'is_public': self.is_public,
                 'show_email': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL),
                 'show_last_seen': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN),
                 'show_member_since': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE)
             }
-        json_serializeable['avatar'] = (
-            None if self.avatar is None
-            else self.avatar.to_json_serializeable(relationships=True)
-        )
+        }
         if backrefs:
             json_serializeable['role'] = \
                 self.role.to_json_serializeable(backrefs=True)
         if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
             json_serializeable['corpora'] = {
                 x.hashid: x.to_json_serializeable(relationships=True)
                 for x in self.corpora
@@ -650,10 +899,6 @@ class User(HashidMixin, UserMixin, db.Model):
                 x.hashid: x.to_json_serializeable(relationships=True)
                 for x in self.spacy_nlp_pipeline_models
             }
-            json_serializeable['followed_corpora'] = {
-                x.hashid: x.to_json_serializeable(relationships=True)
-                for x in self.followed_corpora
-            }
 
         if filter_by_privacy_settings:
             if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL):
@@ -786,6 +1031,8 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
         if backrefs:
             json_serializeable['user'] = \
                 self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
@@ -912,7 +1159,10 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
             **self.file_mixin_to_json_serializeable()
         }
         if backrefs:
-            json_serializeable['user'] = self.user.to_json_serializeable(backrefs=True)
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
@@ -971,6 +1221,8 @@ class JobInput(FileMixin, HashidMixin, db.Model):
         if backrefs:
             json_serializeable['job'] = \
                 self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
@@ -1035,6 +1287,8 @@ class JobResult(FileMixin, HashidMixin, db.Model):
         if backrefs:
             json_serializeable['job'] = \
                 self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
@@ -1114,7 +1368,6 @@ class Job(HashidMixin, db.Model):
             raise e
         return job
 
-
     def delete(self):
         ''' Delete the job and its inputs and results from the database. '''
         if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:  # noqa
@@ -1159,8 +1412,7 @@ class Job(HashidMixin, db.Model):
             'service_args': self.service_args,
             'service_version': self.service_version,
             'status': self.status.name,
-            'title': self.title,
-            'url': self.url
+            'title': self.title
         }
         if backrefs:
             json_serializeable['user'] = \
@@ -1246,9 +1498,9 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
     def to_json_serializeable(self, backrefs=False, relationships=False):
         json_serializeable = {
             'id': self.hashid,
-            'url': self.url,
             'address': self.address,
             'author': self.author,
+            'description': self.description,
             'booktitle': self.booktitle,
             'chapter': self.chapter,
             'editor': self.editor,
@@ -1267,6 +1519,8 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
         if backrefs:
             json_serializeable['corpus'] = \
                 self.corpus.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
         return json_serializeable
 
 
@@ -1297,14 +1551,15 @@ class Corpus(HashidMixin, db.Model):
         lazy='dynamic',
         cascade='all, delete-orphan'
     )
-    following_user_associations = db.relationship(
+    corpus_follower_associations = db.relationship(
         'CorpusFollowerAssociation',
-        back_populates='followed_corpus'
+        back_populates='corpus',
+        cascade='all, delete-orphan'
     )
-    following_users = association_proxy(
-        'following_user_associations',
-        'following_user',
-        creator=lambda u: CorpusFollowerAssociation(following_user=u)
+    followers = association_proxy(
+        'corpus_follower_associations',
+        'follower',
+        creator=lambda u: CorpusFollowerAssociation(follower=u)
     )
     user = db.relationship('User', back_populates='corpora')
     # "static" attributes
@@ -1315,7 +1570,7 @@ class Corpus(HashidMixin, db.Model):
 
     @property
     def analysis_url(self):
-        return url_for('corpora.analyse_corpus', corpus_id=self.id)
+        return url_for('corpora.analysis', corpus_id=self.id)
 
     @property
     def jsonpatch_path(self):
@@ -1403,8 +1658,13 @@ class Corpus(HashidMixin, db.Model):
             'is_public': self.is_public
         }
         if backrefs:
-            json_serializeable['user'] = self.user.to_json_serializeable(backrefs=True)
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
         if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
             json_serializeable['files'] = {
                 x.hashid: x.to_json_serializeable(relationships=True)
                 for x in self.files
@@ -1424,11 +1684,27 @@ class Corpus(HashidMixin, db.Model):
 @db.event.listens_for(JobResult, 'after_delete')
 @db.event.listens_for(SpaCyNLPPipelineModel, 'after_delete')
 @db.event.listens_for(TesseractOCRPipelineModel, 'after_delete')
-def ressource_after_delete(mapper, connection, ressource):
-    jsonpatch = [{'op': 'remove', 'path': ressource.jsonpatch_path}]
-    room = f'users.{ressource.user_hashid}'
-    socketio.emit('users.patch', jsonpatch, room=room)
-    room = f'/users/{ressource.user_hashid}'
+def resource_after_delete(mapper, connection, resource):
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': resource.jsonpatch_path
+        }
+    ]
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+@db.event.listens_for(CorpusFollowerAssociation, 'after_delete')
+def cfa_after_delete_handler(mapper, connection, cfa):
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': jsonpatch_path
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
     socketio.emit('PATCH', jsonpatch, room=room)
 
 
@@ -1439,14 +1715,33 @@ def ressource_after_delete(mapper, connection, ressource):
 @db.event.listens_for(JobResult, 'after_insert')
 @db.event.listens_for(SpaCyNLPPipelineModel, 'after_insert')
 @db.event.listens_for(TesseractOCRPipelineModel, 'after_insert')
-def ressource_after_insert_handler(mapper, connection, ressource):
-    value = ressource.to_json_serializeable()
+def resource_after_insert_handler(mapper, connection, resource):
+    jsonpatch_value = resource.to_json_serializeable()
     for attr in mapper.relationships:
-        value[attr.key] = {}
+        jsonpatch_value[attr.key] = {}
     jsonpatch = [
-        {'op': 'add', 'path': ressource.jsonpatch_path, 'value': value}
+        {
+            'op': 'add',
+            'path': resource.jsonpatch_path,
+            'value': jsonpatch_value
+        }
     ]
-    room = f'/users/{ressource.user_hashid}'
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+@db.event.listens_for(CorpusFollowerAssociation, 'after_insert')
+def cfa_after_insert_handler(mapper, connection, cfa):
+    jsonpatch_value = cfa.to_json_serializeable()
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'add',
+            'path': jsonpatch_path,
+            'value': jsonpatch_value
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
     socketio.emit('PATCH', jsonpatch, room=room)
 
 
@@ -1457,28 +1752,29 @@ def ressource_after_insert_handler(mapper, connection, ressource):
 @db.event.listens_for(JobResult, 'after_update')
 @db.event.listens_for(SpaCyNLPPipelineModel, 'after_update')
 @db.event.listens_for(TesseractOCRPipelineModel, 'after_update')
-def ressource_after_update_handler(mapper, connection, ressource):
+def resource_after_update_handler(mapper, connection, resource):
     jsonpatch = []
-    for attr in db.inspect(ressource).attrs:
+    for attr in db.inspect(resource).attrs:
         if attr.key in mapper.relationships:
             continue
         if not attr.load_history().has_changes():
             continue
+        jsonpatch_path = f'{resource.jsonpatch_path}/{attr.key}'
         if isinstance(attr.value, datetime):
-            value = f'{attr.value.isoformat()}Z'
+            jsonpatch_value = f'{attr.value.isoformat()}Z'
         elif isinstance(attr.value, Enum):
-            value = attr.value.name
+            jsonpatch_value = attr.value.name
         else:
-            value = attr.value
+            jsonpatch_value = attr.value
         jsonpatch.append(
             {
                 'op': 'replace',
-                'path': f'{ressource.jsonpatch_path}/{attr.key}',
-                'value': value
+                'path': jsonpatch_path,
+                'value': jsonpatch_value
             }
         )
     if jsonpatch:
-        room = f'/users/{ressource.user_hashid}'
+        room = f'/users/{resource.user_hashid}'
         socketio.emit('PATCH', jsonpatch, room=room)
 
 

app/services/__init__.py

@@ -1,4 +1,5 @@
 from flask import Blueprint
+from flask_login import login_required
 import os
 import yaml
 
@@ -9,4 +10,16 @@ with open(services_file, 'r') as f:
     SERVICES = yaml.safe_load(f)
 
 bp = Blueprint('services', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
 from . import routes  # noqa

app/services/forms.py

@@ -1,12 +1,17 @@
-from flask_login import current_user
 from flask_wtf import FlaskForm
+from flask_login import current_user
 from flask_wtf.file import FileField, FileRequired
-from wtforms import (BooleanField, DecimalRangeField, MultipleFileField,
-                     SelectField, StringField, SubmitField, ValidationError)
+from wtforms import (
+    BooleanField,
+    DecimalRangeField,
+    MultipleFileField,
+    SelectField,
+    StringField,
+    SubmitField,
+    ValidationError
+)
 from wtforms.validators import InputRequired, Length
-
 from app.models import SpaCyNLPPipelineModel, TesseractOCRPipelineModel
-
 from . import SERVICES
 
 
@@ -33,6 +38,8 @@ class CreateFileSetupPipelineJobForm(CreateJobBaseForm):
                 raise ValidationError('JPEG, PNG and TIFF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-file-setup-pipeline-job-form'
         service_manifest = SERVICES['file-setup-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
@@ -60,6 +67,8 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
             raise ValidationError('PDF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-job-form'
         service_manifest = SERVICES['tesseract-ocr-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
@@ -75,12 +84,18 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
                 del self.binarization.render_kw['disabled']
                 if 'ocropus_nlbin_threshold' in service_info['methods']:
                     del self.ocropus_nlbin_threshold.render_kw['disabled']
+        user_models = [
+            x for x in current_user.tesseract_ocr_pipeline_models.order_by(TesseractOCRPipelineModel.title).all()
+        ]
         models = [
             x for x in TesseractOCRPipelineModel.query.order_by(TesseractOCRPipelineModel.title).all()
             if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
         ]
-        self.model.choices = [('', 'Choose your option')]
-        self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
         self.model.default = ''
         self.version.choices = [(x, x) for x in service_manifest['versions']]
         self.version.data = version
@@ -106,6 +121,8 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
             raise ValidationError('PDF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-transkribus-htr-pipeline-job-form'
         transkribus_htr_pipeline_models = kwargs.pop('transkribus_htr_pipeline_models', [])
         service_manifest = SERVICES['transkribus-htr-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
@@ -144,6 +161,8 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
             raise ValidationError('Plain text files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-job-form'
         service_manifest = SERVICES['spacy-nlp-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
@@ -155,12 +174,18 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
         if 'methods' in service_info:
             if 'encoding_detection' in service_info['methods']:
                 del self.encoding_detection.render_kw['disabled']
-        models = [
-            x for x in SpaCyNLPPipelineModel.query.order_by(SpaCyNLPPipelineModel.title).all()
-            if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
+        user_models = [
+            x for x in current_user.spacy_nlp_pipeline_models.order_by(SpaCyNLPPipelineModel.title).all()
         ]
-        self.model.choices = [('', 'Choose your option')]
-        self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        models = [
+            x for x in SpaCyNLPPipelineModel.query.filter(SpaCyNLPPipelineModel.user != current_user, SpaCyNLPPipelineModel.is_public == True).order_by(SpaCyNLPPipelineModel.title).all()
+            if version in x.compatible_service_versions
+        ]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
         self.model.default = ''
         self.version.choices = [(x, x) for x in service_manifest['versions']]
         self.version.data = version

app/services/routes.py

@@ -1,5 +1,6 @@
-from flask import abort, current_app, flash, make_response, Markup, render_template, request
-from flask_login import current_user, login_required
+from flask import abort, current_app, flash, Markup, redirect, render_template, request, url_for
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
 import requests
 from app import db, hashids
 from app.models import (
@@ -18,8 +19,14 @@ from .forms import (
 )
 
 
+@bp.route('/services')
+@register_breadcrumb(bp, '.', 'Services')
+def services():
+    return redirect(url_for('main.dashboard'))
+
+
 @bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.file_setup_pipeline', '<i class="nopaque-icons service-icons left" data-service="file-setup-pipeline"></i>File Setup')
 def file_setup_pipeline():
     service = 'file-setup-pipeline'
     service_manifest = SERVICES[service]
@@ -54,13 +61,13 @@ def file_setup_pipeline():
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/file_setup_pipeline.html.j2',
-        form=form,
-        title=service_manifest['name']
+        title=service_manifest['name'],
+        form=form
     )
 
 
 @bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.tesseract_ocr_pipeline', '<i class="nopaque-icons service-icons left" data-service="tesseract-ocr-pipeline"></i>Tesseract OCR Pipeline')
 def tesseract_ocr_pipeline():
     service_name = 'tesseract-ocr-pipeline'
     service_manifest = SERVICES[service_name]
@@ -100,16 +107,18 @@ def tesseract_ocr_pipeline():
         x for x in TesseractOCRPipelineModel.query.all()
         if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
     ]
+    user_tesseract_ocr_pipeline_models_count = len(current_user.tesseract_ocr_pipeline_models.all())
     return render_template(
         'services/tesseract_ocr_pipeline.html.j2',
+        title=service_manifest['name'],
         form=form,
         tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models,
-        title=service_manifest['name']
+        user_tesseract_ocr_pipeline_models_count=user_tesseract_ocr_pipeline_models_count
     )
 
 
 @bp.route('/transkribus-htr-pipeline', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.transkribus_htr_pipeline', '<i class="nopaque-icons service-icons left" data-service="transkribus-htr-pipeline"></i>Transkribus HTR Pipeline')
 def transkribus_htr_pipeline():
     if not current_app.config.get('NOPAQUE_TRANSKRIBUS_ENABLED'):
         abort(404)
@@ -126,10 +135,9 @@ def transkribus_htr_pipeline():
         abort(500)
     transkribus_htr_pipeline_models = r.json()['trpModelMetadata']
     transkribus_htr_pipeline_models.append({'modelId': 48513, 'name': 'Caroline Minuscle', 'language': 'lat', 'isoLanguages': ['lat']})
-    print(transkribus_htr_pipeline_models[len(transkribus_htr_pipeline_models)-1])
     form = CreateTranskribusHTRPipelineJobForm(
-        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models,
         prefix='create-job-form',
+        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models,
         version=version
     )
     if form.is_submitted():
@@ -161,14 +169,14 @@ def transkribus_htr_pipeline():
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/transkribus_htr_pipeline.html.j2',
-        form=form,
         title=service_manifest['name'],
+        form=form,
         transkribus_htr_pipeline_models=transkribus_htr_pipeline_models
     )
 
 
 @bp.route('/spacy-nlp-pipeline', methods=['GET', 'POST'])
-@login_required
+@register_breadcrumb(bp, '.spacy_nlp_pipeline', '<i class="nopaque-icons service-icons left" data-service="spacy-nlp-pipeline"></i>SpaCy NLP Pipeline')
 def spacy_nlp_pipeline():
     service = 'spacy-nlp-pipeline'
     service_manifest = SERVICES[service]
@@ -177,6 +185,7 @@ def spacy_nlp_pipeline():
         abort(404)
     form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
     spacy_nlp_pipeline_models = SpaCyNLPPipelineModel.query.all()
+    user_spacy_nlp_pipeline_models_count = len(current_user.spacy_nlp_pipeline_models.all())
     if form.is_submitted():
         if not form.validate():
             response = {'errors': form.errors}
@@ -206,16 +215,17 @@ def spacy_nlp_pipeline():
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/spacy_nlp_pipeline.html.j2',
+        title=service_manifest['name'],
         form=form,
         spacy_nlp_pipeline_models=spacy_nlp_pipeline_models,
-        title=service_manifest['name']
+        user_spacy_nlp_pipeline_models_count=user_spacy_nlp_pipeline_models_count
     )
 
 
 @bp.route('/corpus-analysis')
-@login_required
+@register_breadcrumb(bp, '.corpus_analysis', '<i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus Analysis')
 def corpus_analysis():
     return render_template(
         'services/corpus_analysis.html.j2',
-        title='Corpus analysis'
+        title='Corpus Analysis'
     )

app/settings/__init__.py

@@ -1,5 +1,18 @@
 from flask import Blueprint
+from flask_login import login_required
 
 
 bp = Blueprint('settings', __name__)
-from . import routes  # noqa
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes

app/settings/forms.py

@@ -1,43 +0,0 @@
-from flask_wtf import FlaskForm
-from wtforms import PasswordField, SelectField, SubmitField, ValidationError
-from wtforms.validators import DataRequired, EqualTo
-from app.models import UserSettingJobStatusMailNotificationLevel
-
-
-class ChangePasswordForm(FlaskForm):
-    password = PasswordField('Old password', validators=[DataRequired()])
-    new_password = PasswordField(
-        'New password',
-        validators=[
-            DataRequired(),
-            EqualTo('new_password_2', message='Passwords must match')
-        ]
-    )
-    new_password_2 = PasswordField(
-        'New password confirmation',
-        validators=[
-            DataRequired(),
-            EqualTo('new_password', message='Passwords must match')
-        ]
-    )
-    submit = SubmitField()
-
-    def __init__(self, user, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.user = user
-
-    def validate_current_password(self, field):
-        if not self.user.verify_password(field.data):
-            raise ValidationError('Invalid password')
-
-
-class EditNotificationSettingsForm(FlaskForm):
-    job_status_mail_notification_level = SelectField(
-        'Job status mail notification level',
-        choices=[
-            (x.name, x.name.capitalize())
-            for x in UserSettingJobStatusMailNotificationLevel
-        ],
-        validators=[DataRequired()]
-    )
-    submit = SubmitField()

app/settings/routes.py

@@ -1,39 +1,12 @@
-from flask import flash, redirect, render_template, url_for
-from flask_login import current_user, login_required
-from app import db
-from app.models import UserSettingJobStatusMailNotificationLevel
+from flask import g, url_for
+from flask_breadcrumbs import register_breadcrumb
+from flask_login import current_user
+from app.users.settings.routes import settings as settings_route
 from . import bp
-from .forms import ChangePasswordForm, EditNotificationSettingsForm
 
 
-@bp.route('', methods=['GET', 'POST'])
-@login_required
+@bp.route('/settings', methods=['GET', 'POST'])
+@register_breadcrumb(bp, '.', '<i class="material-icons left">settings</i>Settings')
 def settings():
-    change_password_form = ChangePasswordForm(
-        current_user,
-        prefix='change-password-form'
-    )
-    edit_notification_settings_form = EditNotificationSettingsForm(
-        data=current_user.to_json_serializeable(),
-        prefix='edit-notification-settings-form'
-    )
-    # region handle change_password_form POST
-    if change_password_form.submit.data and change_password_form.validate():
-        current_user.password = change_password_form.new_password.data
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.settings'))
-    # endregion handle change_password_form POST
-    # region handle edit_notification_settings_form POST
-    if edit_notification_settings_form.submit and edit_notification_settings_form.validate():
-        current_user.setting_job_status_mail_notification_level = edit_notification_settings_form.job_status_mail_notification_level.data
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.settings'))
-    # endregion handle edit_notification_settings_form POST
-    return render_template(
-        'settings/settings.html.j2',
-        change_password_form=change_password_form,
-        edit_notification_settings_form=edit_notification_settings_form,
-        title='Settings'
-    )
+    g._nopaque_redirect_location_on_post = url_for('.settings')
+    return settings_route(current_user.id)

app/static/css/colors.scss

@@ -22,6 +22,11 @@ $color: (
     "surface": #ffffff,
     "error": #b00020
   ),
+  "social-area": (
+    "base": #d6ae86,
+    "darken": #C98536,
+    "lighten": #EAE2DB
+  ),
   "service": (
     "corpus-analysis": (
       "base": #aa9cc9,
@@ -108,6 +113,16 @@ $color: (
   }
 }
 
+@each $key, $color-code in map-get($color, "social-area") {
+  .social-area-color-#{$key} {
+    background-color: $color-code !important;
+  }
+
+  .social-area-color-border-#{$key} {
+    border-color: $color-code !important;
+  }
+}
+
 @each $service-name, $color-palette in map-get($color, "service") {
   .service-color[data-service="#{$service-name}"] {
     background-color: map-get($color-palette, "base") !important;

app/static/css/materialize/fixes.css

@@ -1,3 +1,8 @@
 .parallax-container .parallax {
   z-index: 0;
 }
+
+.autocomplete-content {
+  width: 100% !important;
+  left: 0 !important;
+}

app/static/css/style.css

@@ -19,6 +19,10 @@
   height: 30px !important;
 }
 
+#manual-modal .manual-chapter-title {
+  display: none;
+}
+
 .show-if-only-child:not(:only-child) {
   display: none !important;
 }

app/static/js/App.js

@@ -60,6 +60,10 @@ class App {
         iconPrefix = '<i class="left nopaque-icons">J</i>';
         break;
       }
+      case 'settings': {
+        iconPrefix = '<i class="left material-icons">settings</i>';
+        break;
+      }
       default: {
         iconPrefix = '<i class="left material-icons">notifications</i>';
         break;
@@ -91,7 +95,7 @@ class App {
       .filter((operation) => {return subRegExp.test(operation.path);});
     for (let operation of subFilteredPatch) {
       let [match, userId, jobId] = operation.path.match(subRegExp);
-      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
+      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-status="${operation.value}"></span>`, 'job');
     }
 
     // Apply Patch

app/static/js/CorpusAnalysis/CorpusAnalysisApp.js

@@ -7,8 +7,6 @@ class CorpusAnalysisApp {
       container: document.querySelector('#corpus-analysis-app-container'),
       extensionTabs: document.querySelector('#corpus-analysis-app-extension-tabs'),
       initModal: document.querySelector('#corpus-analysis-app-init-modal'),
-      initError: document.querySelector('#corpus-analysis-app-init-error'),
-      initProgress: document.querySelector('#corpus-analysis-app-init-progress'),
       overview: document.querySelector('#corpus-analysis-app-overview')
     };
     // Materialize elements
@@ -27,6 +25,7 @@ class CorpusAnalysisApp {
   init() {
     this.disableActionElements();
     this.elements.m.initModal.open();
+  
     // Init data
     this.data.cQiClient = new CQiClient(this.settings.corpusId);
     this.data.cQiClient.connect()
@@ -43,14 +42,17 @@ class CorpusAnalysisApp {
           this.elements.m.initModal.close();
         },
         cQiError => {
-          this.elements.initError.innerText = JSON.stringify(cQiError);
-          this.elements.initError.classList.remove('hide');
-          this.elements.initProgress.classList.add('hide');
+          let errorsElement = this.elements.initModal.querySelector('.errors');
+          let progressElement = this.elements.initModal.querySelector('.progress');
+          errorsElement.innerText = JSON.stringify(cQiError);
+          errorsElement.classList.remove('hide');
+          progressElement.classList.add('hide');
           if ('payload' in cQiError && 'code' in cQiError.payload && 'msg' in cQiError.payload) {
             app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
           }
         }
       );
+  
     // Add event listeners
     for (let extensionSelectorElement of this.elements.overview.querySelectorAll('.extension-selector')) {
       extensionSelectorElement.addEventListener('click', () => {

app/static/js/CorpusAnalysis/CorpusAnalysisReader.js

@@ -106,41 +106,102 @@ class CorpusAnalysisReader {
   renderCorpusPagination() {
     this.clearCorpusPagination();
     if (this.data.corpus.p.pages === 0) {return;}
-    this.elements.corpusPagination.innerHTML += `
+    let pageElement;
+    // First page button. Disables first page button if on first page
+    pageElement = Utils.HTMLToElement(
+      `
         <li class="${this.data.corpus.p.page === 1 ? 'disabled' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.page === 1 ? '' : 'data-target="1"'}>
             <i class="material-icons">first_page</i>
           </a>
         </li>
-    `.trim();
-    this.elements.corpusPagination.innerHTML += `
+      `
+    );
+    this.elements.corpusPagination.appendChild(pageElement);
+    // Previous page button. Disables previous page button if on first page
+    pageElement = Utils.HTMLToElement(
+      `
         <li class="${this.data.corpus.p.has_prev ? 'waves-effect' : 'disabled'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.has_prev ? 'data-target="' + this.data.corpus.p.prev_num + '"' : ''}>
             <i class="material-icons">chevron_left</i>
           </a>
         </li>
-    `.trim();
-    for (let i = 1; i <= this.data.corpus.p.pages; i++) {
-      this.elements.corpusPagination.innerHTML += `
+      `
+    );
+    this.elements.corpusPagination.appendChild(pageElement);
+    // First page as number. Hides first page button if on first page
+    if (this.data.corpus.p.page > 6) {
+      pageElement = Utils.HTMLToElement(
+        `
+          <li class="waves-effect">
+            <a class="corpus-analysis-action pagination-trigger" data-target="1">1</a>
+          </li>
+        `
+      );
+      this.elements.corpusPagination.appendChild(pageElement);
+      pageElement = Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
+      this.elements.corpusPagination.appendChild(pageElement);
+    }
+
+    // render page buttons (5 before and 5 after current page)
+    for (let i = this.data.corpus.p.page -5; i <= this.data.corpus.p.page; i++) {
+      if (i <= 0) {continue;}
+      pageElement = Utils.HTMLToElement(
+        `
           <li class="${i === this.data.corpus.p.page ? 'active' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${i === this.data.corpus.p.page ? '' : 'data-target="' + i + '"'}>${i}</a>
           </li>
-      `.trim();
+        `
+      );
+      this.elements.corpusPagination.appendChild(pageElement);
+    };
+    for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page +5; i++) {
+      if (i > this.data.corpus.p.pages) {break;}
+      pageElement = Utils.HTMLToElement(
+        `
+          <li class="${i === this.data.corpus.p.page ? 'active' : 'waves-effect'}">
+          <a class="corpus-analysis-action pagination-trigger" ${i === this.data.corpus.p.page ? '' : 'data-target="' + i + '"'}>${i}</a>
+          </li>
+        `
+      );
+      this.elements.corpusPagination.appendChild(pageElement);
+    };
+    // Last page as number. Hides last page button if on last page
+    if (this.data.corpus.p.page < this.data.corpus.p.pages - 6) {
+      pageElement = Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
+      this.elements.corpusPagination.appendChild(pageElement);
+      pageElement = Utils.HTMLToElement(
+        `
+          <li class="waves-effect">
+            <a class="corpus-analysis-action pagination-trigger" data-target="${this.data.corpus.p.pages}">${this.data.corpus.p.pages}</a>
+          </li>
+        `
+      );
+      this.elements.corpusPagination.appendChild(pageElement);
     }
-    this.elements.corpusPagination.innerHTML += `
+    // Next page button. Disables next page button if on last page
+    pageElement = Utils.HTMLToElement(
+      `
         <li class="${this.data.corpus.p.has_next ? 'waves-effect' : 'disabled'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.has_next ? 'data-target="' + this.data.corpus.p.next_num + '"' : ''}>
             <i class="material-icons">chevron_right</i>
           </a>
         </li>
-    `.trim();
-    this.elements.corpusPagination.innerHTML += `
+      `
+    );
+    this.elements.corpusPagination.appendChild(pageElement);
+    // Last page button. Disables last page button if on last page
+    pageElement = Utils.HTMLToElement(
+      `
         <li class="${this.data.corpus.p.page === this.data.corpus.p.pages ? 'disabled' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.page === this.data.corpus.p.pages ? '' : 'data-target="' + this.data.corpus.p.pages + '"'}>
             <i class="material-icons">last_page</i>
           </a>
         </li>
-    `.trim();
+      `
+    );
+    this.elements.corpusPagination.appendChild(pageElement);
+    
     for (let paginateTriggerElement of this.elements.corpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
       paginateTriggerElement.addEventListener('click', event => {
         event.preventDefault();
@@ -182,6 +243,7 @@ class CorpusAnalysisReader {
       return;
     }
     this.app.disableActionElements();
+    window.scrollTo(top);
     this.elements.progress.classList.remove('hide');
     this.data.corpus.o.paginate(pageNum, this.settings.perPage)
       .then(

app/static/js/CorpusAnalysis/QueryBuilder.js

@@ -561,7 +561,6 @@ class ConcordanceQueryBuilder {
       if (tokenIsEmpty === false) {
         tokenQueryText = '[' + tokenQueryText + ']';
         }
-      console.log(tokenQueryText);
       this.queryChipFactory('token', tokenQueryContent, tokenQueryText);
       this.hideEverything();
       this.elements.positionalAttrArea.classList.add('hide');

app/static/js/Forms/Form.js

@@ -92,7 +92,6 @@ class Form {
       }
       if (request.status === 400) {
         let responseJson = JSON.parse(request.responseText);
-        console.log(responseJson);
         for (let [inputName, inputErrors] of Object.entries(responseJson.errors)) {
           let inputFieldElement = this.formElement
             .querySelector(`input[name$="${inputName}"], select[name$="${inputName}"]`)
@@ -122,10 +121,11 @@ class Form {
     request.setRequestHeader('Accept', 'application/json');
     let formData = new FormData(this.formElement);
     switch (this.formElement.enctype) {
-      case 'application/x-www-form-urlencoded':
+      case 'application/x-www-form-urlencoded': {
         let urlSearchParams = new URLSearchParams(formData);
         request.send(urlSearchParams);
         break;
+      }
       case 'multipart/form-data': {
         request.send(formData);
         break;

app/static/js/Requests/Requests.js

@@ -0,0 +1,40 @@
+let Requests = {};
+
+Requests.JSONfetch = (input, init={}) => {
+  return new Promise((resolve, reject) => {
+    let fixedInit = {};
+    fixedInit.headers = {};
+    fixedInit.headers['Accept'] = 'application/json';
+    if (init.hasOwnProperty('body')) {
+      fixedInit.headers['Content-Type'] = 'application/json';
+    }
+    fetch(input, Utils.mergeObjectsDeep(init, fixedInit))
+      .then(
+        (response) => {
+          if (response.ok) {
+            resolve(response.clone());
+          } else {
+            reject(response);
+          }
+          if (response.status === 204) {
+            return;
+          } 
+          response.json()
+            .then(
+              (json) => {
+                let message = json.message || json;
+                let category = json.category || 'message';
+                app.flash(message, category);
+              },
+              (error) => {
+                app.flash(`[${response.status}]: ${response.statusText}`, 'error');
+              }
+            );
+        },
+        (response) => {
+          app.flash('Something went wrong', 'error');
+          reject(response);
+        }
+      );
+  });
+};

app/static/js/Requests/admin/admin.js

@@ -0,0 +1,20 @@
+/*****************************************************************************
+* Admin                                                                      *
+* Fetch requests for /admin routes                                           *
+*****************************************************************************/
+Requests.admin = {};
+
+Requests.admin.users = {};
+
+Requests.admin.users.entity = {};
+
+Requests.admin.users.entity.confirmed = {};
+
+Requests.admin.users.entity.confirmed.update = (userId, value) => {
+  let input = `/admin/users/${userId}/confirmed`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(value)
+  };
+  return Requests.JSONfetch(input, init);
+};

app/static/js/Requests/contributions/contributions.js

@@ -0,0 +1,5 @@
+/*****************************************************************************
+* Contributions                                                              *
+* Fetch requests for /contributions routes                                   *
+*****************************************************************************/
+Requests.contributions = {};

app/static/js/Requests/contributions/spacy_nlp_pipeline_models.js

@@ -0,0 +1,26 @@
+/*****************************************************************************
+* SpaCy NLP Pipeline Models                                                  *
+* Fetch requests for /contributions/spacy-nlp-pipeline-models routes         *
+*****************************************************************************/
+Requests.contributions.spacy_nlp_pipeline_models = {};
+
+Requests.contributions.spacy_nlp_pipeline_models.entity = {};
+
+Requests.contributions.spacy_nlp_pipeline_models.entity.delete = (spacyNlpPipelineModelId) => {
+  let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic = {};
+
+Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic.update = (spacyNlpPipelineModelId, value) => {
+  let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}/is_public`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(value)
+  };
+  return Requests.JSONfetch(input, init);
+};

app/static/js/Requests/contributions/tesseract_ocr_pipeline_models.js

@@ -0,0 +1,26 @@
+/*****************************************************************************
+* Tesseract OCR Pipeline Models                                              *
+* Fetch requests for /contributions/tesseract-ocr-pipeline-models routes     *
+*****************************************************************************/
+Requests.contributions.tesseract_ocr_pipeline_models = {};
+
+Requests.contributions.tesseract_ocr_pipeline_models.entity = {};
+
+Requests.contributions.tesseract_ocr_pipeline_models.entity.delete = (tesseractOcrPipelineModelId) => {
+  let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic = {};
+
+Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic.update = (tesseractOcrPipelineModelId, value) => {
+  let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}/is_public`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(value)
+  };
+  return Requests.JSONfetch(input, init);
+};

app/static/js/Requests/corpora/corpora.js

@@ -0,0 +1,46 @@
+/*****************************************************************************
+* Corpora                                                                    *
+* Fetch requests for /corpora routes                                         *
+*****************************************************************************/
+Requests.corpora = {};
+
+Requests.corpora.entity = {};
+
+Requests.corpora.entity.delete = (corpusId) => {
+  let input = `/corpora/${corpusId}`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.corpora.entity.build = (corpusId) => {
+  let input = `/corpora/${corpusId}/build`;
+  let init = {
+    method: 'POST',
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.corpora.entity.generateShareLink = (corpusId, role, expiration) => {
+  let input = `/corpora/${corpusId}/generate-share-link`;
+  let init = {
+    method: 'POST',
+    body: JSON.stringify({role: role, expiration: expiration})
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.corpora.entity.isPublic = {};
+
+Requests.corpora.entity.isPublic.update = (corpusId, isPublic) => {
+  let input = `/corpora/${corpusId}/is_public`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(isPublic)
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+
+

app/static/js/Requests/corpora/files.js

@@ -0,0 +1,15 @@
+/*****************************************************************************
+* Corpora                                                                    *
+* Fetch requests for /corpora/<entity>/files routes                          *
+*****************************************************************************/
+Requests.corpora.entity.files = {};
+
+Requests.corpora.entity.files.ent = {};
+
+Requests.corpora.entity.files.ent.delete = (corpusId, corpusFileId) => {
+  let input = `/corpora/${corpusId}/files/${corpusFileId}`;
+  let init = {
+    method: 'DELETE',
+  };
+  return Requests.JSONfetch(input, init);
+};

app/static/js/Requests/corpora/followers.js

@@ -0,0 +1,35 @@
+/*****************************************************************************
+* Corpora                                                                    *
+* Fetch requests for /corpora/<entity>/followers routes                      *
+*****************************************************************************/
+Requests.corpora.entity.followers = {};
+
+Requests.corpora.entity.followers.add = (corpusId, usernames) => {
+  let input = `/corpora/${corpusId}/followers`;
+  let init = {
+    method: 'POST',
+    body: JSON.stringify(usernames)
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.corpora.entity.followers.entity = {};
+
+Requests.corpora.entity.followers.entity.delete = (corpusId, followerId) => {
+  let input = `/corpora/${corpusId}/followers/${followerId}`;
+  let init = {
+    method: 'DELETE',
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.corpora.entity.followers.entity.role = {};
+
+Requests.corpora.entity.followers.entity.role.update = (corpusId, followerId, value) => {
+  let input = `/corpora/${corpusId}/followers/${followerId}/role`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(value)
+  };
+  return Requests.JSONfetch(input, init);
+};

app/static/js/Requests/jobs/jobs.js

@@ -0,0 +1,31 @@
+/*****************************************************************************
+* Jobs                                                              *
+* Fetch requests for /jobs routes                                   *
+*****************************************************************************/
+Requests.jobs = {};
+
+Requests.jobs.entity = {};
+
+Requests.jobs.entity.delete = (jobId) => {
+  let input = `/jobs/${jobId}`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+} 
+
+Requests.jobs.entity.log = (jobId) => {
+  let input = `/jobs/${jobId}/log`;
+  let init = {
+    method: 'GET'
+  };
+  return Requests.JSONfetch(input, init);
+}
+
+Requests.jobs.entity.restart = (jobId) => {
+  let input = `/jobs/${jobId}/restart`;
+  let init = {
+    method: 'POST'
+  };
+  return Requests.JSONfetch(input, init);
+} 

app/static/js/Requests/users/settings.js

@@ -0,0 +1,17 @@
+/*****************************************************************************
+* Settings                                                                   *
+* Fetch requests for /users/<entity>/settings routes                         *
+*****************************************************************************/
+Requests.users.entity.settings = {};
+
+Requests.users.entity.settings.profilePrivacy = {};
+
+Requests.users.entity.settings.profilePrivacy.update = (userId, profilePrivacySetting, enabled) => {
+  let input = `/users/${userId}/settings/profile-privacy/${profilePrivacySetting}`;
+  let init = {
+    method: 'PUT',
+    body: JSON.stringify(enabled)
+  };
+  return Requests.JSONfetch(input, init);
+};
+

app/static/js/Requests/users/users.js

@@ -0,0 +1,35 @@
+/*****************************************************************************
+* Users                                                                      *
+* Fetch requests for /users routes                                           *
+*****************************************************************************/
+Requests.users = {};
+
+Requests.users.entity = {};
+
+Requests.users.entity.delete = (userId) => {
+  let input = `/users/${userId}`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+Requests.users.entity.acceptTermsOfUse = () => {
+  let input = `/users/accept-terms-of-use`;
+  let init = {
+    method: 'POST'
+  };
+  return Requests.JSONfetch(input, init);
+};
+
+
+Requests.users.entity.avatar = {};
+
+Requests.users.entity.avatar.delete = (userId) => {
+  let input = `/users/${userId}/avatar`;
+  let init = {
+    method: 'DELETE'
+  };
+  return Requests.JSONfetch(input, init);
+}
+

app/static/js/ResourceDisplays/CorpusDisplay.js

@@ -1,26 +1,11 @@
-class CorpusDisplay extends RessourceDisplay {
+class CorpusDisplay extends ResourceDisplay {
   constructor(displayElement) {
     super(displayElement);
     this.corpusId = displayElement.dataset.corpusId;
     this.displayElement
       .querySelector('.action-button[data-action="build-request"]')
       .addEventListener('click', (event) => {
-        Utils.buildCorpusRequest(this.userId, this.corpusId);
-      });
-    this.displayElement
-      .querySelector('.action-button[data-action="delete-request"]')
-      .addEventListener('click', (event) => {
-        Utils.deleteCorpusRequest(this.userId, this.corpusId);
-      });
-    this.displayElement
-      .querySelector('.action-switch[data-action="toggle-is-public"]')
-      .addEventListener('click', (event) => {
-        if (event.target.tagName !== 'INPUT') {return;}
-        if (event.target.checked) {
-          Utils.enableCorpusIsPublicRequest(this.userId, this.corpusId);
-        } else {
-          Utils.disableCorpusIsPublicRequest(this.userId, this.corpusId);
-        }
+        Requests.corpora.entity.build(this.corpusId);
       });
   }
 
@@ -81,7 +66,7 @@ class CorpusDisplay extends RessourceDisplay {
   }
 
   setStatus(status) {
-    let elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger')
+    let elements = this.displayElement.querySelectorAll('.action-button[data-action="analyze"]');
     for (let element of elements) {
       if (['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
         element.classList.remove('disabled');

app/static/js/ResourceDisplays/JobDisplay.js

@@ -1,22 +1,7 @@
-class JobDisplay extends RessourceDisplay {
+class JobDisplay extends ResourceDisplay {
   constructor(displayElement) {
     super(displayElement);
     this.jobId = this.displayElement.dataset.jobId;
-    this.displayElement
-      .querySelector('.action-button[data-action="delete-request"]')
-      .addEventListener('click', (event) => {
-        Utils.deleteJobRequest(this.userId, this.jobId);
-      });
-    this.displayElement
-      .querySelector('.action-button[data-action="get-log-request"]')
-      .addEventListener('click', (event) => {
-        Utils.getJobLogRequest(this.userId, this.jobId);
-      });
-    this.displayElement
-      .querySelector('.action-button[data-action="restart-request"]')
-      .addEventListener('click', (event) => {
-        Utils.restartJobRequest(this.userId, this.jobId);
-      });
   }
 
   init(user) {

app/static/js/ResourceDisplays/ResourceDisplay.js

@@ -1,4 +1,4 @@
-class RessourceDisplay {
+class ResourceDisplay {
   constructor(displayElement) {
     this.displayElement = displayElement;
     this.userId = this.displayElement.dataset.userId;

app/static/js/ResourceLists/CorpusFileList.js

@@ -8,9 +8,15 @@ class CorpusFileList extends ResourceList {
   constructor(listContainerElement, options = {}) {
     super(listContainerElement, options);
     this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    document.querySelectorAll('.selection-action-trigger[data-selection-action]').forEach((element) => {
+      element.addEventListener('click', (event) => {this.onSelectionAction(event)});
+    });
     this.isInitialized = false;
+    this.selectedItemIds = new Set();
     this.userId = listContainerElement.dataset.userId;
     this.corpusId = listContainerElement.dataset.corpusId;
+    this.hasPermissionView =  listContainerElement.dataset?.hasPermissionView == 'true' || false;
+    this.hasPermissionManageFiles =  listContainerElement.dataset?.hasPermissionManageFiles == 'true' || false;
     if (this.userId === undefined || this.corpusId === undefined) {return;}
     app.subscribeUser(this.userId).then((response) => {
       app.socket.on('PATCH', (patch) => {
@@ -24,20 +30,28 @@ class CorpusFileList extends ResourceList {
   }
 
   get item() {
+    return (values) => {
       return `
         <tr class="list-item clickable hoverable">
+          <td>
+            <label class="list-action-trigger ${this.hasPermissionView ? '' : 'hide'}" data-list-action="select">
+              <input class="select-checkbox" type="checkbox">
+              <span class="disable-on-click"></span>
+            </label>
+          </td>
           <td><span class="filename"></span></td>
           <td><span class="author"></span></td>
           <td><span class="title"></span></td>
           <td><span class="publishing-year"></span></td>
           <td class="right-align">
-          <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete"><i class="material-icons">delete</i></a>
-          <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="download" data-service="corpus-analysis"><i class="material-icons">file_download</i></a>
-          <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
+            <a class="list-action-trigger btn-floating red waves-effect waves-light ${this.hasPermissionManageFiles ? '' : 'hide'}" data-list-action="delete"><i class="material-icons">delete</i></a>
+            <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light ${this.hasPermissionView ? '' : 'hide'}" data-list-action="download" data-service="corpus-analysis"><i class="material-icons">file_download</i></a>
+            <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light ${this.hasPermissionManageFiles ? '' : 'hide'}" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
           </td>
         </tr>
       `.trim();
     }
+  }
 
   get valueNames() {
     return [
@@ -64,11 +78,20 @@ class CorpusFileList extends ResourceList {
       <table>
         <thead>
           <tr>
+            <th>
+              <label class="disable-on-click selection-action-trigger ${this.listContainerElement.dataset?.hasPermissionView == 'true' ? '' : 'hide'}" data-selection-action="select-all">
+                <input class="select-all-checkbox" type="checkbox">
+                <span class="disable-on-click"></span>
+              </label>
+            </th>
             <th>Filename</th>
             <th>Author</th>
             <th>Title</th>
             <th>Publishing year</th>
-            <th></th>
+            <th class="right-align">
+              <a class="selection-action-trigger btn-floating red waves-effect waves-light hide" data-selection-action="delete"><i class="material-icons">delete</i></a>
+              <a class="selection-action-trigger btn-floating service-color darken waves-effect waves-light hide" data-selection-action="download" data-service="corpus-analysis"><i class="material-icons">file_download</i></a>
+            </th>
           </tr>
         </thead>
         <tbody class="list"></tbody>
@@ -93,6 +116,7 @@ class CorpusFileList extends ResourceList {
   }
 
   onClick(event) {
+    if (event.target.closest('.disable-on-click') !== null) {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
@@ -100,7 +124,44 @@ class CorpusFileList extends ResourceList {
     let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
     switch (listAction) {
       case 'delete': {
-        Utils.deleteCorpusFileRequest(this.userId, this.corpusId, itemId);
+        let values = this.listjs.get('id', itemId)[0].values();
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Corpus File deletion</h4>
+                <p>Do you really want to delete the Corpus File <b>${values.title}</b>? All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          if (currentUserId != this.userId) {
+            Requests.corpora.entity.files.ent.delete(this.corpusId, itemId)
+            .then(() => {
+              window.location.reload();
+            });
+          } else {
+            Requests.corpora.entity.files.ent.delete(this.corpusId, itemId)
+          }
+        });
+        modal.open();
         break;
       }
       case 'download': {
@@ -111,12 +172,171 @@ class CorpusFileList extends ResourceList {
         window.location.href = `/corpora/${this.corpusId}/files/${itemId}`;
         break;
       }
+      case 'select': {
+        if (event.target.checked) {
+          this.selectedItemIds.add(itemId);
+        } else {
+          this.selectedItemIds.delete(itemId);
+        }
+        this.renderingItemSelection();
+        break;
+      }
       default: {
         break;
       }
     }
   }
 
+  onSelectionAction(event) {
+    let selectionActionElement = event.target.closest('.selection-action-trigger[data-selection-action]');
+    let selectionAction = selectionActionElement.dataset.selectionAction;
+    let items = this.listjs.items;
+    let selectableItems = Array.from(items)
+      .filter(item => item.elm)
+      .map(item => item.elm.querySelector('.select-checkbox[type="checkbox"]'));
+
+    switch (selectionAction) {
+      case 'select-all': {
+        let selectedIds = new Set(Array.from(items)
+          .map(item => item.values().id))
+        if (event.target.checked !== undefined) {
+          if (event.target.checked) {
+            selectableItems.forEach(selectableItem => selectableItem.checked = true);
+            this.selectedItemIds = selectedIds;
+          } else {
+            selectableItems.forEach(checkbox => checkbox.checked = false);
+            this.selectedItemIds = new Set([...this.selectedItemIds].filter(id => !selectedIds.has(id)));
+          }
+          this.renderingItemSelection();
+        }
+        break;
+      }
+      case 'delete': {
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Corpus File deletion</h4>
+                <p>Do you really want to delete the Corpus Files?</p>
+                  <ul id="selected-items-list"></ul>
+                <p>All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let itemList = document.querySelector('#selected-items-list');
+        this.selectedItemIds.forEach(selectedItemId => {
+          let listItem = this.listjs.get('id', selectedItemId)[0].elm;
+          let values = this.listjs.get('id', listItem.dataset.id)[0].values();
+          let itemElement = Utils.HTMLToElement(`<li> - ${values.title}</li>`);
+          itemList.appendChild(itemElement);
+        });
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          this.selectedItemIds.forEach(selectedItemId => {
+            if (currentUserId != this.userId) {
+              Requests.corpora.entity.files.ent.delete(this.corpusId, selectedItemId)
+              .then(() => {
+                window.location.reload();
+              });
+            } else {
+              Requests.corpora.entity.files.ent.delete(this.corpusId, selectedItemId);
+            }
+          });
+          this.selectedItemIds.clear();
+          this.renderingItemSelection();
+        });
+        modal.open();
+        break;
+      }
+      case 'download': {
+        this.selectedItemIds.forEach(selectedItemId => {
+          let downloadLink = document.createElement('a');
+          downloadLink.href = `/corpora/${this.corpusId}/files/${selectedItemId}/download`;
+          downloadLink.download = '';
+          downloadLink.click();
+        });
+        selectableItems.forEach(checkbox => checkbox.checked = false);
+        this.selectedItemIds.clear();
+        this.renderingItemSelection();
+        break;
+      }  
+      default: {
+        break;
+      }
+    }
+  }
+
+  renderingItemSelection() {
+    let selectionActionButtons;
+    if (this.hasPermissionManageFiles) {
+      selectionActionButtons = document.querySelectorAll('.selection-action-trigger:not([data-selection-action="select-all"])');
+    } else if (this.hasPermissionView) {
+      selectionActionButtons = document.querySelectorAll('.selection-action-trigger:not([data-selection-action="select-all"]):not([data-selection-action="delete"])');
+    }
+    let selectableItems = this.listjs.items;
+    let actionButtons = [];
+
+    Object.values(selectableItems).forEach(selectableItem => {
+      if (selectableItem.elm) {
+        let checkbox = selectableItem.elm.querySelector('.select-checkbox[type="checkbox"]');
+        if (checkbox.checked) {
+          selectableItem.elm.classList.add('grey', 'lighten-3');
+        } else {
+          selectableItem.elm.classList.remove('grey', 'lighten-3');
+        }
+        let itemActionButtons = [];
+        if (this.hasPermissionManageFiles) {
+          itemActionButtons = selectableItem.elm.querySelectorAll('.list-action-trigger:not([data-list-action="select"])');
+        } else if (this.hasPermissionView) {
+          itemActionButtons = selectableItem.elm.querySelectorAll('.list-action-trigger:not([data-list-action="select"]):not([data-list-action="delete"]):not([data-list-action="view"])');
+        }
+        itemActionButtons.forEach(itemActionButton => {
+          actionButtons.push(itemActionButton);
+        });
+      }
+    });
+    // Hide item action buttons if > 0 item is selected and show selection action buttons
+    if (this.selectedItemIds.size > 0) {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.remove('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.add('hide');
+      });
+    } else {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.add('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.remove('hide');
+      });
+    }
+
+    // Check select all checkbox if all items are selected
+    let selectAllCheckbox = document.querySelector('.select-all-checkbox[type="checkbox"]');
+    if (selectableItems.length === this.selectedItemIds.size && selectAllCheckbox.checked === false) {
+      selectAllCheckbox.checked = true;
+    } else if (selectableItems.length !== this.selectedItemIds.size && selectAllCheckbox.checked === true) {
+      selectAllCheckbox.checked = false;
+    }
+  }
+
   onPatch(patch) {
     let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)`);
     let filteredPatch = patch.filter(operation => re.test(operation.path));

app/static/js/ResourceLists/CorpusFollowerList.js

@@ -0,0 +1,199 @@
+class CorpusFollowerList extends ResourceList {
+  static autoInit() {
+    for (let corpusFollowerListElement of document.querySelectorAll('.corpus-follower-list:not(.no-autoinit)')) {
+      new CorpusFollowerList(corpusFollowerListElement);
+    }
+  }
+
+  constructor(listContainerElement, options = {}) {
+    super(listContainerElement, options);
+    this.listjs.on('updated', () => {
+      M.FormSelect.init(this.listjs.list.querySelectorAll('.list-item select'));
+    });
+    this.listjs.list.addEventListener('change', (event) => {this.onChange(event)});
+    this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    this.isInitialized = false;
+    this.userId = listContainerElement.dataset.userId;
+    this.corpusId = listContainerElement.dataset.corpusId;
+    if (this.userId === undefined || this.corpusId === undefined) {return;}
+    app.subscribeUser(this.userId).then((response) => {
+      app.socket.on('PATCH', (patch) => {
+        if (this.isInitialized) {this.onPatch(patch);}
+      });
+    });
+    app.getUser(this.userId).then((user) => {
+      let corpusFollowerAssociations = Object.values(user.corpora[this.corpusId].corpus_follower_associations);
+      // let filteredList = corpusFollowerAssociations.filter(association => association.follower.id != currentUserId);
+      // this.add(filteredList);
+      this.add(Object.values(user.corpora[this.corpusId].corpus_follower_associations));
+      this.isInitialized = true;
+    });
+  }
+
+  get item() {
+    return (values) => {
+      return `
+        <tr class="list-item clickable hoverable">
+          <td><img alt="follower-avatar" class="circle responsive-img follower-avatar" style="width:50%"></td>
+          <td><b class="follower-username"><b></td>
+          <td>
+            <span class="follower-full-name"></span>
+            <br>
+            <i class="follower-about-me"></i>
+          </td>
+          <td>
+            <div class="input-field disable-on-click list-action-trigger" data-list-action="update-role">
+              <select ${values['follower-id'] === currentUserId ? 'disabled' : ''}>
+                <option value="Viewer" ${values['role-name'] === 'Viewer' ? 'selected' : ''}>Viewer</option>
+                <option value="Contributor" ${values['role-name'] === 'Contributor' ? 'selected' : ''}>Contributor</option>
+                <option value="Administrator" ${values['role-name'] === 'Administrator' ? 'selected' : ''}>Administrator</option>
+              </select>
+            </div>
+          </td>
+          <td class="right-align">
+            <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="unfollow-request"><i class="material-icons">delete</i></a>
+            <a class="list-action-trigger btn-floating darken waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
+          </td>
+        </tr>
+      `.trim();
+    }
+  }
+
+  get valueNames() {
+    return [
+      {data: ['id']},
+      {data: ['follower-id']},
+      {name: 'follower-avatar', attr: 'src'},
+      'follower-username',
+      'follower-about-me',
+      'follower-full-name'
+    ];
+  }
+
+  initListContainerElement() {
+    if (!this.listContainerElement.hasAttribute('id')) {
+      this.listContainerElement.id = Utils.generateElementId('corpus-follower-list-');
+    }
+    let listSearchElementId = Utils.generateElementId(`${this.listContainerElement.id}-search-`);
+    this.listContainerElement.innerHTML = `
+      <div class="input-field">
+        <i class="material-icons prefix">search</i>
+        <input id="${listSearchElementId}" class="search" type="text"></input>
+        <label for="${listSearchElementId}">Search corpus follower</label>
+      </div>
+      <table>
+        <thead>
+          <tr>
+            <th style="width:15%;"></th>
+            <th>Username</th>
+            <th>User details</th>
+            <th>Role</th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody class="list"></tbody>
+      </table>
+      <ul class="pagination"></ul>
+    `.trim();
+  }
+
+  mapResourceToValue(corpusFollowerAssociation) {
+    return {
+      'id': corpusFollowerAssociation.id,
+      'follower-id': corpusFollowerAssociation.follower.id,
+      'follower-avatar': corpusFollowerAssociation.follower.avatar ? `/users/${corpusFollowerAssociation.follower.id}/avatar` : '/static/images/user_avatar.png',
+      'follower-username': corpusFollowerAssociation.follower.username,
+      'follower-full-name': corpusFollowerAssociation.follower.full_name ? corpusFollowerAssociation.follower.full_name : '',
+      'follower-about-me': corpusFollowerAssociation.follower.about_me ? corpusFollowerAssociation.follower.about_me : '',
+      'role-name': corpusFollowerAssociation.role.name
+    };
+  }
+
+  sort() {
+    this.listjs.sort('username', {order: 'desc'});
+  }
+
+  onChange(event) {
+    let listItemElement = event.target.closest('.list-item[data-id]');
+    if (listItemElement === null) {return;}
+    let itemId = listItemElement.dataset.id;
+    let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
+    if (listActionElement === null) {return;}
+    let listAction = listActionElement.dataset.listAction;
+    switch (listAction) {
+      case 'update-role': {
+        let followerId = listItemElement.dataset.followerId;
+        let roleName = event.target.value;
+        Requests.corpora.entity.followers.entity.role.update(this.corpusId, followerId, roleName);
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+  
+  onClick(event) {
+    if (event.target.closest('.disable-on-click') !== null) {return;}
+    let listItemElement = event.target.closest('.list-item[data-id]');
+    if (listItemElement === null) {return;}
+    let itemId = listItemElement.dataset.id;
+    let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
+    let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
+    switch (listAction) {
+      case 'unfollow-request': {
+        let followerId = listItemElement.dataset.followerId;
+        if (currentUserId != this.userId) {
+          Requests.corpora.entity.followers.entity.delete(this.corpusId, followerId)
+            .then(() => {
+              window.location.reload();
+            });
+        } else {
+          Requests.corpora.entity.followers.entity.delete(this.corpusId, followerId);
+        }
+        break;
+      }
+      case 'view': {
+        let followerId = listItemElement.dataset.followerId;
+        window.location.href = `/users/${followerId}`;
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+
+  onPatch(patch) {
+    let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/corpus_follower_associations/([A-Za-z0-9]*)`);
+    let filteredPatch = patch.filter(operation => re.test(operation.path));
+    for (let operation of filteredPatch) {
+      switch(operation.op) {
+        case 'add': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/corpus_follower_associations/([A-Za-z0-9]*)$`);
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        }
+        case 'remove': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/corpus_follower_associations/([A-Za-z0-9]*)$`);
+          if (re.test(operation.path)) {
+            let [match, jobId] = operation.path.match(re);
+            this.remove(jobId);
+          }
+          break;
+        }
+        case 'replace': {
+          let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/corpus_follower_associations/([A-Za-z0-9]*)/role$`);
+          if (re.test(operation.path)) {
+            let [match, jobId, valueName] = operation.path.match(re);
+            this.replace(jobId, valueName, operation.value);
+          }
+          break;
+        }
+        default: {
+          break;
+        }
+      }
+    }
+  }
+}

app/static/js/ResourceLists/CorpusList.js

@@ -8,7 +8,11 @@ class CorpusList extends ResourceList {
   constructor(listContainerElement, options = {}) {
     super(listContainerElement, options);
     this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    document.querySelectorAll('.corpus-list-selection-action-trigger[data-selection-action]').forEach((element) => {
+      element.addEventListener('click', (event) => {this.onSelectionAction(event)});
+    });
     this.isInitialized = false
+    this.selectedItemIds = new Set();
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
     app.subscribeUser(this.userId).then((response) => {
@@ -17,24 +21,66 @@ class CorpusList extends ResourceList {
       });
     });
     app.getUser(this.userId).then((user) => {
-      this.add(Object.values(user.corpora));
+      this.add(this.aggregateData(user));
       this.isInitialized = true;
     });
   }
 
+  aggregateData(user) {
+    const aggregatedData = [];
+    for (let corpus of Object.values(user.corpora)) {
+      aggregatedData.push(
+        {
+          'id': corpus.id,
+          'creation-date': corpus.creation_date,
+          'description': corpus.description,
+          'status': corpus.status,
+          'title': corpus.title,
+          'owner': user.username,
+          'is-owner': true,
+          'current-user-is-following': false
+        }
+      );
+    }
+    for (let cfa of Object.values(user.corpus_follower_associations)) {
+      aggregatedData.push(
+        {
+          'id': cfa.corpus.id,
+          'creation-date': cfa.corpus.creation_date,
+          'description': cfa.corpus.description,
+          'status': cfa.corpus.status,
+          'title': cfa.corpus.title,
+          'owner': cfa.corpus.user.username,
+          'is-owner': false,
+          'current-user-is-following': true
+        }
+      );
+    }
+    return aggregatedData;
+  }
+
   // #region Mandatory getters and methods to implement
   get item() {
+    return (values) => {
       return `
         <tr class="list-item clickable hoverable">
-        <td><a class="btn-floating disabled"><i class="material-icons service-color darken" data-service="corpus-analysis">book</i></a></td>
+          <td>
+            <label class="list-action-trigger" data-list-action="select">
+              <input class="select-checkbox" type="checkbox">
+              <span class="disable-on-click"></span>
+            </label>
+          </td>
           <td><b class="title"></b><br><i class="description"></i></td>
+          <td><span class="owner"></span></td>
           <td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
+          <td>${values['current-user-is-following'] ? '<span><i class="left material-icons">visibility</i>Following</span>' : ''}</td>
           <td class="right-align">
             <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete-request"><i class="material-icons">delete</i></a>
             <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
           </td>
         </tr>
       `.trim();
+    };
   }
 
   get valueNames() {
@@ -43,7 +89,9 @@ class CorpusList extends ResourceList {
       {data: ['creation-date']},
       {name: 'status', attr: 'data-status'},
       'description',
-      'title'
+      'title',
+      'owner',
+      'current-user-is-following'
     ];
   }
 
@@ -56,15 +104,24 @@ class CorpusList extends ResourceList {
       <div class="input-field">
         <i class="material-icons prefix">search</i>
         <input id="${listSearchElementId}" class="search" type="text"></input>
-        <label for="${listSearchElementId}">Search corpus</label>
+        <label for="${listSearchElementId}">Search Corpus</label>
       </div>
       <table>
         <thead>
           <tr>
-            <th></th>
+            <th>
+              <label class="corpus-list-selection-action-trigger" data-selection-action="select-all">
+                <input class="corpus-list-select-all-checkbox" type="checkbox">
+                <span></span>
+              </label>
+            </th>
             <th>Title and Description</th>
+            <th>Owner</th>
             <th>Status</th>
             <th></th>
+            <th class="right-align">
+              <a class="corpus-list-selection-action-trigger btn-floating red waves-effect waves-light hide" data-selection-action="delete"><i class="material-icons">delete</i></a>
+            </th>
           </tr>
         </thead>
         <tbody class="list"></tbody>
@@ -73,16 +130,6 @@ class CorpusList extends ResourceList {
     `.trim();
   }
 
-  mapResourceToValue(corpus) {
-    return {
-      'id': corpus.id,
-      'creation-date': corpus.creation_date,
-      'description': corpus.description,
-      'status': corpus.status,
-      'title': corpus.title
-    };
-  }
-
   sort() {
     this.listjs.sort('creation-date', {order: 'desc'});
   }
@@ -95,19 +142,202 @@ class CorpusList extends ResourceList {
     let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
     switch (listAction) {
       case 'delete-request': {
-        Utils.deleteCorpusRequest(this.userId, itemId);
+        let values = this.listjs.get('id', itemId)[0].values();
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Corpus deletion</h4>
+                <p>Do you really want to ${values['is-owner'] ? 'delete' : 'unfollow'} the Corpus <b>${values.title}</b>? ${values['is-owner'] ? 'All files will be permanently deleted!' : ''}</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          if (!values['is-owner']) {
+            Requests.corpora.entity.followers.entity.delete(itemId, currentUserId)
+              .then((response) => {
+                window.location.reload();
+              });
+          } else {
+            Requests.corpora.entity.delete(itemId);
+          }
+        });
+        modal.open();
         break;
       }
       case 'view': {
         window.location.href = `/corpora/${itemId}`;
         break;
       }
+      case 'select': {
+        if (event.target.checked) {
+          this.selectedItemIds.add(itemId);
+        } else {
+          this.selectedItemIds.delete(itemId);
+        }
+        this.renderingItemSelection();
+      }
       default: {
         break;
       }
     }
   }
 
+  onSelectionAction(event) {
+    let selectionActionElement = event.target.closest('.corpus-list-selection-action-trigger[data-selection-action]');
+    let selectionAction = selectionActionElement.dataset.selectionAction;
+    let items = Array.from(this.listjs.items);
+    let selectableItems = Array.from(items)
+      .filter(item => item.elm)
+      .map(item => item.elm.querySelector('.select-checkbox[type="checkbox"]'));
+
+    switch (selectionAction) {
+      case 'select-all': {
+        let selectedIds = new Set(Array.from(items)
+          .map(item => item.values().id))
+        if (event.target.checked !== undefined) {
+          if (event.target.checked) {
+            selectableItems.forEach(selectableItem => selectableItem.checked = true);
+            this.selectedItemIds = selectedIds;
+          } else {
+            selectableItems.forEach(checkbox => checkbox.checked = false);
+            this.selectedItemIds = new Set([...this.selectedItemIds].filter(id => !selectedIds.has(id)));
+          }
+          this.renderingItemSelection();
+        }
+        break;
+      }
+      case 'delete': {
+        // Saved for future use:
+        // <p class="hide">Do you really want to unfollow this Corpora?</p>
+        // <ul id="selected-unfollow-items-list"></ul>
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Corpus deletion</h4>
+                <p>Do you really want to delete this Corpora? <i>All corpora will be permanently deleted!</i></p>
+                <ul id="selected-deletion-items-list"></ul>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let itemDeletionList = document.querySelector('#selected-deletion-items-list');
+        // let itemUnfollowList = document.querySelector('#selected-unfollow-items-list');
+        this.selectedItemIds.forEach(selectedItemId => {
+          let listItem = this.listjs.get('id', selectedItemId)[0].elm;
+          let values = this.listjs.get('id', listItem.dataset.id)[0].values();
+          let itemElement = Utils.HTMLToElement(`<li> - ${values.title}</li>`);
+          // if (!values['is-owner']) { 
+          //   itemUnfollowList.appendChild(itemElement);
+          // } else {
+          itemDeletionList.appendChild(itemElement);
+          // }
+        });
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          this.selectedItemIds.forEach(selectedItemId => {
+            let listItem = this.listjs.get('id', selectedItemId)[0].elm;
+            let values = this.listjs.get('id', listItem.dataset.id)[0].values();
+            if (values['is-owner']) {
+              Requests.corpora.entity.delete(selectedItemId);
+            } else {
+              Requests.corpora.entity.followers.entity.delete(selectedItemId, currentUserId);
+              setTimeout(() => {
+                window.location.reload();
+              }, 1000);
+            }
+          });
+          this.selectedItemIds.clear();
+          this.renderingItemSelection();
+   
+        });
+        modal.open();
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+
+  renderingItemSelection() {
+    let selectionActionButtons = document.querySelectorAll('.corpus-list-selection-action-trigger:not([data-selection-action="select-all"])');
+    let selectableItems = this.listjs.items;
+    let actionButtons = [];
+
+    Object.values(selectableItems).forEach(selectableItem => {
+      if (selectableItem.elm) {
+        let checkbox = selectableItem.elm.querySelector('.select-checkbox[type="checkbox"]');
+        if (checkbox.checked) {
+          selectableItem.elm.classList.add('grey', 'lighten-3');
+        } else {
+          selectableItem.elm.classList.remove('grey', 'lighten-3');
+        }
+        let itemActionButtons = selectableItem.elm.querySelectorAll('.list-action-trigger:not([data-list-action="select"])');
+        itemActionButtons.forEach(itemActionButton => {
+          actionButtons.push(itemActionButton);
+        });
+      }
+    });
+    // Hide item action buttons if > 0 item is selected and show selection action buttons
+    if (this.selectedItemIds.size > 0) {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.remove('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.add('hide');
+      });
+    } else {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.add('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.remove('hide');
+      });
+    }
+
+    // Check select all checkbox if all items are selected
+    let selectAllCheckbox = document.querySelector('.corpus-list-select-all-checkbox[type="checkbox"]');
+    if (selectableItems.length === this.selectedItemIds.size && selectAllCheckbox.checked === false) {
+      selectAllCheckbox.checked = true;
+    } else if (selectableItems.length !== this.selectedItemIds.size && selectAllCheckbox.checked === true) {
+      selectAllCheckbox.checked = false;
+    }
+  }
+
   onPatch(patch) {
     let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)`);
     let filteredPatch = patch.filter(operation => re.test(operation.path));

app/static/js/ResourceLists/DetailledPublicCorpusList.js

@@ -0,0 +1,71 @@
+class DetailledPublicCorpusList extends CorpusList {
+  get item() {
+    return (values) => {
+      return `
+        <tr class="list-item clickable hoverable">
+          <td></td>
+          <td><b class="title"></b><br><i class="description"></i></td>
+          <td><span class="owner"></span></td>
+          <td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
+          <td>${values['current-user-is-following'] ? '<span><i class="left material-icons">visibility</i>Following</span>' : ''}</td>
+          <td class="right-align">
+            <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
+          </td>
+        </tr>
+      `.trim();
+    };
+  }
+
+  get valueNames() {
+    return [
+      {data: ['id']},
+      {data: ['creation-date']},
+      {name: 'status', attr: 'data-status'},
+      'description',
+      'title',
+      'owner',
+      'current-user-is-following'
+    ];
+  }
+
+  initListContainerElement() {
+    if (!this.listContainerElement.hasAttribute('id')) {
+      this.listContainerElement.id = Utils.generateElementId('corpus-list-');
+    }
+    let listSearchElementId = Utils.generateElementId(`${this.listContainerElement.id}-search-`);
+    this.listContainerElement.innerHTML = `
+      <div class="input-field">
+        <i class="material-icons prefix">search</i>
+        <input id="${listSearchElementId}" class="search" type="text"></input>
+        <label for="${listSearchElementId}">Search Corpus</label>
+      </div>
+      <table>
+        <thead>
+          <tr>
+            <th></th>
+            <th>Title and Description</th>
+            <th>Owner</th>
+            <th>Status</th>
+            <th></th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody class="list"></tbody>
+      </table>
+      <ul class="pagination"></ul>
+    `.trim();
+  }
+
+  mapResourceToValue(corpus) {
+    return {
+      'id': corpus.id,
+      'creation-date': corpus.creation_date,
+      'description': corpus.description,
+      'status': corpus.status,
+      'title': corpus.title,
+      'owner': corpus.user.username,
+      'is-owner': corpus.user.id === this.userId,
+      'current-user-is-following': Object.values(corpus.corpus_follower_associations).some(association => association.follower.id === currentUserId)
+    };
+  }
+}

app/static/js/ResourceLists/JobInputList.js

@@ -12,11 +12,7 @@ class JobInputList extends ResourceList {
     this.userId = listContainerElement.dataset.userId;
     this.jobId = listContainerElement.dataset.jobId;
     if (this.userId === undefined || this.jobId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
-    });
+    app.subscribeUser(this.userId);
     app.getUser(this.userId).then((user) => {
       this.add(Object.values(user.jobs[this.jobId].inputs));
       this.isInitialized = true;

app/static/js/ResourceLists/JobList.js

@@ -7,8 +7,13 @@ class JobList extends ResourceList {
 
   constructor(listContainerElement, options = {}) {
     super(listContainerElement, options);
+    this.documentJobArea = document.querySelector('#jobs');
     this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    document.querySelectorAll('.job-list-selection-action-trigger[data-selection-action]').forEach((element) => {
+      element.addEventListener('click', (event) => {this.onSelectionAction(event)});
+    });
     this.isInitialized = false;
+    this.selectedItemIds = new Set();
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
     app.subscribeUser(this.userId).then((response) => {
@@ -24,7 +29,13 @@ class JobList extends ResourceList {
 
   get item() {
     return `
-      <tr class="list-item clickable hoverable service-scheme">
+      <tr class="list-item service-scheme">
+        <td>
+          <label class="list-action-trigger" data-list-action="select">
+            <input class="select-checkbox" type="checkbox">
+            <span class="disable-on-click"></span>
+          </label>
+        </td>
         <td><a class="btn-floating"><i class="nopaque-icons service-icons" data-service="inherit"></i></a></td>
         <td><b class="title"></b><br><i class="description"></i></td>
         <td><span class="badge new job-status-color job-status-text status" data-badge-caption=""></span></td>
@@ -56,15 +67,23 @@ class JobList extends ResourceList {
       <div class="input-field">
         <i class="material-icons prefix">search</i>
         <input id="${listSearchElementId}" class="search" type="text"></input>
-        <label for="${listSearchElementId}">Search job</label>
+        <label for="${listSearchElementId}">Search Job</label>
       </div>
       <table>
         <thead>
           <tr>
+            <th>
+              <label class="job-list-selection-action-trigger" data-selection-action="select-all">
+                <input class="job-list-select-all-checkbox" type="checkbox">
+                <span class="disable-on-click"></span>
+              </label>
+            </th>
             <th>Service</th>
             <th>Title and Description</th>
             <th>Status</th>
-            <th></th>
+            <th class="right-align">
+              <a class="job-list-selection-action-trigger btn-floating red waves-effect waves-light hide" data-selection-action="delete"><i class="material-icons">delete</i></a>
+            </th>
           </tr>
         </thead>
         <tbody class="list"></tbody>
@@ -93,22 +112,185 @@ class JobList extends ResourceList {
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
     let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
-    let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
+    let listAction = listActionElement === null ? '' : listActionElement.dataset.listAction;
     switch (listAction) {
       case 'delete-request': {
-        Utils.deleteJobRequest(this.userId, itemId);
+        let values = this.listjs.get('id', itemId)[0].values();
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Job deletion</h4>
+                <p>Do you really want to delete the Job <b>${values.title}</b>? All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          Requests.jobs.entity.delete(itemId);
+        });
+        modal.open();
         break;
       }
       case 'view': {
         window.location.href = `/jobs/${itemId}`;
         break;
       }
+      case 'select': {
+        if (event.target.checked) {
+          this.selectedItemIds.add(itemId);
+        } else {
+          this.selectedItemIds.delete(itemId);
+        }
+        this.renderingItemSelection();
+        break;
+      }
       default: {
         break;
       }
     }
   }
 
+  onSelectionAction(event) {
+    let selectionActionElement = event.target.closest('.job-list-selection-action-trigger[data-selection-action]');
+    let selectionAction = selectionActionElement.dataset.selectionAction;
+    let items = this.listjs.items;
+    let selectableItems = Array.from(items)
+      .filter(item => item.elm)
+      .map(item => item.elm.querySelector('.select-checkbox[type="checkbox"]'));
+    switch (selectionAction) {
+      case 'select-all': {
+        let selectedIds = new Set(Array.from(items)
+          .map(item => item.values().id))
+        if (event.target.checked !== undefined) {
+          if (event.target.checked) {
+            selectableItems.forEach(selectableItem => selectableItem.checked = true);
+            this.selectedItemIds = selectedIds;
+          } else {
+            selectableItems.forEach(checkbox => checkbox.checked = false);
+            this.selectedItemIds = new Set([...this.selectedItemIds].filter(id => !selectedIds.has(id)));
+          }
+          this.renderingItemSelection();
+        }
+        break;
+      }
+      case 'delete': {
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Corpus File deletion</h4>
+                <p>Do you really want to delete the Jobs?</p>
+                  <ul id="selected-items-list"></ul>
+                <p>All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let itemList = document.querySelector('#selected-items-list');
+        this.selectedItemIds.forEach(selectedItemId => {
+          let listItem = this.listjs.get('id', selectedItemId)[0].elm;
+          let values = this.listjs.get('id', listItem.dataset.id)[0].values();
+          let itemElement = Utils.HTMLToElement(`<li> - ${values.title}</li>`);
+          itemList.appendChild(itemElement);
+        });
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          this.selectedItemIds.forEach(selectedItemId => {
+            Requests.jobs.entity.delete(selectedItemId);
+          });
+          this.selectedItemIds.clear();
+          this.renderingItemSelection();
+        });
+        modal.open();
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+
+  renderingItemSelection() {
+    let selectionActionButtons = document.querySelectorAll('.job-list-selection-action-trigger:not([data-selection-action="select-all"])');
+    let selectableItems = this.listjs.items;
+    let actionButtons = [];
+
+    Object.values(selectableItems).forEach(selectableItem => {
+      if (selectableItem.elm) {
+        let checkbox = selectableItem.elm.querySelector('.select-checkbox[type="checkbox"]');
+        if (checkbox.checked) {
+          selectableItem.elm.classList.add('grey', 'lighten-3');
+        } else {
+          selectableItem.elm.classList.remove('grey', 'lighten-3');
+        }
+        let itemActionButtons = selectableItem.elm.querySelectorAll('.list-action-trigger:not([data-list-action="select"])');
+        itemActionButtons.forEach(itemActionButton => {
+          actionButtons.push(itemActionButton);
+        });
+      }
+    });
+
+
+
+    // Hide item action buttons if > 0 item is selected and show selection action buttons
+    if (this.selectedItemIds.size > 0) {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.remove('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.add('hide');
+      });
+    } else {
+      selectionActionButtons.forEach(selectionActionButton => {
+        selectionActionButton.classList.add('hide');
+      });
+      actionButtons.forEach(actionButton => {
+        actionButton.classList.remove('hide');
+      });
+    }
+
+    // Check select all checkbox if all items are selected
+    let selectAllCheckbox = document.querySelector('.job-list-select-all-checkbox[type="checkbox"]');
+    if (selectableItems.length === this.selectedItemIds.size && selectAllCheckbox.checked === false) {
+      selectAllCheckbox.checked = true;
+    } else if (selectableItems.length !== this.selectedItemIds.size && selectAllCheckbox.checked === true) {
+      selectAllCheckbox.checked = false;
+    }
+
+  }
+
   onPatch(patch) {
     let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)`);
     let filteredPatch = patch.filter(operation => re.test(operation.path));

app/static/js/ResourceLists/PublicCorpusFileList.js

@@ -1,15 +0,0 @@
-class PublicCorpusFileList extends CorpusFileList {
-  get item() {
-    return `
-      <tr class="list-item clickable hoverable">
-        <td><span class="filename"></span></td>
-        <td><span class="author"></span></td>
-        <td><span class="title"></span></td>
-        <td><span class="publishing-year"></span></td>
-        <td class="right-align">
-          <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
-        </td>
-      </tr>
-    `.trim();
-  }
-}

app/static/js/ResourceLists/PublicCorpusList.js

@@ -1,14 +1,55 @@
 class PublicCorpusList extends CorpusList {
   get item() {
+    return (values) => {
       return `
         <tr class="list-item clickable hoverable">
-        <td><a class="btn-floating disabled"><i class="material-icons service-color darken" data-service="corpus-analysis">book</i></a></td>
           <td><b class="title"></b><br><i class="description"></i></td>
-        <td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
+          <td><span class="owner"></span></td>
+          <td>${values['current-user-is-following'] ? '<span><i class="left material-icons">visibility</i></span>' : ''}</td>
           <td class="right-align">
             <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
           </td>
         </tr>
       `.trim();
+    };
+  }
+
+  mapResourceToValue(corpus) {
+    return {
+      'id': corpus.id,
+      'creation-date': corpus.creation_date,
+      'description': corpus.description,
+      'status': corpus.status,
+      'title': corpus.title,
+      'owner': corpus.user.username,
+      'is-owner': corpus.user.id === this.userId ? true : false,
+      'current-user-is-following': Object.values(corpus.corpus_follower_associations).some(association => association.follower.id === currentUserId)
+    };
+  }
+
+  initListContainerElement() {
+    if (!this.listContainerElement.hasAttribute('id')) {
+      this.listContainerElement.id = Utils.generateElementId('corpus-list-');
+    }
+    let listSearchElementId = Utils.generateElementId(`${this.listContainerElement.id}-search-`);
+    this.listContainerElement.innerHTML = `
+      <div class="input-field">
+        <i class="material-icons prefix">search</i>
+        <input id="${listSearchElementId}" class="search" type="text"></input>
+        <label for="${listSearchElementId}">Search Corpus</label>
+      </div>
+      <table>
+        <thead>
+          <tr>
+            <th>Title and Description</th>
+            <th>Owner</th>
+            <th></th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody class="list"></tbody>
+      </table>
+      <ul class="pagination"></ul>
+    `.trim();
   }
 }

app/static/js/ResourceLists/ResourceList.js

@@ -14,6 +14,7 @@ class ResourceList {
     TesseractOCRPipelineModelList.autoInit();
     UserList.autoInit();
     AdminUserList.autoInit();
+    CorpusFollowerList.autoInit();
   }
 
   static defaultOptions = {
@@ -42,7 +43,8 @@ class ResourceList {
   }
 
   add(resources, callback) {
-    let values = resources.map((resource) => {
+    let tmp = Array.isArray(resources) ? resources : [resources];
+    let values = tmp.map((resource) => {
       return this.mapResourceToValue(resource);
     });
     this.listjs.add(values, (items) => {

app/static/js/ResourceLists/SpacyNLPPipelineModelList.js

@@ -30,14 +30,12 @@ class SpaCyNLPPipelineModelList extends ResourceList {
           <td><b><span class="title"></span> <span class="version"></span></b><br><i><span class="description"></span></i></td>
           <td><a class="publisher-url"><span class="publisher"></span></a> (<span class="publishing-year"></span>)<br><a class="publishing-url publishing-url-2"></a></td>
           <td>
-            <div class="list-action-trigger switch center-align" data-list-action="share-request">
-              <span class="share"></span>
+            <span class="disable-on-click">
               <label>
-                <input class="is-public" ${values['is-public'] ? 'checked' : ''} type="checkbox">
-                <span class="lever"></span>
-                public
+                <input ${values['is-public'] ? 'checked' : ''} class="is-public list-action-trigger" data-list-action="toggle-is-public" type="checkbox">
+                <span>Public</span>
               </label>
-            </div>
+            </span>
           </td>
           <td class="right-align">
             <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete-request"><i class="material-icons">delete</i></a>
@@ -80,6 +78,7 @@ class SpaCyNLPPipelineModelList extends ResourceList {
           <tr>
             <th>Title and Description</th>
             <th>Publisher</th>
+            <th>Availability</th>
             <th></th>
           </tr>
         </thead>
@@ -111,6 +110,7 @@ class SpaCyNLPPipelineModelList extends ResourceList {
   }
 
   onChange(event) {
+    if (event.target.tagName !== 'INPUT') {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
@@ -118,8 +118,12 @@ class SpaCyNLPPipelineModelList extends ResourceList {
     if (listActionElement === null) {return;}
     let listAction = listActionElement.dataset.listAction;
     switch (listAction) {
-      case 'share-request': {
-        Utils.spaCyNLPPipelineModelToggleIsPublicRequest(this.userId, itemId);
+      case 'toggle-is-public': {
+        let newIsPublicValue = listActionElement.checked;
+        Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic.update(itemId, newIsPublicValue)
+          .catch((response) => {
+            listActionElement.checked = !newIsPublicValue;
+          });
         break;
       }
       default: {
@@ -129,19 +133,45 @@ class SpaCyNLPPipelineModelList extends ResourceList {
   }
 
   onClick(event) {
+    if (event.target.closest('.disable-on-click') !== null) {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
     let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
-    // ignore switch clicks, handle them by the onChange method instead
-    if (listActionElement.classList.contains('switch')) {
-      event.preventDefault();
-      this.onChange(event);
-    }
     let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
     switch (listAction) {
       case 'delete-request': {
-        Utils.deleteSpaCyNLPPipelineModelRequest(this.userId, itemId);
+        let values = this.listjs.get('id', itemId)[0].values();
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm SpaCy NLP Pipeline Model deletion</h4>
+                <p>Do you really want to delete the SpaCy NLP Pipeline Model <b>${values.title}</b>? All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          Requests.contributions.spacy_nlp_pipeline_models.entity.delete(itemId);
+        });
+        modal.open();
         break;
       }
       case 'view': {

app/static/js/ResourceLists/TesseractOCRPipelineModelList.js

@@ -38,14 +38,12 @@ class TesseractOCRPipelineModelList extends ResourceList {
           <td><b><span class="title"></span> <span class="version"></span></b><br><i><span class="description"></span></i></td>
           <td><a class="publisher-url"><span class="publisher"></span></a> (<span class="publishing-year"></span>)<br><a class="publishing-url"><span class="publishing-url-2"></span></a></td>
           <td>
-            <div class="list-action-trigger switch center-align" data-list-action="share-request">
-              <span class="share"></span>
+            <span class="disable-on-click">
               <label>
-                <input ${values['is-public'] ? 'checked' : ''} class="is-public" type="checkbox">
-                <span class="lever"></span>
-                public
+                <input ${values['is-public'] ? 'checked' : ''} class="is-public list-action-trigger" data-list-action="toggle-is-public" type="checkbox">
+                <span>Public</span>
               </label>
-            </div>
+            </span>
           </td>
           <td class="right-align">
             <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete-request"><i class="material-icons">delete</i></a>
@@ -89,6 +87,7 @@ class TesseractOCRPipelineModelList extends ResourceList {
           <tr>
             <th>Title and Description</th>
             <th>Publisher</th>
+            <th>Availability</th>
             <th></th>
           </tr>
         </thead>
@@ -120,6 +119,7 @@ class TesseractOCRPipelineModelList extends ResourceList {
   }
 
   onChange(event) {
+    if (event.target.tagName !== 'INPUT') {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
@@ -127,8 +127,12 @@ class TesseractOCRPipelineModelList extends ResourceList {
     if (listActionElement === null) {return;}
     let listAction = listActionElement.dataset.listAction;
     switch (listAction) {
-      case 'share-request': {
-        Utils.tesseractOCRPipelineModelToggleIsPublicRequest(this.userId, itemId);
+      case 'toggle-is-public': {
+        let newIsPublicValue = listActionElement.checked;
+        Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic.update(itemId, newIsPublicValue)
+          .catch((response) => {
+            listActionElement.checked = !newIsPublicValue;
+          });
         break;
       }
       default: {
@@ -138,19 +142,45 @@ class TesseractOCRPipelineModelList extends ResourceList {
   }
 
   onClick(event) {
+    if (event.target.closest('.disable-on-click') !== null) {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');
     if (listItemElement === null) {return;}
     let itemId = listItemElement.dataset.id;
     let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
-    // ignore switch clicks, handle them by the onChange method instead
-    if (listActionElement.classList.contains('switch')) {
-      event.preventDefault();
-      this.onChange(event);
-    }
     let listAction = listActionElement === null ? 'view' : listActionElement.dataset.listAction;
     switch (listAction) {
       case 'delete-request': {
-        Utils.deleteTesseractOCRPipelineModelRequest(this.userId, itemId);
+        let values = this.listjs.get('id', itemId)[0].values();
+        let modalElement = Utils.HTMLToElement(
+          `
+            <div class="modal">
+              <div class="modal-content">
+                <h4>Confirm Tesseract OCR Pipeline Model deletion</h4>
+                <p>Do you really want to delete the Tesseract OCR Pipeline Model <b>${values.title}</b>? All files will be permanently deleted!</p>
+              </div>
+              <div class="modal-footer">
+                <a class="btn modal-close waves-effect waves-light">Cancel</a>
+                <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+              </div>
+            </div>
+          `
+        );
+        document.querySelector('#modals').appendChild(modalElement);
+        let modal = M.Modal.init(
+          modalElement,
+          {
+            dismissible: false,
+            onCloseEnd: () => {
+              modal.destroy();
+              modalElement.remove();
+            }
+          }
+        );
+        let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+        confirmElement.addEventListener('click', (event) => {
+          Requests.contributions.tesseract_ocr_pipeline_models.entity.delete(itemId);
+        });
+        modal.open();
         break;
       }
       case 'view': {

app/static/js/ResourceLists/UserList.js

@@ -13,14 +13,14 @@ class UserList extends ResourceList {
   get item() {
     return `
       <tr class="list-item clickable hoverable">
-        <td><img alt="user-image" class="circle responsive-img avatar" style="width:50%"></td>
+        <td><img alt="user-image" class="circle responsive-img avatar" style="width:25%"></td>
         <td><b><span class="username"></span><b></td>
         <td><span class="full-name"></span></td>
         <td><span class="location"></span></td>
         <td><span class="organization"></span></td>
         <td><span class="corpora-online"></span></td>
         <td class="right-align">
-          <a class="list-action-trigger btn-floating waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
+          <a class="list-action-trigger btn-floating waves-effect waves-light social-area-color-darken" data-list-action="view"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim();
@@ -72,12 +72,12 @@ class UserList extends ResourceList {
     return {
       'id': user.id,
       'member-since': user.member_since,
-      'avatar': user.avatar ? `/users/${user.id}/avatar` : '/static/images/user_avatar.png',
+      'avatar': user.avatar,
       'username': user.username,
       'full-name': user.full_name ? user.full_name : '',
       'location': user.location ? user.location : '',
       'organization': user.organization ? user.organization : '',
-      'corpora-online': '-'
+      'corpora-online': Object.values(user.corpora).filter((corpus) => corpus.is_public).length
     };
   };
 

app/static/js/Utils.js

@@ -69,648 +69,4 @@ class Utils {
     return Utils.mergeObjectsDeep(mergedObject, ...objects.slice(2));
   }
 
-  static enableCorpusIsPublicRequest(userId, corpusId) {
-    return new Promise((resolve, reject) => {
-      let corpus;
-      try {
-        corpus = app.data.users[userId].corpora[corpusId];
-      } catch (error) {
-        corpus = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Hier könnte eine Warnung stehen</h4>
-              <p></p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Confirm</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let corpusTitle = corpus?.title;
-        fetch(`/corpora/${corpusId}/enable_is_public`, {method: 'POST', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Corpus "${corpusTitle}" is public now`, 'corpus');
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static disableCorpusIsPublicRequest(userId, corpusId) {
-    return new Promise((resolve, reject) => {
-      let corpus;
-      try {
-        corpus = app.data.users[userId].corpora[corpusId];
-      } catch (error) {
-        corpus = {};
-      }
-
-      let corpusTitle = corpus?.title;
-      fetch(`/corpora/${corpusId}/disable_is_public`, {method: 'POST', headers: {Accept: 'application/json'}})
-        .then(
-          (response) => {
-            if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-            if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-            app.flash(`Corpus "${corpusTitle}" is private now`, 'corpus');
-            resolve(response);
-          },
-          (response) => {
-            app.flash('Something went wrong', 'error');
-            reject(response);
-          }
-        );
-    });
-  }
-
-  static buildCorpusRequest(userId, corpusId) {
-    return new Promise((resolve, reject) => {
-      let corpus;
-      try {
-        corpus = app.data.users[userId].corpora[corpusId];
-      } catch (error) {
-        corpus = {};
-      }
-
-      fetch(`/corpora/${corpusId}/build`, {method: 'POST', headers: {Accept: 'application/json'}})
-        .then(
-          (response) => {
-            if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-            if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-            if (response.status === 409) {app.flash('Conflict', 'error'); reject(response);}
-            app.flash(`Corpus "${corpus?.title}" marked for building`, 'corpus');
-            resolve(response);
-          },
-          (response) => {
-            app.flash('Something went wrong', 'error');
-            reject(response);
-          }
-        );
-    });
-  }
-
-  static deleteCorpusRequest(userId, corpusId) {
-    return new Promise((resolve, reject) => {
-      let corpus;
-      try {
-        corpus = app.data.users[userId].corpora[corpusId];
-      } catch (error) {
-        corpus = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Corpus deletion</h4>
-              <p>Do you really want to delete the Corpus <b>${corpus?.title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-      
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let corpusTitle = corpus?.title;
-        fetch(`/corpora/${corpusId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Corpus "${corpusTitle}" marked for deletion`, 'corpus');
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteCorpusFileRequest(userId, corpusId, corpusFileId) {
-    return new Promise((resolve, reject) => {
-      let corpusFile;
-      try {
-        corpusFile = app.data.users[userId].corpora[corpusId].files[corpusFileId];
-      } catch (error) {
-        corpusFile = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Corpus File deletion</h4>
-              <p>Do you really want to delete the Corpus File <b>${corpusFile?.title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let corpusFileTitle = corpusFile?.title;
-        fetch(`/corpora/${corpusId}/files/${corpusFileId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Corpus File "${corpusFileTitle}" deleted`, 'corpus');
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteSpaCyNLPPipelineModelRequest(userId, spaCyNLPPipelineModelId) {
-    return new Promise((resolve, reject) => {
-      let spaCyNLPPipelineModel;
-      try {
-        spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId];
-      } catch (error) {
-        spaCyNLPPipelineModel = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm SpaCy NLP Pipeline Model deletion</h4>
-              <p>Do you really want to delete the SpaCy NLP Pipeline Model <b>${spaCyNLPPipelineModel?.title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let spaCyNLPPipelineModelTitle = spaCyNLPPipelineModel?.title;
-        fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}`, {method: 'DELETE'})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`SpaCy NLP Pipeline Model "${spaCyNLPPipelineModelTitle}" marked for deletion`);
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteTesseractOCRPipelineModelRequest(userId, tesseractOCRPipelineModelId) {
-    return new Promise((resolve, reject) => {
-      let tesseractOCRPipelineModel;
-      try {
-        tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId];
-      } catch (error) {
-        tesseractOCRPipelineModel = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Tesseract OCR Pipeline Model deletion</h4>
-              <p>Do you really want to delete the Tesseract OCR Pipeline Model <b>${tesseractOCRPipelineModel?.title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let tesseractOCRPipelineModelTitle = tesseractOCRPipelineModel?.title;
-        fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}`, {method: 'DELETE'})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Tesseract OCR Pipeline Model "${tesseractOCRPipelineModelTitle}" marked for deletion`);
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteProfileAvatarRequest(userId) {
-    return new Promise((resolve, reject) => {
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Avatar deletion</h4>
-              <p>Do you really want to delete your Avatar?</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        fetch(`/users/${userId}/avatar`, {method: 'DELETE', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Avatar marked for deletion`);
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteJobRequest(userId, jobId) {
-    return new Promise((resolve, reject) => {
-      let job;
-      try {
-        job = app.data.users[userId].jobs[jobId];
-      } catch (error) {
-        job = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Job deletion</h4>
-              <p>Do you really want to delete the Job <b>${job?.title}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let jobTitle = job?.title;
-        fetch(`/jobs/${jobId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`Job "${jobTitle}" marked for deletion`, 'job');
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static getJobLogRequest(userId, jobId) {
-    return new Promise((resolve, reject) => {
-      fetch(`/jobs/${jobId}/log`, {method: 'GET', headers: {Accept: 'application/json, text/plain'}})
-        .then(
-          (response) => {
-            if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-            if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-            return response.text();
-          },
-          (response) => {
-            app.flash('Something went wrong', 'error');
-            reject(response);
-          }
-        )
-        .then(
-          (text) => {
-            let modalElement = Utils.HTMLToElement(
-              `
-                <div class="modal">
-                  <div class="modal-content">
-                    <h4>Job logs</h4>
-                    <pre><code>${text}</code></pre>
-                  </div>
-                  <div class="modal-footer">
-                    <a class="btn modal-close waves-effect waves-light">Close</a>
-                  </div>
-                </div>
-              `
-            );
-            document.querySelector('#modals').appendChild(modalElement);
-            let modal = M.Modal.init(
-              modalElement,
-              {
-                onCloseEnd: () => {
-                  modal.destroy();
-                  modalElement.remove();
-                }
-              }
-            );
-            modal.open();
-            resolve(text);
-          }
-        );
-    });
-  }
-
-  static restartJobRequest(userId, jobId) {
-    return new Promise((resolve, reject) => {
-      let job;
-      try {
-        job = app.data.users[userId].jobs[jobId];
-      } catch (error) {
-        job = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm Job restart</h4>
-              <p>Do you really want to restart the Job <b>${job?.title}</b>? All Job Results will be permanently deleted.</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Restart</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let jobTitle = job?.title;
-        fetch(`/jobs/${jobId}/restart`, {method: 'POST', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              if (response.status === 409) {app.flash('Conflict', 'error'); reject(response);}
-              app.flash(`Job "${jobTitle}" restarted.`, 'job');
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static deleteUserRequest(userId) {
-    return new Promise((resolve, reject) => {
-      let user;
-      try {
-        user = app.data.users[userId];
-      } catch (error) {
-        user = {};
-      }
-
-      let modalElement = Utils.HTMLToElement(
-        `
-          <div class="modal">
-            <div class="modal-content">
-              <h4>Confirm User deletion</h4>
-              <p>Do you really want to delete the User <b>${user?.username}</b>? All files will be permanently deleted!</p>
-            </div>
-            <div class="modal-footer">
-              <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
-              <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
-            </div>
-          </div>
-        `
-      );
-      document.querySelector('#modals').appendChild(modalElement);
-      let modal = M.Modal.init(
-        modalElement,
-        {
-          dismissible: false,
-          onCloseEnd: () => {
-            modal.destroy();
-            modalElement.remove();
-          }
-        }
-      );
-
-      let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
-      confirmElement.addEventListener('click', (event) => {
-        let userName = user?.username;
-        fetch(`/users/${userId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
-          .then(
-            (response) => {
-              if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
-              if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
-              app.flash(`User "${userName}" marked for deletion`);
-              resolve(response);
-            },
-            (response) => {
-              app.flash('Something went wrong', 'error');
-              reject(response);
-            }
-          );
-      });
-      modal.open();
-    });
-  }
-
-  static tesseractOCRPipelineModelToggleIsPublicRequest(userId, tesseractOCRPipelineModelId, is_public) {
-    return new Promise((resolve, reject) => {
-      let tesseractOCRPipelineModel;
-      try {
-        tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId];
-      } catch (error) {
-        tesseractOCRPipelineModel = {};
-      }
-    
-      fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}})
-        .then(
-          (response) => {
-            if (response.status === 403) {
-              app.flash('Forbidden', 'error');
-              reject(response);
-            }
-            resolve(response);
-          },
-          (response) => {
-            app.flash('Something went wrong', 'error');
-            reject(response);
-          }
-        );
-    });
-  }
-
-  static spaCyNLPPipelineModelToggleIsPublicRequest(userId, spaCyNLPPipelineModelId) {
-    return new Promise((resolve, reject) => {
-      let spaCyNLPPipelineModel;
-      try {
-        spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId];
-      } catch (error) {
-        spaCyNLPPipelineModel = {};
-      }
-
-      fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}})
-        .then(
-          (response) => {
-            if (response.status === 403) {
-              app.flash('Forbidden', 'error');
-              reject(response);
-            }
-            resolve(response);
-          },
-          (response) => {
-            app.flash('Something went wrong', 'error');
-            reject(response);
-          }
-        );
-    });
-  }
 }

app/templates/_navbar.html.j2

@@ -8,29 +8,32 @@
         <img class="hide-on-small-only" src="{{ url_for('static', filename='images/nopaque_-_logo_name_slogan.svg') }}" style="height: 128px; margin-top: -32px; margin-left: -32px;">
         <img class="hide-on-med-and-up" src="{{ url_for('static', filename='images/nopaque_-_logo.svg') }}" style="height: 128px; margin-top: -32px; margin-left: -32px;">
       </a>
-      <ul class="right">
+      <ul class="right hide-on-med-and-down">
+        <li><a href="{{ url_for('main.news') }}"><i class="material-icons left">email</i>News</a></li>
         <li><a class="dropdown-trigger no-autoinit" data-target="nav-more-dropdown" href="#!" id="nav-more-dropdown-trigger"><i class="material-icons">more_vert</i></a></li>
       </ul>
     </div>
     <div class="nav-content primary-variant-color">
       <ul class="tabs tabs-transparent">
-        <li class="tab"><a href="{{ url_for('main.index') }}" target="_self"><i class="material-icons">home</i></a></li>
-        {% if breadcrumbs is defined %}
-        {{ breadcrumbs }}
+        {%- for breadcrumb in breadcrumbs -%}
+        <li class="tab"><a {{ 'class="active"' if loop.last }} href="{{ breadcrumb.url }}" target="_self">{{ breadcrumb.text }}</a></li>
+        {% if not loop.last %}
+        <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
         {% endif %}
+        {%- endfor -%}
       </ul>
-      {% if current_user.is_authenticated %}
+      {# {% if current_user.is_authenticated %}
       <a class="btn-floating btn-large halfway-fab modal-trigger pink tooltipped waves-effect waves-light" data-tooltip="Roadmap" href="#roadmap-modal"><i class="material-icons">explore</i></a>
-      {% endif %}
+      {% endif %} #}
+      <a class="btn-floating btn-large halfway-fab modal-trigger pink tooltipped waves-effect waves-light" data-tooltip="Manual" href="#manual-modal"><i class="material-icons">help</i></a>
     </div>
   </nav>
 </div>
 
 <ul class="dropdown-content" id="nav-more-dropdown">
-  <li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons left">help</i>Manual</a></li>
+  {# <li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons left">help</i>Manual</a></li> #}
   {% if current_user.is_authenticated %}
-  <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>General settings</a></li>
-  <li><a href="{{ url_for('users.edit_profile', user_id=current_user.id) }}"><i class="material-icons left">contact_page</i>Profile settings</a></li>
+  <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>Settings</a></li>
   <li class="divider" tabindex="-1"></li>
   <li><a href="{{ url_for('auth.logout') }}">Log out</a></li>
   {% else %}

app/templates/_roadmap.html.j2

@@ -21,7 +21,7 @@
       <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
       {% if corpus %}
       {% if corpus.files.all() %}
-      <li class="tab"><a{%if request.path == url_for('corpora.analyse_corpus', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" target="_self">Corpus analysis</a></li>
+      <li class="tab"><a{%if request.path == url_for('corpora.analysis', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.analysis', corpus_id=corpus.id) }}" target="_self">Corpus analysis</a></li>
       {% else %}
       <li class="tab disabled tooltipped" data-tooltip="Create at least one corpus file first"><a>Corpus analysis</a></li>
       {% endif %}

app/templates/_scripts.html.j2

@@ -6,21 +6,39 @@
   output='gen/app.%(version)s.js',
   'js/App.js',
   'js/Utils.js',
-  'js/Forms/Form.js',
-  'js/Forms/CreateCorpusFileForm.js',
-  'js/Forms/CreateJobForm.js',
-  'js/Forms/CreateContributionForm.js',
   'js/CorpusAnalysis/CQiClient.js',
   'js/CorpusAnalysis/CorpusAnalysisApp.js',
   'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
   'js/CorpusAnalysis/CorpusAnalysisReader.js',
   'js/CorpusAnalysis/QueryBuilder.js',
-  'js/RessourceDisplays/RessourceDisplay.js',
-  'js/RessourceDisplays/CorpusDisplay.js',
-  'js/RessourceDisplays/JobDisplay.js',
+  'js/XMLtoObject.js'
+%}
+<script src="{{ ASSET_URL }}"></script>
+{%- endassets %}
+{%- assets
+  filters='rjsmin',
+  output='gen/Forms.%(version)s.js',
+  'js/Forms/Form.js',
+  'js/Forms/CreateCorpusFileForm.js',
+  'js/Forms/CreateJobForm.js',
+  'js/Forms/CreateContributionForm.js'
+%}
+<script src="{{ ASSET_URL }}"></script>
+{%- endassets %}
+{%- assets
+  filters='rjsmin',
+  output='gen/ResourceDisplays.%(version)s.js',
+  'js/ResourceDisplays/ResourceDisplay.js',
+  'js/ResourceDisplays/CorpusDisplay.js',
+  'js/ResourceDisplays/JobDisplay.js'
+%}
+<script src="{{ ASSET_URL }}"></script>
+{%- endassets %}
+{%- assets
+  filters='rjsmin',
+  output='gen/ResourceLists.%(version)s.js',
   'js/ResourceLists/ResourceList.js',
   'js/ResourceLists/CorpusFileList.js',
-  'js/ResourceLists/PublicCorpusFileList.js',
   'js/ResourceLists/CorpusList.js',
   'js/ResourceLists/PublicCorpusList.js',
   'js/ResourceLists/JobList.js',
@@ -30,7 +48,25 @@
   'js/ResourceLists/TesseractOCRPipelineModelList.js',
   'js/ResourceLists/UserList.js',
   'js/ResourceLists/AdminUserList.js',
-  'js/XMLtoObject.js'
+  'js/ResourceLists/CorpusFollowerList.js',
+  'js/ResourceLists/DetailledPublicCorpusList.js'
+%}
+<script src="{{ ASSET_URL }}"></script>
+{%- endassets %}
+{%- assets
+  filters='rjsmin',
+  output='gen/Requests.%(version)s.js',
+  'js/Requests/Requests.js',
+  'js/Requests/admin/admin.js',
+  'js/Requests/contributions/contributions.js',
+  'js/Requests/contributions/spacy_nlp_pipeline_models.js',
+  'js/Requests/contributions/tesseract_ocr_pipeline_models.js',
+  'js/Requests/corpora/corpora.js',
+  'js/Requests/corpora/files.js',
+  'js/Requests/corpora/followers.js',
+  'js/Requests/jobs/jobs.js',
+  'js/Requests/users/users.js',
+  'js/Requests/users/settings.js'
 %}
 <script src="{{ ASSET_URL }}"></script>
 {%- endassets %}
@@ -50,7 +86,13 @@
   for (let optionElement of document.querySelectorAll('option[value=""]')) {
     optionElement.disabled = true;
   }
+  for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) {
+    for (let c of optgroupElement.children) {
+      optgroupElement.parentElement.insertAdjacentElement('afterbegin', c);
+    }
+    optgroupElement.remove();
     
+  }
   // Set the data-length attribute on textareas/inputs with the maxlength attribute
   for (let inputElement of document.querySelectorAll('textarea[maxlength], input[maxlength]')) {
     inputElement.dataset.length = inputElement.getAttribute('maxlength');
@@ -70,4 +112,35 @@
   for (let [category, message] of {{ get_flashed_messages(with_categories=True)|tojson }}) {
     app.flash(message, message);
   }
+
+  // Initialize manual modal
+  let manualModalTableOfContentsElement = document.querySelector('#manual-modal-table-of-contents');
+  let manualModalTableOfContents = M.Tabs.init(manualModalTableOfContentsElement);
+  let manualModalElement = document.querySelector('#manual-modal');
+  let manualModal = M.Modal.init(
+    manualModalElement,
+    {
+      onOpenStart: (manualModalElement, modalTriggerElement) => {
+        if ('manualModalChapter' in modalTriggerElement.dataset) {
+          manualModalTableOfContents.select(modalTriggerElement.dataset.manualModalChapter);
+        }
+      }
+    }
+  );
+
+  // Initialize terms of use modal
+  const termsOfUseModal = document.getElementById('terms-of-use-modal');
+  M.Modal.init(
+    termsOfUseModal, 
+    {
+      dismissible: false, 
+      onCloseEnd: () => {
+        Requests.users.entity.acceptTermsOfUse();
+      }
+    }
+  );
+  {% if current_user.is_authenticated and not current_user.terms_of_use_accepted %}
+      termsOfUseModal.M_Modal.open();
+  {% endif %}
+
 </script>