Update Terms of Use Modal and fix message flashing.

- breadcrumbs (!flask-breadcrumbs)
- manual modal button
- api blueprint (!flask-marshmallow/!marshmallow-sqlalchemy)

.dockerignore

@@ -5,8 +5,9 @@
 !app
 !migrations
 !tests
-!.flaskenv
 !boot.sh
 !config.py
-!nopaque.py
+!docker-nopaque-entrypoint.sh
 !requirements.txt
+!requirements.freezed.txt
+!wsgi.py

.env.tpl

@@ -1,204 +1,20 @@
-################################################################################
-# Docker                                                                       #
-################################################################################
-# DEFAULT: ./data
-# NOTE: Use `.` as <project-basedir>
-# HOST_DATA_DIR=
-
-# Example: 1000
+##############################################################################
+# Environment variables used by Docker Compose config files.                 #
+##############################################################################
 # HINT: Use this bash command `id -u`
+# NOTE: 0 (= root user) is not allowed
 HOST_UID=
 
-# Example: 1000
 # HINT: Use this bash command `id -g`
+# NOTE: 0 (= root group) is not allowed
 HOST_GID=
 
-# Example: 999
 # HINT: Use this bash command `getent group docker | cut -d: -f3`
 HOST_DOCKER_GID=
 
-# DEFAULT: ./logs
-# NOTES: Use `.` as <project-basedir>
-# HOST_LOG_DIR=
+# DEFAULT: nopaque
+NOPAQUE_DOCKER_NETWORK_NAME=nopaque
 
-# DEFAULT: nopaque_default
-# DOCKER_NETWORK_NAME=
-
-################################################################################
-# Flask                                                                        #
-# https://flask.palletsprojects.com/en/1.1.x/config/                           #
-################################################################################
-# CHOOSE ONE: http, https
-# DEFAULT: http
-# PREFERRED_URL_SCHEME=
-
-# DEFAULT: hard to guess string
-# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
-# SECRET_KEY=
-
-# DEFAULT: localhost:5000
-# Example: nopaque.example.com/nopaque.example.com:5000
-# HINT: If your instance is publicly available on a different Port then 80/443,
-#       you will have to add this to the server name
-# SERVER_NAME=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# HINT: Set to true if you redirect http to https
-# SESSION_COOKIE_SECURE=
-
-
-################################################################################
-# Flask-Assets                                                                 #
-# https://webassets.readthedocs.io/en/latest/                                  #
-################################################################################
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# ASSETS_DEBUG=
-
-
-################################################################################
-# Flask-Hashids                                                                #
-# https://github.com/Pevtrick/Flask-Hashids                                    #
-################################################################################
-# DEFAULT: 16
-# HASHIDS_MIN_LENGTH=
-
-# NOTE: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
-#       It is strongly recommended that this is NEVER the same as the SECRET_KEY
-HASHIDS_SALT=
-
-
-################################################################################
-# Flask-Login                                                                  #
-# https://flask-login.readthedocs.io/en/latest/                                #
-################################################################################
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# HINT: Set to true if you redirect http to https
-# REMEMBER_COOKIE_SECURE=
-
-
-################################################################################
-# Flask-Mail                                                                   #
-# https://pythonhosted.org/Flask-Mail/                                         #
-################################################################################
-# EXAMPLE: nopaque Admin <nopaque@example.com>
-MAIL_DEFAULT_SENDER=
-
-MAIL_PASSWORD=
-
-# EXAMPLE: smtp.example.com
-MAIL_SERVER=
-
-# EXAMPLE: 587
-MAIL_PORT=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# MAIL_USE_SSL=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# MAIL_USE_TLS=
-
-# EXAMPLE: nopaque@example.com
-MAIL_USERNAME=
-
-
-################################################################################
-# Flask-SQLAlchemy                                                             #
-# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/                  #
-################################################################################
-# DEFAULT: 'sqlite:///<nopaque-basedir>/data.sqlite'
-# NOTE: Use `.` as <nopaque-basedir>,
-#       Don't use a SQLite database when using Docker
-# SQLALCHEMY_DATABASE_URI=
-
-
-################################################################################
-# nopaque                                                                      #
-################################################################################
-# An account is registered with this email adress gets automatically assigned
-# the administrator role.
-# EXAMPLE: admin.nopaque@example.com
-NOPAQUE_ADMIN=
-
-# DEFAULT: /mnt/nopaque
-# NOTE: This must be a network share and it must be available on all Docker
-#       Swarm nodes
-# NOPAQUE_DATA_DIR=
-
-# CHOOSE ONE: False, True
-# DEFAULT: True
-# NOPAQUE_IS_PRIMARY_INSTANCE=
-
-# transport://[userid:password]@hostname[:port]/[virtual_host]
-NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-
-# NOTE: Get these from the nopaque development team
-NOPAQUE_DOCKER_REGISTRY_USERNAME=
-NOPAQUE_DOCKER_REGISTRY_PASSWORD=
-
-# DEFAULT: %Y-%m-%d %H:%M:%S
-# NOPAQUE_LOG_DATE_FORMAT=
-
-# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
-# NOPAQUE_LOG_FORMAT=
-
-# DEFAULT: INFO
-# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
-# NOPAQUE_LOG_LEVEL=
-
-# CHOOSE ONE: False, True
-# DEFAULT: True
-# NOPAQUE_LOG_FILE_ENABLED=
-
-# DEFAULT: <nopaque-basedir>/logs
-# NOTE: Use `.` as <nopaque-basedir>
-# NOPAQUE_LOG_FILE_DIR=
-
-# DEFAULT: NOPAQUE_LOG_LEVEL
-# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
-# NOPAQUE_LOG_FILE_LEVEL=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# NOPAQUE_LOG_STDERR_ENABLED=
-
-# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
-# DEFAULT: NOPAQUE_LOG_LEVEL
-# NOPAQUE_LOG_STDERR_LEVEL=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# HINT: Set this to True only if you are using a proxy in front of nopaque
-# NOPAQUE_PROXY_FIX_ENABLED=
-
-# DEFAULT: 0
-# Number of values to trust for X-Forwarded-For
-# NOPAQUE_PROXY_FIX_X_FOR=
-
-# DEFAULT: 0
-# Number of values to trust for X-Forwarded-Host
-# NOPAQUE_PROXY_FIX_X_HOST=
-
-# DEFAULT: 0
-# Number of values to trust for X-Forwarded-Port
-# NOPAQUE_PROXY_FIX_X_PORT=
-
-# DEFAULT: 0
-# Number of values to trust for X-Forwarded-Prefix
-# NOPAQUE_PROXY_FIX_X_PREFIX=
-
-# DEFAULT: 0
-# Number of values to trust for X-Forwarded-Proto
-# NOPAQUE_PROXY_FIX_X_PROTO=
-
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# NOPAQUE_TRANSKRIBUS_ENABLED=
-
-# READ-COOP account data: https://readcoop.eu/
-# NOPAQUE_READCOOP_USERNAME=
-# NOPAQUE_READCOOP_PASSWORD=
+# NOTE: This must be a network share and it must be available on all
+#       Docker Swarm nodes, mounted to the same path.
+HOST_NOPAQUE_DATA_PATH=/mnt/nopaque

.flaskenv

@@ -1 +0,0 @@
-FLASK_APP=nopaque.py

.gitignore

@@ -1,11 +1,11 @@
 # nopaque specifics
 app/static/gen/
-data/
+volumes/
 docker-compose.override.yml
-logs/
-!logs/dummy
 *.env
 
+*.pjentsch-testing
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.gitlab-ci.yml

@@ -0,0 +1,84 @@
+include:
+  - template: Security/Container-Scanning.gitlab-ci.yml
+
+##############################################################################
+# Pipeline stages in order of execution                                      #
+##############################################################################
+stages:
+  - build
+  - publish
+  - sca
+
+##############################################################################
+# Pipeline behavior                                                          #
+##############################################################################
+workflow:
+  rules:
+    # Run the pipeline on commits to the default branch
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      variables:
+        # Set the Docker image tag to `latest`
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest
+      when: always
+    # Run the pipeline on tag creation
+    - if: $CI_COMMIT_TAG
+      variables:
+        # Set the Docker image tag to the Git tag name
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+      when: always
+    # Don't run the pipeline on all other occasions
+    - when: never
+
+##############################################################################
+# Default values for pipeline jobs                                           #
+##############################################################################
+default:
+  image: docker:24.0.6
+  services:
+    - docker:24.0.6-dind
+  tags:
+    - docker
+
+##############################################################################
+# CI/CD variables for all jobs in the pipeline                               #
+##############################################################################
+variables:
+  DOCKER_TLS_CERTDIR: /certs
+  DOCKER_BUILD_PATH: .
+  DOCKERFILE: Dockerfile
+
+##############################################################################
+# Pipeline jobs                                                              #
+##############################################################################
+build:
+  stage: build
+  script:
+    - docker build --tag $DOCKER_IMAGE --file $DOCKERFILE $DOCKER_BUILD_PATH
+    - docker save $DOCKER_IMAGE > docker_image.tar
+  artifacts:
+    paths:
+      - docker_image.tar
+
+publish:
+  stage: publish
+  before_script:
+    - docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - docker load --input docker_image.tar
+    - docker push $DOCKER_IMAGE
+  after_script:
+    - docker logout $CI_REGISTRY
+
+container_scanning:
+  stage: sca
+  rules:
+    # Run the job on commits to the default branch
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+    # Run the job on tag creation
+    - if: $CI_COMMIT_TAG
+      when: always
+    # Don't run the job on all other occasions
+    - when: never
+  variables:
+    CS_IMAGE: $DOCKER_IMAGE

.vscode/extensions.json

@@ -1,7 +1,8 @@
 {
     "recommendations": [
-        "samuelcolvin.jinjahtml",
+        "irongeek.vscode-env",
         "ms-azuretools.vscode-docker",
-        "ms-python.python"
+        "ms-python.python",
+        "samuelcolvin.jinjahtml"
     ]
 }

.vscode/settings.json

@@ -1 +1,24 @@
-{}
+{
+    "editor.rulers": [79],
+    "editor.tabSize": 4,
+    "emmet.includeLanguages": {
+        "jinja-html": "html"
+    },
+    "files.associations": {
+        ".flaskenv": "env",
+        "*.env.tpl": "env",
+        "*.txt.j2": "jinja"
+    },
+    "files.insertFinalNewline": true,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "[html]": {
+        "editor.tabSize": 2
+    },
+    "[javascript]": {
+        "editor.tabSize": 2
+    },
+    "[jinja-html]": {
+        "editor.tabSize": 2
+    }
+}

Dockerfile

@@ -1,50 +1,57 @@
-FROM python:3.8.10-slim-buster
+FROM python:3.10.13-slim-bookworm
 
 
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
 
 
-ARG DOCKER_GID
-ARG UID
-ARG GID
-
-
+# Set environment variables
 ENV LANG="C.UTF-8"
 ENV PYTHONDONTWRITEBYTECODE="1"
 ENV PYTHONUNBUFFERED="1"
 
 
+# Install system dependencies
 RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
+      gosu \
       libpq-dev \
  && rm --recursive /var/lib/apt/lists/*
 
 
-RUN groupadd --gid "${DOCKER_GID}" docker \
- && groupadd --gid "${GID}" nopaque \
- && useradd --create-home --gid nopaque --groups "${DOCKER_GID}" --no-log-init --uid "${UID}" nopaque
+# Create a non-root user
+RUN useradd --create-home --no-log-init nopaque \
+ && groupadd docker \
+ && usermod --append --groups docker nopaque
+
 USER nopaque
 WORKDIR /home/nopaque
 
 
-ENV PYTHON3_VENV_PATH="/home/nopaque/venv"
-RUN python3 -m venv "${PYTHON3_VENV_PATH}"
-ENV PATH="${PYTHON3_VENV_PATH}/bin:${PATH}"
+# Create a Python virtual environment
+ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv"
+RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}"
+ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}"
 
 
-COPY --chown=nopaque:nopaque requirements.txt .
-RUN python3 -m pip install --requirement requirements.txt \
- && rm requirements.txt
+# Install Python dependencies
+COPY --chown=nopaque:nopaque requirements.freezed.txt requirements.freezed.txt
+RUN python3 -m pip install --requirement requirements.freezed.txt \
+ && rm requirements.freezed.txt
 
 
+# Install the application
+COPY docker-nopaque-entrypoint.sh /usr/local/bin/
 COPY --chown=nopaque:nopaque app app
 COPY --chown=nopaque:nopaque migrations migrations
 COPY --chown=nopaque:nopaque tests tests
-COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py ./
+COPY --chown=nopaque:nopaque boot.sh config.py wsgi.py ./
 
 
 EXPOSE 5000
 
 
-ENTRYPOINT ["./boot.sh"]
+USER root
+
+
+ENTRYPOINT ["docker-nopaque-entrypoint.sh"]

README.md

@@ -1,5 +1,8 @@
 # nopaque
 
+![release badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/-/badges/release.svg)
+![pipeline badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/badges/master/pipeline.svg?ignore_skipped=true)
+
 nopaque bundles various tools and services that provide humanities scholars with DH methods and thus can support their various individual research processes. Using nopaque, researchers can subject digitized sources to Optical Character Recognition (OCR). The resulting text files can then be used as a data basis for Natural Language Processing (NLP). The texts are automatically subjected to various linguistic annotations. The data processed via NLP can then be summarized in the web application as corpora and analyzed by means of an information retrieval system through complex search queries. The range of functions of the web application will be successively extended according to the needs of the researchers.
 
 ## Prerequisites and requirements
@@ -32,7 +35,7 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
 # Create data directories
-username@hostname:~$ mkdir data/{db,logs,mq}
+username@hostname:~$ mkdir -p volumes/{db,mq}
 username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
 # Fill out the variables within these files.

app/__init__.py

@@ -2,6 +2,8 @@ from apifairy import APIFairy
 from config import Config
 from docker import DockerClient
 from flask import Flask
+from flask.logging import default_handler
+from flask_admin import Admin
 from flask_apscheduler import APScheduler
 from flask_assets import Environment
 from flask_login import LoginManager
@@ -12,79 +14,143 @@ from flask_paranoid import Paranoid
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 from flask_hashids import Hashids
+from logging import Formatter, StreamHandler
+from werkzeug.middleware.proxy_fix import ProxyFix
+from .extensions.nopaque_flask_admin_views import AdminIndexView, ModelView
 
 
+docker_client = DockerClient.from_env()
+
+admin = Admin()
 apifairy = APIFairy()
 assets = Environment()
 db = SQLAlchemy()
-docker_client = DockerClient()
 hashids = Hashids()
 login = LoginManager()
-login.login_view = 'auth.login'
-login.login_message = 'Please log in to access this page.'
 ma = Marshmallow()
 mail = Mail()
 migrate = Migrate(compare_type=True)
 paranoid = Paranoid()
-paranoid.redirect_view = '/'
 scheduler = APScheduler()
 socketio = SocketIO()
 
 
 def create_app(config: Config = Config) -> Flask:
-    ''' Creates an initialized Flask (WSGI Application) object. '''
-    app: Flask = Flask(__name__)
+    ''' Creates an initialized Flask object. '''
+
+    app = Flask(__name__)
     app.config.from_object(config)
-    config.init_app(app)
+
+    # region Logging
+    log_formatter = Formatter(
+        fmt=app.config['NOPAQUE_LOG_FORMAT'],
+        datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT']
+    )
+
+    log_handler = StreamHandler()
+    log_handler.setFormatter(log_formatter)
+    log_handler.setLevel(app.config['NOPAQUE_LOG_LEVEL'])
+
+    app.logger.setLevel('DEBUG')
+    app.logger.removeHandler(default_handler)
+    app.logger.addHandler(log_handler)
+    # endregion Logging
+
+    # region Middlewares
+    if app.config['NOPAQUE_PROXY_FIX_ENABLED']:
+        app.wsgi_app = ProxyFix(
+            app.wsgi_app,
+            x_for=app.config['NOPAQUE_PROXY_FIX_X_FOR'],
+            x_host=app.config['NOPAQUE_PROXY_FIX_X_HOST'],
+            x_port=app.config['NOPAQUE_PROXY_FIX_X_PORT'],
+            x_prefix=app.config['NOPAQUE_PROXY_FIX_X_PREFIX'],
+            x_proto=app.config['NOPAQUE_PROXY_FIX_X_PROTO']
+        )
+    # endregion Middlewares
+
+    # region Extensions
     docker_client.login(
         app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
         password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
         registry=app.config['NOPAQUE_DOCKER_REGISTRY']
     )
 
+    from .models import AnonymousUser, User
+
+    admin.init_app(app, index_view=AdminIndexView())
     apifairy.init_app(app)
     assets.init_app(app)
     db.init_app(app)
     hashids.init_app(app)
     login.init_app(app)
+    login.anonymous_user = AnonymousUser
+    login.login_view = 'auth.login'
+    login.user_loader(lambda user_id: User.query.get(int(user_id)))
     ma.init_app(app)
     mail.init_app(app)
     migrate.init_app(app, db)
     paranoid.init_app(app)
+    paranoid.redirect_view = '/'
     scheduler.init_app(app)
-    socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
+    socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
+    # endregion Extensions
 
-    from .admin import bp as admin_blueprint
-    app.register_blueprint(admin_blueprint, url_prefix='/admin')
-
-    from .api import bp as api_blueprint
+    # region Blueprints
+    from .blueprints.api import bp as api_blueprint
     app.register_blueprint(api_blueprint, url_prefix='/api')
 
-    from .auth import bp as auth_blueprint
-    app.register_blueprint(auth_blueprint, url_prefix='/auth')
+    from .blueprints.auth import bp as auth_blueprint
+    app.register_blueprint(auth_blueprint)
 
-    from .contributions import bp as contributions_blueprint
+    from .blueprints.contributions import bp as contributions_blueprint
     app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
-    from .corpora import bp as corpora_blueprint
-    app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
+    from .blueprints.corpora import bp as corpora_blueprint
+    app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
 
-    from .errors import bp as errors_blueprint
-    app.register_blueprint(errors_blueprint)
+    from .blueprints.errors import bp as errors_bp
+    app.register_blueprint(errors_bp)
 
-    from .jobs import bp as jobs_blueprint
+    from .blueprints.jobs import bp as jobs_blueprint
     app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
 
-    from .main import bp as main_blueprint
-    app.register_blueprint(main_blueprint, url_prefix='/')
+    from .blueprints.main import bp as main_blueprint
+    app.register_blueprint(main_blueprint, cli_group=None)
 
-    from .services import bp as services_blueprint
+    from .blueprints.services import bp as services_blueprint
     app.register_blueprint(services_blueprint, url_prefix='/services')
 
-    from .settings import bp as settings_blueprint
+    from .blueprints.settings import bp as settings_blueprint
     app.register_blueprint(settings_blueprint, url_prefix='/settings')
 
-    from .users import bp as users_blueprint
-    app.register_blueprint(users_blueprint, url_prefix='/users')
+    from .blueprints.users import bp as users_blueprint
+    app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
+
+    from .blueprints.workshops import bp as workshops_blueprint
+    app.register_blueprint(workshops_blueprint, url_prefix='/workshops')
+
+    from .models import _models
+    for model in _models:
+        admin.add_view(ModelView(model, db.session, category='Database'))
+    # endregion Blueprints
+
+    # region SocketIO Namespaces
+    from .namespaces.cqi_over_sio import CQiOverSocketIONamespace
+    socketio.on_namespace(CQiOverSocketIONamespace('/cqi_over_sio'))
+    # endregion SocketIO Namespaces
+
+    # region Database event Listeners
+    from .models.event_listeners import register_event_listeners
+    register_event_listeners()
+    # endregion Database event Listeners
+
+    # region Add scheduler jobs
+    if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
+        from .jobs import handle_corpora
+        scheduler.add_job('handle_corpora', handle_corpora, seconds=3, trigger='interval')
+
+        from .jobs import handle_jobs
+        scheduler.add_job('handle_jobs', handle_jobs, seconds=3, trigger='interval')
+    # endregion Add scheduler jobs
 
     return app

app/admin/forms.py

@@ -1,13 +0,0 @@
-from app.models import Role
-from flask_wtf import FlaskForm
-from wtforms import BooleanField, SelectField, SubmitField
-
-
-class AdminEditUserForm(FlaskForm):
-    confirmed = BooleanField('Confirmed')
-    role = SelectField('Role')
-    submit = SubmitField('Submit')
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.role.choices = [(x.hashid, x.name) for x in Role.query.all()]

app/admin/routes.py

@@ -1,111 +0,0 @@
-from flask import current_app, flash, redirect, render_template, url_for
-from flask_login import login_required
-from threading import Thread
-from app import db, hashids
-from app.decorators import admin_required
-from app.models import Role, User, UserSettingJobStatusMailNotificationLevel
-from app.settings.forms import (
-    EditNotificationSettingsForm
-)
-from app.users.forms import EditProfileSettingsForm
-from . import bp
-from .forms import AdminEditUserForm
-
-
-@bp.before_request
-@login_required
-@admin_required
-def before_request():
-    '''
-    Ensures that the routes in this package can be visited only by users with
-    administrator privileges (login_required and admin_required).
-    '''
-    pass
-
-
-@bp.route('')
-def index():
-    return redirect(url_for('.users'))
-
-
-@bp.route('/users')
-def users():
-    users = [x.to_json_serializeable(backrefs=True) for x in User.query.all()]
-    return render_template(
-        'admin/users.html.j2',
-        users=users,
-        title='Users'
-    )
-
-
-@bp.route('/users/<hashid:user_id>')
-def user(user_id):
-    user = User.query.get_or_404(user_id)
-    return render_template('admin/user.html.j2', title='User', user=user)
-
-
-@bp.route('/users/<hashid:user_id>/edit', methods=['GET', 'POST'])
-def edit_user(user_id):
-    user = User.query.get_or_404(user_id)
-    admin_edit_user_form = AdminEditUserForm(
-        data={'confirmed': user.confirmed, 'role': user.role.hashid},
-        prefix='admin-edit-user-form'
-    )
-    edit_profile_settings_form = EditProfileSettingsForm(
-        user,
-        data=user.to_json_serializeable(),
-        prefix='edit-profile-settings-form'
-    )
-    edit_notification_settings_form = EditNotificationSettingsForm(
-        data=user.to_json_serializeable(),
-        prefix='edit-notification-settings-form'
-    )
-    if (admin_edit_user_form.submit.data
-            and admin_edit_user_form.validate()):
-        user.confirmed = admin_edit_user_form.confirmed.data
-        role_id = hashids.decode(admin_edit_user_form.role.data)
-        user.role = Role.query.get(role_id)
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
-    if (edit_profile_settings_form.submit.data
-            and edit_profile_settings_form.validate()):
-        user.email = edit_profile_settings_form.email.data
-        user.username = edit_profile_settings_form.username.data
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
-    if (edit_notification_settings_form.submit.data
-            and edit_notification_settings_form.validate()):
-        user.setting_job_status_mail_notification_level = \
-            UserSettingJobStatusMailNotificationLevel[
-                edit_notification_settings_form.job_status_mail_notification_level.data  # noqa
-            ]
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.edit_user', user_id=user.id))
-    return render_template(
-        'admin/edit_user.html.j2',
-        admin_edit_user_form=admin_edit_user_form,
-        edit_profile_settings_form=edit_profile_settings_form,
-        edit_notification_settings_form=edit_notification_settings_form,
-        title='Edit user',
-        user=user
-    )
-
-
-@bp.route('/users/<hashid:user_id>/delete', methods=['DELETE'])
-def delete_user(user_id):
-    def _delete_user(app, user_id):
-        with app.app_context():
-            user = User.query.get(user_id)
-            user.delete()
-            db.session.commit()
-
-    User.query.get_or_404(user_id)
-    thread = Thread(
-        target=_delete_user,
-        args=(current_app._get_current_object(), user_id)
-    )
-    thread.start()
-    return {}, 202

app/auth/__init__.py

@@ -1,8 +0,0 @@
-from flask import Blueprint
-
-
-USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
-
-
-bp = Blueprint('auth', __name__)
-from . import routes

app/blueprints/api/jobs.py

@@ -5,8 +5,8 @@ from flask import abort, Blueprint
 from werkzeug.exceptions import InternalServerError
 from app import db, hashids
 from app.models import Job, JobInput, JobStatus, TesseractOCRPipelineModel
-from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
 from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
 
 
 bp = Blueprint('jobs', __name__)
@@ -77,7 +77,7 @@ def delete_job(job_id):
     job = Job.query.get(job_id)
     if job is None:
         abort(404)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     try:
         job.delete()
@@ -97,6 +97,6 @@ def get_job(job_id):
     job = Job.query.get(job_id)
     if job is None:
         abort(404)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     return job

app/blueprints/api/schemas.py

@@ -2,7 +2,6 @@ from apifairy.fields import FileField
 from marshmallow import validate, validates, ValidationError
 from marshmallow.decorators import post_dump
 from app import ma
-from app.auth import USERNAME_REGEX
 from app.models import (
     Job,
     JobStatus,
@@ -11,7 +10,7 @@ from app.models import (
     User,
     UserSettingJobStatusMailNotificationLevel
 )
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 
 
 
@@ -142,7 +141,10 @@ class UserSchema(ma.SQLAlchemySchema):
     username = ma.auto_field(
         validate=[
             validate.Length(min=1, max=64),
-            validate.Regexp(USERNAME_REGEX, error='Usernames must have only letters, numbers, dots or underscores')
+            validate.Regexp(
+                User.username_pattern,
+                error='Usernames must have only letters, numbers, dots or underscores'
+            )
         ]
     )
     email = ma.auto_field(validate=validate.Email())

app/blueprints/api/users.py

@@ -3,11 +3,11 @@ from apifairy import authenticate, body, response
 from apifairy.decorators import other_responses
 from flask import abort, Blueprint
 from werkzeug.exceptions import InternalServerError
-from app import db
 from app.email import create_message, send
+from app import db
 from app.models import User
-from .schemas import EmptySchema, UserSchema
 from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, UserSchema
 
 
 bp = Blueprint('users', __name__)
@@ -60,7 +60,7 @@ def delete_user(user_id):
     user = User.query.get(user_id)
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     user.delete()
     db.session.commit()
@@ -78,7 +78,7 @@ def get_user(user_id):
     user = User.query.get(user_id)
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     return user
 
@@ -94,6 +94,6 @@ def get_user_by_username(username):
     user = User.query.filter(User.username == username).first()
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     return user

app/blueprints/auth/__init__.py

@@ -0,0 +1,27 @@
+from flask import Blueprint, redirect, request, url_for
+from flask_login import current_user
+from app import db
+
+
+bp = Blueprint('auth', __name__)
+
+
+@bp.before_app_request
+def before_request():
+    if not current_user.is_authenticated:
+        return
+
+    current_user.ping()
+    db.session.commit()
+
+    if (
+        not current_user.confirmed
+        and request.endpoint
+        and request.blueprint != 'auth'
+        and request.endpoint != 'static'
+        and request.endpoint != 'main.accept_terms_of_use'
+    ):
+        return redirect(url_for('auth.unconfirmed'))
+
+
+from . import routes

app/blueprints/auth/forms.py

@@ -8,7 +8,6 @@ from wtforms import (
 )
 from wtforms.validators import InputRequired, Email, EqualTo, Length, Regexp
 from app.models import User
-from . import USERNAME_REGEX
 
 
 class RegistrationForm(FlaskForm):
@@ -22,7 +21,7 @@ class RegistrationForm(FlaskForm):
             InputRequired(),
             Length(max=64),
             Regexp(
-                USERNAME_REGEX,
+                User.username_pattern,
                 message=(
                     'Usernames must have only letters, numbers, dots or '
                     'underscores'
@@ -44,15 +43,28 @@ class RegistrationForm(FlaskForm):
             EqualTo('password', message='Passwords must match')
         ]
     )
+    terms_of_use_accepted = BooleanField(
+        'I have read and accept the terms of use',
+        validators=[InputRequired()]
+    )
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'registration-form'
+        super().__init__(*args, **kwargs)
+
     def validate_email(self, field):
         if User.query.filter_by(email=field.data.lower()).first():
             raise ValidationError('Email already registered')
 
     def validate_username(self, field):
         if User.query.filter_by(username=field.data).first():
-            raise ValidationError('Username already in use')
+            raise ValidationError('Username already registered')
+
+    def validate_terms_of_use_accepted(self, field):
+        if not field.data:
+            raise ValidationError('Terms of Use not accepted')
 
 
 class LoginForm(FlaskForm):
@@ -61,11 +73,21 @@ class LoginForm(FlaskForm):
     remember_me = BooleanField('Keep me logged in')
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'login-form'
+        super().__init__(*args, **kwargs)
+
 
 class ResetPasswordRequestForm(FlaskForm):
     email = StringField('Email', validators=[InputRequired(), Email()])
     submit = SubmitField()
 
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-request-form'
+        super().__init__(*args, **kwargs)
+
 
 class ResetPasswordForm(FlaskForm):
     password = PasswordField(
@@ -83,3 +105,8 @@ class ResetPasswordForm(FlaskForm):
         ]
     )
     submit = SubmitField()
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'reset-password-form'
+        super().__init__(*args, **kwargs)

app/blueprints/auth/routes.py

@@ -1,11 +1,4 @@
-from flask import (
-    abort,
-    flash,
-    redirect,
-    render_template,
-    request,
-    url_for
-)
+from flask import abort, flash, redirect, render_template, request, url_for
 from flask_login import current_user, login_user, login_required, logout_user
 from app import db
 from app.email import create_message, send
@@ -19,33 +12,18 @@ from .forms import (
 )
 
 
-@bp.before_app_request
-def before_request():
-    """
-    Checks if a user is unconfirmed when visiting specific sites. Redirects to
-    unconfirmed view if user is unconfirmed.
-    """
-    if current_user.is_authenticated:
-        current_user.ping()
-        db.session.commit()
-        if (not current_user.confirmed
-                and request.endpoint
-                and request.blueprint != 'auth'
-                and request.endpoint != 'static'):
-            return redirect(url_for('auth.unconfirmed'))
-
-
 @bp.route('/register', methods=['GET', 'POST'])
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = RegistrationForm(prefix='registration-form')
+    form = RegistrationForm()
     if form.validate_on_submit():
         try:
             user = User.create(
                 email=form.email.data.lower(),
                 password=form.password.data,
-                username=form.username.data
+                username=form.username.data,
+                terms_of_use_accepted=form.terms_of_use_accepted.data
             )
         except OSError:
             flash('Internal Server Error', category='error')
@@ -65,8 +43,8 @@ def register():
         return redirect(url_for('.login'))
     return render_template(
         'auth/register.html.j2',
-        form=form,
-        title='Register'
+        title='Register',
+        form=form
     )
 
 
@@ -74,7 +52,7 @@ def register():
 def login():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = LoginForm(prefix='login-form')
+    form = LoginForm()
     if form.validate_on_submit():
         user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
         if user and user.verify_password(form.password.data):
@@ -85,7 +63,11 @@ def login():
             flash('You have been logged in')
             return redirect(next)
         flash('Invalid email/username or password', category='error')
-    return render_template('auth/login.html.j2', form=form, title='Log in')
+    return render_template(
+        'auth/login.html.j2',
+        title='Log in',
+        form=form
+    )
 
 
 @bp.route('/logout')
@@ -101,10 +83,13 @@ def logout():
 def unconfirmed():
     if current_user.confirmed:
         return redirect(url_for('main.dashboard'))
-    return render_template('auth/unconfirmed.html.j2', title='Unconfirmed')
+    return render_template(
+        'auth/unconfirmed.html.j2',
+        title='Unconfirmed'
+    )
 
 
-@bp.route('/confirm')
+@bp.route('/confirm-request')
 @login_required
 def confirm_request():
     if current_user.confirmed:
@@ -135,11 +120,11 @@ def confirm(token):
     return redirect(url_for('.unconfirmed'))
 
 
-@bp.route('/reset_password', methods=['GET', 'POST'])
+@bp.route('/reset-password-request', methods=['GET', 'POST'])
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = ResetPasswordRequestForm(prefix='reset-password-request-form')
+    form = ResetPasswordRequestForm()
     if form.validate_on_submit():
         user = User.query.filter_by(email=form.email.data.lower()).first()
         if user is not None:
@@ -159,16 +144,16 @@ def reset_password_request():
         return redirect(url_for('.login'))
     return render_template(
         'auth/reset_password_request.html.j2',
-        form=form,
-        title='Password Reset'
+        title='Password Reset',
+        form=form
     )
 
 
-@bp.route('/reset_password/<token>', methods=['GET', 'POST'])
+@bp.route('/reset-password/<token>', methods=['GET', 'POST'])
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    form = ResetPasswordForm(prefix='reset-password-form')
+    form = ResetPasswordForm()
     if form.validate_on_submit():
         if User.reset_password(token, form.password.data):
             db.session.commit()
@@ -177,7 +162,7 @@ def reset_password(token):
         return redirect(url_for('main.index'))
     return render_template(
         'auth/reset_password.html.j2',
-        form=form,
         title='Password Reset',
+        form=form,
         token=token
     )

app/blueprints/contributions/__init__.py

@@ -0,0 +1,25 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('contributions', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes
+
+
+from .spacy_nlp_pipeline_models import bp as spacy_nlp_pipeline_models_bp
+bp.register_blueprint(spacy_nlp_pipeline_models_bp, url_prefix='/spacy-nlp-pipeline-models')
+
+from .tesseract_ocr_pipeline_models import bp as tesseract_ocr_pipeline_models_bp
+bp.register_blueprint(tesseract_ocr_pipeline_models_bp, url_prefix='/tesseract-ocr-pipeline-models')

app/blueprints/contributions/forms.py

@@ -0,0 +1,47 @@
+from flask_wtf import FlaskForm
+from wtforms import (
+    StringField,
+    SubmitField,
+    SelectMultipleField,
+    IntegerField
+)
+from wtforms.validators import InputRequired, Length
+
+
+class ContributionBaseForm(FlaskForm):
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=64)]
+    )
+    description = StringField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    version = StringField(
+        'Version',
+        validators=[InputRequired(), Length(max=16)]
+    )
+    publisher = StringField(
+        'Publisher',
+        validators=[InputRequired(), Length(max=128)]
+    )
+    publisher_url = StringField(
+        'Publisher URL',
+        validators=[InputRequired(), Length(max=512)]
+    )
+    publishing_url = StringField(
+        'Publishing URL',
+        validators=[InputRequired(), Length(max=512)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    compatible_service_versions = SelectMultipleField(
+        'Compatible service versions'
+    )
+    submit = SubmitField()
+
+
+class UpdateContributionBaseForm(ContributionBaseForm):
+    pass

app/blueprints/contributions/routes.py

@@ -0,0 +1,7 @@
+from flask import render_template
+from . import bp
+
+
+@bp.route('')
+def index():
+    return render_template('contributions/index.html.j2', title='Contributions')

app/blueprints/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -0,0 +1,18 @@
+from flask import current_app, Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('spacy_nlp_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/contributions/spacy_nlp_pipeline_models/forms.py

@@ -0,0 +1,48 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import StringField, ValidationError
+from wtforms.validators import InputRequired, Length
+from app.blueprints.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
+    spacy_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+
+    def validate_spacy_model_file(self, field):
+        if not field.data.filename.lower().endswith(('.tar.gz', ('.whl'))):
+            raise ValidationError('.tar.gz or .whl files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateSpaCyNLPPipelineModelForm(UpdateContributionBaseForm):
+    pipeline_name = StringField(
+        'Pipeline name',
+        validators=[InputRequired(), Length(max=64)]
+    )
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-spacy-nlp-pipeline-model-form'
+        super().__init__(*args, **kwargs)
+        service_manifest = SERVICES['spacy-nlp-pipeline']
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/blueprints/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -0,0 +1,53 @@
+from flask import abort, current_app, request
+from flask_login import current_user, login_required
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import SpaCyNLPPipelineModel
+from . import bp
+
+
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@login_required
+@content_negotiation(produces='application/json')
+def delete_spacy_model(spacy_nlp_pipeline_model_id):
+    def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
+        with app.app_context():
+            snpm = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
+            snpm.delete()
+            db.session.commit()
+
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_spacy_model,
+        args=(current_app._get_current_object(), snpm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'SpaCy NLP Pipeline Model "{snpm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    snpm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'SpaCy NLP Pipeline Model "{snpm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/blueprints/contributions/spacy_nlp_pipeline_models/routes.py

@@ -0,0 +1,70 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user, login_required
+from app import db
+from app.models import SpaCyNLPPipelineModel
+from . import bp
+from .forms import (
+    CreateSpaCyNLPPipelineModelForm,
+    UpdateSpaCyNLPPipelineModelForm
+)
+
+
+@bp.route('/')
+@login_required
+def index():
+    return redirect(url_for('contributions.index', _anchor='spacy-nlp-pipeline-models'))
+
+
+@bp.route('/create', methods=['GET', 'POST'])
+@login_required
+def create():
+    form = CreateSpaCyNLPPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            snpm = SpaCyNLPPipelineModel.create(
+                form.spacy_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                pipeline_name=form.pipeline_name.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
+        return {}, 201, {'Location': url_for('.index')}
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/create.html.j2',
+        title='Create SpaCy NLP Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+@login_required
+def entity(spacy_nlp_pipeline_model_id):
+    snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
+    if not (snpm.user == current_user or current_user.is_administrator):
+        abort(403)
+    form = UpdateSpaCyNLPPipelineModelForm(data=snpm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(snpm)
+        if db.session.is_modified(snpm):
+            flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.index'))
+    return render_template(
+        'contributions/spacy_nlp_pipeline_models/entity.html.j2',
+        title=f'{snpm.title} {snpm.version}',
+        form=form,
+        spacy_nlp_pipeline_model=snpm
+    )

app/blueprints/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('tesseract_ocr_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import json_routes, routes

app/blueprints/contributions/tesseract_ocr_pipeline_models/forms.py

@@ -0,0 +1,39 @@
+from flask_wtf.file import FileField, FileRequired
+from wtforms import ValidationError
+from app.blueprints.services import SERVICES
+from ..forms import ContributionBaseForm, UpdateContributionBaseForm
+
+
+class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
+    tesseract_model_file = FileField(
+        'File',
+        validators=[FileRequired()]
+    )
+
+    def validate_tesseract_model_file(self, field):
+        if not field.data.filename.lower().endswith('.traineddata'):
+            raise ValidationError('traineddata files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''
+
+
+class UpdateTesseractOCRPipelineModelForm(UpdateContributionBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'edit-tesseract-ocr-pipeline-model-form'
+        service_manifest = SERVICES['tesseract-ocr-pipeline']
+        super().__init__(*args, **kwargs)
+        self.compatible_service_versions.choices = [('', 'Choose your option')]
+        self.compatible_service_versions.choices += [
+            (x, x) for x in service_manifest['versions'].keys()
+        ]
+        self.compatible_service_versions.default = ''

app/blueprints/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -0,0 +1,52 @@
+from flask import abort, current_app, request
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation, permission_required
+from app.models import TesseractOCRPipelineModel
+from . import bp
+
+
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@content_negotiation(produces='application/json')
+def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
+    def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
+        with app.app_context():
+            topm = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
+            topm.delete()
+            db.session.commit()
+
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    thread = Thread(
+        target=_delete_tesseract_ocr_pipeline_model,
+        args=(current_app._get_current_object(), topm.id)
+    )
+    thread.start()
+    response_data = {
+        'message': \
+            f'Tesseract OCR Pipeline Model "{topm.title}" marked for deletion'
+    }
+    return response_data, 202
+
+
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@permission_required('CONTRIBUTE')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    topm.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Tesseract OCR Pipeline Model "{topm.title}"'
+            f' is now {"public" if is_public else "private"}'
+        )
+    }
+    return response_data, 200

app/blueprints/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -0,0 +1,66 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import TesseractOCRPipelineModel
+from . import bp
+from .forms import (
+    CreateTesseractOCRPipelineModelForm,
+    UpdateTesseractOCRPipelineModelForm
+)
+
+
+@bp.route('/')
+def index():
+    return redirect(url_for('contributions.index', _anchor='tesseract-ocr-pipeline-models'))
+
+
+@bp.route('/create', methods=['GET', 'POST'])
+def create():
+    form = CreateTesseractOCRPipelineModelForm()
+    if form.is_submitted():
+        if not form.validate():
+            return {'errors': form.errors}, 400
+        try:
+            topm = TesseractOCRPipelineModel.create(
+                form.tesseract_model_file.data,
+                compatible_service_versions=form.compatible_service_versions.data,
+                description=form.description.data,
+                publisher=form.publisher.data,
+                publisher_url=form.publisher_url.data,
+                publishing_url=form.publishing_url.data,
+                publishing_year=form.publishing_year.data,
+                is_public=False,
+                title=form.title.data,
+                version=form.version.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
+        return {}, 201, {'Location': url_for('.index')}
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/create.html.j2',
+        title='Create Tesseract OCR Pipeline Model',
+        form=form
+    )
+
+
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+def entity(tesseract_ocr_pipeline_model_id):
+    topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
+    if not (topm.user == current_user or current_user.is_administrator):
+        abort(403)
+    form = UpdateTesseractOCRPipelineModelForm(data=topm.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(topm)
+        if db.session.is_modified(topm):
+            flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
+            db.session.commit()
+        return redirect(url_for('.index'))
+    return render_template(
+        'contributions/tesseract_ocr_pipeline_models/entity.html.j2',
+        title=f'{topm.title} {topm.version}',
+        form=form,
+        tesseract_ocr_pipeline_model=topm
+    )

app/blueprints/corpora/__init__.py

@@ -0,0 +1,19 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('corpora', __name__)
+bp.cli.short_help = 'Corpus commands.'
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import cli, files, followers, routes

app/blueprints/corpora/cli.py

@@ -0,0 +1,34 @@
+from flask import current_app
+import shutil
+from app import db
+from app.models import Corpus, CorpusStatus
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset built corpora. '''
+    status = [
+        CorpusStatus.QUEUED,
+        CorpusStatus.BUILDING,
+        CorpusStatus.BUILT,
+        CorpusStatus.STARTING_ANALYSIS_SESSION,
+        CorpusStatus.RUNNING_ANALYSIS_SESSION,
+        CorpusStatus.CANCELING_ANALYSIS_SESSION
+    ]
+    for corpus in [x for x in Corpus.query.all() if x.status in status]:
+        print(f'Resetting corpus {corpus}')
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        corpus.status = CorpusStatus.UNPREPARED
+        corpus.num_analysis_sessions = 0
+    db.session.commit()

app/blueprints/corpora/decorators.py

@@ -0,0 +1,33 @@
+from flask import abort
+from flask_login import current_user
+from functools import wraps
+from app.models import Corpus, CorpusFollowerAssociation
+
+
+def corpus_follower_permission_required(*permissions):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            corpus_id = kwargs.get('corpus_id')
+            corpus = Corpus.query.get_or_404(corpus_id)
+            if not (corpus.user == current_user or current_user.is_administrator):
+                cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+                if cfa is None:
+                    abort(403)
+                if not all([cfa.role.has_permission(p) for p in permissions]):
+                    abort(403)
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator
+
+
+def corpus_owner_or_admin_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        corpus_id = kwargs.get('corpus_id')
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if not (corpus.user == current_user or current_user.is_administrator):
+            abort(403)
+        return f(*args, **kwargs)
+    return decorated_function
+

app/blueprints/corpora/files/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/corpora/files/forms.py

@@ -1,39 +1,14 @@
 from flask_wtf import FlaskForm
 from flask_wtf.file import FileField, FileRequired
 from wtforms import (
-    BooleanField,
     StringField,
     SubmitField,
-    TextAreaField,
     ValidationError,
     IntegerField
 )
 from wtforms.validators import InputRequired, Length
 
 
-class CorpusBaseForm(FlaskForm):
-    description = TextAreaField(
-        'Description',
-        validators=[InputRequired(), Length(max=255)]
-    )
-    title = StringField('Title', validators=[InputRequired(), Length(max=32)])
-    submit = SubmitField()
-
-
-class CreateCorpusForm(CorpusBaseForm):
-    def __init__(self, *args, **kwargs):
-        if 'prefix' not in kwargs:
-            kwargs['prefix'] = 'create-corpus-form'
-        super().__init__(*args, **kwargs)
-
-
-class UpdateCorpusForm(CorpusBaseForm):
-    def __init__(self, *args, **kwargs):
-        if 'prefix' not in kwargs:
-            kwargs['prefix'] = 'update-corpus-form'
-        super().__init__(*args, **kwargs)
-
-
 class CorpusFileBaseForm(FlaskForm):
     author = StringField(
         'Author',
@@ -62,25 +37,18 @@ class CorpusFileBaseForm(FlaskForm):
 class CreateCorpusFileForm(CorpusFileBaseForm):
     vrt = FileField('File', validators=[FileRequired()])
 
+    def validate_vrt(self, field):
+        if not field.data.filename.lower().endswith('.vrt'):
+            raise ValidationError('VRT files only!')
+
     def __init__(self, *args, **kwargs):
         if 'prefix' not in kwargs:
             kwargs['prefix'] = 'create-corpus-file-form'
         super().__init__(*args, **kwargs)
 
-    def validate_vrt(self, field):
-        if not field.data.filename.lower().endswith('.vrt'):
-            raise ValidationError('VRT files only!')
-
 
 class UpdateCorpusFileForm(CorpusFileBaseForm):
     def __init__(self, *args, **kwargs):
         if 'prefix' not in kwargs:
             kwargs['prefix'] = 'update-corpus-file-form'
         super().__init__(*args, **kwargs)
-
-class ChangeCorpusSettingsForm(FlaskForm):
-    is_public = BooleanField('Public Corpus')
-    submit = SubmitField()
-
-class ImportCorpusForm(FlaskForm):
-    pass

app/blueprints/corpora/files/json_routes.py

@@ -0,0 +1,30 @@
+from flask import current_app
+from threading import Thread
+from app.decorators import content_negotiation
+from app import db
+from app.models import CorpusFile
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def delete_corpus_file(corpus_id, corpus_file_id):
+    def _delete_corpus_file(app, corpus_file_id):
+        with app.app_context():
+            corpus_file = CorpusFile.query.get(corpus_file_id)
+            corpus_file.delete()
+            db.session.commit()
+
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    thread = Thread(
+        target=_delete_corpus_file,
+        args=(current_app._get_current_object(), corpus_file.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus File "{corpus_file.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 202

app/blueprints/corpora/files/routes.py

@@ -0,0 +1,93 @@
+from flask import (
+    abort,
+    flash,
+    redirect,
+    render_template,
+    send_from_directory,
+    url_for
+)
+from app import db
+from app.models import Corpus, CorpusFile, CorpusStatus
+from ..decorators import corpus_follower_permission_required
+from . import bp
+from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
+
+
+@bp.route('/<hashid:corpus_id>/files')
+def corpus_files(corpus_id):
+    return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
+
+
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def create_corpus_file(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    form = CreateCorpusFileForm()
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            corpus_file = CorpusFile.create(
+                form.vrt.data,
+                address=form.address.data,
+                author=form.author.data,
+                booktitle=form.booktitle.data,
+                chapter=form.chapter.data,
+                editor=form.editor.data,
+                institution=form.institution.data,
+                journal=form.journal.data,
+                pages=form.pages.data,
+                publisher=form.publisher.data,
+                publishing_year=form.publishing_year.data,
+                school=form.school.data,
+                title=form.title.data,
+                mimetype='application/vrt+xml',
+                corpus=corpus
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        corpus.status = CorpusStatus.UNPREPARED
+        db.session.commit()
+        flash(f'Corpus File "{corpus_file.filename}" added', category='corpus')
+        return '', 201, {'Location': corpus.url}
+    return render_template(
+        'corpora/files/create.html.j2',
+        title='Add corpus file',
+        form=form,
+        corpus=corpus
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(corpus_file)
+        if db.session.is_modified(corpus_file):
+            corpus_file.corpus.status = CorpusStatus.UNPREPARED
+            db.session.commit()
+            flash(f'Corpus file "{corpus_file.filename}" updated', category='corpus')
+        return redirect(corpus_file.corpus.url)
+    return render_template(
+        'corpora/files/corpus_file.html.j2',
+        title='Edit corpus file',
+        form=form,
+        corpus=corpus_file.corpus,
+        corpus_file=corpus_file
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
+@corpus_follower_permission_required('VIEW')
+def download_corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    return send_from_directory(
+        corpus_file.path.parent,
+        corpus_file.path.name,
+        as_attachment=True,
+        download_name=corpus_file.filename,
+        mimetype=corpus_file.mimetype
+    )

app/blueprints/corpora/followers/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes

app/blueprints/corpora/followers/json_routes.py

@@ -0,0 +1,76 @@
+from flask import abort, flash, jsonify, make_response, request
+from flask_login import current_user
+from app import db
+from app.decorators import content_negotiation
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def create_corpus_followers(corpus_id):
+    usernames = request.json
+    if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    for username in usernames:
+        user = User.query.filter_by(username=username, is_public=True).first_or_404()
+        user.follow_corpus(corpus)
+    db.session.commit()
+    response_data = {
+        'message': f'Users are now following "{corpus.title}"',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_follower_role(corpus_id, follower_id):
+    role_name = request.json
+    if not isinstance(role_name, str):
+        abort(400)
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    cfa.role = cfr
+    db.session.commit()
+    response_data = {
+        'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
+def delete_corpus_follower(corpus_id, follower_id):
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    if not (
+        current_user.id == follower_id
+        or current_user == cfa.corpus.user 
+        or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
+        or current_user.is_administrator):
+        abort(403)
+    if current_user.id == follower_id:
+        flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
+        response = make_response()
+        response.status_code = 204
+    else:
+        response_data = {
+            'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
+            'category': 'corpus'
+        }
+        response = jsonify(response_data)
+        response.status_code = 200
+    cfa.follower.unfollow_corpus(cfa.corpus)
+    db.session.commit()
+    return response

app/blueprints/corpora/forms.py

@@ -0,0 +1,33 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, SubmitField, TextAreaField
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusBaseForm(FlaskForm):
+    description = TextAreaField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    title = StringField('Title', validators=[InputRequired(), Length(max=32)])
+    submit = SubmitField()
+
+
+class CreateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class ImportCorpusForm(FlaskForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'import-corpus-form'
+        super().__init__(*args, **kwargs)

app/blueprints/corpora/routes.py

@@ -0,0 +1,299 @@
+from datetime import datetime
+from flask import (
+    abort,
+    current_app,
+    flash,
+    Flask,
+    jsonify,
+    redirect,
+    request,
+    render_template,
+    url_for
+)
+from flask_login import current_user
+from string import punctuation
+from threading import Thread
+import nltk
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from . import bp
+from .decorators import corpus_follower_permission_required
+from .forms import CreateCorpusForm
+
+
+
+def _delete_corpus(app: Flask, corpus_id: int):
+    with app.app_context():
+        corpus: Corpus = Corpus.query.get(corpus_id)
+        corpus.delete()
+        db.session.commit()
+
+
+def _build_corpus(app: Flask, corpus_id: int):
+    with app.app_context():
+        corpus = Corpus.query.get(corpus_id)
+        corpus.build()
+        db.session.commit()
+
+
+@bp.route('')
+def corpora():
+    return redirect(url_for('main.dashboard', _anchor='corpora'))
+
+
+@bp.route('/create', methods=['GET', 'POST'])
+def create_corpus():
+    form = CreateCorpusForm()
+
+    if form.validate_on_submit():
+        try:
+            corpus = Corpus.create(
+                title=form.title.data,
+                description=form.description.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+
+        flash(f'Corpus "{corpus.title}" created', 'corpus')
+        return redirect(corpus.url)
+
+    return render_template(
+        'corpora/create.html.j2',
+        title='Create corpus',
+        form=form
+    )
+
+
+@bp.route('/<hashid:corpus_id>')
+def corpus(corpus_id: int):
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    cfa = CorpusFollowerAssociation.query.filter_by(
+        corpus_id=corpus_id,
+        follower_id=current_user.id
+    ).first()
+
+    if cfa is None:
+        if corpus.user == current_user or current_user.is_administrator:
+            cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
+        else:
+            cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
+    else:
+        cfr = cfa.role
+
+    cfrs = CorpusFollowerRole.query.all()
+
+    # TODO: Better solution for filtering admin
+    users = User.query.filter(
+        User.is_public == True,
+        User.id != current_user.id,
+        User.id != corpus.user.id,
+        User.role_id < 4
+    ).all()
+
+    if (
+        corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        return render_template(
+            'corpora/corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfr=cfr,
+            cfrs=cfrs,
+            users=users
+        )
+
+    if (
+        current_user.is_following_corpus(corpus)
+        or corpus.is_public
+    ):
+        cfas = CorpusFollowerAssociation.query.filter(
+            Corpus.id == corpus_id,
+            CorpusFollowerAssociation.follower_id != corpus.user.id
+        ).all()
+        return render_template(
+            'corpora/public_corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfrs=cfrs,
+            cfr=cfr,
+            cfas=cfas,
+            users=users
+        )
+
+    abort(403)
+
+
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+def delete_corpus(corpus_id: int):
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    if not (
+        corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    thread = Thread(
+        target=_delete_corpus,
+        args=(current_app._get_current_object(), corpus.id)
+    )
+    thread.start()
+
+    return jsonify(f'Corpus "{corpus.title}" marked for deletion.'), 202
+
+
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
+def build_corpus(corpus_id: int):
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    cfa = CorpusFollowerAssociation.query.filter_by(
+        corpus_id=corpus_id,
+        follower_id=current_user.id
+    ).first()
+
+    if not (
+        cfa is not None and cfa.role.has_permission('MANAGE_FILES')
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    if len(corpus.files.all()) == 0:
+        abort(409)
+
+    thread = Thread(
+        target=_build_corpus,
+        args=(current_app._get_current_object(), corpus.id)
+    )
+    thread.start()
+
+    return jsonify(f'Corpus "{corpus.title}" marked for building.'), 202
+
+
+@bp.route('/<hashid:corpus_id>/create-share-link', methods=['POST'])
+def create_share_link(corpus_id: int):
+    data = request.json
+
+    expiration_date = data['expiration_date']
+    if not isinstance(expiration_date, str):
+        abort(400)
+
+    role_name = data['role_name']
+    if not isinstance(role_name, str):
+        abort(400)
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    cfa = CorpusFollowerAssociation.query.filter_by(
+        corpus_id=corpus_id,
+        follower_id=current_user.id
+    ).first()
+
+    if not (
+        cfa is not None and cfa.role.has_permission('MANAGE_FOLLOWERS')
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    _expiration_date = datetime.strptime(expiration_date, '%b %d, %Y')
+
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+
+    token = current_user.generate_follow_corpus_token(
+        corpus.hashid,
+        role_name,
+        _expiration_date
+    )
+
+    corpus_share_link = url_for(
+        'corpora.follow_corpus',
+        corpus_id=corpus_id,
+        token=token,
+        _external=True
+    )
+
+    return jsonify(corpus_share_link)
+
+
+@bp.route('/<hashid:corpus_id>/analysis')
+@corpus_follower_permission_required('VIEW')
+def analysis(corpus_id: int):
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    return render_template(
+        'corpora/analysis.html.j2',
+        corpus=corpus,
+        title=f'Analyse Corpus {corpus.title}'
+    )
+
+
+@bp.route('/<hashid:corpus_id>/analysis/stopwords')
+def get_stopwords(corpus_id: int):
+        languages = [
+            'german',
+            'english',
+            'catalan',
+            'greek',
+            'spanish',
+            'french',
+            'italian',
+            'russian',
+            'chinese'
+        ]
+
+        nltk.download('stopwords', quiet=True)
+        stopwords = {
+            language: nltk.corpus.stopwords.words(language)
+            for language in languages
+        }
+        stopwords['punctuation'] = list(punctuation)
+        stopwords['punctuation'] += ['—', '|', '–', '“', '„', '--']
+        stopwords['user_stopwords'] = []
+
+        return jsonify(stopwords)
+
+
+@bp.route('/<hashid:corpus_id>/follow/<token>')
+def follow_corpus(corpus_id: int, token: str):
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    if not current_user.follow_corpus_by_token(token):
+        abort(403)
+
+    db.session.commit()
+
+    flash(f'You are following "{corpus.title}" now', category='corpus')
+    return redirect(corpus.url)
+
+
+@bp.route('/<hashid:corpus_id>/is-public', methods=['PUT'])
+def update_is_public(corpus_id):
+    new_value = request.json
+    if not isinstance(new_value, bool):
+        abort(400)
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+
+    if not (
+        corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+
+    corpus.is_public = new_value
+    db.session.commit()
+
+    return jsonify(f'Corpus "{corpus.title}" is now {"public" if new_value else "private"}'), 200

app/blueprints/errors/handlers.py

@@ -0,0 +1,20 @@
+from flask import jsonify, render_template, request
+from werkzeug.exceptions import HTTPException
+from . import bp
+
+
+@bp.app_errorhandler(HTTPException)
+def handle_http_exception(e: HTTPException):
+    ''' Generic HTTP exception handler '''
+    accept_json = request.accept_mimetypes.accept_json
+    accept_html = request.accept_mimetypes.accept_html
+
+    if accept_json and not accept_html:
+        error = {
+            'code': e.code,
+            'name': e.name,
+            'description': e.description
+        }
+        return jsonify(error), e.code
+
+    return render_template('errors/error.html.j2', error=e), e.code

app/blueprints/jobs/__init__.py

@@ -0,0 +1,13 @@
+from flask import Blueprint
+
+
+bp = Blueprint('jobs', __name__)
+
+
+from . import routes
+
+from .inputs import bp as inputs_bp
+bp.register_blueprint(inputs_bp, url_prefix='/<hashid:job_id>/inputs')
+
+from .results import bp as results_bp
+bp.register_blueprint(results_bp, url_prefix='/<hashid:job_id>/results')

app/blueprints/jobs/inputs/__init__.py

@@ -1,5 +1,7 @@
 from flask import Blueprint
 
 
-bp = Blueprint('main', __name__)
+bp = Blueprint('inputs', __name__)
+
+
 from . import routes

app/blueprints/jobs/inputs/routes.py

@@ -0,0 +1,27 @@
+from flask import abort, send_from_directory
+from flask_login import current_user, login_required
+from app.models import JobInput
+from . import bp
+
+
+@bp.route('/<hashid:job_input_id>/download')
+@login_required
+def download_job_input(job_id: int, job_input_id: int):
+    job_input = JobInput.query.filter_by(
+        job_id=job_id,
+        id=job_input_id
+    ).first_or_404()
+
+    if not (
+        job_input.job.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    return send_from_directory(
+        job_input.path.parent,
+        job_input.path.name,
+        as_attachment=True,
+        download_name=job_input.filename,
+        mimetype=job_input.mimetype
+    )

app/blueprints/jobs/results/__init__.py

@@ -1,5 +1,7 @@
 from flask import Blueprint
 
 
-bp = Blueprint('admin', __name__)
+bp = Blueprint('results', __name__)
+
+
 from . import routes

app/blueprints/jobs/results/routes.py

@@ -0,0 +1,27 @@
+from flask import abort, send_from_directory
+from flask_login import current_user, login_required
+from app.models import JobResult
+from . import bp
+
+
+@bp.route('/<hashid:job_result_id>/download')
+@login_required
+def download_job_result(job_id: int, job_result_id: int):
+    job_result = JobResult.query.filter_by(
+        job_id=job_id,
+        id=job_result_id
+    ).first_or_404()
+
+    if not (
+        job_result.job.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    return send_from_directory(
+        job_result.path.parent,
+        job_result.path.name,
+        as_attachment=True,
+        download_name=job_result.filename,
+        mimetype=job_result.mimetype
+    )

app/blueprints/jobs/routes.py

@@ -0,0 +1,111 @@
+from flask import (
+    abort,
+    current_app,
+    Flask,
+    jsonify,
+    redirect,
+    render_template,
+    url_for
+)
+from flask_login import current_user, login_required
+from threading import Thread
+from app import db
+from app.decorators import admin_required
+from app.models import Job, JobStatus
+from . import bp
+
+
+@bp.route('')
+@login_required
+def index():
+    return redirect(url_for('main.dashboard', _anchor='jobs'))
+
+
+@bp.route('/<hashid:job_id>')
+@login_required
+def job(job_id: int):
+    job = Job.query.get_or_404(job_id)
+
+    if not (
+        job.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    return render_template(
+        'jobs/job.html.j2',
+        title='Job',
+        job=job
+    )
+
+
+def _delete_job(app: Flask, job_id: int):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.delete()
+        db.session.commit()
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@login_required
+def delete_job(job_id: int):
+    job = Job.query.get_or_404(job_id)
+
+    if not (
+        job.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    thread = Thread(
+        target=_delete_job,
+        args=(current_app._get_current_object(), job.id)
+    )
+    thread.start()
+
+    return jsonify(f'Job "{job.title}" marked for deletion.'), 202
+
+
+@bp.route('/<hashid:job_id>/log')
+@admin_required
+def job_log(job_id: int):
+    job = Job.query.get_or_404(job_id)
+
+    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+        abort(409)
+
+    log_file_path = job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt'
+    with log_file_path.open() as log_file:
+        log = log_file.read()
+
+    return jsonify(log)
+
+
+def _restart_job(app: Flask, job_id: int):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.restart()
+        db.session.commit()
+
+
+@bp.route('/<hashid:job_id>/restart', methods=['POST'])
+@login_required
+def restart_job(job_id: int):
+    job = Job.query.get_or_404(job_id)
+
+    if not (
+        job.user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    if job.status != JobStatus.FAILED:
+        abort(409)
+
+    thread = Thread(
+        target=_restart_job,
+        args=(current_app._get_current_object(), job.id)
+    )
+    thread.start()
+
+    return jsonify(f'Job "{job.title}" marked for restarting.'), 202

app/blueprints/main/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('main', __name__, cli_group=None)
+from . import cli, routes

app/blueprints/main/cli.py

@@ -0,0 +1,51 @@
+from flask import current_app
+from flask_migrate import upgrade
+from pathlib import Path
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerRole,
+    Role,
+    SpaCyNLPPipelineModel,
+    TesseractOCRPipelineModel,
+    User
+)
+from . import bp
+
+
+@bp.cli.command('deploy')
+def deploy():
+    ''' Run deployment tasks. '''
+
+    print('Make default directories')
+    base_dir = current_app.config['NOPAQUE_DATA_DIR']
+    default_dirs: list[Path] = [
+        base_dir / 'tmp',
+        base_dir / 'users'
+    ]
+    for default_dir in default_dirs:
+        if not default_dir.exists():
+            default_dir.mkdir()
+        if not default_dir.is_dir():
+            raise NotADirectoryError(f'{default_dir} is not a directory')
+
+    print('Migrate database to latest revision')
+    upgrade()
+
+    print('Insert/Update default Roles')
+    Role.insert_defaults()
+    print('Insert/Update default Users')
+    User.insert_defaults()
+    print('Insert/Update default CorpusFollowerRoles')
+    CorpusFollowerRole.insert_defaults()
+    print('Insert/Update default SpaCyNLPPipelineModels')
+    SpaCyNLPPipelineModel.insert_defaults()
+    print('Insert/Update default TesseractOCRPipelineModels')
+    TesseractOCRPipelineModel.insert_defaults()
+
+    print('Stop running analysis sessions')
+    for corpus in Corpus.query.all():
+        corpus.num_analysis_sessions = 0
+    db.session.commit()
+
+    # TODO: Implement checks for if the nopaque network exists

app/blueprints/main/routes.py

@@ -0,0 +1,96 @@
+from flask import abort, flash, jsonify, redirect, render_template, url_for
+from flask_login import current_user, login_required, login_user
+from app.blueprints.auth.forms import LoginForm
+from app.models import Corpus, User
+from . import bp
+from app import db
+
+
+@bp.route('/', methods=['GET', 'POST'])
+def index():
+    form = LoginForm()
+    if form.validate_on_submit():
+        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+        if user and user.verify_password(form.password.data):
+            login_user(user, form.remember_me.data)
+            flash('You have been logged in')
+            return redirect(url_for('.dashboard'))
+        flash('Invalid email/username or password', category='error')
+        redirect(url_for('.index'))
+    return render_template(
+        'main/index.html.j2',
+        title='nopaque',
+        form=form
+    )
+
+
+@bp.route('/faq')
+def faq():
+    return render_template(
+        'main/faq.html.j2',
+        title='Frequently Asked Questions'
+    )
+
+
+@bp.route('/dashboard')
+@login_required
+def dashboard():
+    return render_template(
+        'main/dashboard.html.j2',
+        title='Dashboard'
+    )
+
+
+@bp.route('/manual')
+def manual():
+    return render_template(
+        'main/manual.html.j2',
+        title='Manual'
+    )
+
+
+@bp.route('/news')
+def news():
+    return render_template(
+        'main/news.html.j2',
+        title='News'
+    )
+
+
+@bp.route('/privacy-policy')
+def privacy_policy():
+    return render_template(
+        'main/privacy_policy.html.j2',
+        title='Privacy statement (GDPR)'
+    )
+
+
+@bp.route('/terms-of-use')
+def terms_of_use():
+    return render_template(
+        'main/terms_of_use.html.j2',
+        title='Terms of use'
+    )
+
+
+@bp.route('/accept-terms-of-use', methods=['POST'])
+@login_required
+def accept_terms_of_use():
+    current_user.terms_of_use_accepted = True
+    db.session.commit()
+
+    return jsonify('You accepted the terms of use'), 202
+
+
+
+@bp.route('/social')
+@login_required
+def social():
+    corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
+    users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
+    return render_template(
+        'main/social.html.j2',
+        title='Social',
+        corpora=corpora,
+        users=users
+    )

app/blueprints/services/__init__.py

@@ -0,0 +1,24 @@
+from flask import Blueprint
+from flask_login import login_required
+from pathlib import Path
+import yaml
+
+
+services_file = Path(__file__).parent / 'services.yml'
+with services_file.open('r') as f:
+    SERVICES = yaml.safe_load(f)
+
+bp = Blueprint('services', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes  # noqa

app/blueprints/services/forms.py

@@ -1,12 +1,17 @@
-from flask_login import current_user
 from flask_wtf import FlaskForm
+from flask_login import current_user
 from flask_wtf.file import FileField, FileRequired
-from wtforms import (BooleanField, DecimalRangeField, MultipleFileField,
-                     SelectField, StringField, SubmitField, ValidationError)
+from wtforms import (
+    BooleanField,
+    DecimalRangeField,
+    MultipleFileField,
+    SelectField,
+    StringField,
+    SubmitField,
+    ValidationError
+)
 from wtforms.validators import InputRequired, Length
-
 from app.models import SpaCyNLPPipelineModel, TesseractOCRPipelineModel
-
 from . import SERVICES
 
 
@@ -33,6 +38,8 @@ class CreateFileSetupPipelineJobForm(CreateJobBaseForm):
                 raise ValidationError('JPEG, PNG and TIFF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-file-setup-pipeline-job-form'
         service_manifest = SERVICES['file-setup-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
@@ -54,12 +61,14 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
         if field.data:
             if not('methods' in service_info and 'binarization' in service_info['methods']):
                 raise ValidationError('Binarization is not available')
-              
+
     def validate_pdf(self, field):
         if field.data.mimetype != 'application/pdf':
             raise ValidationError('PDF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-tesseract-ocr-pipeline-job-form'
         service_manifest = SERVICES['tesseract-ocr-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
@@ -75,12 +84,18 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
                 del self.binarization.render_kw['disabled']
                 if 'ocropus_nlbin_threshold' in service_info['methods']:
                     del self.ocropus_nlbin_threshold.render_kw['disabled']
+        user_models = [
+            x for x in current_user.tesseract_ocr_pipeline_models.order_by(TesseractOCRPipelineModel.title).all()
+        ]
         models = [
             x for x in TesseractOCRPipelineModel.query.order_by(TesseractOCRPipelineModel.title).all()
             if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
         ]
-        self.model.choices = [('', 'Choose your option')]
-        self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
         self.model.default = ''
         self.version.choices = [(x, x) for x in service_manifest['versions']]
         self.version.data = version
@@ -106,6 +121,8 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
             raise ValidationError('PDF files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-transkribus-htr-pipeline-job-form'
         transkribus_htr_pipeline_models = kwargs.pop('transkribus_htr_pipeline_models', [])
         service_manifest = SERVICES['transkribus-htr-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
@@ -129,7 +146,7 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
     encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
     txt = FileField('File', validators=[FileRequired()])
     model = SelectField('Model', validators=[InputRequired()])
-    
+
     def validate_encoding_detection(self, field):
         service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]
         if field.data:
@@ -144,23 +161,30 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
             raise ValidationError('Plain text files only!')
 
     def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-spacy-nlp-pipeline-job-form'
         service_manifest = SERVICES['spacy-nlp-pipeline']
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
         service_info = service_manifest['versions'][version]
-        print(service_info)
         if self.encoding_detection.render_kw is None:
             self.encoding_detection.render_kw = {}
         self.encoding_detection.render_kw['disabled'] = True
         if 'methods' in service_info:
             if 'encoding_detection' in service_info['methods']:
                 del self.encoding_detection.render_kw['disabled']
-        models = [
-            x for x in SpaCyNLPPipelineModel.query.order_by(SpaCyNLPPipelineModel.title).all()
-            if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
+        user_models = [
+            x for x in current_user.spacy_nlp_pipeline_models.order_by(SpaCyNLPPipelineModel.title).all()
         ]
-        self.model.choices = [('', 'Choose your option')]
-        self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        models = [
+            x for x in SpaCyNLPPipelineModel.query.filter(SpaCyNLPPipelineModel.user != current_user, SpaCyNLPPipelineModel.is_public == True).order_by(SpaCyNLPPipelineModel.title).all()
+            if version in x.compatible_service_versions
+        ]
+        self.model.choices = {
+            '': [('', 'Choose your option')],
+            'Your models': [(x.hashid, f'{x.title} [{x.version}]') for x in user_models] if user_models else [(0, 'Nothing here yet...')],
+            'Public models': [(x.hashid, f'{x.title} [{x.version}]') for x in models]
+        }
         self.model.default = ''
         self.version.choices = [(x, x) for x in service_manifest['versions']]
         self.version.data = version

app/blueprints/services/routes.py

@@ -1,5 +1,5 @@
-from flask import abort, current_app, flash, make_response, Markup, render_template, request
-from flask_login import current_user, login_required
+from flask import abort, current_app, flash, redirect, render_template, request, url_for
+from flask_login import current_user
 import requests
 from app import db, hashids
 from app.models import (
@@ -18,8 +18,12 @@ from .forms import (
 )
 
 
+@bp.route('/services')
+def services():
+    return redirect(url_for('main.dashboard'))
+
+
 @bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
-@login_required
 def file_setup_pipeline():
     service = 'file-setup-pipeline'
     service_manifest = SERVICES[service]
@@ -49,18 +53,17 @@ def file_setup_pipeline():
                 abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/file_setup_pipeline.html.j2',
-        form=form,
-        title=service_manifest['name']
+        title=service_manifest['name'],
+        form=form
     )
 
 
 @bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
-@login_required
 def tesseract_ocr_pipeline():
     service_name = 'tesseract-ocr-pipeline'
     service_manifest = SERVICES[service_name]
@@ -93,23 +96,24 @@ def tesseract_ocr_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     tesseract_ocr_pipeline_models = [
         x for x in TesseractOCRPipelineModel.query.all()
         if version in x.compatible_service_versions and (x.is_public == True or x.user == current_user)
     ]
+    user_tesseract_ocr_pipeline_models_count = len(current_user.tesseract_ocr_pipeline_models.all())
     return render_template(
         'services/tesseract_ocr_pipeline.html.j2',
+        title=service_manifest['name'],
         form=form,
         tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models,
-        title=service_manifest['name']
+        user_tesseract_ocr_pipeline_models_count=user_tesseract_ocr_pipeline_models_count
     )
 
 
 @bp.route('/transkribus-htr-pipeline', methods=['GET', 'POST'])
-@login_required
 def transkribus_htr_pipeline():
     if not current_app.config.get('NOPAQUE_TRANSKRIBUS_ENABLED'):
         abort(404)
@@ -126,10 +130,9 @@ def transkribus_htr_pipeline():
         abort(500)
     transkribus_htr_pipeline_models = r.json()['trpModelMetadata']
     transkribus_htr_pipeline_models.append({'modelId': 48513, 'name': 'Caroline Minuscle', 'language': 'lat', 'isoLanguages': ['lat']})
-    print(transkribus_htr_pipeline_models[len(transkribus_htr_pipeline_models)-1])
     form = CreateTranskribusHTRPipelineJobForm(
-        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models,
         prefix='create-job-form',
+        transkribus_htr_pipeline_models=transkribus_htr_pipeline_models,
         version=version
     )
     if form.is_submitted():
@@ -156,19 +159,18 @@ def transkribus_htr_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/transkribus_htr_pipeline.html.j2',
-        form=form,
         title=service_manifest['name'],
+        form=form,
         transkribus_htr_pipeline_models=transkribus_htr_pipeline_models
     )
 
 
 @bp.route('/spacy-nlp-pipeline', methods=['GET', 'POST'])
-@login_required
 def spacy_nlp_pipeline():
     service = 'spacy-nlp-pipeline'
     service_manifest = SERVICES[service]
@@ -177,6 +179,7 @@ def spacy_nlp_pipeline():
         abort(404)
     form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
     spacy_nlp_pipeline_models = SpaCyNLPPipelineModel.query.all()
+    user_spacy_nlp_pipeline_models_count = len(current_user.spacy_nlp_pipeline_models.all())
     if form.is_submitted():
         if not form.validate():
             response = {'errors': form.errors}
@@ -201,21 +204,21 @@ def spacy_nlp_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
         'services/spacy_nlp_pipeline.html.j2',
+        title=service_manifest['name'],
         form=form,
         spacy_nlp_pipeline_models=spacy_nlp_pipeline_models,
-        title=service_manifest['name']
+        user_spacy_nlp_pipeline_models_count=user_spacy_nlp_pipeline_models_count
     )
 
 
 @bp.route('/corpus-analysis')
-@login_required
 def corpus_analysis():
     return render_template(
         'services/corpus_analysis.html.j2',
-        title='Corpus analysis'
+        title='Corpus Analysis'
     )

app/blueprints/services/services.yml

@@ -10,7 +10,7 @@ file-setup-pipeline:
 tesseract-ocr-pipeline:
   name: 'Tesseract OCR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.1'
+  latest_version: '0.1.2'
   versions:
     0.1.0:
       methods:
@@ -23,6 +23,12 @@ tesseract-ocr-pipeline:
         - 'ocropus_nlbin_threshold'
       publishing_year: 2022
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
+    0.1.2:
+      methods:
+        - 'binarization'
+        - 'ocropus_nlbin_threshold'
+      publishing_year: 2023
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.2'
 transkribus-htr-pipeline:
   name: 'Transkribus HTR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
@@ -41,7 +47,7 @@ transkribus-htr-pipeline:
 spacy-nlp-pipeline:
   name: 'SpaCy NLP Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.2'
+  latest_version: '0.1.1'
   versions:
     0.1.0:
       methods:
@@ -56,5 +62,5 @@ spacy-nlp-pipeline:
     0.1.2:
       methods:
         - 'encoding_detection'
-      publishing_year: 2022
+      publishing_year: 2024
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2'

app/blueprints/settings/__init__.py

@@ -2,4 +2,6 @@ from flask import Blueprint
 
 
 bp = Blueprint('settings', __name__)
-from . import routes  # noqa
+
+
+from . import routes

app/blueprints/settings/forms.py

@@ -0,0 +1,160 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired, FileSize
+from wtforms import (
+    PasswordField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    ValidationError
+)
+from wtforms.validators import (
+    DataRequired,
+    Email,
+    EqualTo,
+    Length,
+    Regexp
+)
+from app.models import User, UserSettingJobStatusMailNotificationLevel
+
+
+class UpdateAccountInformationForm(FlaskForm):
+    email = StringField(
+        'E-Mail',
+        validators=[DataRequired(), Length(max=254), Email()]
+    )
+    username = StringField(
+        'Username',
+        validators=[
+            DataRequired(),
+            Length(max=64),
+            Regexp(
+                User.username_pattern,
+                message=(
+                    'Usernames must have only letters, numbers, dots or '
+                    'underscores'
+                )
+            )
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user: User, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-account-information-form'
+        super().__init__(*args, **kwargs)
+        self.user = user
+
+    def validate_email(self, field):
+        if (field.data != self.user.email
+                and User.query.filter_by(email=field.data).first()):
+            raise ValidationError('Email already registered')
+
+    def validate_username(self, field):
+        if (field.data != self.user.username
+                and User.query.filter_by(username=field.data).first()):
+            raise ValidationError('Username already in use')
+
+
+class UpdateProfileInformationForm(FlaskForm):
+    full_name = StringField(
+        'Full name',
+        validators=[Length(max=128)]
+    )
+    about_me = TextAreaField(
+        'About me',
+        validators=[
+            Length(max=254)
+        ]
+    )
+    website = StringField(
+        'Website',
+        validators=[
+            Length(max=254)
+        ]
+    )
+    organization = StringField(
+        'Organization',
+        validators=[
+            Length(max=128)
+        ]
+    )
+    location = StringField(
+        'Location',
+        validators=[
+            Length(max=128)
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user: User, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-profile-information-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateAvatarForm(FlaskForm):
+    avatar = FileField('File', validators=[FileRequired(), FileSize(2_000_000)])
+    submit = SubmitField()
+
+    def validate_avatar(self, field):
+        valid_mimetypes = ['image/jpeg', 'image/png']
+        if field.data.mimetype not in valid_mimetypes:
+            raise ValidationError('JPEG and PNG files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-avatar-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdatePasswordForm(FlaskForm):
+    password = PasswordField('Old password', validators=[DataRequired()])
+    new_password = PasswordField(
+        'New password',
+        validators=[
+            DataRequired(),
+            EqualTo('new_password_2', message='Passwords must match')
+        ]
+    )
+    new_password_2 = PasswordField(
+        'New password confirmation',
+        validators=[
+            DataRequired(),
+            EqualTo('new_password', message='Passwords must match')
+        ]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user: User, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-password-form'
+        super().__init__(*args, **kwargs)
+        self.user = user
+
+    def validate_current_password(self, field):
+        if not self.user.verify_password(field.data):
+            raise ValidationError('Invalid password')
+
+
+class UpdateNotificationsForm(FlaskForm):
+    job_status_mail_notification_level = SelectField(
+        'Job status mail notification level',
+        choices=[
+            (x.name, x.name.capitalize())
+            for x in UserSettingJobStatusMailNotificationLevel
+        ],
+        validators=[DataRequired()]
+    )
+    submit = SubmitField()
+
+    def __init__(self, user: User, *args, **kwargs):
+        if 'data' not in kwargs:
+            kwargs['data'] = user.to_json_serializeable()
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-notifications-form'
+        super().__init__(*args, **kwargs)

app/blueprints/settings/routes.py

@@ -0,0 +1,158 @@
+from flask import (
+    abort,
+    flash,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    url_for
+)
+from flask_login import current_user, login_required
+from app import db
+from app.models import Avatar
+from . import bp
+from .forms import (
+    UpdateAvatarForm,
+    UpdatePasswordForm,
+    UpdateNotificationsForm,
+    UpdateAccountInformationForm,
+    UpdateProfileInformationForm
+)
+
+
+@bp.route('', methods=['GET', 'POST'])
+@login_required
+def index():
+    update_account_information_form = UpdateAccountInformationForm(current_user)
+    update_profile_information_form = UpdateProfileInformationForm(current_user)
+    update_avatar_form = UpdateAvatarForm()
+    update_password_form = UpdatePasswordForm(current_user)
+    update_notifications_form = UpdateNotificationsForm(current_user)
+
+    # region handle update profile information form
+    if update_profile_information_form.submit.data and update_profile_information_form.validate():
+        current_user.about_me = update_profile_information_form.about_me.data
+        current_user.location = update_profile_information_form.location.data
+        current_user.organization = update_profile_information_form.organization.data
+        current_user.website = update_profile_information_form.website.data
+        current_user.full_name = update_profile_information_form.full_name.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.index'))
+    # endregion handle update profile information form
+
+    # region handle update avatar form
+    if update_avatar_form.submit.data and update_avatar_form.validate():
+        try:
+            Avatar.create(
+                update_avatar_form.avatar.data,
+                user=current_user
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.index'))
+    # endregion handle update avatar form
+
+    # region handle update account information form
+    if update_account_information_form.submit.data and update_account_information_form.validate():
+        current_user.email = update_account_information_form.email.data
+        current_user.username = update_account_information_form.username.data
+        db.session.commit()
+        flash('Profile settings updated')
+        return redirect(url_for('.index'))
+    # endregion handle update account information form
+
+    # region handle update password form
+    if update_password_form.submit.data and update_password_form.validate():
+        current_user.password = update_password_form.new_password.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.index'))
+    # endregion handle update password form
+
+    # region handle update notifications form
+    if update_notifications_form.submit.data and update_notifications_form.validate():
+        current_user.setting_job_status_mail_notification_level = \
+            update_notifications_form.job_status_mail_notification_level.data
+        db.session.commit()
+        flash('Your changes have been saved')
+        return redirect(url_for('.index'))
+    # endregion handle update notifications form
+
+    return render_template(
+        'settings/index.html.j2',
+        title='Settings',
+        update_account_information_form=update_account_information_form,
+        update_avatar_form=update_avatar_form,
+        update_notifications_form=update_notifications_form,
+        update_password_form=update_password_form,
+        update_profile_information_form=update_profile_information_form,
+        user=current_user
+    )
+
+
+@bp.route('/profile-is-public', methods=['PUT'])
+@login_required
+def update_profile_is_public():
+    new_value = request.json
+
+    if not isinstance(new_value, bool):
+        abort(400)
+
+    current_user.is_public = new_value
+    db.session.commit()
+
+    return jsonify('Your changes have been saved'), 200
+
+
+@bp.route('/profile-show-email', methods=['PUT'])
+@login_required
+def update_profile_show_email():
+    new_value = request.json
+
+    if not isinstance(new_value, bool):
+        abort(400)
+
+    if new_value:
+        current_user.add_profile_privacy_setting('SHOW_EMAIL')
+    else:
+        current_user.remove_profile_privacy_setting('SHOW_EMAIL')
+    db.session.commit()
+
+    return jsonify('Your changes have been saved'), 200
+
+
+@bp.route('/profile-show-last-seen', methods=['PUT'])
+@login_required
+def update_profile_show_last_seen():
+    new_value = request.json
+
+    if not isinstance(new_value, bool):
+        abort(400)
+
+    if new_value:
+        current_user.add_profile_privacy_setting('SHOW_LAST_SEEN')
+    else:
+        current_user.remove_profile_privacy_setting('SHOW_LAST_SEEN')
+    db.session.commit()
+
+    return jsonify('Your changes have been saved'), 200
+
+
+@bp.route('/profile-show-member-since', methods=['PUT'])
+@login_required
+def update_profile_show_member_since():
+    new_value = request.json
+
+    if not isinstance(new_value, bool):
+        abort(400)
+
+    if new_value:
+        current_user.add_profile_privacy_setting('SHOW_MEMBER_SINCE')
+    else:
+        current_user.remove_profile_privacy_setting('SHOW_MEMBER_SINCE')
+    db.session.commit()
+
+    return jsonify('Your changes have been saved'), 200

app/blueprints/users/__init__.py

@@ -2,4 +2,6 @@ from flask import Blueprint
 
 
 bp = Blueprint('users', __name__)
-from . import events, routes
+
+
+from . import cli, events, routes

app/blueprints/users/cli.py

@@ -0,0 +1,12 @@
+from app.models import User
+from app import db
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset terms of use accept '''
+    for user in [x for x in User.query.all() if x.terms_of_use_accepted]:
+        print(f'Resetting user {user.username}')
+        user.terms_of_use_accepted = False
+    db.session.commit()

app/blueprints/users/events.py

@@ -0,0 +1,91 @@
+from flask_login import current_user
+from flask_socketio import join_room, leave_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import User
+
+
+@socketio.on('SUBSCRIBE User')
+@socketio_login_required
+def subscribe(user_hashid: str) -> dict:
+    if not isinstance(user_hashid, str):
+        return {
+            'code': 400,
+            'name': 'Bad Request',
+            'description': 'Invalid User ID.'
+        }
+
+    user_id = hashids.decode(user_hashid)
+
+    if not isinstance(user_id, int):
+        return {
+            'code': 400,
+            'name': 'Bad Request',
+            'description': 'Invalid User ID.'
+        }
+
+    user = User.query.get(user_id)
+
+    if user is None:
+        return {
+            'code': 404,
+            'name': 'Not Found',
+            'description': 'User not found.'
+        }
+
+    if not (
+        user == current_user
+        or current_user.is_administrator
+    ):
+        return {
+            'code': 403,
+            'name': 'Forbidden',
+            'description': 'Not allowed to subscribe to this user.'
+        }
+
+    join_room(f'/users/{user.hashid}')
+
+    return {'code': 204, 'name': 'No Content'}
+
+
+@socketio.on('UNSUBSCRIBE User')
+@socketio_login_required
+def unsubscribe(user_hashid: str) -> dict:
+    if not isinstance(user_hashid, str):
+        return {
+            'code': 400,
+            'name': 'Bad Request',
+            'description': 'Invalid User ID.'
+        }
+
+    user_id = hashids.decode(user_hashid)
+
+    if not isinstance(user_id, int):
+        return {
+            'code': 400,
+            'name': 'Bad Request',
+            'description': 'Invalid User ID.'
+        }
+
+    user = User.query.get(user_id)
+
+    if user is None:
+        return {
+            'code': 404,
+            'name': 'Not Found',
+            'description': 'User not found.'
+        }
+
+    if not (
+        user == current_user
+        or current_user.is_administrator
+    ):
+        return {
+            'code': 403,
+            'name': 'Forbidden',
+            'description': 'Not allowed to unsubscribe from this user.'
+        }
+
+    leave_room(f'/users/{user.hashid}')
+
+    return {'code': 204, 'name': 'No Content'}

app/blueprints/users/routes.py

@@ -0,0 +1,134 @@
+from flask import (
+    abort,
+    current_app,
+    Flask,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for
+)
+from flask_login import current_user, login_required, logout_user
+from threading import Thread
+from app import db
+from app.models import Avatar, User
+from . import bp
+
+
+@bp.route('')
+@login_required
+def index():
+    return redirect(url_for('main.social_area', _anchor='users'))
+
+
+@bp.route('/<hashid:user_id>')
+@login_required
+def user(user_id: int):
+    user = User.query.get_or_404(user_id)
+
+    if not (
+        user.is_public
+        or user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    accept_json = request.accept_mimetypes.accept_json
+    accept_html = request.accept_mimetypes.accept_html
+
+    if accept_json and not accept_html:
+        return user.to_json_serializeable(
+            backrefs=True,
+            relationships=True
+        )
+
+    return render_template(
+        'users/user.html.j2',
+        title=user.username,
+        user=user
+    )
+
+
+def _delete_user(app: Flask, user_id: int):
+    with app.app_context():
+        user = User.query.get(user_id)
+        user.delete()
+        db.session.commit()
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@login_required
+def delete_user(user_id: int):
+    user = User.query.get_or_404(user_id)
+
+    if not (
+        user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    if user == current_user:
+        logout_user()
+
+    thread = Thread(
+        target=_delete_user,
+        args=(current_app._get_current_object(), user.id)
+    )
+    thread.start()
+
+    return jsonify(f'User "{user.username}" marked for deletion'), 202
+
+
+@bp.route('/<hashid:user_id>/avatar')
+@login_required
+def user_avatar(user_id: int):
+    user = User.query.get_or_404(user_id)
+
+    if not (
+        user.is_public
+        or user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    if user.avatar is None:
+        return redirect(url_for('static', filename='images/user_avatar.png'))
+
+    return send_from_directory(
+        user.avatar.path.parent,
+        user.avatar.path.name,
+        as_attachment=True,
+        download_name=user.avatar.filename,
+        mimetype=user.avatar.mimetype
+    )
+
+
+def _delete_avatar(app: Flask, avatar_id: int):
+    with app.app_context():
+        avatar = Avatar.query.get(avatar_id)
+        avatar.delete()
+        db.session.commit()
+
+
+@bp.route('/<hashid:user_id>/avatar', methods=['DELETE'])
+@login_required
+def delete_user_avatar(user_id: int):
+    user = User.query.get_or_404(user_id)
+
+    if user.avatar is None:
+        abort(409)
+
+    if not (
+        user == current_user
+        or current_user.is_administrator
+    ):
+        abort(403)
+
+    thread = Thread(
+        target=_delete_avatar,
+        args=(current_app._get_current_object(), user.avatar.id)
+    )
+    thread.start()
+
+    return jsonify('Avatar marked for deletion'), 202

app/blueprints/workshops/__init__.py

@@ -1,5 +1,5 @@
 from flask import Blueprint
 
 
-bp = Blueprint('contributions', __name__)
+bp = Blueprint('workshops', __name__)
 from . import routes

app/blueprints/workshops/routes.py

@@ -0,0 +1,15 @@
+from flask import redirect, render_template, url_for
+from . import bp
+
+
+@bp.route('')
+def workshops():
+    return redirect(url_for('main.dashboard'))
+
+
+@bp.route('/fgho_sommerschule_2023')
+def fgho_sommerschule_2023():
+    return render_template(
+        'workshops/fgho_sommerschule_2023.html.j2',
+        title='FGHO Sommerschule 2023',
+    )

app/cli.py

@@ -1,72 +0,0 @@
-from flask import current_app
-from flask_migrate import upgrade
-import click
-import os
-from app.models import (
-    Role,
-    User,
-    TesseractOCRPipelineModel,
-    SpaCyNLPPipelineModel
-)
-
-
-def _make_default_dirs():
-    base_dir = current_app.config['NOPAQUE_DATA_DIR']
-
-    default_directories = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
-    ]
-    for directory in default_directories:
-        if os.path.exists(directory):
-            if not os.path.isdir(directory):
-                raise NotADirectoryError(f'{directory} is not a directory')
-        else:
-            os.mkdir(directory)
-
-
-def register(app):
-    @app.cli.command()
-    def deploy():
-        ''' Run deployment tasks. '''
-        # Make default directories
-        _make_default_dirs()
-
-        # migrate database to latest revision
-        upgrade()
-
-        # Insert/Update default database values
-        current_app.logger.info('Insert/Update default roles')
-        Role.insert_defaults()
-        current_app.logger.info('Insert/Update default users')
-        User.insert_defaults()
-        current_app.logger.info('Insert/Update default SpaCyNLPPipelineModels')
-        SpaCyNLPPipelineModel.insert_defaults()
-        current_app.logger.info('Insert/Update default TesseractOCRPipelineModels')
-        TesseractOCRPipelineModel.insert_defaults()
-
-    @app.cli.group()
-    def converter():
-        ''' Converter commands. '''
-        pass
-
-    @converter.command()
-    @click.argument('json_db')
-    @click.argument('data_dir')
-    def sandpaper(json_db, data_dir):
-        ''' Sandpaper converter '''
-        from app.converters.sandpaper import convert
-        convert(json_db, data_dir)
-
-    @app.cli.group()
-    def test():
-        ''' Test commands. '''
-        pass
-
-    @test.command('run')
-    def run_test():
-        ''' Run unit tests. '''
-        from unittest import TestLoader, TextTestRunner
-        from unittest.suite import TestSuite
-        tests: TestSuite = TestLoader().discover('tests')
-        TextTestRunner(verbosity=2).run(tests)

app/contributions/forms.py

@@ -1,121 +0,0 @@
-from flask import current_app
-from flask_wtf import FlaskForm
-from flask_wtf.file import FileField, FileRequired
-from wtforms import (
-    BooleanField,
-    StringField,
-    SubmitField,
-    SelectMultipleField,
-    IntegerField,
-    ValidationError
-)
-from wtforms.validators import InputRequired, Length
-from app.services import SERVICES
-
-
-class ContributionBaseForm(FlaskForm):
-    title = StringField(
-        'Title',
-        validators=[InputRequired(), Length(max=64)]
-    )
-    description = StringField(
-        'Description',
-        validators=[InputRequired(), Length(max=255)]
-    )
-    version = StringField(
-        'Version',
-        validators=[InputRequired(), Length(max=16)]
-    )
-    publisher = StringField(
-        'Publisher',
-        validators=[InputRequired(), Length(max=128)]
-    )
-    publisher_url = StringField(
-        'Publisher URL',
-        validators=[InputRequired(), Length(max=512)]
-    )
-    publishing_url = StringField(
-        'Publishing URL',
-        validators=[InputRequired(), Length(max=512)]
-    )
-    publishing_year = IntegerField(
-        'Publishing year',
-        validators=[InputRequired()]
-    )
-    compatible_service_versions = SelectMultipleField(
-        'Compatible service versions'
-    )
-    submit = SubmitField()
-
-
-class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
-    tesseract_model_file = FileField(
-        'File',
-        validators=[FileRequired()]
-    )
-    
-    def validate_tesseract_model_file(self, field):
-        if not field.data.filename.lower().endswith('.traineddata'):
-            raise ValidationError('traineddata files only!')
-
-    def __init__(self, *args, **kwargs):
-        service_manifest = SERVICES['tesseract-ocr-pipeline']
-        super().__init__(*args, **kwargs)
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
-    spacy_model_file = FileField(
-        'File',
-        validators=[FileRequired()]
-    )
-    pipeline_name = StringField(
-        'Pipeline name',
-        validators=[InputRequired(), Length(max=64)]
-    )
-
-    def validate_spacy_model_file(self, field):
-        if not field.data.filename.lower().endswith('.tar.gz'):
-            raise ValidationError('.tar.gz files only!')
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        service_manifest = SERVICES['spacy-nlp-pipeline']
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class EditContributionBaseForm(ContributionBaseForm):
-    pass
-
-class EditTesseractOCRPipelineModelForm(EditContributionBaseForm):
-    def __init__(self, *args, **kwargs):
-        service_manifest = SERVICES['tesseract-ocr-pipeline']
-        super().__init__(*args, **kwargs)
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''
-
-
-class EditSpaCyNLPPipelineModelForm(EditContributionBaseForm):
-    pipeline_name = StringField(
-        'Pipeline name',
-        validators=[InputRequired(), Length(max=64)]
-    )
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        service_manifest = SERVICES['spacy-nlp-pipeline']
-        self.compatible_service_versions.choices = [('', 'Choose your option')]
-        self.compatible_service_versions.choices += [
-            (x, x) for x in service_manifest['versions'].keys()
-        ]
-        self.compatible_service_versions.default = ''

app/contributions/routes.py

@@ -1,233 +0,0 @@
-from flask import (
-    abort,
-    current_app,
-    flash,
-    Markup,
-    redirect,
-    render_template,
-    url_for
-)
-from flask_login import login_required, current_user
-from threading import Thread
-from app import db
-from app.decorators import permission_required 
-from app.models import (
-    Permission,
-    SpaCyNLPPipelineModel,
-    TesseractOCRPipelineModel
-)
-from . import bp
-from .forms import (
-    CreateSpaCyNLPPipelineModelForm,
-    CreateTesseractOCRPipelineModelForm,
-    EditSpaCyNLPPipelineModelForm,
-    EditTesseractOCRPipelineModelForm
-)
-
-
-@bp.before_request
-@login_required
-def before_request():
-    pass
-
-
-@bp.route('/')
-def contributions():
-    return render_template(
-        'contributions/contributions.html.j2',
-        title='Contributions'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models')
-def tesseract_ocr_pipeline_models():
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_models.html.j2',
-        title='Tesseract OCR Pipeline Models'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
-def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    form = EditTesseractOCRPipelineModelForm(
-        data=tesseract_ocr_pipeline_model.to_json_serializeable(),
-        prefix='edit-tesseract-ocr-pipeline-model-form'
-    )
-    if form.validate_on_submit():
-        form.populate_obj(tesseract_ocr_pipeline_model)
-        if db.session.is_modified(tesseract_ocr_pipeline_model):
-            message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model.url}">{tesseract_ocr_pipeline_model.title}</a>" updated')
-            flash(message)
-            db.session.commit()
-        return redirect(url_for('.tesseract_ocr_pipeline_models'))
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_model.html.j2',
-        form=form,
-        tesseract_ocr_pipeline_model=tesseract_ocr_pipeline_model,
-        title=f'{tesseract_ocr_pipeline_model.title} {tesseract_ocr_pipeline_model.version}'
-    )
-
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
-def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
-    def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
-        with app.app_context():
-            tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
-            tesseract_ocr_pipeline_model.delete()
-            db.session.commit()
-
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_tesseract_ocr_pipeline_model,
-        args=(current_app._get_current_object(), tesseract_ocr_pipeline_model_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
-def create_tesseract_ocr_pipeline_model():
-    form = CreateTesseractOCRPipelineModelForm(prefix='create-tesseract-ocr-pipeline-model-form')
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.create(
-                form.tesseract_model_file.data,
-                compatible_service_versions=form.compatible_service_versions.data,
-                description=form.description.data,
-                publisher=form.publisher.data,
-                publisher_url=form.publisher_url.data,
-                publishing_url=form.publishing_url.data,
-                publishing_year=form.publishing_year.data,
-                is_public=False,
-                title=form.title.data,
-                version=form.version.data,
-                user=current_user
-            )
-        except OSError:
-            abort(500)
-        db.session.commit()
-        tesseract_ocr_pipeline_model_url = url_for(
-            '.tesseract_ocr_pipeline_model',
-            tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id
-        )
-        message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model_url}">{tesseract_ocr_pipeline_model.title}</a>" created')
-        flash(message)
-        return {}, 201, {'Location': tesseract_ocr_pipeline_model_url}
-    return render_template(
-        'contributions/create_tesseract_ocr_pipeline_model.html.j2',
-        form=form,
-        title='Create Tesseract OCR Pipeline Model'
-    )
-
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/toggle-public-status', methods=['POST'])
-@permission_required(Permission.CONTRIBUTE)
-def toggle_tesseract_ocr_pipeline_model_public_status(tesseract_ocr_pipeline_model_id):
-    tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    tesseract_ocr_pipeline_model.is_public = not tesseract_ocr_pipeline_model.is_public
-    db.session.commit()
-    return {}, 201
-
-
-@bp.route('/spacy-nlp-pipeline-models')
-def spacy_nlp_pipeline_models():
-    return render_template(
-        'contributions/spacy_nlp_pipeline_models.html.j2',
-        title='SpaCy NLP Pipeline Models'
-    )
-
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
-def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    form = EditSpaCyNLPPipelineModelForm(
-        data=spacy_nlp_pipeline_model.to_json_serializeable(),
-        prefix='edit-spacy-nlp-pipeline-model-form'
-    )
-    if form.validate_on_submit():
-        form.populate_obj(spacy_nlp_pipeline_model)
-        if db.session.is_modified(spacy_nlp_pipeline_model):
-            message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model.url}">{spacy_nlp_pipeline_model.title}</a>" updated')
-            flash(message)
-            db.session.commit()
-        return redirect(url_for('.spacy_nlp_pipeline_models'))
-    return render_template(
-        'contributions/spacy_nlp_pipeline_model.html.j2',
-        form=form,
-        spacy_nlp_pipeline_model=spacy_nlp_pipeline_model,
-        title=f'{spacy_nlp_pipeline_model.title} {spacy_nlp_pipeline_model.version}'
-    )
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
-def delete_spacy_model(spacy_nlp_pipeline_model_id):
-    def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
-        with app.app_context():
-            spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
-            spacy_nlp_pipeline_model.delete()
-            db.session.commit()
-    
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_spacy_model,
-        args=(current_app._get_current_object(), spacy_nlp_pipeline_model_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
-def create_spacy_nlp_pipeline_model():
-    form = CreateSpaCyNLPPipelineModelForm(prefix='create-spacy-nlp-pipeline-model-form')
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.create(
-                form.spacy_model_file.data,
-                compatible_service_versions=form.compatible_service_versions.data,
-                description=form.description.data,
-                pipeline_name=form.pipeline_name.data,
-                publisher=form.publisher.data,
-                publisher_url=form.publisher_url.data,
-                publishing_url=form.publishing_url.data,
-                publishing_year=form.publishing_year.data,
-                is_public=False,
-                title=form.title.data,
-                version=form.version.data,
-                user=current_user
-            )
-        except OSError:
-            abort(500)
-        db.session.commit()
-        spacy_nlp_pipeline_model_url = url_for(
-            '.spacy_nlp_pipeline_model',
-            spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id
-        )
-        message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model_url}">{spacy_nlp_pipeline_model.title}</a>" created')
-        flash(message)
-        return {}, 201, {'Location': spacy_nlp_pipeline_model_url}
-    return render_template(
-        'contributions/create_spacy_nlp_pipeline_model.html.j2',
-        form=form,
-        title='Create SpaCy NLP Pipeline Model'
-    )
-
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/toggle-public-status', methods=['POST'])
-@permission_required(Permission.CONTRIBUTE)
-def toggle_spacy_nlp_pipeline_model_public_status(spacy_nlp_pipeline_model_id):
-    spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
-        abort(403)
-    spacy_nlp_pipeline_model.is_public = not spacy_nlp_pipeline_model.is_public
-    db.session.commit()
-    return {}, 201

app/converters/cli.py

@@ -0,0 +1,22 @@
+import click
+from . import bp
+from .sandpaper import SandpaperConverter
+
+
+@bp.cli.group('converter')
+def converter():
+    ''' Converter commands. '''
+    pass
+
+@converter.group('sandpaper')
+def sandpaper_converter():
+    ''' Sandpaper converter commands. '''
+    pass
+
+@sandpaper_converter.command('run')
+@click.argument('json_db_file')
+@click.argument('data_dir')
+def run_sandpaper_converter(json_db_file, data_dir):
+    ''' Run the sandpaper converter. '''
+    sandpaper_converter = SandpaperConverter(json_db_file, data_dir)
+    sandpaper_converter.run()

app/converters/sandpaper.py

@@ -1,107 +1,100 @@
+from datetime import datetime
 from flask import current_app
+from pathlib import Path
+import json
+import shutil
 from app import db
 from app.models import User, Corpus, CorpusFile
-from datetime import datetime
-import json
-import os
-import shutil
 
 
-def convert(json_db_file, data_dir):
-    with open(json_db_file, 'r') as f:
-        json_db = json.loads(f.read())
+class SandpaperConverter:
+    def __init__(self, json_db_file: Path, data_dir: Path):
+        self.json_db_file = json_db_file
+        self.data_dir = data_dir
 
-    for json_user in json_db:
-        if not json_user['confirmed']:
-            current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
-            continue
-        user_dir = os.path.join(data_dir, str(json_user['id']))
-        convert_user(json_user, user_dir)
-        db.session.commit()
+    def run(self):
+        with self.json_db_file.open('r') as f:
+            json_db: list[dict] = json.load(f)
+
+        for json_user in json_db:
+            if not json_user['confirmed']:
+                current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
+                continue
+            user_dir = self.data_dir / f'{json_user["id"]}'
+            self.convert_user(json_user, user_dir)
+            db.session.commit()
 
 
-def convert_user(json_user, user_dir):
-    current_app.logger.info(f'Create User {json_user["username"]}...')
-    user = User(
-        confirmed=json_user['confirmed'],
-        email=json_user['email'],
-        last_seen=datetime.fromtimestamp(json_user['last_seen']),
-        member_since=datetime.fromtimestamp(json_user['member_since']),
-        password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
-        username=json_user['username']
-    )
-    db.session.add(user)
-    db.session.flush(objects=[user])
-    db.session.refresh(user)
-    try:
-        user.makedirs()
-    except OSError as e:
-        current_app.logger.error(e)
-        db.session.rollback()
-        raise Exception('Internal Server Error')
-    for json_corpus in json_user['corpora'].values():
-        if not json_corpus['files'].values():
-            current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
-            continue
-        corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
-        convert_corpus(json_corpus, user, corpus_dir)
-    current_app.logger.info('Done')
+    def convert_user(self, json_user: dict, user_dir: Path):
+        current_app.logger.info(f'Create User {json_user["username"]}...')
+        try:
+            user = User.create(
+                confirmed=json_user['confirmed'],
+                email=json_user['email'],
+                last_seen=datetime.fromtimestamp(json_user['last_seen']),
+                member_since=datetime.fromtimestamp(json_user['member_since']),
+                password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
+                username=json_user['username']
+            )
+        except OSError:
+            raise Exception('Internal Server Error')
+        for json_corpus in json_user['corpora'].values():
+            if not json_corpus['files'].values():
+                current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
+                continue
+            corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
+            self.convert_corpus(json_corpus, user, corpus_dir)
+        current_app.logger.info('Done')
 
 
-def convert_corpus(json_corpus, user, corpus_dir):
-    current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
-    corpus = Corpus(
-        user=user,
-        creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
-        description=json_corpus['description'],
-        title=json_corpus['title']
-    )
-    db.session.add(corpus)
-    db.session.flush(objects=[corpus])
-    db.session.refresh(corpus)
-    try:
-        corpus.makedirs()
-    except OSError as e:
-        current_app.logger.error(e)
-        db.session.rollback()
-        raise Exception('Internal Server Error')
-    for json_corpus_file in json_corpus['files'].values():
-        convert_corpus_file(json_corpus_file, corpus, corpus_dir)
-    current_app.logger.info('Done')
+    def convert_corpus(self, json_corpus: dict, user: User, corpus_dir: Path):
+        current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
+        try:
+            corpus = Corpus.create(
+                user=user,
+                creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
+                description=json_corpus['description'],
+                title=json_corpus['title']
+            )
+        except OSError:
+            raise Exception('Internal Server Error')
+        for json_corpus_file in json_corpus['files'].values():
+            self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
+        current_app.logger.info('Done')
 
 
-def convert_corpus_file(json_corpus_file, corpus, corpus_dir):
-    current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
-    corpus_file = CorpusFile(
-        corpus=corpus,
-        address=json_corpus_file['address'],
-        author=json_corpus_file['author'],
-        booktitle=json_corpus_file['booktitle'],
-        chapter=json_corpus_file['chapter'],
-        editor=json_corpus_file['editor'],
-        filename=json_corpus_file['filename'],
-        institution=json_corpus_file['institution'],
-        journal=json_corpus_file['journal'],
-        mimetype='application/vrt+xml',
-        pages=json_corpus_file['pages'],
-        publisher=json_corpus_file['publisher'],
-        publishing_year=json_corpus_file['publishing_year'],
-        school=json_corpus_file['school'],
-        title=json_corpus_file['title']
-    )
-    db.session.add(corpus_file)
-    db.session.flush(objects=[corpus_file])
-    db.session.refresh(corpus_file)
-    try:
-        shutil.copy2(
-            os.path.join(corpus_dir, json_corpus_file['filename']),
-            corpus_file.path
+    def convert_corpus_file(self, json_corpus_file: dict, corpus: Corpus, corpus_dir: Path):
+        current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
+        corpus_file = CorpusFile(
+            corpus=corpus,
+            address=json_corpus_file['address'],
+            author=json_corpus_file['author'],
+            booktitle=json_corpus_file['booktitle'],
+            chapter=json_corpus_file['chapter'],
+            editor=json_corpus_file['editor'],
+            filename=json_corpus_file['filename'],
+            institution=json_corpus_file['institution'],
+            journal=json_corpus_file['journal'],
+            mimetype='application/vrt+xml',
+            pages=json_corpus_file['pages'],
+            publisher=json_corpus_file['publisher'],
+            publishing_year=json_corpus_file['publishing_year'],
+            school=json_corpus_file['school'],
+            title=json_corpus_file['title']
         )
-    except:
-        current_app.logger.warning(
-            'Can not convert corpus file: '
-            f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
-            ' -> '
-            f'{corpus_file.path}'
-        )
-    current_app.logger.info('Done')
+        db.session.add(corpus_file)
+        db.session.flush(objects=[corpus_file])
+        db.session.refresh(corpus_file)
+        try:
+            shutil.copy2(
+                corpus_dir / json_corpus_file['filename'],
+                corpus_file.path
+            )
+        except:
+            current_app.logger.warning(
+                'Can not convert corpus file: '
+                f'{corpus_dir / json_corpus_file["filename"]}'
+                ' -> '
+                f'{corpus_file.path}'
+            )
+        current_app.logger.info('Done')

app/converters/vrt.py

@@ -1,69 +1,25 @@
 from flask import current_app
+from pathlib import Path
 
 
-def normalize_vrt_file(input_file, output_file):
-    def check_pos_attribute_order(vrt_lines):
-        # The following orders are possible:
-        # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
-        # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
-        # since 27.01.2022: 'word,pos,lemma,simple_pos'
-        # This Function tries to find out which order we have by looking at the
-        # number of attributes and the position of the simple_pos attribute
-        SIMPLE_POS_LABELS = [
-            'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ',
-            'DET', 'INTJ', 'NOUN', 'NUM', 'PART',
-            'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',
-            'VERB', 'X'
-        ]
-        for line in vrt_lines:
-            if line.startswith('<'):
-                continue
-            pos_attrs = line.rstrip('\n').split('\t')
-            num_pos_attrs = len(pos_attrs)
-            if num_pos_attrs == 4:
-                if pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos']
-                continue
-            elif num_pos_attrs == 5:
-                if pos_attrs[2] in SIMPLE_POS_LABELS:
-                    return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
-                elif pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
-                continue
-        return None
-
-
-    def check_has_ent_as_s_attr(vrt_lines):
-        for line in vrt_lines:
-            if line.startswith('<ent'):
-                return True
-        return False
-
-
-    def pos_attrs_to_string_1(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
-
-
-    def pos_attrs_to_string_2(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'
-
+def normalize_vrt_file(input_file: Path, output_file: Path):
     current_app.logger.info(f'Converting {input_file}...')
 
-    with open(input_file) as f:
+    with input_file.open() as f:
         input_vrt_lines = f.readlines()
 
-    pos_attr_order = check_pos_attribute_order(input_vrt_lines)
-    has_ent_as_s_attr = check_has_ent_as_s_attr(input_vrt_lines)
+    pos_attr_order = _check_pos_attribute_order(input_vrt_lines)
+    has_ent_as_s_attr = _check_has_ent_as_s_attr(input_vrt_lines)
 
     current_app.logger.info(f'Detected pos_attr_order: [{",".join(pos_attr_order)}]')
     current_app.logger.info(f'Detected has_ent_as_s_attr: {has_ent_as_s_attr}')
 
     if pos_attr_order == ['word', 'lemma', 'simple_pos', 'pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_1
+        pos_attrs_to_string_function = _pos_attrs_to_string_1
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     else:
         raise Exception('Can not handle format')
 
@@ -113,5 +69,49 @@ def normalize_vrt_file(input_file, output_file):
                     current_ent = pos_attrs[4]
         output_vrt += pos_attrs_to_string_function(pos_attrs)
 
-    with open(output_file, 'w') as f:
+    with output_file.open(mode='w') as f:
         f.write(output_vrt)
+
+
+def _check_pos_attribute_order(vrt_lines: list[str]) -> list[str]:
+    # The following orders are possible:
+    # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
+    # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
+    # since 27.01.2022: 'word,pos,lemma,simple_pos'
+    # This Function tries to find out which order we have by looking at the
+    # number of attributes and the position of the simple_pos attribute
+    SIMPLE_POS_LABELS = [
+        'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'DET', 'INTJ', 'NOUN', 'NUM',
+        'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X'
+    ]
+    for line in vrt_lines:
+        if line.startswith('<'):
+            continue
+        pos_attrs = line.rstrip('\n').split('\t')
+        num_pos_attrs = len(pos_attrs)
+        if num_pos_attrs == 4:
+            if pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos']
+            continue
+        elif num_pos_attrs == 5:
+            if pos_attrs[2] in SIMPLE_POS_LABELS:
+                return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
+            elif pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
+            continue
+    # TODO: raise exception "can't determine attribute order"
+
+
+def _check_has_ent_as_s_attr(vrt_lines: list[str]) -> bool:
+    for line in vrt_lines:
+        if line.startswith('<ent'):
+            return True
+    return False
+
+
+def _pos_attrs_to_string_1(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
+
+
+def _pos_attrs_to_string_2(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'

app/corpora/__init__.py

@@ -1,5 +0,0 @@
-from flask import Blueprint
-
-
-bp = Blueprint('corpora', __name__)
-from . import cqi_over_socketio, routes  # noqa

app/corpora/cqi_over_socketio/__init__.py

@@ -1,115 +0,0 @@
-from flask import session
-from flask_login import current_user
-from flask_socketio import ConnectionRefusedError
-from threading import Lock
-import cqi
-from app import db, hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus, CorpusStatus
-
-
-'''
-This package tunnels the Corpus Query interface (CQi) protocol through
-Socket.IO (SIO) by wrapping each CQi function in a seperate SIO event.
-
-This module only handles the SIO connect/disconnect, which handles the setup
-and teardown of necessary ressources for later use. Each CQi function has a
-corresponding SIO event. The event handlers are spread across the different
-modules within this package.
-
-Basic concept:
-1. A client connects to the SIO namespace and provides the id of a corpus to be
-   analysed.
-     1.1 The analysis session counter of the corpus is incremented.
-     1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
-     1.3 Wait until the CQP server is running.
-     1.4 Connect the CQiClient to the server.
-     1.5 Save the CQiClient and the Lock in the session for subsequential use.
-2. A client emits an event and may provide a single json object with necessary
-   arguments for the targeted CQi function.
-3. A SIO event handler (decorated with cqi_over_socketio) gets executed.
-     - The event handler function defines all arguments. Hence the client
-       is sent as a single json object, the decorator decomposes it to fit
-       the functions signature. This also includes type checking and proper
-       use of the lock (acquire/release) mechanism.
-4. Wait for more events
-5. The client disconnects from the SIO namespace
-     1.1 The analysis session counter of the corpus is decremented.
-     1.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
-'''
-
-
-NAMESPACE = '/corpora/corpus/corpus_analysis'
-
-
-# Import all CQi over Socket.IO event handlers
-from .cqi_corpora_corpus_subcorpora import *  # noqa
-from .cqi_corpora_corpus_structural_attributes import *  # noqa
-from .cqi_corpora_corpus_positional_attributes import *  # noqa
-from .cqi_corpora_corpus_alignment_attributes import *  # noqa
-from .cqi_corpora_corpus import *  # noqa
-from .cqi_corpora import *  # noqa
-from .cqi import *  # noqa
-
-
-@socketio.on('connect', namespace=NAMESPACE)
-@socketio_login_required
-def connect(auth):
-    # the auth variable is used in a hacky way. It contains the corpus id for
-    # which a corpus analysis session should be started.
-    corpus_id = hashids.decode(auth['corpus_id'])
-    corpus = Corpus.query.get(corpus_id)
-    if corpus is None:
-        # return {'code': 404, 'msg': 'Not Found'}
-        raise ConnectionRefusedError('Not Found')
-    if not (corpus.user == current_user
-            or current_user.is_following_corpus(corpus)
-            or current_user.is_administrator()):
-        # return {'code': 403, 'msg': 'Forbidden'}
-        raise ConnectionRefusedError('Forbidden')
-    if corpus.status not in [
-        CorpusStatus.BUILT,
-        CorpusStatus.STARTING_ANALYSIS_SESSION,
-        CorpusStatus.RUNNING_ANALYSIS_SESSION,
-        CorpusStatus.CANCELING_ANALYSIS_SESSION
-    ]:
-        # return {'code': 424, 'msg': 'Failed Dependency'}
-        raise ConnectionRefusedError('Failed Dependency')
-    if corpus.num_analysis_sessions is None:
-        corpus.num_analysis_sessions = 0
-        db.session.commit()
-    corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
-    db.session.commit()
-    retry_counter = 20
-    while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
-        if retry_counter == 0:
-            corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
-            db.session.commit()
-            return {'code': 408, 'msg': 'Request Timeout'}
-        socketio.sleep(3)
-        retry_counter -= 1
-        db.session.refresh(corpus)
-    cqi_client = cqi.CQiClient(f'cqpserver_{corpus_id}')
-    session['d'] = {
-        'corpus_id': corpus_id,
-        'cqi_client': cqi_client,
-        'cqi_client_lock': Lock(),
-    }
-    # return {'code': 200, 'msg': 'OK'}
-
-
-@socketio.on('disconnect', namespace=NAMESPACE)
-def disconnect():
-    if 'd' not in session:
-        return
-    session['d']['cqi_client_lock'].acquire()
-    try:
-        session['d']['cqi_client'].disconnect()
-    except (BrokenPipeError, cqi.errors.CQiException):
-        pass
-    session['d']['cqi_client_lock'].release()
-    corpus = Corpus.query.get(session['d']['corpus_id'])
-    corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
-    db.session.commit()
-    session.pop('d')
-    # return {'code': 200, 'msg': 'OK'}

app/corpora/cqi_over_socketio/cqi.py

@@ -1,43 +0,0 @@
-from socket import gaierror
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.connect', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_connect(cqi_client: cqi.CQiClient):
-    try:
-        cqi_status = cqi_client.connect()
-    except gaierror as e:
-        return {
-            'code': 500,
-            'msg': 'Internal Server Error',
-            'payload': {'code': e.args[0], 'desc': e.args[1]}
-        }
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.disconnect', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_disconnect(cqi_client: cqi.CQiClient):
-    cqi_status = cqi_client.disconnect()
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.ping', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_ping(cqi_client: cqi.CQiClient):
-    cqi_status = cqi_client.ping()
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora.py

@@ -1,22 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_get(cqi_client: cqi.CQiClient, corpus_name: str):
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = {**cqi_corpus.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_list(cqi_client: cqi.CQiClient):
-    payload = [{**x.attrs} for x in cqi_client.corpora.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus.py

@@ -1,85 +0,0 @@
-from flask import session
-import cqi
-import math
-from app import db, socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, lookups_by_cpos
-
-
-@socketio.on('cqi.corpora.corpus.drop', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_drop(cqi_client: cqi.CQiClient, corpus_name: str):
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_status = cqi_corpus.drop()
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.query', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_query(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, query: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_status = cqi_corpus.query(subcorpus_name, query)
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-###############################################################################
-# nopaque specific CQi extensions                                             #
-###############################################################################
-@socketio.on('cqi.corpora.corpus.update_db', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_update_db(cqi_client: cqi.CQiClient, corpus_name: str):
-    corpus = Corpus.query.get(session['d']['corpus_id'])
-    corpus.num_tokens = cqi_client.corpora.get('CORPUS').attrs['size']
-    db.session.commit()
-
-
-@socketio.on('cqi.corpora.corpus.paginate', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, page: int = 1, per_page: int = 20):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    # Sanity checks
-    if (
-        per_page < 1
-        or page < 1
-        or (
-            cqi_corpus.attrs['size'] > 0
-            and page > math.ceil(cqi_corpus.attrs['size'] / per_page)
-        )
-    ):
-        return {'code': 416, 'msg': 'Range Not Satisfiable'}
-    first_cpos = (page - 1) * per_page
-    last_cpos = min(cqi_corpus.attrs['size'], first_cpos + per_page)
-    cpos_list = [*range(first_cpos, last_cpos)]
-    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
-    payload = {}
-    # the items for the current page
-    payload['items'] = [cpos_list]
-    # the lookups for the items
-    payload['lookups'] = lookups
-    # the total number of items matching the query
-    payload['total'] = cqi_corpus.attrs['size']
-    # the number of items to be displayed on a page.
-    payload['per_page'] = per_page
-    # The total number of pages
-    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
-    # the current page number (1 indexed)
-    payload['page'] = page if payload['pages'] > 0 else None
-    # True if a previous page exists
-    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
-    # True if a next page exists.
-    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
-    # Number of the previous page.
-    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
-    # Number of the next page
-    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.alignment_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_alignment_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, alignment_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_alignment_attribute = cqi_corpus.alignment_attributes.get(alignment_attribute_name)  # noqa
-    payload = {**cqi_alignment_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.alignment_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_alignment_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.alignment_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.positional_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_positional_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, positional_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_positional_attribute = cqi_corpus.positional_attributes.get(positional_attribute_name)  # noqa
-    payload = {**cqi_positional_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.positional_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_positional_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.positional_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.structural_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_structural_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, structural_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_structural_attribute = cqi_corpus.structural_attributes.get(structural_attribute_name)  # noqa
-    payload = {**cqi_structural_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.structural_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_structural_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.structural_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py

@@ -1,125 +0,0 @@
-import cqi
-import math
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, export_subcorpus, partial_export_subcorpus
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_get(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    payload = {**cqi_subcorpus.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.subcorpora.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.drop', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_drop(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_status = cqi_subcorpus.drop()
-    payload = {'code': cqi_status,
-               'msg': cqi.api.specification.lookup[cqi_status]}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.dump', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_dump(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_1', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_fdist_1(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_2', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_fdist_2(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-###############################################################################
-# nopaque specific CQi extensions                                             #
-###############################################################################
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.paginate', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50, page: int = 1, per_page: int = 20):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    # Sanity checks
-    if (
-        per_page < 1
-        or page < 1
-        or (
-            cqi_subcorpus.attrs['size'] > 0
-            and page > math.ceil(cqi_subcorpus.attrs['size'] / per_page)
-        )
-    ):
-        return {'code': 416, 'msg': 'Range Not Satisfiable'}
-    offset = (page - 1) * per_page
-    cutoff = per_page
-    cqi_results_export = export_subcorpus(
-        cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
-    payload = {}
-    # the items for the current page
-    payload['items'] = cqi_results_export.pop('matches')
-    # the lookups for the items
-    payload['lookups'] = cqi_results_export
-    # the total number of items matching the query
-    payload['total'] = cqi_subcorpus.attrs['size']
-    # the number of items to be displayed on a page.
-    payload['per_page'] = per_page
-    # The total number of pages
-    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
-    # the current page number (1 indexed)
-    payload['page'] = page if payload['pages'] > 0 else None
-    # True if a previous page exists
-    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
-    # True if a next page exists.
-    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
-    # Number of the previous page.
-    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
-    # Number of the next page
-    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.partial_export', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_partial_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, match_id_list: list, context: int = 50):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
-    return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_partial_export}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.export', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
-    return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_export}

app/corpora/cqi_over_socketio/utils.py

@@ -1,178 +0,0 @@
-from flask import session
-from functools import wraps
-from inspect import signature
-import cqi
-
-
-def cqi_over_socketio(f):
-    @wraps(f)
-    def wrapped(*args):
-        if 'd' not in session:
-            return {'code': 424, 'msg': 'Failed Dependency'}
-        f_args = {}
-        # Check for missing args and if all provided args are of the right type
-        for param in signature(f).parameters.values():
-            if param.name == 'corpus_name':
-                f_args[param.name] = f'NOPAQUE_{session["d"]["corpus_id"]}'
-                continue
-            if param.name == 'cqi_client':
-                f_args[param.name] = session['d']['cqi_client']
-                continue
-            if param.default is param.empty:
-                # args
-                if param.name not in args[0]:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                arg = args[0][param.name]
-                if type(arg) is not param.annotation:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                f_args[param.name] = arg
-            else:
-                # kwargs
-                if param.name not in args[0]:
-                    continue
-                arg = args[0][param.name]
-                if type(arg) is not param.annotation:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                f_args[param.name] = arg
-        session['d']['cqi_client_lock'].acquire()
-        try:
-            return_value = f(**f_args)
-        except BrokenPipeError:
-            return_value = {
-                'code': 500,
-                'msg': 'Internal Server Error'
-            }
-        except cqi.errors.CQiException as e:
-            return_value = {
-                'code': 500,
-                'msg': 'Internal Server Error',
-                'payload': {
-                    'code': e.code,
-                    'desc': e.description,
-                    'msg': e.name
-                }
-            }
-        finally:
-            session['d']['cqi_client_lock'].release()
-        return return_value
-    return wrapped
-
-
-def lookups_by_cpos(corpus, cpos_list):
-    lookups = {}
-    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
-    for attr in corpus.positional_attributes.list():
-        cpos_attr_values = attr.values_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            lookups['cpos_lookup'][cpos][attr.attrs['name']] = \
-                cpos_attr_values[i]
-    for attr in corpus.structural_attributes.list():
-        # We only want to iterate over non subattributes, identifiable by
-        # attr.attrs['has_values'] == False
-        if attr.attrs['has_values']:
-            continue
-        cpos_attr_ids = attr.ids_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            if cpos_attr_ids[i] == -1:
-                continue
-            lookups['cpos_lookup'][cpos][attr.attrs['name']] = cpos_attr_ids[i]
-        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
-        if not occured_attr_ids:
-            continue
-        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
-        if not subattrs:
-            continue
-        lookup_name = f'{attr.attrs["name"]}_lookup'
-        lookups[lookup_name] = {}
-        for attr_id in occured_attr_ids:
-            lookups[lookup_name][attr_id] = {}
-        for subattr in subattrs:
-            subattr_name = subattr.attrs['name'][(len(attr.attrs['name']) + 1):]  # noqa
-            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
-                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
-    return lookups
-
-
-def partial_export_subcorpus(subcorpus, match_id_list, context=25):
-    if subcorpus.attrs['size'] == 0:
-        return {"matches": []}
-    match_boundaries = []
-    for match_id in match_id_list:
-        if match_id < 0 or match_id >= subcorpus.attrs['size']:
-            continue
-        match_boundaries.append(
-            (
-                match_id,
-                subcorpus.dump(subcorpus.attrs['fields']['match'], match_id, match_id)[0],
-                subcorpus.dump(subcorpus.attrs['fields']['matchend'], match_id, match_id)[0]
-            )
-        )
-    cpos_set = set()
-    matches = []
-    for match_boundary in match_boundaries:
-        match_num, match_start, match_end = match_boundary
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.attrs['size'] - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}
-
-
-def export_subcorpus(subcorpus, context=25, cutoff=float('inf'), offset=0):
-    if subcorpus.attrs['size'] == 0:
-        return {"matches": []}
-    first_match = max(0, offset)
-    last_match = min((offset + cutoff - 1), (subcorpus.attrs['size'] - 1))
-    match_boundaries = zip(
-        list(range(first_match, last_match + 1)),
-        subcorpus.dump(subcorpus.attrs['fields']['match'], first_match, last_match),
-        subcorpus.dump(subcorpus.attrs['fields']['matchend'], first_match, last_match)
-    )
-    cpos_set = set()
-    matches = []
-    for match_num, match_start, match_end in match_boundaries:
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.attrs['size'] - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}

app/corpora/routes.py

@@ -1,332 +0,0 @@
-from flask import (
-    abort,
-    current_app,
-    flash,
-    Markup,
-    redirect,
-    render_template,
-    request,
-    send_from_directory,
-    url_for
-)
-from flask_login import current_user, login_required
-from threading import Thread
-import os
-from app import db, hashids
-from app.models import Corpus, CorpusFile, CorpusStatus, CorpusFollowerAssociation, User
-from . import bp
-from .forms import ChangeCorpusSettingsForm, CreateCorpusFileForm, CreateCorpusForm, UpdateCorpusFileForm
-
-
-@bp.route('')
-@login_required
-def corpora():
-    query = Corpus.query.filter(
-        (Corpus.user_id == current_user.id) | (Corpus.is_public == True)
-    )
-    corpora = [c.to_json_serializeable() for c in query.all()]
-    return render_template('corpora/corpora.html.j2', corpora=corpora, title='Corpora')
-
-
-@bp.route('/create', methods=['GET', 'POST'])
-@login_required
-def create_corpus():
-    form = CreateCorpusForm()
-    if form.validate_on_submit():
-        try:
-            corpus = Corpus.create(
-                title=form.title.data,
-                description=form.description.data,
-                user=current_user
-            )
-        except OSError:
-            abort(500)
-        db.session.commit()
-        message = Markup(
-            f'Corpus "<a href="{corpus.url}">{corpus.title}</a>" created'
-        )
-        flash(message, 'corpus')
-        return redirect(corpus.url)
-    return render_template(
-        'corpora/create_corpus.html.j2',
-        form=form,
-        title='Create corpus'
-    )
-
-
-@bp.route('/<hashid:corpus_id>', methods=['GET', 'POST'])
-@login_required
-def corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user
-            or current_user.is_administrator()
-            or current_user.is_following_corpus(corpus)
-            or corpus.is_public):
-        abort(403)
-    corpus_settings_form = ChangeCorpusSettingsForm(
-        data=corpus.to_json_serializeable(),
-        prefix='corpus-settings-form'
-    )
-    if corpus_settings_form.validate_on_submit():
-        corpus.is_public = corpus_settings_form.is_public.data
-        db.session.commit()
-        flash('Your changes have been saved')
-        return redirect(url_for('.corpus', corpus_id=corpus.id))
-    if corpus.user == current_user or current_user.is_administrator():
-        return render_template(
-            'corpora/corpus.html.j2',
-            corpus_settings_form=corpus_settings_form,
-            corpus=corpus,
-            title='Corpus'
-        )
-    else:
-        print('public')
-        return render_template(
-            'corpora/corpus_public.html.j2',
-            corpus=corpus,
-            title='Corpus'
-        )
-
-
-
-# @bp.route('/<hashid:corpus_id>/update')
-# @login_required
-# def update_corpus(corpus_id):
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     if not (corpus.user == current_user or current_user.is_administrator()):
-#         abort(403)
-#     return render_template(
-#         'corpora/update_corpus.html.j2',
-#         corpus=corpus,
-#         title='Corpus'
-#     )
-
-
-@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
-@login_required
-def delete_corpus(corpus_id):
-    def _delete_corpus(app, corpus_id):
-        with app.app_context():
-            corpus = Corpus.query.get(corpus_id)
-            corpus.delete()
-            db.session.commit()
-
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_corpus,
-        args=(current_app._get_current_object(), corpus_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/analyse')
-@login_required
-def analyse_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user
-            or current_user.is_administrator()
-            or current_user.is_following_corpus(corpus)
-            or corpus.is_public):
-        abort(403)
-    return render_template(
-        'corpora/analyse_corpus.html.j2',
-        corpus=corpus,
-        title=f'Analyse Corpus {corpus.title}'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
-@login_required
-def build_corpus(corpus_id):
-    def _build_corpus(app, corpus_id):
-        with app.app_context():
-            corpus = Corpus.query.get(corpus_id)
-            corpus.build()
-            db.session.commit()
-
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    # Check if the corpus has corpus files
-    if not corpus.files.all():
-        response = {'errors': {'message': 'Corpus file(s) required'}}
-        return response, 409
-    thread = Thread(
-        target=_build_corpus,
-        args=(current_app._get_current_object(), corpus_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
-@login_required
-def create_corpus_file(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    if not (corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    form = CreateCorpusFileForm()
-    if form.is_submitted():
-        if not form.validate():
-            response = {'errors': form.errors}
-            return response, 400
-        try:
-            corpus_file = CorpusFile.create(
-                form.vrt.data,
-                address=form.address.data,
-                author=form.author.data,
-                booktitle=form.booktitle.data,
-                chapter=form.chapter.data,
-                editor=form.editor.data,
-                institution=form.institution.data,
-                journal=form.journal.data,
-                pages=form.pages.data,
-                publisher=form.publisher.data,
-                publishing_year=form.publishing_year.data,
-                school=form.school.data,
-                title=form.title.data,
-                mimetype='application/vrt+xml',
-                corpus=corpus
-            )
-        except (AttributeError, OSError):
-            abort(500)
-        corpus.status = CorpusStatus.UNPREPARED
-        db.session.commit()
-        message = Markup(
-            'Corpus file'
-            f'"<a href="{corpus_file.url}">{corpus_file.filename}</a>" added'
-        )
-        flash(message, category='corpus')
-        return {}, 201, {'Location': corpus.url}
-    return render_template(
-        'corpora/create_corpus_file.html.j2',
-        corpus=corpus,
-        form=form,
-        title='Add corpus file'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
-@login_required
-def corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
-    if form.validate_on_submit():
-        form.populate_obj(corpus_file)
-        if db.session.is_modified(corpus_file):
-            corpus_file.corpus.status = CorpusStatus.UNPREPARED
-            db.session.commit()
-            message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
-            flash(message, category='corpus')
-        return redirect(corpus_file.corpus.url)
-    return render_template(
-        'corpora/corpus_file.html.j2',
-        corpus=corpus_file.corpus,
-        corpus_file=corpus_file,
-        form=form,
-        title='Edit corpus file'
-    )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
-@login_required
-def delete_corpus_file(corpus_id, corpus_file_id):
-    def _delete_corpus_file(app, corpus_file_id):
-        with app.app_context():
-            corpus_file = CorpusFile.query.get(corpus_file_id)
-            corpus_file.delete()
-            db.session.commit()
-
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_corpus_file,
-        args=(current_app._get_current_object(), corpus_file_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
-@login_required
-def download_corpus_file(corpus_id, corpus_file_id):
-    corpus_file = CorpusFile.query.filter_by(corpus_id = corpus_id, id=corpus_file_id).first_or_404()
-    if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
-        abort(403)
-    return send_from_directory(
-        os.path.dirname(corpus_file.path),
-        os.path.basename(corpus_file.path),
-        as_attachment=True,
-        attachment_filename=corpus_file.filename,
-        mimetype=corpus_file.mimetype
-    )
-
-
-@bp.route('/import', methods=['GET', 'POST'])
-@login_required
-def import_corpus():
-    abort(503)
-
-
-@bp.route('/<hashid:corpus_id>/export')
-@login_required
-def export_corpus(corpus_id):
-    abort(503)
-
-@bp.route('/<hashid:corpus_id>/follow', methods=['GET', 'POST'])
-@login_required
-def follow_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    user_hashid = request.args.get('user_id')
-    if user_hashid is None:
-        user = current_user
-    else:
-        if not current_user.is_administrator():
-            abort(403)
-        else:
-            user_id = hashids.decode(user_hashid)
-            user = User.query.get_or_404(user_id)
-    if not user.is_following_corpus(corpus):
-        user.follow_corpus(corpus)
-    db.session.commit()
-    return {}, 202
-
-@bp.route('/<hashid:corpus_id>/unfollow', methods=['GET', 'POST'])
-@login_required
-def unfollow_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
-    user_hashid = request.args.get('user_id')
-    if user_hashid is None:
-        user = current_user
-    else:
-        if not current_user.is_administrator():
-            abort(403)
-        else:
-            user_id = hashids.decode(user_hashid)
-            user = User.query.get_or_404(user_id)
-    if user.is_following_corpus(corpus):
-        user.unfollow_corpus(corpus)
-    db.session.commit()
-    return {}, 202
-
-@bp.route('/add_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
-def add_permission(corpus_id, user_id, permission):
-    a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
-    a.add_permission(permission)
-    db.session.commit()
-    return 'ok'
-
-
-@bp.route('/remove_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
-def remove_permission(corpus_id, user_id, permission):
-    a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
-    a.remove_permission(permission)
-    db.session.commit()
-    return 'ok'
-

app/daemon/__init__.py

@@ -1,11 +0,0 @@
-from app import db
-from flask import Flask
-from .corpus_utils import check_corpora
-from .job_utils import check_jobs
-
-
-def daemon(app: Flask):
-    with app.app_context():
-        check_corpora()
-        check_jobs()
-        db.session.commit()

app/decorators.py

@@ -1,7 +1,8 @@
-from flask import abort, current_app
+from flask import abort, request
 from flask_login import current_user
 from functools import wraps
-from threading import Thread
+from typing import Optional
+from werkzeug.exceptions import NotAcceptable
 from app.models import Permission
 
 
@@ -22,22 +23,21 @@ def admin_required(f):
 
 def socketio_login_required(f):
     @wraps(f)
-    def decorated_function(*args, **kwargs):
+    def wrapper(*args, **kwargs):
         if current_user.is_authenticated:
             return f(*args, **kwargs)
-        else:
-            return {'code': 401, 'msg': 'Unauthorized'}
-    return decorated_function
+        return {'status': 401, 'statusText': 'Unauthorized'}
+    return wrapper
 
 
 def socketio_permission_required(permission):
     def decorator(f):
         @wraps(f)
-        def decorated_function(*args, **kwargs):
+        def wrapper(*args, **kwargs):
             if not current_user.can(permission):
-                return {'code': 403, 'msg': 'Forbidden'}
+                return {'status': 403, 'statusText': 'Forbidden'}
             return f(*args, **kwargs)
-        return decorated_function
+        return wrapper
     return decorator
 
 
@@ -45,19 +45,35 @@ def socketio_admin_required(f):
     return socketio_permission_required(Permission.ADMINISTRATE)(f)
 
 
-def background(f):
-    '''
-    ' This decorator executes a function in a Thread.
-    ' Decorated functions need to be executed within a code block where an
-    ' app context exists.
-    '
-    ' NOTE: An app object is passed as a keyword argument to the decorated
-    '       function.
-    '''
-    @wraps(f)
-    def wrapped(*args, **kwargs):
-        kwargs['app'] = current_app._get_current_object()
-        thread = Thread(target=f, args=args, kwargs=kwargs)
-        thread.start()
-        return thread
-    return wrapped
+def content_negotiation(
+    produces: Optional[str | list[str]] = None,
+    consumes: Optional[str | list[str]] = None
+):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            provided = request.mimetype
+            if consumes is None:
+                consumeables = None
+            elif isinstance(consumes, str):
+                consumeables = {consumes}
+            elif isinstance(consumes, list) and all(isinstance(x, str) for x in consumes):
+                consumeables = {*consumes}
+            else:
+                raise TypeError()
+            accepted = {*request.accept_mimetypes.values()}
+            if produces is None:
+                produceables = None
+            elif isinstance(produces, str):
+                produceables = {produces}
+            elif isinstance(produces, list) and all(isinstance(x, str) for x in produces):
+                produceables = {*produces}
+            else:
+                raise TypeError()
+            if produceables is not None and len(produceables & accepted) == 0:
+                raise NotAcceptable()
+            if consumeables is not None and provided not in consumeables:
+                raise NotAcceptable()
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator

app/email.py

@@ -1,25 +1,32 @@
-from flask import current_app, render_template
+from flask import current_app, Flask, render_template
 from flask_mail import Message
 from threading import Thread
 from app import mail
 
 
-def create_message(recipient, subject, template, **kwargs):
-    subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
-    msg: Message = Message(
-        body=render_template(f'{template}.txt.j2', **kwargs),
-        html=render_template(f'{template}.html.j2', **kwargs),
+def create_message(
+    recipient: str,
+    subject: str,
+    template: str,
+    **context
+) -> Message:
+    message = Message(
+        body=render_template(f'{template}.txt.j2', **context),
+        html=render_template(f'{template}.html.j2', **context),
         recipients=[recipient],
-        subject=f'{subject_prefix} {subject}'
+        subject=f'[nopaque] {subject}'
     )
-    return msg
+    return message
 
 
-def send(msg, *args, **kwargs):
-    def _send(app, msg):
+def send(message: Message) -> Thread:
+    def _send(app: Flask, message: Message):
         with app.app_context():
-            mail.send(msg)
+            mail.send(message)
 
-    thread = Thread(target=_send, args=[current_app._get_current_object(), msg])
+    thread = Thread(
+        target=_send,
+        args=[current_app._get_current_object(), message]
+    )
     thread.start()
     return thread

app/errors/handlers.py

@@ -1,11 +0,0 @@
-from flask import render_template, request
-from werkzeug.exceptions import HTTPException
-from . import bp
-
-
-@bp.errorhandler(HTTPException)
-def generic_error_handler(e):
-    if (request.accept_mimetypes.accept_json
-            and not request.accept_mimetypes.accept_html):
-        return {'errors': {'message': e.description}}, e.code
-    return render_template('errors/error.html.j2', error=e), e.code

app/extensions/nopaque_flask_admin_views.py

@@ -0,0 +1,20 @@
+from flask import abort
+from flask_admin import (
+    AdminIndexView as _AdminIndexView,
+    expose
+)
+from flask_admin.contrib.sqla import ModelView as _ModelView
+from flask_login import current_user
+
+
+class AdminIndexView(_AdminIndexView):
+    @expose('/')
+    def index(self):
+        if not current_user.is_administrator:
+            abort(403)
+        return super().index()
+
+
+class ModelView(_ModelView):
+    def is_accessible(self):
+        return current_user.is_administrator

app/extensions/nopaque_sqlalchemy_type_decorators.py

@@ -0,0 +1,42 @@
+import json
+from app import db
+
+
+class ContainerColumn(db.TypeDecorator):
+    impl = db.String
+
+    def __init__(self, container_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.container_type = container_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.container_type):
+            return json.dumps(value)
+        elif isinstance(value, str) and isinstance(json.loads(value), self.container_type):
+            return value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return json.loads(value)
+
+
+class IntEnumColumn(db.TypeDecorator):
+    impl = db.Integer
+
+    def __init__(self, enum_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_type = enum_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.enum_type) and isinstance(value.value, int):
+            return value.value
+        elif isinstance(value, int):
+            return self.enum_type(value).value
+        elif isinstance(value, str):
+            return self.enum_type[value].value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return self.enum_type(value)

app/jobs/__init__.py

@@ -1,5 +1,2 @@
-from flask import Blueprint
-
-
-bp = Blueprint('jobs', __name__)
-from . import routes
+from .handle_corpora import handle_corpora
+from .handle_jobs import handle_jobs

app/jobs/handle_corpora.py

@@ -1,12 +1,16 @@
-from app import docker_client
-from app.models import Corpus, CorpusStatus
 from flask import current_app
 import docker
 import os
 import shutil
+from app import db, docker_client, scheduler
+from app.models import Corpus, CorpusStatus
 
 
-def check_corpora():
+def handle_corpora():
+    with scheduler.app.app_context():
+        _handle_corpora()
+
+def _handle_corpora():
     corpora = Corpus.query.all()
     for corpus in [x for x in corpora if x.status == CorpusStatus.SUBMITTED]:
         _create_build_corpus_service(corpus)
@@ -17,40 +21,39 @@ def check_corpora():
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
         corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
-        _checkout_analysing_corpus_container(corpus)
+        _checkout_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
         _create_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
         _remove_cqpserver_container(corpus)
+    db.session.commit()
 
-def _create_build_corpus_service(corpus):
+def _create_build_corpus_service(corpus: Corpus):
     ''' # Docker service settings # '''
     ''' ## Command ## '''
     command = ['bash', '-c']
     command.append(
-        f'mkdir /corpora/data/nopaque_{corpus.id}'
+        f'mkdir /corpora/data/nopaque-{corpus.hashid.lower()}'
         ' && '
         'cwb-encode'
         ' -c utf8'
-        f' -d /corpora/data/nopaque_{corpus.id}'
+        f' -d /corpora/data/nopaque-{corpus.hashid.lower()}'
         ' -f /root/files/corpus.vrt'
-        f' -R /usr/local/share/cwb/registry/nopaque_{corpus.id}'
+        f' -R /usr/local/share/cwb/registry/nopaque-{corpus.hashid.lower()}'
         ' -P pos -P lemma -P simple_pos'
         ' -S ent:0+type -S s:0'
         ' -S text:0+address+author+booktitle+chapter+editor+institution+journal+pages+publisher+publishing_year+school+title'
         ' -xsB -9'
         ' && '
-        f'cwb-make -V NOPAQUE_{corpus.id}'
+        f'cwb-make -V NOPAQUE-{corpus.hashid.upper()}'
     )
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'corpus.build',
-        'corpus_id': str(corpus.id)
+        'nopaque.server_name': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -95,7 +98,7 @@ def _create_build_corpus_service(corpus):
         return
     corpus.status = CorpusStatus.QUEUED
 
-def _checkout_build_corpus_service(corpus):
+def _checkout_build_corpus_service(corpus: Corpus):
     service_name = f'build-corpus_{corpus.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -123,8 +126,7 @@ def _checkout_build_corpus_service(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _create_cqpserver_container(corpus):
-    ''' # Docker container settings # '''
+def _create_cqpserver_container(corpus: Corpus):
     ''' ## Command ## '''
     command = []
     command.append(
@@ -139,21 +141,25 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Name ## '''
-    name = f'cqpserver_{corpus.id}'
+    name = f'nopaque-cqpserver-{corpus.id}'
     ''' ## Network ## '''
-    network = f'{current_app.config["DOCKER_NETWORK_NAME"]}'
+    network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}'
     ''' ## Volumes ## '''
     volumes = []
     ''' ### Corpus data volume ### '''
     data_volume_source = os.path.join(corpus.path, 'cwb', 'data')
     data_volume_target = '/corpora/data'
+    # data_volume_source = os.path.join(corpus.path, 'cwb', 'data', f'nopaque_{corpus.id}')
+    # data_volume_target = f'/corpora/data/nopaque_{corpus.hashid.lower()}'
     data_volume = f'{data_volume_source}:{data_volume_target}:rw'
     volumes.append(data_volume)
     ''' ### Corpus registry volume ### '''
     registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry')
     registry_volume_target = '/usr/local/share/cwb/registry'
+    # registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry', f'nopaque_{corpus.id}')
+    # registry_volume_target = f'/usr/local/share/cwb/registry/nopaque_{corpus.hashid.lower()}'
     registry_volume = f'{registry_volume_source}:{registry_volume_target}:rw'
     volumes.append(registry_volume)
     # Check if a cqpserver container already exists. If this is the case,
@@ -194,8 +200,8 @@ def _create_cqpserver_container(corpus):
         return
     corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
 
-def _checkout_analysing_corpus_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _checkout_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         docker_client.containers.get(container_name)
     except docker.errors.NotFound as e:
@@ -205,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Get container "{container_name}" failed: {e}')
 
-def _remove_cqpserver_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _remove_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:

app/jobs/handle_jobs.py

@@ -1,11 +1,3 @@
-from app import db, docker_client, hashids
-from app.models import (
-    Job,
-    JobResult,
-    JobStatus,
-    TesseractOCRPipelineModel,
-    SpaCyNLPPipelineModel
-)
 from datetime import datetime
 from flask import current_app
 from werkzeug.utils import secure_filename
@@ -13,9 +5,21 @@ import docker
 import json
 import os
 import shutil
+from app import db, docker_client, hashids, scheduler
+from app.models import (
+    Job,
+    JobResult,
+    JobStatus,
+    TesseractOCRPipelineModel,
+    SpaCyNLPPipelineModel
+)
 
 
-def check_jobs():
+def handle_jobs():
+    with scheduler.app.app_context():
+        _handle_jobs()
+
+def _handle_jobs():
     jobs = Job.query.all()
     for job in [x for x in jobs if x.status == JobStatus.SUBMITTED]:
         _create_job_service(job)
@@ -23,8 +27,9 @@ def check_jobs():
         _checkout_job_service(job)
     for job in [x for x in jobs if x.status == JobStatus.CANCELING]:
         _remove_job_service(job)
+    db.session.commit()
 
-def _create_job_service(job):
+def _create_job_service(job: Job):
     ''' # Docker service settings # '''
     ''' ## Service specific settings ## '''
     if job.service == 'file-setup-pipeline':
@@ -81,9 +86,7 @@ def _create_job_service(job):
     constraints = ['node.role==worker']
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'job',
-        'job_id': str(job.id)
+        'origin': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -164,7 +167,7 @@ def _create_job_service(job):
         return
     job.status = JobStatus.QUEUED
 
-def _checkout_job_service(job):
+def _checkout_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -213,7 +216,7 @@ def _checkout_job_service(job):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _remove_job_service(job):
+def _remove_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)

app/jobs/routes.py

@@ -1,116 +0,0 @@
-from flask import (
-    abort,
-    current_app,
-    render_template,
-    send_from_directory
-)
-from flask_login import current_user, login_required
-from threading import Thread
-import os
-from app import db
-from app.decorators import admin_required
-from app.models import Job, JobInput, JobResult, JobStatus
-from . import bp
-
-
-@bp.route('/<hashid:job_id>')
-@login_required
-def job(job_id):
-    job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    return render_template(
-        'jobs/job.html.j2',
-        job=job,
-        title='Job'
-    )
-
-
-@bp.route('/<hashid:job_id>', methods=['DELETE'])
-@login_required
-def delete_job(job_id):
-    def _delete_job(app, job_id):
-        with app.app_context():
-            job = Job.query.get(job_id)
-            job.delete()
-            db.session.commit()
-
-    job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    thread = Thread(
-        target=_delete_job,
-        args=(current_app._get_current_object(), job_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:job_id>/log')
-@login_required
-@admin_required
-def job_log(job_id):
-    job = Job.query.get_or_404(job_id)
-    if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-        response = {'errors': {'message': 'Job status is not completed or failed'}}
-        return response, 409
-    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
-        log = log_file.read()
-    return log, 200, {'Content-Type': 'text/plain; charset=utf-8'}
-
-
-@bp.route('/<hashid:job_id>/restart', methods=['POST'])
-@login_required
-def restart_job(job_id):
-    def _restart_job(app, job_id):
-        with app.app_context():
-            job = Job.query.get(job_id)
-            job.restart()
-            db.session.commit()
-
-    job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    if job.status == JobStatus.FAILED:
-        response = {'errors': {'message': 'Job status is not "failed"'}}
-        return response, 409
-    thread = Thread(
-        target=_restart_job,
-        args=(current_app._get_current_object(), job_id)
-    )
-    thread.start()
-    return {}, 202
-
-
-@bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
-@login_required
-def download_job_input(job_id, job_input_id):
-    job_input = JobInput.query.get_or_404(job_input_id)
-    if job_input.job.id != job_id:
-        abort(404)
-    if not (job_input.job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    return send_from_directory(
-        os.path.dirname(job_input.path),
-        os.path.basename(job_input.path),
-        as_attachment=True,
-        attachment_filename=job_input.filename,
-        mimetype=job_input.mimetype
-    )
-
-
-@bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
-@login_required
-def download_job_result(job_id, job_result_id):
-    job_result = JobResult.query.get_or_404(job_result_id)
-    if job_result.job.id != job_id:
-        abort(404)
-    if not (job_result.job.user == current_user or current_user.is_administrator()):
-        abort(403)
-    return send_from_directory(
-        os.path.dirname(job_result.path),
-        os.path.basename(job_result.path),
-        as_attachment=True,
-        attachment_filename=job_result.filename,
-        mimetype=job_result.mimetype
-    )

app/main/routes.py

@@ -1,64 +0,0 @@
-from flask import flash, redirect, render_template, url_for
-from flask_login import current_user, login_required, login_user
-from app.auth.forms import LoginForm
-from app.models import Corpus, User
-from . import bp
-
-
-@bp.route('', methods=['GET', 'POST'])
-def index():
-    form = LoginForm(prefix='login-form')
-    if form.validate_on_submit():
-        user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
-        if user and user.verify_password(form.password.data):
-            login_user(user, form.remember_me.data)
-            flash('You have been logged in')
-            return redirect(url_for('.dashboard'))
-        flash('Invalid email/username or password', category='error')
-        redirect(url_for('.index'))
-    return render_template('main/index.html.j2', form=form, title='nopaque')
-
-
-@bp.route('/faq')
-def faq():
-    return render_template('main/faq.html.j2', title='Frequently Asked Questions')
-
-
-@bp.route('/dashboard')
-@login_required
-def dashboard():
-    users = [
-        u.to_json_serializeable(filter_by_privacy_settings=True) for u
-        in User.query.filter(User.is_public == True, User.id != current_user.id).all()
-    ]
-    corpora = [
-        c.to_json_serializeable() for c
-        in Corpus.query.filter(Corpus.is_public == True).all()
-    ]
-    return render_template('main/dashboard.html.j2', title='Dashboard', users=users, corpora=corpora)
-
-
-@bp.route('/dashboard2')
-@login_required
-def dashboard2():
-    return render_template('main/dashboard2.html.j2', title='Dashboard')
-
-
-@bp.route('/user_manual')
-def user_manual():
-    return render_template('main/user_manual.html.j2', title='User manual')
-
-
-@bp.route('/news')
-def news():
-    return render_template('main/news.html.j2', title='News')
-
-
-@bp.route('/privacy_policy')
-def privacy_policy():
-    return render_template('main/privacy_policy.html.j2', title='Privacy statement (GDPR)')
-
-
-@bp.route('/terms_of_use')
-def terms_of_use():
-    return render_template('main/terms_of_use.html.j2', title='Terms of Use')

app/models/__init__.py

@@ -0,0 +1,45 @@
+from .anonymous_user import AnonymousUser
+from .avatar import Avatar
+from .corpus_file import CorpusFile
+from .corpus_follower_association import CorpusFollowerAssociation
+from .corpus_follower_role import CorpusFollowerPermission, CorpusFollowerRole
+from .corpus import CorpusStatus, Corpus
+from .job_input import JobInput
+from .job_result import JobResult
+from .job import JobStatus, Job
+from .role import Permission, Role
+from .spacy_nlp_pipeline_model import SpaCyNLPPipelineModel
+from .tesseract_ocr_pipeline_model import TesseractOCRPipelineModel
+from .token import Token
+from .user import (
+    ProfilePrivacySettings,
+    UserSettingJobStatusMailNotificationLevel,
+    User
+)
+
+
+_models = [
+    Avatar,
+    CorpusFile,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    Corpus,
+    JobInput,
+    JobResult,
+    Job,
+    Role,
+    SpaCyNLPPipelineModel,
+    TesseractOCRPipelineModel,
+    Token,
+    User
+]
+
+
+_enums = [
+    CorpusFollowerPermission,
+    CorpusStatus,
+    JobStatus,
+    Permission,
+    ProfilePrivacySettings,
+    UserSettingJobStatusMailNotificationLevel
+]

app/models/anonymous_user.py

@@ -0,0 +1,10 @@
+from flask_login import AnonymousUserMixin
+
+
+class AnonymousUser(AnonymousUserMixin):
+    def can(self, permissions):
+        return False
+
+    @property
+    def is_administrator(self):
+        return False

app/models/avatar.py

@@ -0,0 +1,40 @@
+from flask import current_app
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class Avatar(HashidMixin, FileMixin, db.Model):
+    __tablename__ = 'avatars'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Relationships
+    user = db.relationship('User', back_populates='avatar')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'avatar'
+        # return os.path.join(self.user.path, 'avatar')
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable