Bump nopaque version

- breadcrumbs (!flask-breadcrumbs)
- manual modal button
- api blueprint (!flask-marshmallow/!marshmallow-sqlalchemy)

.dockerignore

@@ -5,9 +5,9 @@
 !app
 !migrations
 !tests
-!.flaskenv
 !boot.sh
 !config.py
 !docker-nopaque-entrypoint.sh
-!nopaque.py
 !requirements.txt
+!requirements.freezed.txt
+!wsgi.py

.env.tpl

@@ -1,32 +1,20 @@
 ##############################################################################
-# Variables for use in Docker Compose YAML files                             #
+# Environment variables used by Docker Compose config files.                 #
 ##############################################################################
 # HINT: Use this bash command `id -u`
 # NOTE: 0 (= root user) is not allowed
 HOST_UID=
 
 # HINT: Use this bash command `id -g`
+# NOTE: 0 (= root group) is not allowed
 HOST_GID=
 
 # HINT: Use this bash command `getent group docker | cut -d: -f3`
 HOST_DOCKER_GID=
 
 # DEFAULT: nopaque
-# DOCKER_DEFAULT_NETWORK_NAME=
-
-# DEFAULT: ./volumes/db/data
-# NOTE: Use `.` as <project-basedir>
-# DOCKER_DB_SERVICE_DATA_VOLUME_SOURCE_PATH=
-
-# DEFAULT: ./volumes/mq/data
-# NOTE: Use `.` as <project-basedir>
-# DOCKER_MQ_SERVICE_DATA_VOLUME_SOURCE_PATH=
+NOPAQUE_DOCKER_NETWORK_NAME=nopaque
 
 # NOTE: This must be a network share and it must be available on all
-#       Docker Swarm nodes, mounted to the same path with the same
-#       user and group ownership.
-DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH=
-
-# DEFAULT: ./volumes/nopaque/logs
-# NOTE: Use `.` as <project-basedir>
-# DOCKER_NOPAQUE_SERVICE_LOGS_VOLUME_SOURCE_PATH=.
+#       Docker Swarm nodes, mounted to the same path.
+HOST_NOPAQUE_DATA_PATH=/mnt/nopaque

.flaskenv

@@ -1 +0,0 @@
-FLASK_APP=nopaque.py

.gitignore

@@ -2,8 +2,6 @@
 app/static/gen/
 volumes/
 docker-compose.override.yml
-logs/
-!logs/dummy
 *.env
 
 *.pjentsch-testing

.vscode/settings.json

@@ -1,9 +1,17 @@
 {
     "editor.rulers": [79],
-    "files.insertFinalNewline": true,
-    "[css]": {
-        "editor.tabSize": 2
+    "editor.tabSize": 4,
+    "emmet.includeLanguages": {
+        "jinja-html": "html"
     },
+    "files.associations": {
+        ".flaskenv": "env",
+        "*.env.tpl": "env",
+        "*.txt.j2": "jinja"
+    },
+    "files.insertFinalNewline": true,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
     "[html]": {
         "editor.tabSize": 2
     },
@@ -12,8 +20,5 @@
     },
     "[jinja-html]": {
         "editor.tabSize": 2
-    },
-    "[scss]": {
-        "editor.tabSize": 2
     }
 }

Dockerfile

@@ -4,11 +4,13 @@ FROM python:3.10.13-slim-bookworm
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
 
 
+# Set environment variables
 ENV LANG="C.UTF-8"
 ENV PYTHONDONTWRITEBYTECODE="1"
 ENV PYTHONUNBUFFERED="1"
 
 
+# Install system dependencies
 RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
@@ -17,37 +19,39 @@ RUN apt-get update \
  && rm --recursive /var/lib/apt/lists/*
 
 
+# Create a non-root user
 RUN useradd --create-home --no-log-init nopaque \
  && groupadd docker \
  && usermod --append --groups docker nopaque
 
-
 USER nopaque
 WORKDIR /home/nopaque
 
 
+# Create a Python virtual environment
 ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv"
 RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}"
 ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}"
 
 
+# Install Python dependencies
+COPY --chown=nopaque:nopaque requirements.freezed.txt requirements.freezed.txt
+RUN python3 -m pip install --requirement requirements.freezed.txt \
+ && rm requirements.freezed.txt
+
+
+# Install the application
+COPY docker-nopaque-entrypoint.sh /usr/local/bin/
 COPY --chown=nopaque:nopaque app app
 COPY --chown=nopaque:nopaque migrations migrations
 COPY --chown=nopaque:nopaque tests tests
-COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py requirements.txt ./
-
-
-RUN python3 -m pip install --requirement requirements.txt \
- && mkdir logs
-
-
-USER root
-
-
-COPY docker-nopaque-entrypoint.sh /usr/local/bin/
+COPY --chown=nopaque:nopaque boot.sh config.py wsgi.py ./
 
 
 EXPOSE 5000
 
 
+USER root
+
+
 ENTRYPOINT ["docker-nopaque-entrypoint.sh"]

README.md

@@ -35,7 +35,7 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
 # Create data directories
-username@hostname:~$ mkdir data/{db,logs,mq}
+username@hostname:~$ mkdir -p volumes/{db,mq}
 username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
 # Fill out the variables within these files.

app/__init__.py

@@ -2,9 +2,9 @@ from apifairy import APIFairy
 from config import Config
 from docker import DockerClient
 from flask import Flask
+from flask.logging import default_handler
 from flask_apscheduler import APScheduler
 from flask_assets import Environment
-from flask_breadcrumbs import Breadcrumbs, default_breadcrumb_root
 from flask_login import LoginManager
 from flask_mail import Mail
 from flask_marshmallow import Marshmallow
@@ -13,95 +13,142 @@ from flask_paranoid import Paranoid
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 from flask_hashids import Hashids
+from logging import Formatter, StreamHandler
+from werkzeug.middleware.proxy_fix import ProxyFix
 
 
+docker_client = DockerClient.from_env()
+
 apifairy = APIFairy()
 assets = Environment()
-breadcrumbs = Breadcrumbs()
 db = SQLAlchemy()
-docker_client = DockerClient()
 hashids = Hashids()
 login = LoginManager()
-login.login_view = 'auth.login'
-login.login_message = 'Please log in to access this page.'
 ma = Marshmallow()
 mail = Mail()
 migrate = Migrate(compare_type=True)
 paranoid = Paranoid()
-paranoid.redirect_view = '/'
 scheduler = APScheduler()
 socketio = SocketIO()
 
 
 def create_app(config: Config = Config) -> Flask:
-    ''' Creates an initialized Flask (WSGI Application) object. '''
+    ''' Creates an initialized Flask object. '''
+
     app = Flask(__name__)
     app.config.from_object(config)
-    config.init_app(app)
+
+    # region Logging
+    log_formatter = Formatter(
+        fmt=app.config['NOPAQUE_LOG_FORMAT'],
+        datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT']
+    )
+
+    log_handler = StreamHandler()
+    log_handler.setFormatter(log_formatter)
+    log_handler.setLevel(app.config['NOPAQUE_LOG_LEVEL'])
+
+    app.logger.setLevel('DEBUG')
+    app.logger.removeHandler(default_handler)
+    app.logger.addHandler(log_handler)
+    # endregion Logging
+
+    # region Middlewares
+    if app.config['NOPAQUE_PROXY_FIX_ENABLED']:
+        app.wsgi_app = ProxyFix(
+            app.wsgi_app,
+            x_for=app.config['NOPAQUE_PROXY_FIX_X_FOR'],
+            x_host=app.config['NOPAQUE_PROXY_FIX_X_HOST'],
+            x_port=app.config['NOPAQUE_PROXY_FIX_X_PORT'],
+            x_prefix=app.config['NOPAQUE_PROXY_FIX_X_PREFIX'],
+            x_proto=app.config['NOPAQUE_PROXY_FIX_X_PROTO']
+        )
+    # endregion Middlewares
+
+    # region Extensions
     docker_client.login(
         app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
         password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
         registry=app.config['NOPAQUE_DOCKER_REGISTRY']
     )
 
+    from .models import AnonymousUser, User
+
     apifairy.init_app(app)
     assets.init_app(app)
-    breadcrumbs.init_app(app)
     db.init_app(app)
     hashids.init_app(app)
     login.init_app(app)
+    login.anonymous_user = AnonymousUser
+    login.login_view = 'auth.login'
+    login.user_loader(lambda user_id: User.query.get(int(user_id)))
     ma.init_app(app)
     mail.init_app(app)
     migrate.init_app(app, db)
     paranoid.init_app(app)
+    paranoid.redirect_view = '/'
     scheduler.init_app(app)
-    socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
+    socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
+    # endregion Extensions
 
-    from .admin import bp as admin_blueprint
-    default_breadcrumb_root(admin_blueprint, '.admin')
+    # region Blueprints
+    from .blueprints.admin import bp as admin_blueprint
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
 
-    from .api import bp as api_blueprint
+    from .blueprints.api import bp as api_blueprint
     app.register_blueprint(api_blueprint, url_prefix='/api')
 
-    from .auth import bp as auth_blueprint
-    default_breadcrumb_root(auth_blueprint, '.')
+    from .blueprints.auth import bp as auth_blueprint
     app.register_blueprint(auth_blueprint)
 
-    from .contributions import bp as contributions_blueprint
-    default_breadcrumb_root(contributions_blueprint, '.contributions')
+    from .blueprints.contributions import bp as contributions_blueprint
     app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
-    from .corpora import bp as corpora_blueprint
-    from .corpora.cqi_over_sio import CQiNamespace
-    default_breadcrumb_root(corpora_blueprint, '.corpora')
+    from .blueprints.corpora import bp as corpora_blueprint
     app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
-    socketio.on_namespace(CQiNamespace('/cqi_over_sio'))
 
-    from .errors import bp as errors_bp
+    from .blueprints.errors import bp as errors_bp
     app.register_blueprint(errors_bp)
 
-    from .jobs import bp as jobs_blueprint
-    default_breadcrumb_root(jobs_blueprint, '.jobs')
+    from .blueprints.jobs import bp as jobs_blueprint
     app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
 
-    from .main import bp as main_blueprint
-    default_breadcrumb_root(main_blueprint, '.')
+    from .blueprints.main import bp as main_blueprint
     app.register_blueprint(main_blueprint, cli_group=None)
 
-    from .services import bp as services_blueprint
-    default_breadcrumb_root(services_blueprint, '.services')
+    from .blueprints.services import bp as services_blueprint
     app.register_blueprint(services_blueprint, url_prefix='/services')
 
-    from .settings import bp as settings_blueprint
-    default_breadcrumb_root(settings_blueprint, '.settings')
+    from .blueprints.settings import bp as settings_blueprint
     app.register_blueprint(settings_blueprint, url_prefix='/settings')
 
-    from .users import bp as users_blueprint
-    default_breadcrumb_root(users_blueprint, '.users')
+    from .blueprints.users import bp as users_blueprint
     app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
 
-    from .workshops import bp as workshops_blueprint
+    from .blueprints.workshops import bp as workshops_blueprint
     app.register_blueprint(workshops_blueprint, url_prefix='/workshops')
+    # endregion Blueprints
+
+    # region SocketIO Namespaces
+    from .namespaces.cqi_over_sio import CQiOverSocketIONamespace
+    socketio.on_namespace(CQiOverSocketIONamespace('/cqi_over_sio'))
+
+    from .namespaces.users import UsersNamespace
+    socketio.on_namespace(UsersNamespace('/users'))
+    # endregion SocketIO Namespaces
+
+    # region Database event Listeners
+    from .models.event_listeners import register_event_listeners
+    register_event_listeners()
+    # endregion Database event Listeners
+
+    # region Add scheduler jobs
+    if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
+        from .jobs import handle_corpora
+        scheduler.add_job('handle_corpora', handle_corpora, seconds=3, trigger='interval')
+
+        from .jobs import handle_jobs
+        scheduler.add_job('handle_jobs', handle_jobs, seconds=3, trigger='interval')
+    # endregion Add scheduler jobs
 
     return app

app/auth/__init__.py

@@ -1,5 +0,0 @@
-from flask import Blueprint
-
-
-bp = Blueprint('auth', __name__)
-from . import routes

app/blueprints/admin/json_routes.py

@@ -1,6 +1,6 @@
 from flask import abort, request
-from app import db
 from app.decorators import content_negotiation
+from app import db
 from app.models import User
 from . import bp
 

app/blueprints/admin/routes.py

@@ -1,8 +1,7 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from app import db, hashids
 from app.models import Avatar, Corpus, Role, User
-from app.users.settings.forms import (
+from app.blueprints.users.settings.forms import (
     UpdateAvatarForm,
     UpdatePasswordForm,
     UpdateNotificationsForm,
@@ -11,14 +10,9 @@ from app.users.settings.forms import (
 )
 from . import bp
 from .forms import UpdateUserForm
-from app.users.utils import (
-    user_endpoint_arguments_constructor as user_eac,
-    user_dynamic_list_constructor as user_dlc
-)
 
 
 @bp.route('')
-@register_breadcrumb(bp, '.', '<i class="material-icons left">admin_panel_settings</i>Administration')
 def admin():
     return render_template(
         'admin/admin.html.j2',
@@ -27,7 +21,6 @@ def admin():
 
 
 @bp.route('/corpora')
-@register_breadcrumb(bp, '.corpora', 'Corpora')
 def corpora():
     corpora = Corpus.query.all()
     return render_template(
@@ -38,7 +31,6 @@ def corpora():
 
 
 @bp.route('/users')
-@register_breadcrumb(bp, '.users', '<i class="material-icons left">group</i>Users')
 def users():
     users = User.query.all()
     return render_template(
@@ -49,7 +41,6 @@ def users():
 
 
 @bp.route('/users/<hashid:user_id>')
-@register_breadcrumb(bp, '.users.entity', '', dynamic_list_constructor=user_dlc)
 def user(user_id):
     user = User.query.get_or_404(user_id)
     corpora = Corpus.query.filter(Corpus.user == user).all()
@@ -62,7 +53,6 @@ def user(user_id):
 
 
 @bp.route('/users/<hashid:user_id>/settings', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.users.entity.settings', '<i class="material-icons left">settings</i>Settings')
 def user_settings(user_id):
     user = User.query.get_or_404(user_id)
     update_account_information_form = UpdateAccountInformationForm(user)

app/blueprints/api/jobs.py

@@ -5,8 +5,8 @@ from flask import abort, Blueprint
 from werkzeug.exceptions import InternalServerError
 from app import db, hashids
 from app.models import Job, JobInput, JobStatus, TesseractOCRPipelineModel
-from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
 from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema
 
 
 bp = Blueprint('jobs', __name__)
@@ -77,7 +77,7 @@ def delete_job(job_id):
     job = Job.query.get(job_id)
     if job is None:
         abort(404)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     try:
         job.delete()
@@ -97,6 +97,6 @@ def get_job(job_id):
     job = Job.query.get(job_id)
     if job is None:
         abort(404)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     return job

app/blueprints/api/schemas.py

@@ -10,7 +10,7 @@ from app.models import (
     User,
     UserSettingJobStatusMailNotificationLevel
 )
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 
 
 

app/blueprints/api/users.py

@@ -3,11 +3,11 @@ from apifairy import authenticate, body, response
 from apifairy.decorators import other_responses
 from flask import abort, Blueprint
 from werkzeug.exceptions import InternalServerError
-from app import db
 from app.email import create_message, send
+from app import db
 from app.models import User
-from .schemas import EmptySchema, UserSchema
 from .auth import auth_error_responses, token_auth
+from .schemas import EmptySchema, UserSchema
 
 
 bp = Blueprint('users', __name__)
@@ -60,7 +60,7 @@ def delete_user(user_id):
     user = User.query.get(user_id)
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     user.delete()
     db.session.commit()
@@ -78,7 +78,7 @@ def get_user(user_id):
     user = User.query.get(user_id)
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     return user
 
@@ -94,6 +94,6 @@ def get_user_by_username(username):
     user = User.query.filter(User.username == username).first()
     if user is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     return user

app/blueprints/auth/__init__.py

@@ -0,0 +1,29 @@
+from flask import Blueprint, redirect, request, url_for
+from flask_login import current_user
+from app import db
+
+
+bp = Blueprint('auth', __name__)
+
+
+@bp.before_app_request
+def before_request():
+    if not current_user.is_authenticated:
+        return
+
+    current_user.ping()
+    db.session.commit()
+
+    if (
+        not current_user.confirmed
+        and request.endpoint
+        and request.blueprint != 'auth'
+        and request.endpoint != 'static'
+    ):
+        return redirect(url_for('auth.unconfirmed'))
+
+    if not current_user.terms_of_use_accepted:
+        return redirect(url_for('main.terms_of_use'))
+
+
+from . import routes

app/blueprints/auth/forms.py

@@ -60,7 +60,11 @@ class RegistrationForm(FlaskForm):
 
     def validate_username(self, field):
         if User.query.filter_by(username=field.data).first():
-            raise ValidationError('Username already in use')
+            raise ValidationError('Username already registered')
+
+    def validate_terms_of_use_accepted(self, field):
+        if not field.data:
+            raise ValidationError('Terms of Use not accepted')
 
 
 class LoginForm(FlaskForm):

app/blueprints/auth/routes.py

@@ -1,5 +1,4 @@
 from flask import abort, flash, redirect, render_template, request, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user, login_user, login_required, logout_user
 from app import db
 from app.email import create_message, send
@@ -13,24 +12,7 @@ from .forms import (
 )
 
 
-@bp.before_app_request
-def before_request():
-    """
-    Checks if a user is unconfirmed when visiting specific sites. Redirects to
-    unconfirmed view if user is unconfirmed.
-    """
-    if current_user.is_authenticated:
-        current_user.ping()
-        db.session.commit()
-        if (not current_user.confirmed
-                and request.endpoint
-                and request.blueprint != 'auth'
-                and request.endpoint != 'static'):
-            return redirect(url_for('auth.unconfirmed'))
-
-
 @bp.route('/register', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.register', 'Register')
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
@@ -67,7 +49,6 @@ def register():
 
 
 @bp.route('/login', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.login', 'Login')
 def login():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
@@ -98,7 +79,6 @@ def logout():
 
 
 @bp.route('/unconfirmed')
-@register_breadcrumb(bp, '.unconfirmed', 'Unconfirmed')
 @login_required
 def unconfirmed():
     if current_user.confirmed:
@@ -141,7 +121,6 @@ def confirm(token):
 
 
 @bp.route('/reset-password-request', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.reset_password_request', 'Password Reset')
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
@@ -171,7 +150,6 @@ def reset_password_request():
 
 
 @bp.route('/reset-password/<token>', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.reset_password', 'Password Reset')
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))

app/blueprints/contributions/__init__.py

@@ -0,0 +1,25 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('contributions', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes
+
+
+from .spacy_nlp_pipeline_models import bp as spacy_nlp_pipeline_models_bp
+bp.register_blueprint(spacy_nlp_pipeline_models_bp, url_prefix='/spacy-nlp-pipeline-models')
+
+from .tesseract_ocr_pipeline_models import bp as tesseract_ocr_pipeline_models_bp
+bp.register_blueprint(tesseract_ocr_pipeline_models_bp, url_prefix='/tesseract-ocr-pipeline-models')

app/blueprints/contributions/routes.py

@@ -0,0 +1,7 @@
+from flask import render_template
+from . import bp
+
+
+@bp.route('')
+def index():
+    return render_template('contributions/index.html.j2', title='Contributions')

app/blueprints/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -0,0 +1,18 @@
+from flask import current_app, Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('spacy_nlp_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/contributions/spacy_nlp_pipeline_models/forms.py

@@ -1,7 +1,7 @@
 from flask_wtf.file import FileField, FileRequired
 from wtforms import StringField, ValidationError
 from wtforms.validators import InputRequired, Length
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 from ..forms import ContributionBaseForm, UpdateContributionBaseForm
 
 

app/blueprints/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -1,13 +1,14 @@
 from flask import abort, current_app, request
-from flask_login import current_user
+from flask_login import current_user, login_required
 from threading import Thread
 from app import db
 from app.decorators import content_negotiation, permission_required
 from app.models import SpaCyNLPPipelineModel
-from .. import bp
+from . import bp
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@login_required
 @content_negotiation(produces='application/json')
 def delete_spacy_model(spacy_nlp_pipeline_model_id):
     def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
@@ -17,7 +18,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
             db.session.commit()
 
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (snpm.user == current_user or current_user.is_administrator()):
+    if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
     thread = Thread(
         target=_delete_spacy_model,
@@ -31,7 +32,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
@@ -39,7 +40,7 @@ def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):
     if not isinstance(is_public, bool):
         abort(400)
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (snpm.user == current_user or current_user.is_administrator()):
+    if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
     snpm.is_public = is_public
     db.session.commit()

app/blueprints/contributions/spacy_nlp_pipeline_models/routes.py

@@ -1,6 +1,5 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
-from flask_login import current_user
+from flask_login import current_user, login_required
 from app import db
 from app.models import SpaCyNLPPipelineModel
 from . import bp
@@ -8,23 +7,17 @@ from .forms import (
     CreateSpaCyNLPPipelineModelForm,
     UpdateSpaCyNLPPipelineModelForm
 )
-from .utils import (
-    spacy_nlp_pipeline_model_dlc as spacy_nlp_pipeline_model_dlc
-)
 
 
-@bp.route('/spacy-nlp-pipeline-models')
-@register_breadcrumb(bp, '.spacy_nlp_pipeline_models', 'SpaCy NLP Pipeline Models')
-def spacy_nlp_pipeline_models():
-    return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
-        title='SpaCy NLP Pipeline Models'
-    )
+@bp.route('/')
+@login_required
+def index():
+    return redirect(url_for('contributions.index', _anchor='spacy-nlp-pipeline-models'))
 
 
-@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.create', 'Create')
-def create_spacy_nlp_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+@login_required
+def create():
     form = CreateSpaCyNLPPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -48,7 +41,7 @@ def create_spacy_nlp_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
-        return {}, 201, {'Location': url_for('.spacy_nlp_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/spacy_nlp_pipeline_models/create.html.j2',
         title='Create SpaCy NLP Pipeline Model',
@@ -56,11 +49,11 @@ def create_spacy_nlp_pipeline_model():
     )
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.spacy_nlp_pipeline_models.entity', '', dynamic_list_constructor=spacy_nlp_pipeline_model_dlc)
-def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+@login_required
+def entity(spacy_nlp_pipeline_model_id):
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
-    if not (snpm.user == current_user or current_user.is_administrator()):
+    if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
     form = UpdateSpaCyNLPPipelineModelForm(data=snpm.to_json_serializeable())
     if form.validate_on_submit():
@@ -68,9 +61,9 @@ def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
         if db.session.is_modified(snpm):
             flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.spacy_nlp_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_model.html.j2',
+        'contributions/spacy_nlp_pipeline_models/entity.html.j2',
         title=f'{snpm.title} {snpm.version}',
         form=form,
         spacy_nlp_pipeline_model=snpm

app/blueprints/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -2,7 +2,7 @@ from flask import Blueprint
 from flask_login import login_required
 
 
-bp = Blueprint('contributions', __name__)
+bp = Blueprint('tesseract_ocr_pipeline_models', __name__)
 
 
 @bp.before_request
@@ -15,9 +15,4 @@ def before_request():
     pass
 
 
-from . import (
-    routes,
-    spacy_nlp_pipeline_models,
-    tesseract_ocr_pipeline_models,
-    transkribus_htr_pipeline_models
-)
+from . import json_routes, routes

app/blueprints/contributions/tesseract_ocr_pipeline_models/forms.py

@@ -1,6 +1,6 @@
 from flask_wtf.file import FileField, FileRequired
 from wtforms import ValidationError
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 from ..forms import ContributionBaseForm, UpdateContributionBaseForm
 
 

app/blueprints/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -7,7 +7,7 @@ from app.models import TesseractOCRPipelineModel
 from . import bp
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
 @content_negotiation(produces='application/json')
 def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
@@ -17,7 +17,7 @@ def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
             db.session.commit()
 
     topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (topm.user == current_user or current_user.is_administrator()):
+    if not (topm.user == current_user or current_user.is_administrator):
         abort(403)
     thread = Thread(
         target=_delete_tesseract_ocr_pipeline_model,
@@ -31,7 +31,7 @@ def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):
@@ -39,7 +39,7 @@ def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_i
     if not isinstance(is_public, bool):
         abort(400)
     topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (topm.user == current_user or current_user.is_administrator()):
+    if not (topm.user == current_user or current_user.is_administrator):
         abort(403)
     topm.is_public = is_public
     db.session.commit()

app/blueprints/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -1,5 +1,4 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
 from app import db
 from app.models import TesseractOCRPipelineModel
@@ -8,23 +7,15 @@ from .forms import (
     CreateTesseractOCRPipelineModelForm,
     UpdateTesseractOCRPipelineModelForm
 )
-from .utils import (
-    tesseract_ocr_pipeline_model_dlc as tesseract_ocr_pipeline_model_dlc
-)
 
 
-@bp.route('/tesseract-ocr-pipeline-models')
-@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models', 'Tesseract OCR Pipeline Models')
-def tesseract_ocr_pipeline_models():
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
-        title='Tesseract OCR Pipeline Models'
-    )
+@bp.route('/')
+def index():
+    return redirect(url_for('contributions.index', _anchor='tesseract-ocr-pipeline-models'))
 
 
-@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.create', 'Create')
-def create_tesseract_ocr_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+def create():
     form = CreateTesseractOCRPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -47,7 +38,7 @@ def create_tesseract_ocr_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
-        return {}, 201, {'Location': url_for('.tesseract_ocr_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/tesseract_ocr_pipeline_models/create.html.j2',
         title='Create Tesseract OCR Pipeline Model',
@@ -55,11 +46,10 @@ def create_tesseract_ocr_pipeline_model():
     )
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.tesseract_ocr_pipeline_models.entity', '', dynamic_list_constructor=tesseract_ocr_pipeline_model_dlc)
-def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+def entity(tesseract_ocr_pipeline_model_id):
     topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
-    if not (topm.user == current_user or current_user.is_administrator()):
+    if not (topm.user == current_user or current_user.is_administrator):
         abort(403)
     form = UpdateTesseractOCRPipelineModelForm(data=topm.to_json_serializeable())
     if form.validate_on_submit():
@@ -67,9 +57,9 @@ def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
         if db.session.is_modified(topm):
             flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.tesseract_ocr_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_model.html.j2',
+        'contributions/tesseract_ocr_pipeline_models/entity.html.j2',
         title=f'{topm.title} {topm.version}',
         form=form,
         tesseract_ocr_pipeline_model=topm

app/blueprints/corpora/cli.py

@@ -1,7 +1,7 @@
-from app.models import Corpus, CorpusStatus
-import os
+from flask import current_app
 import shutil
 from app import db
+from app.models import Corpus, CorpusStatus
 from . import bp
 
 
@@ -18,10 +18,17 @@ def reset():
     ]
     for corpus in [x for x in Corpus.query.all() if x.status in status]:
         print(f'Resetting corpus {corpus}')
-        shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
-        os.mkdir(os.path.join(corpus.path, 'cwb'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'data'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'registry'))
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
         corpus.status = CorpusStatus.UNPREPARED
         corpus.num_analysis_sessions = 0
     db.session.commit()

app/blueprints/corpora/decorators.py

@@ -10,7 +10,7 @@ def corpus_follower_permission_required(*permissions):
         def decorated_function(*args, **kwargs):
             corpus_id = kwargs.get('corpus_id')
             corpus = Corpus.query.get_or_404(corpus_id)
-            if not (corpus.user == current_user or current_user.is_administrator()):
+            if not (corpus.user == current_user or current_user.is_administrator):
                 cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
                 if cfa is None:
                     abort(403)
@@ -26,7 +26,7 @@ def corpus_owner_or_admin_required(f):
     def decorated_function(*args, **kwargs):
         corpus_id = kwargs.get('corpus_id')
         corpus = Corpus.query.get_or_404(corpus_id)
-        if not (corpus.user == current_user or current_user.is_administrator()):
+        if not (corpus.user == current_user or current_user.is_administrator):
             abort(403)
         return f(*args, **kwargs)
     return decorated_function

app/blueprints/corpora/events.py

@@ -15,7 +15,7 @@ def get_corpus(corpus_hashid):
     if not (
         corpus.is_public
         or corpus.user == current_user
-        or current_user.is_administrator()
+        or current_user.is_administrator
     ):
         return {'options': {'status': 403, 'statusText': 'Forbidden'}}
     return {
@@ -38,7 +38,7 @@ def subscribe_corpus(corpus_hashid):
     if not (
         corpus.is_public
         or corpus.user == current_user
-        or current_user.is_administrator()
+        or current_user.is_administrator
     ):
         return {'options': {'status': 403, 'statusText': 'Forbidden'}}
     join_room(f'/corpora/{corpus.hashid}')

app/blueprints/corpora/files/json_routes.py

@@ -1,7 +1,7 @@
-from flask import abort, current_app
+from flask import current_app
 from threading import Thread
-from app import db
 from app.decorators import content_negotiation
+from app import db
 from app.models import CorpusFile
 from ..decorators import corpus_follower_permission_required
 from . import bp

app/blueprints/corpora/files/routes.py

@@ -6,25 +6,19 @@ from flask import (
     send_from_directory,
     url_for
 )
-from flask_breadcrumbs import register_breadcrumb
-import os
 from app import db
 from app.models import Corpus, CorpusFile, CorpusStatus
 from ..decorators import corpus_follower_permission_required
-from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
 from . import bp
 from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
-from .utils import corpus_file_dynamic_list_constructor as corpus_file_dlc
 
 
 @bp.route('/<hashid:corpus_id>/files')
-@register_breadcrumb(bp, '.entity.files', 'Files', endpoint_arguments_constructor=corpus_eac)
 def corpus_files(corpus_id):
     return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
 
 
 @bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.entity.files.create', 'Create', endpoint_arguments_constructor=corpus_eac)
 @corpus_follower_permission_required('MANAGE_FILES')
 def create_corpus_file(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
@@ -66,7 +60,6 @@ def create_corpus_file(corpus_id):
 
 
 @bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.entity.files.entity', '', dynamic_list_constructor=corpus_file_dlc)
 @corpus_follower_permission_required('MANAGE_FILES')
 def corpus_file(corpus_id, corpus_file_id):
     corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
@@ -92,9 +85,9 @@ def corpus_file(corpus_id, corpus_file_id):
 def download_corpus_file(corpus_id, corpus_file_id):
     corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
     return send_from_directory(
-        os.path.dirname(corpus_file.path),
-        os.path.basename(corpus_file.path),
+        corpus_file.path.parent,
+        corpus_file.path.name,
         as_attachment=True,
-        attachment_filename=corpus_file.filename,
+        download_name=corpus_file.filename,
         mimetype=corpus_file.mimetype
     )

app/blueprints/corpora/followers/json_routes.py

@@ -58,7 +58,7 @@ def delete_corpus_follower(corpus_id, follower_id):
         current_user.id == follower_id
         or current_user == cfa.corpus.user 
         or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
-        or current_user.is_administrator()):
+        or current_user.is_administrator):
         abort(403)
     if current_user.id == follower_id:
         flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')

app/blueprints/corpora/routes.py

@@ -1,5 +1,4 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
 from app import db
 from app.models import (
@@ -11,20 +10,14 @@ from app.models import (
 from . import bp
 from .decorators import corpus_follower_permission_required
 from .forms import CreateCorpusForm
-from .utils import (
-    corpus_endpoint_arguments_constructor as corpus_eac,
-    corpus_dynamic_list_constructor as corpus_dlc
-)
 
 
 @bp.route('')
-@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">I</i>My Corpora')
 def corpora():
     return redirect(url_for('main.dashboard', _anchor='corpora'))
 
 
 @bp.route('/create', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.create', 'Create')
 def create_corpus():
     form = CreateCorpusForm()
     if form.validate_on_submit():
@@ -47,7 +40,6 @@ def create_corpus():
 
 
 @bp.route('/<hashid:corpus_id>')
-@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=corpus_dlc)
 def corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     cfrs = CorpusFollowerRole.query.all()
@@ -55,13 +47,13 @@ def corpus(corpus_id):
     users = User.query.filter(User.is_public == True, User.id != current_user.id, User.id != corpus.user.id, User.role_id < 4).all()
     cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
     if cfa is None:
-        if corpus.user == current_user or current_user.is_administrator():
+        if corpus.user == current_user or current_user.is_administrator:
             cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
         else:
             cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
     else:
         cfr = cfa.role
-    if corpus.user == current_user or current_user.is_administrator():
+    if corpus.user == current_user or current_user.is_administrator:
         return render_template(
             'corpora/corpus.html.j2',
             title=corpus.title,
@@ -87,7 +79,6 @@ def corpus(corpus_id):
 
 @bp.route('/<hashid:corpus_id>/analysis')
 @corpus_follower_permission_required('VIEW')
-@register_breadcrumb(bp, '.entity.analysis', 'Analysis', endpoint_arguments_constructor=corpus_eac)
 def analysis(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     return render_template(
@@ -108,13 +99,11 @@ def follow_corpus(corpus_id, token):
 
 
 @bp.route('/import', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.import', 'Import')
 def import_corpus():
     abort(503)
 
 
 @bp.route('/<hashid:corpus_id>/export')
 @corpus_follower_permission_required('VIEW')
-@register_breadcrumb(bp, '.entity.export', 'Export', endpoint_arguments_constructor=corpus_eac)
 def export_corpus(corpus_id):
     abort(503)

app/blueprints/jobs/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('jobs', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/jobs/json_routes.py

@@ -1,7 +1,6 @@
 from flask import abort, current_app
 from flask_login import current_user
 from threading import Thread
-import os
 from app import db
 from app.decorators import admin_required, content_negotiation
 from app.models import Job, JobStatus
@@ -18,7 +17,7 @@ def delete_job(job_id):
             db.session.commit()
 
     job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     thread = Thread(
         target=_delete_job,
@@ -39,7 +38,7 @@ def job_log(job_id):
     if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
         response = {'errors': {'message': 'Job status is not completed or failed'}}
         return response, 409
-    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+    with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
         log = log_file.read()
     response_data = {
         'jobLog': log
@@ -57,7 +56,7 @@ def restart_job(job_id):
             db.session.commit()
 
     job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     if job.status == JobStatus.FAILED:
         response = {'errors': {'message': 'Job status is not "failed"'}}

app/blueprints/jobs/routes.py

@@ -5,25 +5,20 @@ from flask import (
     send_from_directory,
     url_for
 )
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
-import os
 from app.models import Job, JobInput, JobResult
 from . import bp
-from .utils import job_dynamic_list_constructor as job_dlc
 
 
 @bp.route('')
-@register_breadcrumb(bp, '.', '<i class="nopaque-icons left">J</i>My Jobs')
-def corpora():
+def jobs():
     return redirect(url_for('main.dashboard', _anchor='jobs'))
 
 
 @bp.route('/<hashid:job_id>')
-@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=job_dlc)
 def job(job_id):
     job = Job.query.get_or_404(job_id)
-    if not (job.user == current_user or current_user.is_administrator()):
+    if not (job.user == current_user or current_user.is_administrator):
         abort(403)
     return render_template(
         'jobs/job.html.j2',
@@ -35,13 +30,13 @@ def job(job_id):
 @bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
 def download_job_input(job_id, job_input_id):
     job_input = JobInput.query.filter_by(job_id=job_id, id=job_input_id).first_or_404()
-    if not (job_input.job.user == current_user or current_user.is_administrator()):
+    if not (job_input.job.user == current_user or current_user.is_administrator):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_input.path),
-        os.path.basename(job_input.path),
+        job_input.path.parent,
+        job_input.path.name,
         as_attachment=True,
-        attachment_filename=job_input.filename,
+        download_name=job_input.filename,
         mimetype=job_input.mimetype
     )
 
@@ -49,12 +44,12 @@ def download_job_input(job_id, job_input_id):
 @bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
 def download_job_result(job_id, job_result_id):
     job_result = JobResult.query.filter_by(job_id=job_id, id=job_result_id).first_or_404()
-    if not (job_result.job.user == current_user or current_user.is_administrator()):
+    if not (job_result.job.user == current_user or current_user.is_administrator):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_result.path),
-        os.path.basename(job_result.path),
+        job_result.path.parent,
+        job_result.path.name,
         as_attachment=True,
-        attachment_filename=job_result.filename,
+        download_name=job_result.filename,
         mimetype=job_result.mimetype
     )

app/blueprints/main/cli.py

@@ -1,7 +1,9 @@
 from flask import current_app
 from flask_migrate import upgrade
-import os
+from pathlib import Path
+from app import db
 from app.models import (
+    Corpus,
     CorpusFollowerRole,
     Role,
     SpaCyNLPPipelineModel,
@@ -14,25 +16,22 @@ from . import bp
 @bp.cli.command('deploy')
 def deploy():
     ''' Run deployment tasks. '''
-    # Make default directories
+
     print('Make default directories')
     base_dir = current_app.config['NOPAQUE_DATA_DIR']
-    default_dirs = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
+    default_dirs: list[Path] = [
+        base_dir / 'tmp',
+        base_dir / 'users'
     ]
-    for dir in default_dirs:
-        if os.path.exists(dir):
-            if not os.path.isdir(dir):
-                raise NotADirectoryError(f'{dir} is not a directory')
-        else:
-            os.mkdir(dir)
+    for default_dir in default_dirs:
+        if not default_dir.exists():
+            default_dir.mkdir()
+        if not default_dir.is_dir():
+            raise NotADirectoryError(f'{default_dir} is not a directory')
 
-    # migrate database to latest revision
     print('Migrate database to latest revision')
     upgrade()
 
-    # Insert/Update default database values
     print('Insert/Update default Roles')
     Role.insert_defaults()
     print('Insert/Update default Users')
@@ -44,4 +43,9 @@ def deploy():
     print('Insert/Update default TesseractOCRPipelineModels')
     TesseractOCRPipelineModel.insert_defaults()
 
+    print('Stop running analysis sessions')
+    for corpus in Corpus.query.all():
+        corpus.num_analysis_sessions = 0
+    db.session.commit()
+
     # TODO: Implement checks for if the nopaque network exists

app/blueprints/main/routes.py

@@ -1,14 +1,11 @@
 from flask import flash, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user, login_required, login_user
-from app.auth.forms import LoginForm
+from app.blueprints.auth.forms import LoginForm
 from app.models import Corpus, User
-from sqlalchemy import or_
 from . import bp
 
 
 @bp.route('/', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.', '<i class="material-icons">home</i>')
 def index():
     form = LoginForm()
     if form.validate_on_submit():
@@ -27,7 +24,6 @@ def index():
 
 
 @bp.route('/faq')
-@register_breadcrumb(bp, '.faq', 'Frequently Asked Questions')
 def faq():
     return render_template(
         'main/faq.html.j2',
@@ -36,7 +32,6 @@ def faq():
 
 
 @bp.route('/dashboard')
-@register_breadcrumb(bp, '.dashboard', '<i class="material-icons left">dashboard</i>Dashboard')
 @login_required
 def dashboard():
     return render_template(
@@ -45,8 +40,15 @@ def dashboard():
     )
 
 
+@bp.route('/manual')
+def manual():
+    return render_template(
+        'main/manual.html.j2',
+        title='Manual'
+    )
+
+
 @bp.route('/news')
-@register_breadcrumb(bp, '.news', '<i class="material-icons left">email</i>News')
 def news():
     return render_template(
         'main/news.html.j2',
@@ -55,7 +57,6 @@ def news():
 
 
 @bp.route('/privacy_policy')
-@register_breadcrumb(bp, '.privacy_policy', 'Private statement (GDPR)')
 def privacy_policy():
     return render_template(
         'main/privacy_policy.html.j2',
@@ -64,7 +65,6 @@ def privacy_policy():
 
 
 @bp.route('/terms_of_use')
-@register_breadcrumb(bp, '.terms_of_use', 'Terms of Use')
 def terms_of_use():
     return render_template(
         'main/terms_of_use.html.j2',
@@ -72,17 +72,14 @@ def terms_of_use():
     )
 
 
-@bp.route('/social-area')
-@register_breadcrumb(bp, '.social_area', '<i class="material-icons left">group</i>Social Area')
+@bp.route('/social')
 @login_required
-def social_area():
-    print('test')
+def social():
     corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
-    print(corpora)
     users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
     return render_template(
-        'main/social_area.html.j2',
-        title='Social Area',
+        'main/social.html.j2',
+        title='Social',
         corpora=corpora,
         users=users
     )

app/blueprints/services/__init__.py

@@ -1,12 +1,11 @@
 from flask import Blueprint
 from flask_login import login_required
-import os
+from pathlib import Path
 import yaml
 
 
-services_file = \
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'services.yml')
-with open(services_file, 'r') as f:
+services_file = Path(__file__).parent / 'services.yml'
+with services_file.open('r') as f:
     SERVICES = yaml.safe_load(f)
 
 bp = Blueprint('services', __name__)

app/blueprints/services/forms.py

@@ -167,7 +167,6 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
         service_info = service_manifest['versions'][version]
-        print(service_info)
         if self.encoding_detection.render_kw is None:
             self.encoding_detection.render_kw = {}
         self.encoding_detection.render_kw['disabled'] = True

app/blueprints/services/routes.py

@@ -1,5 +1,4 @@
-from flask import abort, current_app, flash, Markup, redirect, render_template, request, url_for
-from flask_breadcrumbs import register_breadcrumb
+from flask import abort, current_app, flash, redirect, render_template, request, url_for
 from flask_login import current_user
 import requests
 from app import db, hashids
@@ -20,13 +19,11 @@ from .forms import (
 
 
 @bp.route('/services')
-@register_breadcrumb(bp, '.', 'Services')
 def services():
     return redirect(url_for('main.dashboard'))
 
 
 @bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.file_setup_pipeline', '<i class="nopaque-icons service-icons left" data-service="file-setup-pipeline"></i>File Setup')
 def file_setup_pipeline():
     service = 'file-setup-pipeline'
     service_manifest = SERVICES[service]
@@ -56,7 +53,7 @@ def file_setup_pipeline():
                 abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
@@ -67,7 +64,6 @@ def file_setup_pipeline():
 
 
 @bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.tesseract_ocr_pipeline', '<i class="nopaque-icons service-icons left" data-service="tesseract-ocr-pipeline"></i>Tesseract OCR Pipeline')
 def tesseract_ocr_pipeline():
     service_name = 'tesseract-ocr-pipeline'
     service_manifest = SERVICES[service_name]
@@ -100,7 +96,7 @@ def tesseract_ocr_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     tesseract_ocr_pipeline_models = [
@@ -118,7 +114,6 @@ def tesseract_ocr_pipeline():
 
 
 @bp.route('/transkribus-htr-pipeline', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.transkribus_htr_pipeline', '<i class="nopaque-icons service-icons left" data-service="transkribus-htr-pipeline"></i>Transkribus HTR Pipeline')
 def transkribus_htr_pipeline():
     if not current_app.config.get('NOPAQUE_TRANSKRIBUS_ENABLED'):
         abort(404)
@@ -164,7 +159,7 @@ def transkribus_htr_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
@@ -176,7 +171,6 @@ def transkribus_htr_pipeline():
 
 
 @bp.route('/spacy-nlp-pipeline', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.spacy_nlp_pipeline', '<i class="nopaque-icons service-icons left" data-service="spacy-nlp-pipeline"></i>SpaCy NLP Pipeline')
 def spacy_nlp_pipeline():
     service = 'spacy-nlp-pipeline'
     service_manifest = SERVICES[service]
@@ -210,7 +204,7 @@ def spacy_nlp_pipeline():
             abort(500)
         job.status = JobStatus.SUBMITTED
         db.session.commit()
-        message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+        message = f'Job "<a href="{job.url}">{job.title}</a>" created'
         flash(message, 'job')
         return {}, 201, {'Location': job.url}
     return render_template(
@@ -223,7 +217,6 @@ def spacy_nlp_pipeline():
 
 
 @bp.route('/corpus-analysis')
-@register_breadcrumb(bp, '.corpus_analysis', '<i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus Analysis')
 def corpus_analysis():
     return render_template(
         'services/corpus_analysis.html.j2',

app/blueprints/settings/routes.py

@@ -1,12 +1,10 @@
 from flask import g, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
-from app.users.settings.routes import settings as settings_route
+from app.blueprints.users.settings.routes import settings as settings_route
 from . import bp
 
 
 @bp.route('/settings', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.', '<i class="material-icons left">settings</i>Settings')
 def settings():
     g._nopaque_redirect_location_on_post = url_for('.settings')
     return settings_route(current_user.id)

app/blueprints/users/__init__.py

@@ -15,4 +15,4 @@ def before_request():
     pass
 
 
-from . import cli, events, json_routes, routes, settings
+from . import cli, json_routes, routes, settings

app/blueprints/users/cli.py

@@ -1,6 +1,4 @@
 from app.models import User
-import os
-import shutil
 from app import db
 from . import bp
 

app/blueprints/users/json_routes.py

@@ -17,7 +17,7 @@ def delete_user(user_id):
             db.session.commit()
 
     user = User.query.get_or_404(user_id)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     thread = Thread(
         target=_delete_user,
@@ -44,7 +44,7 @@ def delete_user_avatar(user_id):
     user = User.query.get_or_404(user_id)
     if user.avatar is None:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     thread = Thread(
         target=_delete_avatar,

app/blueprints/users/routes.py

@@ -5,25 +5,20 @@ from flask import (
     send_from_directory,
     url_for
 )
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
-import os
 from app.models import User
 from . import bp
-from .utils import user_dynamic_list_constructor as user_dlc
 
 
 @bp.route('')
-@register_breadcrumb(bp, '.', '<i class="material-icons left">group</i>Users')
 def users():
     return redirect(url_for('main.social_area', _anchor='users'))
 
 
 @bp.route('/<hashid:user_id>')
-@register_breadcrumb(bp, '.entity', '', dynamic_list_constructor=user_dlc)
 def user(user_id):
     user = User.query.get_or_404(user_id)
-    if not (user.is_public or user == current_user or current_user.is_administrator()):
+    if not (user.is_public or user == current_user or current_user.is_administrator):
         abort(403)
     return render_template(
         'users/user.html.j2',
@@ -35,14 +30,14 @@ def user(user_id):
 @bp.route('/<hashid:user_id>/avatar')
 def user_avatar(user_id):
     user = User.query.get_or_404(user_id)
-    if not (user.is_public or user == current_user or current_user.is_administrator()):
+    if not (user.is_public or user == current_user or current_user.is_administrator):
         abort(403)
     if user.avatar is None:
         return redirect(url_for('static', filename='images/user_avatar.png'))
     return send_from_directory(
-        os.path.dirname(user.avatar.path),
-        os.path.basename(user.avatar.path),
+        user.avatar.path.parent,
+        user.avatar.path.name,
         as_attachment=True,
-        attachment_filename=user.avatar.filename,
+        download_name=user.avatar.filename,
         mimetype=user.avatar.mimetype
     )

app/blueprints/users/settings/forms.py

@@ -1,6 +1,5 @@
-from flask_login import current_user
 from flask_wtf import FlaskForm
-from flask_wtf.file import FileField, FileRequired
+from flask_wtf.file import FileField, FileRequired, FileSize
 from wtforms import (
     PasswordField,
     SelectField,
@@ -17,7 +16,6 @@ from wtforms.validators import (
     Regexp
 )
 from app.models import User, UserSettingJobStatusMailNotificationLevel
-from app.wtforms.validators import FileSize
 
 
 class UpdateAccountInformationForm(FlaskForm):
@@ -100,7 +98,7 @@ class UpdateProfileInformationForm(FlaskForm):
 
 
 class UpdateAvatarForm(FlaskForm):
-    avatar = FileField('File', validators=[FileRequired(), FileSize(2)])
+    avatar = FileField('File', validators=[FileRequired(), FileSize(2_000_000)])
     submit = SubmitField()
 
     def validate_avatar(self, field):

app/blueprints/users/settings/json_routes.py

@@ -10,7 +10,7 @@ from . import bp
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_user_profile_privacy_setting_is_public(user_id):
     user = User.query.get_or_404(user_id)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     enabled = request.json
     if not isinstance(enabled, bool):
@@ -32,7 +32,7 @@ def update_user_profile_privacy_settings(user_id, profile_privacy_setting_name):
         profile_privacy_setting = ProfilePrivacySettings[profile_privacy_setting_name]
     except KeyError:
         abort(404)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
     enabled = request.json
     if not isinstance(enabled, bool):

app/blueprints/users/settings/routes.py

@@ -1,9 +1,7 @@
 from flask import abort, flash, g, redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
 from app import db
 from app.models import Avatar, User
-from ..utils import user_endpoint_arguments_constructor as user_eac
 from . import bp
 from .forms import (
     UpdateAvatarForm,
@@ -15,10 +13,9 @@ from .forms import (
 
 
 @bp.route('/<hashid:user_id>/settings', methods=['GET', 'POST'])
-@register_breadcrumb(bp, '.entity.settings', '<i class="material-icons left">settings</i>Settings', endpoint_arguments_constructor=user_eac)
 def settings(user_id):
     user = User.query.get_or_404(user_id)
-    if not (user == current_user or current_user.is_administrator()):
+    if not (user == current_user or current_user.is_administrator):
         abort(403)
 
     redirect_location_on_post = g.pop(

app/blueprints/workshops/routes.py

@@ -1,16 +1,13 @@
 from flask import redirect, render_template, url_for
-from flask_breadcrumbs import register_breadcrumb
 from . import bp
 
 
 @bp.route('')
-@register_breadcrumb(bp, '.', '<i class="material-icons left">business_center</i>Workshops')
 def workshops():
     return redirect(url_for('main.dashboard'))
 
 
 @bp.route('/fgho_sommerschule_2023')
-@register_breadcrumb(bp, '.fgho_sommerschule_2023', 'FGHO Sommerschule 2023')
 def fgho_sommerschule_2023():
     return render_template(
         'workshops/fgho_sommerschule_2023.html.j2',

app/contributions/routes.py

@@ -1,9 +0,0 @@
-from flask import redirect, url_for
-from flask_breadcrumbs import register_breadcrumb
-from . import bp
-
-
-@bp.route('')
-@register_breadcrumb(bp, '.', '<i class="material-icons left">new_label</i>My Contributions')
-def contributions():
-    return redirect(url_for('main.dashboard', _anchor='contributions'))

app/contributions/spacy_nlp_pipeline_models/utils.py

@@ -1,13 +0,0 @@
-from flask import request, url_for
-from app.models import SpaCyNLPPipelineModel
-
-
-def spacy_nlp_pipeline_model_dlc():
-    snpm_id = request.view_args['spacy_nlp_pipeline_model_id']
-    snpm = SpaCyNLPPipelineModel.query.get_or_404(snpm_id)
-    return [
-        {
-            'text': f'{snpm.title} {snpm.version}',
-            'url': url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=snpm_id)
-        }
-    ]

app/contributions/tesseract_ocr_pipeline_models/utils.py

@@ -1,13 +0,0 @@
-from flask import request, url_for
-from app.models import TesseractOCRPipelineModel
-
-
-def tesseract_ocr_pipeline_model_dlc():
-    topm_id = request.view_args['tesseract_ocr_pipeline_model_id']
-    topm = TesseractOCRPipelineModel.query.get_or_404(topm_id)
-    return [
-        {
-            'text': f'{topm.title} {topm.version}',
-            'url': url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=topm_id)
-        }
-    ]

app/contributions/transkribus_htr_pipeline_models/__init__.py

@@ -1,2 +0,0 @@
-from .. import bp
-from . import routes

app/contributions/transkribus_htr_pipeline_models/routes.py

@@ -1,7 +0,0 @@
-from flask import abort
-from . import bp
-
-
-@bp.route('/transkribus_htr_pipeline_models')
-def transkribus_htr_pipeline_models():
-    return abort(503)

app/converters/sandpaper.py

@@ -1,81 +1,69 @@
+from datetime import datetime
 from flask import current_app
+from pathlib import Path
+import json
+import shutil
 from app import db
 from app.models import User, Corpus, CorpusFile
-from datetime import datetime
-import json
-import os
-import shutil
 
 
 class SandpaperConverter:
-    def __init__(self, json_db_file, data_dir):
+    def __init__(self, json_db_file: Path, data_dir: Path):
         self.json_db_file = json_db_file
         self.data_dir = data_dir
 
     def run(self):
-        with open(self.json_db_file, 'r') as f:
-            json_db = json.loads(f.read())
+        with self.json_db_file.open('r') as f:
+            json_db: list[dict] = json.load(f)
 
         for json_user in json_db:
             if not json_user['confirmed']:
                 current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
                 continue
-            user_dir = os.path.join(self.data_dir, str(json_user['id']))
+            user_dir = self.data_dir / f'{json_user["id"]}'
             self.convert_user(json_user, user_dir)
             db.session.commit()
 
 
-    def convert_user(self, json_user, user_dir):
+    def convert_user(self, json_user: dict, user_dir: Path):
         current_app.logger.info(f'Create User {json_user["username"]}...')
-        user = User(
-            confirmed=json_user['confirmed'],
-            email=json_user['email'],
-            last_seen=datetime.fromtimestamp(json_user['last_seen']),
-            member_since=datetime.fromtimestamp(json_user['member_since']),
-            password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
-            username=json_user['username']
-        )
-        db.session.add(user)
-        db.session.flush(objects=[user])
-        db.session.refresh(user)
         try:
-            user.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+            user = User.create(
+                confirmed=json_user['confirmed'],
+                email=json_user['email'],
+                last_seen=datetime.fromtimestamp(json_user['last_seen']),
+                member_since=datetime.fromtimestamp(json_user['member_since']),
+                password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
+                username=json_user['username']
+            )
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus in json_user['corpora'].values():
             if not json_corpus['files'].values():
                 current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
                 continue
-            corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
+            corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
             self.convert_corpus(json_corpus, user, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus(self, json_corpus, user, corpus_dir):
+    def convert_corpus(self, json_corpus: dict, user: User, corpus_dir: Path):
         current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
-        corpus = Corpus(
-            user=user,
-            creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
-            description=json_corpus['description'],
-            title=json_corpus['title']
-        )
-        db.session.add(corpus)
-        db.session.flush(objects=[corpus])
-        db.session.refresh(corpus)
         try:
-            corpus.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+            corpus = Corpus.create(
+                user=user,
+                creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
+                description=json_corpus['description'],
+                title=json_corpus['title']
+            )
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus_file in json_corpus['files'].values():
             self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
+    def convert_corpus_file(self, json_corpus_file: dict, corpus: Corpus, corpus_dir: Path):
         current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
         corpus_file = CorpusFile(
             corpus=corpus,
@@ -99,13 +87,13 @@ class SandpaperConverter:
         db.session.refresh(corpus_file)
         try:
             shutil.copy2(
-                os.path.join(corpus_dir, json_corpus_file['filename']),
+                corpus_dir / json_corpus_file['filename'],
                 corpus_file.path
             )
         except:
             current_app.logger.warning(
                 'Can not convert corpus file: '
-                f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
+                f'{corpus_dir / json_corpus_file["filename"]}'
                 ' -> '
                 f'{corpus_file.path}'
             )

app/converters/vrt.py

@@ -1,69 +1,25 @@
 from flask import current_app
+from pathlib import Path
 
 
-def normalize_vrt_file(input_file, output_file):
-    def check_pos_attribute_order(vrt_lines):
-        # The following orders are possible:
-        # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
-        # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
-        # since 27.01.2022: 'word,pos,lemma,simple_pos'
-        # This Function tries to find out which order we have by looking at the
-        # number of attributes and the position of the simple_pos attribute
-        SIMPLE_POS_LABELS = [
-            'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ',
-            'DET', 'INTJ', 'NOUN', 'NUM', 'PART',
-            'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',
-            'VERB', 'X'
-        ]
-        for line in vrt_lines:
-            if line.startswith('<'):
-                continue
-            pos_attrs = line.rstrip('\n').split('\t')
-            num_pos_attrs = len(pos_attrs)
-            if num_pos_attrs == 4:
-                if pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos']
-                continue
-            elif num_pos_attrs == 5:
-                if pos_attrs[2] in SIMPLE_POS_LABELS:
-                    return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
-                elif pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
-                continue
-        return None
-
-
-    def check_has_ent_as_s_attr(vrt_lines):
-        for line in vrt_lines:
-            if line.startswith('<ent'):
-                return True
-        return False
-
-
-    def pos_attrs_to_string_1(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
-
-
-    def pos_attrs_to_string_2(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'
-
+def normalize_vrt_file(input_file: Path, output_file: Path):
     current_app.logger.info(f'Converting {input_file}...')
 
-    with open(input_file) as f:
+    with input_file.open() as f:
         input_vrt_lines = f.readlines()
 
-    pos_attr_order = check_pos_attribute_order(input_vrt_lines)
-    has_ent_as_s_attr = check_has_ent_as_s_attr(input_vrt_lines)
+    pos_attr_order = _check_pos_attribute_order(input_vrt_lines)
+    has_ent_as_s_attr = _check_has_ent_as_s_attr(input_vrt_lines)
 
     current_app.logger.info(f'Detected pos_attr_order: [{",".join(pos_attr_order)}]')
     current_app.logger.info(f'Detected has_ent_as_s_attr: {has_ent_as_s_attr}')
 
     if pos_attr_order == ['word', 'lemma', 'simple_pos', 'pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_1
+        pos_attrs_to_string_function = _pos_attrs_to_string_1
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     else:
         raise Exception('Can not handle format')
 
@@ -113,5 +69,49 @@ def normalize_vrt_file(input_file, output_file):
                     current_ent = pos_attrs[4]
         output_vrt += pos_attrs_to_string_function(pos_attrs)
 
-    with open(output_file, 'w') as f:
+    with output_file.open(mode='w') as f:
         f.write(output_vrt)
+
+
+def _check_pos_attribute_order(vrt_lines: list[str]) -> list[str]:
+    # The following orders are possible:
+    # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
+    # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
+    # since 27.01.2022: 'word,pos,lemma,simple_pos'
+    # This Function tries to find out which order we have by looking at the
+    # number of attributes and the position of the simple_pos attribute
+    SIMPLE_POS_LABELS = [
+        'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'DET', 'INTJ', 'NOUN', 'NUM',
+        'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X'
+    ]
+    for line in vrt_lines:
+        if line.startswith('<'):
+            continue
+        pos_attrs = line.rstrip('\n').split('\t')
+        num_pos_attrs = len(pos_attrs)
+        if num_pos_attrs == 4:
+            if pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos']
+            continue
+        elif num_pos_attrs == 5:
+            if pos_attrs[2] in SIMPLE_POS_LABELS:
+                return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
+            elif pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
+            continue
+    # TODO: raise exception "can't determine attribute order"
+
+
+def _check_has_ent_as_s_attr(vrt_lines: list[str]) -> bool:
+    for line in vrt_lines:
+        if line.startswith('<ent'):
+            return True
+    return False
+
+
+def _pos_attrs_to_string_1(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
+
+
+def _pos_attrs_to_string_2(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'

app/corpora/cqi_over_sio/utils.py

@@ -1,131 +0,0 @@
-from cqi.models.corpora import Corpus as CQiCorpus
-from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
-from typing import Dict, List
-
-
-def lookups_by_cpos(corpus: CQiCorpus, cpos_list: List[int]) -> Dict:
-    lookups = {}
-    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
-    for attr in corpus.positional_attributes.list():
-        cpos_attr_values: List[str] = attr.values_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
-    for attr in corpus.structural_attributes.list():
-        # We only want to iterate over non subattributes, identifiable by
-        # attr.has_values == False
-        if attr.has_values:
-            continue
-        cpos_attr_ids: List[int] = attr.ids_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            if cpos_attr_ids[i] == -1:
-                continue
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
-        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
-        if len(occured_attr_ids) == 0:
-            continue
-        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
-        if len(subattrs) == 0:
-            continue
-        lookup_name: str = f'{attr.name}_lookup'
-        lookups[lookup_name] = {}
-        for attr_id in occured_attr_ids:
-            lookups[lookup_name][attr_id] = {}
-        for subattr in subattrs:
-            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
-            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
-                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
-    return lookups
-
-
-def partial_export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    match_id_list: List[int],
-    context: int = 25
-) -> Dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    match_boundaries = []
-    for match_id in match_id_list:
-        if match_id < 0 or match_id >= subcorpus.size:
-            continue
-        match_boundaries.append(
-            (
-                match_id,
-                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
-                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
-            )
-        )
-    cpos_set = set()
-    matches = []
-    for match_boundary in match_boundaries:
-        match_num, match_start, match_end = match_boundary
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}
-
-
-def export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    context: int = 25,
-    cutoff: float = float('inf'),
-    offset: int = 0
-) -> Dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    first_match = max(0, offset)
-    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
-    match_boundaries = zip(
-        range(first_match, last_match + 1),
-        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
-        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
-    )
-    cpos_set = set()
-    matches = []
-    for match_num, match_start, match_end in match_boundaries:
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}

app/corpora/files/__init__.py

@@ -1,2 +0,0 @@
-from .. import bp
-from . import json_routes, routes

app/corpora/files/utils.py

@@ -1,15 +0,0 @@
-from flask import request, url_for
-from app.models import CorpusFile
-from ..utils import corpus_endpoint_arguments_constructor as corpus_eac
-
-
-def corpus_file_dynamic_list_constructor():
-    corpus_id = request.view_args['corpus_id']
-    corpus_file_id = request.view_args['corpus_file_id']
-    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
-    return [
-        {
-            'text': f'{corpus_file.author}: {corpus_file.title} ({corpus_file.publishing_year})',
-            'url': url_for('.corpus_file', corpus_id=corpus_id, corpus_file_id=corpus_file_id)
-        }
-    ]

app/corpora/utils.py

@@ -1,17 +0,0 @@
-from flask import request, url_for
-from app.models import Corpus
-
-
-def corpus_endpoint_arguments_constructor():
-    return {'corpus_id': request.view_args['corpus_id']}
-
-
-def corpus_dynamic_list_constructor():
-    corpus_id = request.view_args['corpus_id']
-    corpus = Corpus.query.get_or_404(corpus_id)
-    return [
-        {
-            'text': f'<i class="material-icons left">book</i>{corpus.title}',
-            'url': url_for('.corpus', corpus_id=corpus_id)
-        }
-    ]

app/daemon/__init__.py

@@ -1,11 +0,0 @@
-from app import db
-from flask import Flask
-from .corpus_utils import check_corpora
-from .job_utils import check_jobs
-
-
-def daemon(app: Flask):
-    with app.app_context():
-        check_corpora()
-        check_jobs()
-        db.session.commit()

app/decorators.py

@@ -1,8 +1,7 @@
-from flask import abort, current_app, request
+from flask import abort, request
 from flask_login import current_user
 from functools import wraps
-from threading import Thread
-from typing import List, Union
+from typing import Optional
 from werkzeug.exceptions import NotAcceptable
 from app.models import Permission
 
@@ -24,22 +23,21 @@ def admin_required(f):
 
 def socketio_login_required(f):
     @wraps(f)
-    def decorated_function(*args, **kwargs):
+    def wrapper(*args, **kwargs):
         if current_user.is_authenticated:
             return f(*args, **kwargs)
-        else:
-            return {'code': 401, 'msg': 'Unauthorized'}
-    return decorated_function
+        return {'status': 401, 'statusText': 'Unauthorized'}
+    return wrapper
 
 
 def socketio_permission_required(permission):
     def decorator(f):
         @wraps(f)
-        def decorated_function(*args, **kwargs):
+        def wrapper(*args, **kwargs):
             if not current_user.can(permission):
-                return {'code': 403, 'msg': 'Forbidden'}
+                return {'status': 403, 'statusText': 'Forbidden'}
             return f(*args, **kwargs)
-        return decorated_function
+        return wrapper
     return decorator
 
 
@@ -47,27 +45,9 @@ def socketio_admin_required(f):
     return socketio_permission_required(Permission.ADMINISTRATE)(f)
 
 
-def background(f):
-    '''
-    ' This decorator executes a function in a Thread.
-    ' Decorated functions need to be executed within a code block where an
-    ' app context exists.
-    '
-    ' NOTE: An app object is passed as a keyword argument to the decorated
-    '       function.
-    '''
-    @wraps(f)
-    def wrapped(*args, **kwargs):
-        kwargs['app'] = current_app._get_current_object()
-        thread = Thread(target=f, args=args, kwargs=kwargs)
-        thread.start()
-        return thread
-    return wrapped
-
-
 def content_negotiation(
-    produces: Union[str, List[str], None] = None,
-    consumes: Union[str, List[str], None] = None
+    produces: Optional[str | list[str]] = None,
+    consumes: Optional[str | list[str]] = None
 ):
     def decorator(f):
         @wraps(f)

app/email.py

@@ -1,25 +1,32 @@
-from flask import current_app, render_template
+from flask import current_app, Flask, render_template
 from flask_mail import Message
 from threading import Thread
 from app import mail
 
 
-def create_message(recipient, subject, template, **kwargs):
-    subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
-    msg: Message = Message(
-        body=render_template(f'{template}.txt.j2', **kwargs),
-        html=render_template(f'{template}.html.j2', **kwargs),
+def create_message(
+    recipient: str,
+    subject: str,
+    template: str,
+    **context
+) -> Message:
+    message = Message(
+        body=render_template(f'{template}.txt.j2', **context),
+        html=render_template(f'{template}.html.j2', **context),
         recipients=[recipient],
-        subject=f'{subject_prefix} {subject}'
+        subject=f'[nopaque] {subject}'
     )
-    return msg
+    return message
 
 
-def send(msg, *args, **kwargs):
-    def _send(app, msg):
+def send(message: Message) -> Thread:
+    def _send(app: Flask, message: Message):
         with app.app_context():
-            mail.send(msg)
+            mail.send(message)
 
-    thread = Thread(target=_send, args=[current_app._get_current_object(), msg])
+    thread = Thread(
+        target=_send,
+        args=[current_app._get_current_object(), message]
+    )
     thread.start()
     return thread

app/extensions/nopaque_sqlalchemy_extras/__init__.py

@@ -0,0 +1,2 @@
+from .types import ContainerColumn
+from .types import IntEnumColumn

app/extensions/nopaque_sqlalchemy_extras/types.py

@@ -0,0 +1,42 @@
+import json
+from app import db
+
+
+class ContainerColumn(db.TypeDecorator):
+    impl = db.String
+
+    def __init__(self, container_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.container_type = container_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.container_type):
+            return json.dumps(value)
+        elif isinstance(value, str) and isinstance(json.loads(value), self.container_type):
+            return value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return json.loads(value)
+
+
+class IntEnumColumn(db.TypeDecorator):
+    impl = db.Integer
+
+    def __init__(self, enum_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_type = enum_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.enum_type) and isinstance(value.value, int):
+            return value.value
+        elif isinstance(value, int):
+            return self.enum_type(value).value
+        elif isinstance(value, str):
+            return self.enum_type[value].value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return self.enum_type(value)

app/jobs/__init__.py

@@ -1,18 +1,2 @@
-from flask import Blueprint
-from flask_login import login_required
-
-
-bp = Blueprint('jobs', __name__)
-
-
-@bp.before_request
-@login_required
-def before_request():
-    '''
-    Ensures that the routes in this package can only be visited by users that
-    are logged in.
-    '''
-    pass
-
-
-from . import routes, json_routes
+from .handle_corpora import handle_corpora
+from .handle_jobs import handle_jobs

app/jobs/handle_corpora.py

@@ -1,12 +1,16 @@
-from app import docker_client
-from app.models import Corpus, CorpusStatus
 from flask import current_app
 import docker
 import os
 import shutil
+from app import db, docker_client, scheduler
+from app.models import Corpus, CorpusStatus
 
 
-def check_corpora():
+def handle_corpora():
+    with scheduler.app.app_context():
+        _handle_corpora()
+
+def _handle_corpora():
     corpora = Corpus.query.all()
     for corpus in [x for x in corpora if x.status == CorpusStatus.SUBMITTED]:
         _create_build_corpus_service(corpus)
@@ -17,13 +21,14 @@ def check_corpora():
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
         corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
-        _checkout_analysing_corpus_container(corpus)
+        _checkout_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
         _create_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
         _remove_cqpserver_container(corpus)
+    db.session.commit()
 
-def _create_build_corpus_service(corpus):
+def _create_build_corpus_service(corpus: Corpus):
     ''' # Docker service settings # '''
     ''' ## Command ## '''
     command = ['bash', '-c']
@@ -45,12 +50,10 @@ def _create_build_corpus_service(corpus):
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'corpus.build',
-        'corpus_id': str(corpus.id)
+        'nopaque.server_name': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -95,7 +98,7 @@ def _create_build_corpus_service(corpus):
         return
     corpus.status = CorpusStatus.QUEUED
 
-def _checkout_build_corpus_service(corpus):
+def _checkout_build_corpus_service(corpus: Corpus):
     service_name = f'build-corpus_{corpus.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -123,8 +126,7 @@ def _checkout_build_corpus_service(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _create_cqpserver_container(corpus):
-    ''' # Docker container settings # '''
+def _create_cqpserver_container(corpus: Corpus):
     ''' ## Command ## '''
     command = []
     command.append(
@@ -139,9 +141,9 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Name ## '''
-    name = f'cqpserver_{corpus.id}'
+    name = f'nopaque-cqpserver-{corpus.id}'
     ''' ## Network ## '''
     network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}'
     ''' ## Volumes ## '''
@@ -198,8 +200,8 @@ def _create_cqpserver_container(corpus):
         return
     corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
 
-def _checkout_analysing_corpus_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _checkout_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         docker_client.containers.get(container_name)
     except docker.errors.NotFound as e:
@@ -209,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Get container "{container_name}" failed: {e}')
 
-def _remove_cqpserver_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _remove_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:

app/jobs/handle_jobs.py

@@ -1,11 +1,3 @@
-from app import db, docker_client, hashids
-from app.models import (
-    Job,
-    JobResult,
-    JobStatus,
-    TesseractOCRPipelineModel,
-    SpaCyNLPPipelineModel
-)
 from datetime import datetime
 from flask import current_app
 from werkzeug.utils import secure_filename
@@ -13,9 +5,21 @@ import docker
 import json
 import os
 import shutil
+from app import db, docker_client, hashids, scheduler
+from app.models import (
+    Job,
+    JobResult,
+    JobStatus,
+    TesseractOCRPipelineModel,
+    SpaCyNLPPipelineModel
+)
 
 
-def check_jobs():
+def handle_jobs():
+    with scheduler.app.app_context():
+        _handle_jobs()
+
+def _handle_jobs():
     jobs = Job.query.all()
     for job in [x for x in jobs if x.status == JobStatus.SUBMITTED]:
         _create_job_service(job)
@@ -23,8 +27,9 @@ def check_jobs():
         _checkout_job_service(job)
     for job in [x for x in jobs if x.status == JobStatus.CANCELING]:
         _remove_job_service(job)
+    db.session.commit()
 
-def _create_job_service(job):
+def _create_job_service(job: Job):
     ''' # Docker service settings # '''
     ''' ## Service specific settings ## '''
     if job.service == 'file-setup-pipeline':
@@ -81,9 +86,7 @@ def _create_job_service(job):
     constraints = ['node.role==worker']
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'job',
-        'job_id': str(job.id)
+        'origin': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -164,7 +167,7 @@ def _create_job_service(job):
         return
     job.status = JobStatus.QUEUED
 
-def _checkout_job_service(job):
+def _checkout_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -213,7 +216,7 @@ def _checkout_job_service(job):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _remove_job_service(job):
+def _remove_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)

app/jobs/utils.py

@@ -1,13 +0,0 @@
-from flask import request, url_for
-from app.models import Job
-
-
-def job_dynamic_list_constructor():
-    job_id = request.view_args['job_id']
-    job = Job.query.get_or_404(job_id)
-    return [
-        {
-            'text': f'<i class="nopaque-icons left service-icons" data-service="{job.service}"></i>{job.title}',
-            'url': url_for('.job', job_id=job_id)
-        }
-    ]

app/models/__init__.py

@@ -0,0 +1,14 @@
+from .anonymous_user import *
+from .avatar import *
+from .corpus_file import *
+from .corpus_follower_association import *
+from .corpus_follower_role import *
+from .corpus import *
+from .job_input import *
+from .job_result import *
+from .job import *
+from .role import *
+from .spacy_nlp_pipeline_model import *
+from .tesseract_ocr_pipeline_model import *
+from .token import *
+from .user import *

app/models/anonymous_user.py

@@ -0,0 +1,10 @@
+from flask_login import AnonymousUserMixin
+
+
+class AnonymousUser(AnonymousUserMixin):
+    def can(self, permissions):
+        return False
+
+    @property
+    def is_administrator(self):
+        return False

app/models/avatar.py

@@ -0,0 +1,40 @@
+from flask import current_app
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class Avatar(HashidMixin, FileMixin, db.Model):
+    __tablename__ = 'avatars'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Relationships
+    user = db.relationship('User', back_populates='avatar')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'avatar'
+        # return os.path.join(self.user.path, 'avatar')
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus.py

@@ -0,0 +1,199 @@
+from datetime import datetime
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from sqlalchemy.ext.associationproxy import association_proxy
+from pathlib import Path
+import shutil
+import xml.etree.ElementTree as ET
+from app import db
+from app.converters.vrt import normalize_vrt_file
+from app.extensions.nopaque_sqlalchemy_extras import IntEnumColumn
+from .corpus_follower_association import CorpusFollowerAssociation
+
+
+class CorpusStatus(IntEnum):
+    UNPREPARED = 1
+    SUBMITTED = 2
+    QUEUED = 3
+    BUILDING = 4
+    BUILT = 5
+    FAILED = 6
+    STARTING_ANALYSIS_SESSION = 7
+    RUNNING_ANALYSIS_SESSION = 8
+    CANCELING_ANALYSIS_SESSION = 9
+
+    @staticmethod
+    def get(corpus_status: 'CorpusStatus | int | str') -> 'CorpusStatus':
+        if isinstance(corpus_status, CorpusStatus):
+            return corpus_status
+        if isinstance(corpus_status, int):
+            return CorpusStatus(corpus_status)
+        if isinstance(corpus_status, str):
+            return CorpusStatus[corpus_status]
+        raise TypeError('corpus_status must be CorpusStatus, int, or str')
+
+
+class Corpus(HashidMixin, db.Model):
+    '''
+    Class to define a corpus.
+    '''
+    __tablename__ = 'corpora'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
+    description = db.Column(db.String(255))
+    status = db.Column(
+        IntEnumColumn(CorpusStatus),
+        default=CorpusStatus.UNPREPARED
+    )
+    title = db.Column(db.String(32))
+    num_analysis_sessions = db.Column(db.Integer, default=0)
+    num_tokens = db.Column(db.Integer, default=0)
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    files = db.relationship(
+        'CorpusFile',
+        back_populates='corpus',
+        lazy='dynamic',
+        cascade='all, delete-orphan'
+    )
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='corpus',
+        cascade='all, delete-orphan'
+    )
+    followers = association_proxy(
+        'corpus_follower_associations',
+        'follower',
+        creator=lambda u: CorpusFollowerAssociation(follower=u)
+    )
+    user = db.relationship('User', back_populates='corpora')
+    # "static" attributes
+    max_num_tokens = 2_147_483_647
+
+    def __repr__(self):
+        return f'<Corpus {self.title}>'
+
+    @property
+    def analysis_url(self):
+        return url_for('corpora.analysis', corpus_id=self.id)
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/corpora/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'corpora' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for('corpora.corpus', corpus_id=self.id)
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def create(**kwargs):
+        corpus = Corpus(**kwargs)
+        db.session.add(corpus)
+        db.session.flush(objects=[corpus])
+        db.session.refresh(corpus)
+        corpus_files_dir = corpus.path / 'files'
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            corpus.path.mkdir()
+            corpus_files_dir.mkdir()
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return corpus
+
+    def build(self):
+        corpus_cwb_dir = self.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus_cwb_dir, ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            self.status = CorpusStatus.FAILED
+            raise
+        corpus_element = ET.fromstring('<corpus>\n</corpus>')
+        for corpus_file in self.files:
+            normalized_vrt_path = corpus_cwb_dir / f'{corpus_file.id}.norm.vrt'
+            try:
+                normalize_vrt_file(corpus_file.path, normalized_vrt_path)
+            except:
+                self.status = CorpusStatus.FAILED
+                return
+            element_tree = ET.parse(normalized_vrt_path)
+            text_element = element_tree.getroot()
+            text_element.set('author', corpus_file.author)
+            text_element.set('title', corpus_file.title)
+            text_element.set(
+                'publishing_year',
+                f'{corpus_file.publishing_year}'
+            )
+            text_element.set('address', corpus_file.address or 'NULL')
+            text_element.set('booktitle', corpus_file.booktitle or 'NULL')
+            text_element.set('chapter', corpus_file.chapter or 'NULL')
+            text_element.set('editor', corpus_file.editor or 'NULL')
+            text_element.set('institution', corpus_file.institution or 'NULL')
+            text_element.set('journal', corpus_file.journal or 'NULL')
+            text_element.set('pages', f'{corpus_file.pages}' or 'NULL')
+            text_element.set('publisher', corpus_file.publisher or 'NULL')
+            text_element.set('school', corpus_file.school or 'NULL')
+            text_element.tail = '\n'
+            # corpus_element.insert(1, text_element)
+            corpus_element.append(text_element)
+        ET.ElementTree(corpus_element).write(
+            corpus_cwb_dir / 'corpus.vrt',
+            encoding='utf-8'
+        )
+        self.status = CorpusStatus.SUBMITTED
+
+    def delete(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'description': self.description,
+            'max_num_tokens': self.max_num_tokens,
+            'num_analysis_sessions': self.num_analysis_sessions,
+            'num_tokens': self.num_tokens,
+            'status': self.status.name,
+            'title': self.title,
+            'is_public': self.is_public
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
+            json_serializeable['files'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.files
+            }
+        return json_serializeable

app/models/corpus_file.py

@@ -0,0 +1,102 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .corpus import CorpusStatus
+from .file_mixin import FileMixin
+
+
+class CorpusFile(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'corpus_files'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    # Fields
+    author = db.Column(db.String(255))
+    description = db.Column(db.String(255))
+    publishing_year = db.Column(db.Integer)
+    title = db.Column(db.String(255))
+    address = db.Column(db.String(255))
+    booktitle = db.Column(db.String(255))
+    chapter = db.Column(db.String(255))
+    editor = db.Column(db.String(255))
+    institution = db.Column(db.String(255))
+    journal = db.Column(db.String(255))
+    pages = db.Column(db.String(255))
+    publisher = db.Column(db.String(255))
+    school = db.Column(db.String(255))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='files'
+    )
+
+    @property
+    def download_url(self):
+        return url_for(
+            'corpora.download_corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.corpus.jsonpatch_path}/files/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.corpus.path / 'files' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'corpora.corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.corpus.user.hashid
+
+    @property
+    def user_id(self):
+        return self.corpus.user_id
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+        self.corpus.status = CorpusStatus.UNPREPARED
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'address': self.address,
+            'author': self.author,
+            'description': self.description,
+            'booktitle': self.booktitle,
+            'chapter': self.chapter,
+            'editor': self.editor,
+            'institution': self.institution,
+            'journal': self.journal,
+            'pages': self.pages,
+            'publisher': self.publisher,
+            'publishing_year': self.publishing_year,
+            'school': self.school,
+            'title': self.title,
+            **self.file_mixin_to_json_serializeable(
+                backrefs=backrefs,
+                relationships=relationships
+            )
+        }
+        if backrefs:
+            json_serializeable['corpus'] = \
+                self.corpus.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_association.py

@@ -0,0 +1,47 @@
+from flask_hashids import HashidMixin
+from app import db
+from .corpus_follower_role import CorpusFollowerRole
+
+
+class CorpusFollowerAssociation(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_associations'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    follower_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    role_id = db.Column(db.Integer, db.ForeignKey('corpus_follower_roles.id'))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='corpus_follower_associations'
+    )
+    follower = db.relationship(
+        'User',
+        back_populates='corpus_follower_associations'
+    )
+    role = db.relationship(
+        'CorpusFollowerRole',
+        back_populates='corpus_follower_associations'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = CorpusFollowerRole.query.filter_by(default=True).first()
+        super().__init__(**kwargs)
+
+    def __repr__(self):
+        return f'<CorpusFollowerAssociation {self.follower.__repr__()} ~ {self.role.__repr__()} ~ {self.corpus.__repr__()}>'
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'corpus': self.corpus.to_json_serializeable(backrefs=True),
+            'follower': self.follower.to_json_serializeable(),
+            'role': self.role.to_json_serializeable()
+        }
+        if backrefs:
+            pass
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_role.py

@@ -0,0 +1,106 @@
+from flask_hashids import HashidMixin
+from enum import IntEnum
+from app import db
+
+
+class CorpusFollowerPermission(IntEnum):
+    VIEW = 1
+    MANAGE_FILES = 2
+    MANAGE_FOLLOWERS = 4
+    MANAGE_CORPUS = 8
+
+    @staticmethod
+    def get(corpus_follower_permission: 'CorpusFollowerPermission | int | str') -> 'CorpusFollowerPermission':
+        if isinstance(corpus_follower_permission, CorpusFollowerPermission):
+            return corpus_follower_permission
+        if isinstance(corpus_follower_permission, int):
+            return CorpusFollowerPermission(corpus_follower_permission)
+        if isinstance(corpus_follower_permission, str):
+            return CorpusFollowerPermission[corpus_follower_permission]
+        raise TypeError('corpus_follower_permission must be CorpusFollowerPermission, int, or str')
+
+
+class CorpusFollowerRole(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='role'
+    )
+
+    def __repr__(self):
+        return f'<CorpusFollowerRole {self.name}>'
+
+    def has_permission(self, permission: CorpusFollowerPermission | int | str):
+        perm = CorpusFollowerPermission.get(permission)
+        return self.permissions & perm.value == perm.value
+
+    def add_permission(self, permission: CorpusFollowerPermission | int | str):
+        perm = CorpusFollowerPermission.get(permission)
+        if not self.has_permission(perm):
+            self.permissions += perm.value
+
+    def remove_permission(self, permission: CorpusFollowerPermission | int | str):
+        perm = CorpusFollowerPermission.get(permission)
+        if self.has_permission(perm):
+            self.permissions -= perm.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name
+                for x in CorpusFollowerPermission
+                if self.has_permission(x)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['corpus_follower_association'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpus_follower_association
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'Anonymous': [],
+            'Viewer': [
+                CorpusFollowerPermission.VIEW
+            ],
+            'Contributor': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES
+            ],
+            'Administrator': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES,
+                CorpusFollowerPermission.MANAGE_FOLLOWERS,
+                CorpusFollowerPermission.MANAGE_CORPUS
+
+            ]
+        }
+        default_role_name = 'Viewer'
+        for role_name, permissions in roles.items():
+            role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+            if role is None:
+                role = CorpusFollowerRole(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()

app/models/default_records/tesseract_ocr_pipeline_model.yml

@@ -10,6 +10,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Amharic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata'
@@ -22,6 +23,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Arabic'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata'
@@ -34,6 +36,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Assamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata'
@@ -46,6 +49,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata'
@@ -58,6 +62,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata'
@@ -70,6 +75,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Belarusian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata'
@@ -82,6 +88,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bengali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata'
@@ -94,6 +101,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tibetan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata'
@@ -106,6 +114,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bosnian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata'
@@ -118,6 +127,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bulgarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata'
@@ -130,6 +140,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Catalan; Valencian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata'
@@ -142,6 +153,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Cebuano'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata'
@@ -154,6 +166,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Czech'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata'
@@ -166,6 +179,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Chinese - Simplified'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata'
@@ -178,6 +192,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Chinese - Traditional'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata'
@@ -190,6 +205,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Cherokee'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata'
@@ -202,6 +218,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Welsh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata'
@@ -214,6 +231,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Danish'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata'
@@ -226,6 +244,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata'
@@ -238,6 +257,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Dzongkha'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata'
@@ -250,6 +270,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Modern (1453-)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata'
@@ -262,6 +283,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata'
@@ -274,6 +296,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English, Middle (1100-1500)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata'
@@ -286,6 +309,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Esperanto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata'
@@ -298,6 +322,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Estonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata'
@@ -310,6 +335,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Basque'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata'
@@ -322,6 +348,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Persian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata'
@@ -334,6 +361,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Finnish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata'
@@ -346,6 +374,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'French'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata'
@@ -358,6 +387,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German Fraktur'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata'
@@ -370,6 +400,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'French, Middle (ca. 1400-1600)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata'
@@ -382,6 +413,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Irish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata'
@@ -394,6 +426,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Galician'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata'
@@ -406,6 +439,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Ancient (-1453)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata'
@@ -418,6 +452,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Gujarati'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata'
@@ -430,6 +465,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Haitian; Haitian Creole'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata'
@@ -442,6 +478,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hebrew'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata'
@@ -454,6 +491,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hindi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata'
@@ -466,6 +504,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Croatian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata'
@@ -478,6 +517,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hungarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata'
@@ -490,6 +530,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Inuktitut'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata'
@@ -502,6 +543,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Indonesian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata'
@@ -514,6 +556,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Icelandic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata'
@@ -526,6 +569,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Italian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata'
@@ -538,6 +582,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Italian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata'
@@ -550,6 +595,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Javanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata'
@@ -562,6 +608,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Japanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata'
@@ -574,6 +621,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kannada'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata'
@@ -586,6 +634,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata'
@@ -598,6 +647,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian - Old'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata'
@@ -610,6 +660,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kazakh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata'
@@ -622,6 +673,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Central Khmer'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata'
@@ -634,6 +686,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kirghiz; Kyrgyz'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata'
@@ -646,6 +699,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Korean'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata'
@@ -658,6 +712,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kurdish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata'
@@ -670,6 +725,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lao'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata'
@@ -682,6 +738,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata'
@@ -694,6 +751,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latvian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata'
@@ -706,6 +764,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lithuanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata'
@@ -718,6 +777,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malayalam'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata'
@@ -730,6 +790,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Marathi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata'
@@ -742,6 +803,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Macedonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata'
@@ -754,6 +816,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Maltese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata'
@@ -766,6 +829,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malay'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata'
@@ -778,6 +842,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Burmese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata'
@@ -790,6 +855,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Nepali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata'
@@ -802,6 +868,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Dutch; Flemish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata'
@@ -814,6 +881,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Norwegian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata'
@@ -826,6 +894,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Oriya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata'
@@ -838,6 +907,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Panjabi; Punjabi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata'
@@ -850,6 +920,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Polish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata'
@@ -862,6 +933,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Portuguese'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata'
@@ -874,6 +946,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Pushto; Pashto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata'
@@ -886,6 +959,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Romanian; Moldavian; Moldovan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata'
@@ -898,6 +972,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Russian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata'
@@ -910,6 +985,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Sanskrit'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata'
@@ -922,6 +998,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Sinhala; Sinhalese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata'
@@ -934,6 +1011,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovak'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata'
@@ -946,6 +1024,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovenian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata'
@@ -958,6 +1037,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Spanish; Castilian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata'
@@ -970,6 +1050,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Spanish; Castilian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata'
@@ -982,6 +1063,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Albanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata'
@@ -994,6 +1076,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata'
@@ -1006,6 +1089,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian - Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata'
@@ -1018,6 +1102,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swahili'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata'
@@ -1030,6 +1115,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swedish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata'
@@ -1042,6 +1128,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Syriac'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata'
@@ -1054,6 +1141,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tamil'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata'
@@ -1066,6 +1154,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Telugu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata'
@@ -1078,6 +1167,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tajik'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata'
@@ -1090,6 +1180,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tagalog'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata'
@@ -1102,6 +1193,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Thai'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata'
@@ -1114,6 +1206,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tigrinya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata'
@@ -1126,6 +1219,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Turkish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata'
@@ -1138,6 +1232,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uighur; Uyghur'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata'
@@ -1150,6 +1245,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Ukrainian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata'
@@ -1162,6 +1258,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Urdu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata'
@@ -1174,6 +1271,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata'
@@ -1186,6 +1284,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata'
@@ -1198,6 +1297,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Vietnamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata'
@@ -1210,6 +1310,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Yiddish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata'
@@ -1222,3 +1323,4 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'