Bump nopaque version

.vscode/settings.json

@@ -1,7 +1,24 @@
 {
     "editor.rulers": [79],
-    "editor.tabSize": 2,
+    "editor.tabSize": 4,
+    "emmet.includeLanguages": {
+        "jinja-html": "html"
+    },
+    "files.associations": {
+        ".flaskenv": "env",
+        "*.env.tpl": "env",
+        "*.txt.j2": "jinja"
+    },
     "files.insertFinalNewline": true,
     "files.trimFinalNewlines": true,
-    "files.trimTrailingWhitespace": true
+    "files.trimTrailingWhitespace": true,
+    "[html]": {
+        "editor.tabSize": 2
+    },
+    "[javascript]": {
+        "editor.tabSize": 2
+    },
+    "[jinja-html]": {
+        "editor.tabSize": 2
+    }
 }

README.md

@@ -35,7 +35,7 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
 # Create data directories
-username@hostname:~$ mkdir volumes/{db,mq}
+username@hostname:~$ mkdir -p volumes/{db,mq}
 username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
 # Fill out the variables within these files.

app/__init__.py

@@ -92,46 +92,49 @@ def create_app(config: Config = Config) -> Flask:
     # endregion Extensions
 
     # region Blueprints
-    from .admin import bp as admin_blueprint
+    from .blueprints.admin import bp as admin_blueprint
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
 
-    from .api import bp as api_blueprint
+    from .blueprints.api import bp as api_blueprint
     app.register_blueprint(api_blueprint, url_prefix='/api')
 
-    from .auth import bp as auth_blueprint
+    from .blueprints.auth import bp as auth_blueprint
     app.register_blueprint(auth_blueprint)
 
-    from .contributions import bp as contributions_blueprint
+    from .blueprints.contributions import bp as contributions_blueprint
     app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
-    from .corpora import bp as corpora_blueprint
+    from .blueprints.corpora import bp as corpora_blueprint
     app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
 
-    from .errors import bp as errors_bp
+    from .blueprints.errors import bp as errors_bp
     app.register_blueprint(errors_bp)
 
-    from .jobs import bp as jobs_blueprint
+    from .blueprints.jobs import bp as jobs_blueprint
     app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
 
-    from .main import bp as main_blueprint
+    from .blueprints.main import bp as main_blueprint
     app.register_blueprint(main_blueprint, cli_group=None)
 
-    from .services import bp as services_blueprint
+    from .blueprints.services import bp as services_blueprint
     app.register_blueprint(services_blueprint, url_prefix='/services')
 
-    from .settings import bp as settings_blueprint
+    from .blueprints.settings import bp as settings_blueprint
     app.register_blueprint(settings_blueprint, url_prefix='/settings')
 
-    from .users import bp as users_blueprint
+    from .blueprints.users import bp as users_blueprint
     app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
 
-    from .workshops import bp as workshops_blueprint
+    from .blueprints.workshops import bp as workshops_blueprint
     app.register_blueprint(workshops_blueprint, url_prefix='/workshops')
     # endregion Blueprints
 
     # region SocketIO Namespaces
-    from .corpora.cqi_over_sio import CQiOverSocketIO
-    socketio.on_namespace(CQiOverSocketIO('/cqi_over_sio'))
+    from .namespaces.cqi_over_sio import CQiOverSocketIONamespace
+    socketio.on_namespace(CQiOverSocketIONamespace('/cqi_over_sio'))
+
+    from .namespaces.users import UsersNamespace
+    socketio.on_namespace(UsersNamespace('/users'))
     # endregion SocketIO Namespaces
 
     # region Database event Listeners
@@ -141,25 +144,11 @@ def create_app(config: Config = Config) -> Flask:
 
     # region Add scheduler jobs
     if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
-        from .tasks import handle_corpora
+        from .jobs import handle_corpora
         scheduler.add_job('handle_corpora', handle_corpora, seconds=3, trigger='interval')
 
-        from .tasks import handle_jobs
+        from .jobs import handle_jobs
         scheduler.add_job('handle_jobs', handle_jobs, seconds=3, trigger='interval')
     # endregion Add scheduler jobs
 
     return app
-
-
-# def _add_admin_views():
-#     from flask_admin.contrib.sqla import ModelView
-#     from . import models
-
-#     for v in models.__dict__.values():
-#         # Check if v is a class
-#         if not isinstance(v, type):
-#             continue
-#         # Check if v is a subclass of db.Model
-#         if not issubclass(v, db.Model):
-#             continue
-#         admin.add_view(ModelView(v, db.session, category='Database'))

app/admin/events.py

@@ -1,49 +0,0 @@
-from flask_login import current_user
-from flask_socketio import disconnect, Namespace
-from app import db, hashids
-from app.decorators import socketio_admin_required
-from app.models import User
-
-
-class AdminNamespace(Namespace):
-    def on_connect(self):
-        # Check if the user is authenticated and is an administrator
-        if not (current_user.is_authenticated and current_user.is_administrator):
-            disconnect()
-
-
-    @socketio_admin_required
-    def on_set_user_confirmed(self, user_hashid: str, confirmed_value: bool):
-        # Decode the user hashid
-        user_id = hashids.decode(user_hashid)
-
-        # Validate user_id
-        if not isinstance(user_id, int):
-            return {
-                'code': 400,
-                'body': 'user_id is invalid'
-            }
-
-        # Validate confirmed_value
-        if not isinstance(confirmed_value, bool):
-            return {
-                'code': 400,
-                'body': 'confirmed_value is invalid'
-            }
-
-        # Load user from database
-        user = User.query.get(user_id)
-        if user is None:
-            return {
-                'code': 404,
-                'body': 'User not found'
-            }
-
-        # Update user confirmed status
-        user.confirmed = confirmed_value
-        db.session.commit()
-
-        return {
-            'code': 200,
-            'body': f'User "{user.username}" is now {"confirmed" if confirmed_value else "unconfirmed"}'
-        }

app/auth/__init__.py

@@ -1,5 +0,0 @@
-from flask import Blueprint
-
-
-bp = Blueprint('auth', __name__)
-from . import routes

app/blueprints/admin/routes.py

@@ -1,7 +1,7 @@
 from flask import abort, flash, redirect, render_template, url_for
 from app import db, hashids
 from app.models import Avatar, Corpus, Role, User
-from app.users.settings.forms import (
+from app.blueprints.users.settings.forms import (
     UpdateAvatarForm,
     UpdatePasswordForm,
     UpdateNotificationsForm,

app/blueprints/api/schemas.py

@@ -10,7 +10,7 @@ from app.models import (
     User,
     UserSettingJobStatusMailNotificationLevel
 )
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 
 
 

app/blueprints/auth/__init__.py

@@ -0,0 +1,29 @@
+from flask import Blueprint, redirect, request, url_for
+from flask_login import current_user
+from app import db
+
+
+bp = Blueprint('auth', __name__)
+
+
+@bp.before_app_request
+def before_request():
+    if not current_user.is_authenticated:
+        return
+
+    current_user.ping()
+    db.session.commit()
+
+    if (
+        not current_user.confirmed
+        and request.endpoint
+        and request.blueprint != 'auth'
+        and request.endpoint != 'static'
+    ):
+        return redirect(url_for('auth.unconfirmed'))
+
+    if not current_user.terms_of_use_accepted:
+        return redirect(url_for('main.terms_of_use'))
+
+
+from . import routes

app/blueprints/auth/forms.py

@@ -60,7 +60,11 @@ class RegistrationForm(FlaskForm):
 
     def validate_username(self, field):
         if User.query.filter_by(username=field.data).first():
-            raise ValidationError('Username already in use')
+            raise ValidationError('Username already registered')
+
+    def validate_terms_of_use_accepted(self, field):
+        if not field.data:
+            raise ValidationError('Terms of Use not accepted')
 
 
 class LoginForm(FlaskForm):

app/blueprints/auth/routes.py

@@ -12,26 +12,6 @@ from .forms import (
 )
 
 
-@bp.before_app_request
-def before_request():
-    """
-    Checks if a user is unconfirmed when visiting specific sites. Redirects to
-    unconfirmed view if user is unconfirmed.
-    """
-    if not current_user.is_authenticated:
-        return
-
-    current_user.ping()
-    db.session.commit()
-    if (not current_user.confirmed
-            and request.endpoint
-            and request.blueprint != 'auth'
-            and request.endpoint != 'static'):
-        return redirect(url_for('auth.unconfirmed'))
-    if not current_user.terms_of_use_accepted:
-        return redirect(url_for('main.terms_of_use'))
-
-
 @bp.route('/register', methods=['GET', 'POST'])
 def register():
     if current_user.is_authenticated:

app/blueprints/contributions/__init__.py

@@ -0,0 +1,25 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('contributions', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes
+
+
+from .spacy_nlp_pipeline_models import bp as spacy_nlp_pipeline_models_bp
+bp.register_blueprint(spacy_nlp_pipeline_models_bp, url_prefix='/spacy-nlp-pipeline-models')
+
+from .tesseract_ocr_pipeline_models import bp as tesseract_ocr_pipeline_models_bp
+bp.register_blueprint(tesseract_ocr_pipeline_models_bp, url_prefix='/tesseract-ocr-pipeline-models')

app/blueprints/contributions/routes.py

@@ -0,0 +1,7 @@
+from flask import render_template
+from . import bp
+
+
+@bp.route('')
+def index():
+    return render_template('contributions/index.html.j2', title='Contributions')

app/blueprints/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -0,0 +1,18 @@
+from flask import current_app, Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('spacy_nlp_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/contributions/spacy_nlp_pipeline_models/forms.py

@@ -1,7 +1,7 @@
 from flask_wtf.file import FileField, FileRequired
 from wtforms import StringField, ValidationError
 from wtforms.validators import InputRequired, Length
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 from ..forms import ContributionBaseForm, UpdateContributionBaseForm
 
 

app/blueprints/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -1,5 +1,5 @@
 from flask import abort, current_app, request
-from flask_login import current_user
+from flask_login import current_user, login_required
 from threading import Thread
 from app import db
 from app.decorators import content_negotiation, permission_required
@@ -7,7 +7,8 @@ from app.models import SpaCyNLPPipelineModel
 from . import bp
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@login_required
 @content_negotiation(produces='application/json')
 def delete_spacy_model(spacy_nlp_pipeline_model_id):
     def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
@@ -31,7 +32,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):

app/blueprints/contributions/spacy_nlp_pipeline_models/routes.py

@@ -1,5 +1,5 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_login import current_user
+from flask_login import current_user, login_required
 from app import db
 from app.models import SpaCyNLPPipelineModel
 from . import bp
@@ -9,16 +9,15 @@ from .forms import (
 )
 
 
-@bp.route('/spacy-nlp-pipeline-models')
-def spacy_nlp_pipeline_models():
-    return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
-        title='SpaCy NLP Pipeline Models'
-    )
+@bp.route('/')
+@login_required
+def index():
+    return redirect(url_for('contributions.index', _anchor='spacy-nlp-pipeline-models'))
 
 
-@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
-def create_spacy_nlp_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+@login_required
+def create():
     form = CreateSpaCyNLPPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -42,7 +41,7 @@ def create_spacy_nlp_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
-        return {}, 201, {'Location': url_for('.spacy_nlp_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/spacy_nlp_pipeline_models/create.html.j2',
         title='Create SpaCy NLP Pipeline Model',
@@ -50,8 +49,9 @@ def create_spacy_nlp_pipeline_model():
     )
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
-def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+@login_required
+def entity(spacy_nlp_pipeline_model_id):
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
     if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
@@ -61,9 +61,9 @@ def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
         if db.session.is_modified(snpm):
             flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.spacy_nlp_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_model.html.j2',
+        'contributions/spacy_nlp_pipeline_models/entity.html.j2',
         title=f'{snpm.title} {snpm.version}',
         form=form,
         spacy_nlp_pipeline_model=snpm

app/blueprints/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -2,7 +2,7 @@ from flask import Blueprint
 from flask_login import login_required
 
 
-bp = Blueprint('contributions', __name__)
+bp = Blueprint('tesseract_ocr_pipeline_models', __name__)
 
 
 @bp.before_request
@@ -15,9 +15,4 @@ def before_request():
     pass
 
 
-from . import (
-    routes,
-    spacy_nlp_pipeline_models,
-    tesseract_ocr_pipeline_models,
-    transkribus_htr_pipeline_models
-)
+from . import json_routes, routes

app/blueprints/contributions/tesseract_ocr_pipeline_models/forms.py

@@ -1,6 +1,6 @@
 from flask_wtf.file import FileField, FileRequired
 from wtforms import ValidationError
-from app.services import SERVICES
+from app.blueprints.services import SERVICES
 from ..forms import ContributionBaseForm, UpdateContributionBaseForm
 
 

app/blueprints/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -7,7 +7,7 @@ from app.models import TesseractOCRPipelineModel
 from . import bp
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
 @content_negotiation(produces='application/json')
 def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
@@ -31,7 +31,7 @@ def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):

app/blueprints/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -9,16 +9,13 @@ from .forms import (
 )
 
 
-@bp.route('/tesseract-ocr-pipeline-models')
-def tesseract_ocr_pipeline_models():
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
-        title='Tesseract OCR Pipeline Models'
-    )
+@bp.route('/')
+def index():
+    return redirect(url_for('contributions.index', _anchor='tesseract-ocr-pipeline-models'))
 
 
-@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
-def create_tesseract_ocr_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+def create():
     form = CreateTesseractOCRPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -41,7 +38,7 @@ def create_tesseract_ocr_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
-        return {}, 201, {'Location': url_for('.tesseract_ocr_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/tesseract_ocr_pipeline_models/create.html.j2',
         title='Create Tesseract OCR Pipeline Model',
@@ -49,8 +46,8 @@ def create_tesseract_ocr_pipeline_model():
     )
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
-def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+def entity(tesseract_ocr_pipeline_model_id):
     topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
     if not (topm.user == current_user or current_user.is_administrator):
         abort(403)
@@ -60,9 +57,9 @@ def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
         if db.session.is_modified(topm):
             flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.tesseract_ocr_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_model.html.j2',
+        'contributions/tesseract_ocr_pipeline_models/entity.html.j2',
         title=f'{topm.title} {topm.version}',
         form=form,
         tesseract_ocr_pipeline_model=topm

app/blueprints/jobs/__init__.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('jobs', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/main/cli.py

@@ -44,7 +44,7 @@ def deploy():
     TesseractOCRPipelineModel.insert_defaults()
 
     print('Stop running analysis sessions')
-    for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0).all():
+    for corpus in Corpus.query.all():
         corpus.num_analysis_sessions = 0
     db.session.commit()
 

app/blueprints/main/routes.py

@@ -1,6 +1,6 @@
 from flask import flash, redirect, render_template, url_for
 from flask_login import current_user, login_required, login_user
-from app.auth.forms import LoginForm
+from app.blueprints.auth.forms import LoginForm
 from app.models import Corpus, User
 from . import bp
 
@@ -72,16 +72,14 @@ def terms_of_use():
     )
 
 
-@bp.route('/social-area')
+@bp.route('/social')
 @login_required
-def social_area():
-    print('test')
+def social():
     corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
-    print(corpora)
     users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
     return render_template(
-        'main/social_area.html.j2',
-        title='Social Area',
+        'main/social.html.j2',
+        title='Social',
         corpora=corpora,
         users=users
     )

app/blueprints/services/forms.py

@@ -167,7 +167,6 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
         service_info = service_manifest['versions'][version]
-        print(service_info)
         if self.encoding_detection.render_kw is None:
             self.encoding_detection.render_kw = {}
         self.encoding_detection.render_kw['disabled'] = True

app/blueprints/settings/routes.py

@@ -1,6 +1,6 @@
 from flask import g, url_for
 from flask_login import current_user
-from app.users.settings.routes import settings as settings_route
+from app.blueprints.users.settings.routes import settings as settings_route
 from . import bp
 
 

app/blueprints/users/__init__.py

@@ -15,4 +15,4 @@ def before_request():
     pass
 
 
-from . import cli, events, json_routes, routes, settings
+from . import cli, json_routes, routes, settings

app/contributions/routes.py

@@ -1,7 +0,0 @@
-from flask import redirect, url_for
-from . import bp
-
-
-@bp.route('')
-def contributions():
-    return redirect(url_for('main.dashboard', _anchor='contributions'))

app/contributions/transkribus_htr_pipeline_models/__init__.py

@@ -1,2 +0,0 @@
-from .. import bp
-from . import routes

app/contributions/transkribus_htr_pipeline_models/routes.py

@@ -1,7 +0,0 @@
-from flask import abort
-from . import bp
-
-
-@bp.route('/transkribus_htr_pipeline_models')
-def transkribus_htr_pipeline_models():
-    return abort(503)

app/converters/vrt.py

@@ -1,69 +1,25 @@
 from flask import current_app
+from pathlib import Path
 
 
-def normalize_vrt_file(input_file, output_file):
-    def check_pos_attribute_order(vrt_lines):
-        # The following orders are possible:
-        # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
-        # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
-        # since 27.01.2022: 'word,pos,lemma,simple_pos'
-        # This Function tries to find out which order we have by looking at the
-        # number of attributes and the position of the simple_pos attribute
-        SIMPLE_POS_LABELS = [
-            'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ',
-            'DET', 'INTJ', 'NOUN', 'NUM', 'PART',
-            'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',
-            'VERB', 'X'
-        ]
-        for line in vrt_lines:
-            if line.startswith('<'):
-                continue
-            pos_attrs = line.rstrip('\n').split('\t')
-            num_pos_attrs = len(pos_attrs)
-            if num_pos_attrs == 4:
-                if pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos']
-                continue
-            elif num_pos_attrs == 5:
-                if pos_attrs[2] in SIMPLE_POS_LABELS:
-                    return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
-                elif pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
-                continue
-        return None
-
-
-    def check_has_ent_as_s_attr(vrt_lines):
-        for line in vrt_lines:
-            if line.startswith('<ent'):
-                return True
-        return False
-
-
-    def pos_attrs_to_string_1(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
-
-
-    def pos_attrs_to_string_2(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'
-
+def normalize_vrt_file(input_file: Path, output_file: Path):
     current_app.logger.info(f'Converting {input_file}...')
 
-    with open(input_file) as f:
+    with input_file.open() as f:
         input_vrt_lines = f.readlines()
 
-    pos_attr_order = check_pos_attribute_order(input_vrt_lines)
-    has_ent_as_s_attr = check_has_ent_as_s_attr(input_vrt_lines)
+    pos_attr_order = _check_pos_attribute_order(input_vrt_lines)
+    has_ent_as_s_attr = _check_has_ent_as_s_attr(input_vrt_lines)
 
     current_app.logger.info(f'Detected pos_attr_order: [{",".join(pos_attr_order)}]')
     current_app.logger.info(f'Detected has_ent_as_s_attr: {has_ent_as_s_attr}')
 
     if pos_attr_order == ['word', 'lemma', 'simple_pos', 'pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_1
+        pos_attrs_to_string_function = _pos_attrs_to_string_1
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     else:
         raise Exception('Can not handle format')
 
@@ -113,5 +69,49 @@ def normalize_vrt_file(input_file, output_file):
                     current_ent = pos_attrs[4]
         output_vrt += pos_attrs_to_string_function(pos_attrs)
 
-    with open(output_file, 'w') as f:
+    with output_file.open(mode='w') as f:
         f.write(output_vrt)
+
+
+def _check_pos_attribute_order(vrt_lines: list[str]) -> list[str]:
+    # The following orders are possible:
+    # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
+    # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
+    # since 27.01.2022: 'word,pos,lemma,simple_pos'
+    # This Function tries to find out which order we have by looking at the
+    # number of attributes and the position of the simple_pos attribute
+    SIMPLE_POS_LABELS = [
+        'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'DET', 'INTJ', 'NOUN', 'NUM',
+        'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X'
+    ]
+    for line in vrt_lines:
+        if line.startswith('<'):
+            continue
+        pos_attrs = line.rstrip('\n').split('\t')
+        num_pos_attrs = len(pos_attrs)
+        if num_pos_attrs == 4:
+            if pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos']
+            continue
+        elif num_pos_attrs == 5:
+            if pos_attrs[2] in SIMPLE_POS_LABELS:
+                return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
+            elif pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
+            continue
+    # TODO: raise exception "can't determine attribute order"
+
+
+def _check_has_ent_as_s_attr(vrt_lines: list[str]) -> bool:
+    for line in vrt_lines:
+        if line.startswith('<ent'):
+            return True
+    return False
+
+
+def _pos_attrs_to_string_1(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
+
+
+def _pos_attrs_to_string_2(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'

app/corpora/cqi_over_sio/utils.py

@@ -1,130 +0,0 @@
-from cqi.models.corpora import Corpus as CQiCorpus
-from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
-
-
-def lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
-    lookups = {}
-    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
-    for attr in corpus.positional_attributes.list():
-        cpos_attr_values: list[str] = attr.values_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
-    for attr in corpus.structural_attributes.list():
-        # We only want to iterate over non subattributes, identifiable by
-        # attr.has_values == False
-        if attr.has_values:
-            continue
-        cpos_attr_ids: list[int] = attr.ids_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            if cpos_attr_ids[i] == -1:
-                continue
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
-        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
-        if len(occured_attr_ids) == 0:
-            continue
-        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
-        if len(subattrs) == 0:
-            continue
-        lookup_name: str = f'{attr.name}_lookup'
-        lookups[lookup_name] = {}
-        for attr_id in occured_attr_ids:
-            lookups[lookup_name][attr_id] = {}
-        for subattr in subattrs:
-            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
-            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
-                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
-    return lookups
-
-
-def partial_export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    match_id_list: list[int],
-    context: int = 25
-) -> dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    match_boundaries = []
-    for match_id in match_id_list:
-        if match_id < 0 or match_id >= subcorpus.size:
-            continue
-        match_boundaries.append(
-            (
-                match_id,
-                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
-                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
-            )
-        )
-    cpos_set = set()
-    matches = []
-    for match_boundary in match_boundaries:
-        match_num, match_start, match_end = match_boundary
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}
-
-
-def export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    context: int = 25,
-    cutoff: float = float('inf'),
-    offset: int = 0
-) -> dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    first_match = max(0, offset)
-    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
-    match_boundaries = zip(
-        range(first_match, last_match + 1),
-        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
-        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
-    )
-    cpos_set = set()
-    matches = []
-    for match_num, match_start, match_end in match_boundaries:
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}

app/corpora/files/__init__.py

@@ -1,2 +0,0 @@
-from .. import bp
-from . import json_routes, routes

app/decorators.py

@@ -26,7 +26,7 @@ def socketio_login_required(f):
     def wrapper(*args, **kwargs):
         if current_user.is_authenticated:
             return f(*args, **kwargs)
-        return {'code': 401, 'body': 'Unauthorized'}
+        return {'status': 401, 'statusText': 'Unauthorized'}
     return wrapper
 
 
@@ -35,7 +35,7 @@ def socketio_permission_required(permission):
         @wraps(f)
         def wrapper(*args, **kwargs):
             if not current_user.can(permission):
-                return {'code': 403, 'body': 'Forbidden'}
+                return {'status': 403, 'statusText': 'Forbidden'}
             return f(*args, **kwargs)
         return wrapper
     return decorator

app/email.py

@@ -1,25 +1,32 @@
-from flask import current_app, render_template
+from flask import current_app, Flask, render_template
 from flask_mail import Message
 from threading import Thread
 from app import mail
 
 
-def create_message(recipient, subject, template, **kwargs):
-    subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
-    msg: Message = Message(
-        body=render_template(f'{template}.txt.j2', **kwargs),
-        html=render_template(f'{template}.html.j2', **kwargs),
+def create_message(
+    recipient: str,
+    subject: str,
+    template: str,
+    **context
+) -> Message:
+    message = Message(
+        body=render_template(f'{template}.txt.j2', **context),
+        html=render_template(f'{template}.html.j2', **context),
         recipients=[recipient],
-        subject=f'{subject_prefix} {subject}'
+        subject=f'[nopaque] {subject}'
     )
-    return msg
+    return message
 
 
-def send(msg, *args, **kwargs):
-    def _send(app, msg):
+def send(message: Message) -> Thread:
+    def _send(app: Flask, message: Message):
         with app.app_context():
-            mail.send(msg)
+            mail.send(message)
 
-    thread = Thread(target=_send, args=[current_app._get_current_object(), msg])
+    thread = Thread(
+        target=_send,
+        args=[current_app._get_current_object(), message]
+    )
     thread.start()
     return thread

app/jobs/__init__.py

@@ -1,18 +1,2 @@
-from flask import Blueprint
-from flask_login import login_required
-
-
-bp = Blueprint('jobs', __name__)
-
-
-@bp.before_request
-@login_required
-def before_request():
-    '''
-    Ensures that the routes in this package can only be visited by users that
-    are logged in.
-    '''
-    pass
-
-
-from . import routes, json_routes
+from .handle_corpora import handle_corpora
+from .handle_jobs import handle_jobs

app/jobs/events.py

@@ -1,138 +0,0 @@
-from flask import current_app
-from flask_login import current_user
-from flask_socketio import Namespace
-from app import db, hashids, socketio
-from app.extensions.flask_socketio import admin_required, login_required
-from app.models import Job, JobStatus
-
-
-class JobsNamespace(Namespace):
-    @login_required
-    def on_delete(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the current user is allowed to delete the job
-        if not (job.user == current_user or current_user.is_administrator):
-            return {
-                'code': 403,
-                'body': 'Forbidden'
-            }
-
-        # TODO: This should be a method in the Job model
-        def _delete_job(app, job_id):
-            with app.app_context():
-                job = Job.query.get(job_id)
-                job.delete()
-                db.session.commit()
-
-        # Delete the job in a background task
-        socketio.start_background_task(
-            target=_delete_job,
-            app=current_app._get_current_object(),
-            job_id=job_id
-        )
-
-        return {
-            'code': 202,
-            'body': f'Job "{job.title}" marked for deletion'
-        }
-
-
-    @admin_required
-    def on_get_log(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the job is already processed
-        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-            return {
-                'code': 409,
-                'body': 'Job is not done processing'
-            }
-
-        # Read the log file
-        with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
-            job_log = log_file.read()
-
-        return {
-            'code': 200,
-            'body': job_log
-        }
-
-
-    @login_required
-    def on_restart(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the current user is allowed to restart the job
-        if not (job.user == current_user or current_user.is_administrator):
-            return {
-                'code': 403,
-                'body': 'Forbidden'
-            }
-
-        # TODO: This should be a method in the Job model
-        def _restart_job(app, job_id):
-            with app.app_context():
-                job = Job.query.get(job_id)
-                job.restart()
-                db.session.commit()
-
-        # Restart the job in a background task
-        socketio.start_background_task(
-            target=_restart_job,
-            app=current_app._get_current_object(),
-            job_id=job_id
-        )
-
-        return {
-            'code': 202,
-            'body': f'Job "{job.title}" restarted'
-        }

app/jobs/handle_corpora.py

@@ -1,12 +1,12 @@
-from app import db, docker_client, scheduler
-from app.models import Corpus, CorpusStatus
 from flask import current_app
 import docker
 import os
 import shutil
+from app import db, docker_client, scheduler
+from app.models import Corpus, CorpusStatus
 
 
-def task():
+def handle_corpora():
     with scheduler.app.app_context():
         _handle_corpora()
 
@@ -21,14 +21,14 @@ def _handle_corpora():
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
         corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
-        _checkout_analysing_corpus_container(corpus)
+        _checkout_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
         _create_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
         _remove_cqpserver_container(corpus)
     db.session.commit()
 
-def _create_build_corpus_service(corpus):
+def _create_build_corpus_service(corpus: Corpus):
     ''' # Docker service settings # '''
     ''' ## Command ## '''
     command = ['bash', '-c']
@@ -50,12 +50,10 @@ def _create_build_corpus_service(corpus):
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'corpus.build',
-        'corpus_id': str(corpus.id)
+        'nopaque.server_name': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -100,7 +98,7 @@ def _create_build_corpus_service(corpus):
         return
     corpus.status = CorpusStatus.QUEUED
 
-def _checkout_build_corpus_service(corpus):
+def _checkout_build_corpus_service(corpus: Corpus):
     service_name = f'build-corpus_{corpus.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -128,8 +126,7 @@ def _checkout_build_corpus_service(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _create_cqpserver_container(corpus):
-    ''' # Docker container settings # '''
+def _create_cqpserver_container(corpus: Corpus):
     ''' ## Command ## '''
     command = []
     command.append(
@@ -144,9 +141,9 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Name ## '''
-    name = f'cqpserver_{corpus.id}'
+    name = f'nopaque-cqpserver-{corpus.id}'
     ''' ## Network ## '''
     network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}'
     ''' ## Volumes ## '''
@@ -203,8 +200,8 @@ def _create_cqpserver_container(corpus):
         return
     corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
 
-def _checkout_analysing_corpus_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _checkout_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         docker_client.containers.get(container_name)
     except docker.errors.NotFound as e:
@@ -214,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Get container "{container_name}" failed: {e}')
 
-def _remove_cqpserver_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _remove_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:

app/jobs/handle_jobs.py

@@ -1,3 +1,10 @@
+from datetime import datetime
+from flask import current_app
+from werkzeug.utils import secure_filename
+import docker
+import json
+import os
+import shutil
 from app import db, docker_client, hashids, scheduler
 from app.models import (
     Job,
@@ -6,16 +13,9 @@ from app.models import (
     TesseractOCRPipelineModel,
     SpaCyNLPPipelineModel
 )
-from datetime import datetime
-from flask import current_app
-from werkzeug.utils import secure_filename
-import docker
-import json
-import os
-import shutil
 
 
-def task():
+def handle_jobs():
     with scheduler.app.app_context():
         _handle_jobs()
 
@@ -29,7 +29,7 @@ def _handle_jobs():
         _remove_job_service(job)
     db.session.commit()
 
-def _create_job_service(job):
+def _create_job_service(job: Job):
     ''' # Docker service settings # '''
     ''' ## Service specific settings ## '''
     if job.service == 'file-setup-pipeline':
@@ -86,9 +86,7 @@ def _create_job_service(job):
     constraints = ['node.role==worker']
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'job',
-        'job_id': str(job.id)
+        'origin': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -169,7 +167,7 @@ def _create_job_service(job):
         return
     job.status = JobStatus.QUEUED
 
-def _checkout_job_service(job):
+def _checkout_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -218,7 +216,7 @@ def _checkout_job_service(job):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _remove_job_service(job):
+def _remove_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)

app/models/corpus.py

@@ -8,7 +8,7 @@ import shutil
 import xml.etree.ElementTree as ET
 from app import db
 from app.converters.vrt import normalize_vrt_file
-from app.extensions.sqlalchemy_extras import IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import IntEnumColumn
 from .corpus_follower_association import CorpusFollowerAssociation
 
 

app/models/default_records/tesseract_ocr_pipeline_model.yml

@@ -10,6 +10,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Amharic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata'
@@ -22,6 +23,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Arabic'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata'
@@ -34,6 +36,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Assamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata'
@@ -46,6 +49,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata'
@@ -58,6 +62,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata'
@@ -70,6 +75,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Belarusian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata'
@@ -82,6 +88,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bengali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata'
@@ -94,6 +101,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tibetan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata'
@@ -106,6 +114,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bosnian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata'
@@ -118,6 +127,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bulgarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata'
@@ -130,6 +140,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Catalan; Valencian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata'
@@ -142,6 +153,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Cebuano'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata'
@@ -154,6 +166,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Czech'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata'
@@ -166,6 +179,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Chinese - Simplified'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata'
@@ -178,6 +192,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Chinese - Traditional'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata'
@@ -190,6 +205,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Cherokee'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata'
@@ -202,6 +218,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Welsh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata'
@@ -214,6 +231,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Danish'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata'
@@ -226,6 +244,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata'
@@ -238,6 +257,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Dzongkha'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata'
@@ -250,6 +270,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Modern (1453-)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata'
@@ -262,6 +283,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata'
@@ -274,6 +296,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English, Middle (1100-1500)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata'
@@ -286,6 +309,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Esperanto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata'
@@ -298,6 +322,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Estonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata'
@@ -310,6 +335,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Basque'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata'
@@ -322,6 +348,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Persian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata'
@@ -334,6 +361,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Finnish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata'
@@ -346,6 +374,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'French'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata'
@@ -358,6 +387,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German Fraktur'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata'
@@ -370,6 +400,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'French, Middle (ca. 1400-1600)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata'
@@ -382,6 +413,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Irish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata'
@@ -394,6 +426,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Galician'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata'
@@ -406,6 +439,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Ancient (-1453)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata'
@@ -418,6 +452,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Gujarati'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata'
@@ -430,6 +465,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Haitian; Haitian Creole'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata'
@@ -442,6 +478,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hebrew'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata'
@@ -454,6 +491,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hindi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata'
@@ -466,6 +504,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Croatian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata'
@@ -478,6 +517,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hungarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata'
@@ -490,6 +530,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Inuktitut'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata'
@@ -502,6 +543,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Indonesian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata'
@@ -514,6 +556,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Icelandic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata'
@@ -526,6 +569,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Italian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata'
@@ -538,6 +582,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Italian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata'
@@ -550,6 +595,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Javanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata'
@@ -562,6 +608,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Japanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata'
@@ -574,6 +621,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kannada'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata'
@@ -586,6 +634,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata'
@@ -598,6 +647,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian - Old'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata'
@@ -610,6 +660,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kazakh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata'
@@ -622,6 +673,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Central Khmer'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata'
@@ -634,6 +686,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kirghiz; Kyrgyz'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata'
@@ -646,6 +699,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Korean'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata'
@@ -658,6 +712,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kurdish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata'
@@ -670,6 +725,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lao'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata'
@@ -682,6 +738,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata'
@@ -694,6 +751,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latvian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata'
@@ -706,6 +764,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lithuanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata'
@@ -718,6 +777,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malayalam'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata'
@@ -730,6 +790,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Marathi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata'
@@ -742,6 +803,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Macedonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata'
@@ -754,6 +816,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Maltese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata'
@@ -766,6 +829,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malay'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata'
@@ -778,6 +842,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Burmese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata'
@@ -790,6 +855,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Nepali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata'
@@ -802,6 +868,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Dutch; Flemish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata'
@@ -814,6 +881,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Norwegian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata'
@@ -826,6 +894,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Oriya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata'
@@ -838,6 +907,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Panjabi; Punjabi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata'
@@ -850,6 +920,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Polish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata'
@@ -862,6 +933,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Portuguese'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata'
@@ -874,6 +946,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Pushto; Pashto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata'
@@ -886,6 +959,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Romanian; Moldavian; Moldovan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata'
@@ -898,6 +972,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Russian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata'
@@ -910,6 +985,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Sanskrit'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata'
@@ -922,6 +998,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Sinhala; Sinhalese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata'
@@ -934,6 +1011,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovak'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata'
@@ -946,6 +1024,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovenian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata'
@@ -958,6 +1037,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Spanish; Castilian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata'
@@ -970,6 +1050,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Spanish; Castilian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata'
@@ -982,6 +1063,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Albanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata'
@@ -994,6 +1076,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata'
@@ -1006,6 +1089,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian - Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata'
@@ -1018,6 +1102,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swahili'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata'
@@ -1030,6 +1115,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swedish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata'
@@ -1042,6 +1128,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Syriac'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata'
@@ -1054,6 +1141,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tamil'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata'
@@ -1066,6 +1154,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Telugu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata'
@@ -1078,6 +1167,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tajik'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata'
@@ -1090,6 +1180,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tagalog'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata'
@@ -1102,6 +1193,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Thai'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata'
@@ -1114,6 +1206,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tigrinya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata'
@@ -1126,6 +1219,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Turkish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata'
@@ -1138,6 +1232,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uighur; Uyghur'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata'
@@ -1150,6 +1245,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Ukrainian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata'
@@ -1162,6 +1258,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Urdu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata'
@@ -1174,6 +1271,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata'
@@ -1186,6 +1284,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata'
@@ -1198,6 +1297,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Vietnamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata'
@@ -1210,6 +1310,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Yiddish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata'
@@ -1222,3 +1323,4 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'

app/models/event_listeners.py

@@ -42,8 +42,9 @@ def resource_after_delete(mapper, connection, resource):
             'path': resource.jsonpatch_path
         }
     ]
+    namespace = '/users'
     room = f'/users/{resource.user_hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def cfa_after_delete(mapper, connection, cfa):
@@ -54,8 +55,9 @@ def cfa_after_delete(mapper, connection, cfa):
             'path': jsonpatch_path
         }
     ]
+    namespace = '/users'
     room = f'/users/{cfa.corpus.user.hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def resource_after_insert(mapper, connection, resource):
@@ -69,8 +71,9 @@ def resource_after_insert(mapper, connection, resource):
             'value': jsonpatch_value
         }
     ]
+    namespace = '/users'
     room = f'/users/{resource.user_hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def cfa_after_insert(mapper, connection, cfa):
@@ -83,8 +86,9 @@ def cfa_after_insert(mapper, connection, cfa):
             'value': jsonpatch_value
         }
     ]
+    namespace = '/users'
     room = f'/users/{cfa.corpus.user.hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def resource_after_update(mapper, connection, resource):
@@ -109,8 +113,9 @@ def resource_after_update(mapper, connection, resource):
             }
         )
     if jsonpatch:
+        namespace = '/users'
         room = f'/users/{resource.user_hashid}'
-        socketio.emit('PATCH', jsonpatch, room=room)
+        socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def job_after_update(mapper, connection, job):

app/models/job.py

@@ -6,7 +6,7 @@ from time import sleep
 from pathlib import Path
 import shutil
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn, IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn, IntEnumColumn
 
 
 class JobStatus(IntEnum):

app/models/spacy_nlp_pipeline_model.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import requests
 import yaml
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn
 from .file_mixin import FileMixin
 from .user import User
 
@@ -41,7 +41,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
     @property
     def url(self):
         return url_for(
-            'contributions.spacy_nlp_pipeline_model',
+            'contributions.spacy_nlp_pipeline_models.entity',
             spacy_nlp_pipeline_model_id=self.id
         )
 

app/models/tesseract_ocr_pipeline_model.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import requests
 import yaml
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn
 from .file_mixin import FileMixin
 from .user import User
 
@@ -40,7 +40,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
     @property
     def url(self):
         return url_for(
-            'contributions.tesseract_ocr_pipeline_model',
+            'contributions.tesseract_ocr_pipeline_models.entity',
             tesseract_ocr_pipeline_model_id=self.id
         )
 

app/models/user.py

@@ -11,7 +11,7 @@ import re
 import secrets
 import shutil
 from app import db, hashids
-from app.extensions.sqlalchemy_extras import IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import IntEnumColumn
 from .corpus import Corpus
 from .corpus_follower_association import CorpusFollowerAssociation
 from .corpus_follower_role import CorpusFollowerRole

app/namespaces/cqi_over_sio/__init__.py

@@ -1,17 +1,16 @@
 from cqi import CQiClient
 from cqi.errors import CQiException
 from cqi.status import CQiStatus
-from docker.models.containers import Container
-from flask import current_app, session
+from flask import current_app
 from flask_login import current_user
 from flask_socketio import Namespace
 from inspect import signature
 from threading import Lock
-from typing import Callable
 from app import db, docker_client, hashids, socketio
 from app.decorators import socketio_login_required
 from app.models import Corpus, CorpusStatus
-from . import extensions
+from . import cqi_extension_functions
+from .utils import SessionManager
 
 
 '''
@@ -38,7 +37,7 @@ Basic concept:
 '''
 
 
-CQI_API_FUNCTION_NAMES: list[str] = [
+CQI_API_FUNCTION_NAMES = [
     'ask_feature_cl_2_3',
     'ask_feature_cqi_1_0',
     'ask_feature_cqp_2_3',
@@ -86,68 +85,90 @@ CQI_API_FUNCTION_NAMES: list[str] = [
 ]
 
 
-class CQiOverSocketIO(Namespace):
+CQI_EXTENSION_FUNCTION_NAMES = [
+    'ext_corpus_update_db',
+    'ext_corpus_static_data',
+    'ext_corpus_paginate_corpus',
+    'ext_cqp_paginate_subcorpus',
+    'ext_cqp_partial_export_subcorpus',
+    'ext_cqp_export_subcorpus',
+]
+
+
+class CQiOverSocketIONamespace(Namespace):
     @socketio_login_required
     def on_connect(self):
         pass
 
     @socketio_login_required
-    def on_init(self, db_corpus_hashid: str):
-        db_corpus_id: int = hashids.decode(db_corpus_hashid)
-        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
-        if db_corpus is None:
+    def on_init(self, corpus_hashid: str) -> dict:
+        corpus_id = hashids.decode(corpus_hashid)
+
+        if not isinstance(corpus_id, int):
+            return {'code': 400, 'msg': 'Bad Request'}
+
+        corpus = Corpus.query.get(corpus_id)
+
+        if corpus is None:
             return {'code': 404, 'msg': 'Not Found'}
-        if not (db_corpus.user == current_user
-                or current_user.is_following_corpus(db_corpus)
-                or current_user.is_administrator):
+
+        if not (
+            corpus.user == current_user
+            or current_user.is_following_corpus(corpus)
+            or current_user.is_administrator
+        ):
             return {'code': 403, 'msg': 'Forbidden'}
-        if db_corpus.status not in [
+
+        if corpus.status not in [
             CorpusStatus.BUILT,
             CorpusStatus.STARTING_ANALYSIS_SESSION,
             CorpusStatus.RUNNING_ANALYSIS_SESSION,
             CorpusStatus.CANCELING_ANALYSIS_SESSION
         ]:
             return {'code': 424, 'msg': 'Failed Dependency'}
-        if db_corpus.num_analysis_sessions is None:
-            db_corpus.num_analysis_sessions = 0
-            db.session.commit()
-        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+
+        corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
         db.session.commit()
-        retry_counter: int = 20
-        while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+        retry_counter = 20
+        while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
             if retry_counter == 0:
-                db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+                corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
                 db.session.commit()
                 return {'code': 408, 'msg': 'Request Timeout'}
             socketio.sleep(3)
             retry_counter -= 1
-            db.session.refresh(db_corpus)
-        # cqi_client: CQiClient = CQiClient(f'cqpserver_{db_corpus_id}')
-        cqpserver_container_name: str = f'cqpserver_{db_corpus_id}'
-        cqpserver_container: Container = docker_client.containers.get(cqpserver_container_name)
-        cqpserver_host: str = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
-        cqi_client: CQiClient = CQiClient(cqpserver_host)
-        session['cqi_over_sio'] = {
-            'cqi_client': cqi_client,
-            'cqi_client_lock': Lock(),
-            'db_corpus_id': db_corpus_id
-        }
+            db.session.refresh(corpus)
+
+        cqpserver_container_name = f'nopaque-cqpserver-{corpus_id}'
+        cqpserver_container = docker_client.containers.get(cqpserver_container_name)
+        cqpserver_ip_address = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
+        cqi_client = CQiClient(cqpserver_ip_address)
+        cqi_client_lock = Lock()
+
+        SessionManager.setup()
+        SessionManager.set_corpus_id(corpus_id)
+        SessionManager.set_cqi_client(cqi_client)
+        SessionManager.set_cqi_client_lock(cqi_client_lock)
+
         return {'code': 200, 'msg': 'OK'}
 
     @socketio_login_required
-    def on_exec(self, fn_name: str, fn_args: dict = {}):
+    def on_exec(self, fn_name: str, fn_args: dict = {}) -> dict:
         try:
-            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+            cqi_client = SessionManager.get_cqi_client()
+            cqi_client_lock = SessionManager.get_cqi_client_lock()
         except KeyError:
             return {'code': 424, 'msg': 'Failed Dependency'}
+
         if fn_name in CQI_API_FUNCTION_NAMES:
-            fn: Callable = getattr(cqi_client.api, fn_name)
-        elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
-            fn: Callable = getattr(extensions, fn_name)
+            fn = getattr(cqi_client.api, fn_name)
+        elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
+            fn = getattr(cqi_extension_functions, fn_name)
         else:
             return {'code': 400, 'msg': 'Bad Request'}
+
         for param in signature(fn).parameters.values():
+            # Check if the parameter is optional or required
             if param.default is param.empty:
                 if param.name not in fn_args:
                     return {'code': 400, 'msg': 'Bad Request'}
@@ -156,6 +177,7 @@ class CQiOverSocketIO(Namespace):
                     continue
             if type(fn_args[param.name]) is not param.annotation:
                 return {'code': 400, 'msg': 'Bad Request'}
+
         cqi_client_lock.acquire()
         try:
             fn_return_value = fn(**fn_args)
@@ -173,6 +195,7 @@ class CQiOverSocketIO(Namespace):
             }
         finally:
             cqi_client_lock.release()
+
         if isinstance(fn_return_value, CQiStatus):
             payload = {
                 'code': fn_return_value.code,
@@ -180,27 +203,31 @@ class CQiOverSocketIO(Namespace):
             }
         else:
             payload = fn_return_value
+
         return {'code': 200, 'msg': 'OK', 'payload': payload}
 
     def on_disconnect(self):
         try:
-            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
-            db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+            corpus_id = SessionManager.get_corpus_id()
+            cqi_client = SessionManager.get_cqi_client()
+            cqi_client_lock = SessionManager.get_cqi_client_lock()
+            SessionManager.teardown()
         except KeyError:
             return
+
         cqi_client_lock.acquire()
-        try:
-            session.pop('cqi_over_sio')
-        except KeyError:
-            pass
+
         try:
             cqi_client.api.ctrl_bye()
         except (BrokenPipeError, CQiException):
             pass
+
         cqi_client_lock.release()
-        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
-        if db_corpus is None:
+
+        corpus = Corpus.query.get(corpus_id)
+
+        if corpus is None:
             return
-        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+
+        corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
         db.session.commit()

app/namespaces/cqi_over_sio/cqi_extension_functions.py

@@ -1,54 +1,39 @@
 from collections import Counter
-from cqi import CQiClient
 from cqi.models.corpora import Corpus as CQiCorpus
 from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
-from cqi.models.attributes import (
-    PositionalAttribute as CQiPositionalAttribute,
-    StructuralAttribute as CQiStructuralAttribute
-)
 from cqi.status import StatusOk as CQiStatusOk
-from flask import session
+from flask import current_app
 import gzip
 import json
 import math
 from app import db
 from app.models import Corpus
-from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
-
-
-CQI_EXTENSION_FUNCTION_NAMES: list[str] = [
-    'ext_corpus_update_db',
-    'ext_corpus_static_data',
-    'ext_corpus_paginate_corpus',
-    'ext_cqp_paginate_subcorpus',
-    'ext_cqp_partial_export_subcorpus',
-    'ext_cqp_export_subcorpus',
-]
+from .utils import SessionManager
 
 
 def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
-    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
-    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    corpus_id = SessionManager.get_corpus_id()
+    cqi_client = SessionManager.get_cqi_client()
+    db_corpus = Corpus.query.get(corpus_id)
+    cqi_corpus = cqi_client.corpora.get(corpus)
     db_corpus.num_tokens = cqi_corpus.size
     db.session.commit()
     return CQiStatusOk()
 
 
 def ext_corpus_static_data(corpus: str) -> dict:
-    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
-    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+    corpus_id = SessionManager.get_corpus_id()
+    db_corpus = Corpus.query.get(corpus_id)
 
     static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
     if static_data_file_path.exists():
         with static_data_file_path.open('rb') as f:
             return f.read()
 
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
-    cqi_p_attrs: list[CQiPositionalAttribute] = cqi_corpus.positional_attributes.list()
-    cqi_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list()
+    cqi_client = SessionManager.get_cqi_client()
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    cqi_p_attrs = cqi_corpus.positional_attributes.list()
+    cqi_s_attrs = cqi_corpus.structural_attributes.list()
 
     static_data = {
         'corpus': {
@@ -61,21 +46,21 @@ def ext_corpus_static_data(corpus: str) -> dict:
     }
 
     for p_attr in cqi_p_attrs:
-        print(f'corpus.freqs.{p_attr.name}')
+        current_app.logger.info(f'corpus.freqs.{p_attr.name}')
         static_data['corpus']['freqs'][p_attr.name] = []
-        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        p_attr_id_list = list(range(p_attr.lexicon_size))
         static_data['corpus']['freqs'][p_attr.name].extend(p_attr.freqs_by_ids(p_attr_id_list))
         del p_attr_id_list
 
-        print(f'p_attrs.{p_attr.name}')
+        current_app.logger.info(f'p_attrs.{p_attr.name}')
         static_data['p_attrs'][p_attr.name] = []
-        cpos_list: list[int] = list(range(cqi_corpus.size))
+        cpos_list = list(range(cqi_corpus.size))
         static_data['p_attrs'][p_attr.name].extend(p_attr.ids_by_cpos(cpos_list))
         del cpos_list
 
-        print(f'values.p_attrs.{p_attr.name}')
+        current_app.logger.info(f'values.p_attrs.{p_attr.name}')
         static_data['values']['p_attrs'][p_attr.name] = []
-        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        p_attr_id_list = list(range(p_attr.lexicon_size))
         static_data['values']['p_attrs'][p_attr.name].extend(p_attr.values_by_ids(p_attr_id_list))
         del p_attr_id_list
 
@@ -91,9 +76,9 @@ def ext_corpus_static_data(corpus: str) -> dict:
             # Note: Needs more testing, don't use it in production       #
             ##############################################################
             cqi_corpus.query('Last', f'<{s_attr.name}> []* </{s_attr.name}>;')
-            cqi_subcorpus: CQiSubcorpus = cqi_corpus.subcorpora.get('Last')
-            first_match: int = 0
-            last_match: int = cqi_subcorpus.size - 1
+            cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
+            first_match = 0
+            last_match = cqi_subcorpus.size - 1
             match_boundaries = zip(
                 range(first_match, last_match + 1),
                 cqi_subcorpus.dump(
@@ -111,7 +96,7 @@ def ext_corpus_static_data(corpus: str) -> dict:
             del cqi_subcorpus, first_match, last_match
             for id, lbound, rbound in match_boundaries:
                 static_data['s_attrs'][s_attr.name]['lexicon'].append({})
-                print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+                current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
                 static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
             del match_boundaries
 
@@ -123,33 +108,33 @@ def ext_corpus_static_data(corpus: str) -> dict:
             # This is a very slow operation, thats why we only use it for
             # the text attribute
             lbound, rbound = s_attr.cpos_by_id(id)
-            print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+            current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
             static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
             static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
-            cpos_list: list[int] = list(range(lbound, rbound + 1))
+            cpos_list = list(range(lbound, rbound + 1))
             for p_attr in cqi_p_attrs:
-                p_attr_ids: list[int] = []
+                p_attr_ids = []
                 p_attr_ids.extend(p_attr.ids_by_cpos(cpos_list))
-                print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
+                current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
                 static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids))
                 del p_attr_ids
             del cpos_list
 
-        sub_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
-        print(f's_attrs.{s_attr.name}.values')
+        sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
+        current_app.logger.info(f's_attrs.{s_attr.name}.values')
         static_data['s_attrs'][s_attr.name]['values'] = [
             sub_s_attr.name[(len(s_attr.name) + 1):]
             for sub_s_attr in sub_s_attrs
         ]
-        s_attr_id_list: list[int] = list(range(s_attr.size))
-        sub_s_attr_values: list[str] = []
+        s_attr_id_list = list(range(s_attr.size))
+        sub_s_attr_values = []
         for sub_s_attr in sub_s_attrs:
             tmp = []
             tmp.extend(sub_s_attr.values_by_ids(s_attr_id_list))
             sub_s_attr_values.append(tmp)
             del tmp
         del s_attr_id_list
-        print(f'values.s_attrs.{s_attr.name}')
+        current_app.logger.info(f'values.s_attrs.{s_attr.name}')
         static_data['values']['s_attrs'][s_attr.name] = [
             {
                 s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id]
@@ -159,11 +144,11 @@ def ext_corpus_static_data(corpus: str) -> dict:
             } for s_attr_id in range(0, s_attr.size)
         ]
         del sub_s_attr_values
-    print('Saving static data to file')
+    current_app.logger.info('Saving static data to file')
     with gzip.open(static_data_file_path, 'wt') as f:
         json.dump(static_data, f)
     del static_data
-    print('Sending static data to client')
+    current_app.logger.info('Sending static data to client')
     with open(static_data_file_path, 'rb') as f:
         return f.read()
 
@@ -173,7 +158,7 @@ def ext_corpus_paginate_corpus(
     page: int = 1,
     per_page: int = 20
 ) -> dict:
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus)
     # Sanity checks
     if (
@@ -188,7 +173,7 @@ def ext_corpus_paginate_corpus(
     first_cpos = (page - 1) * per_page
     last_cpos = min(cqi_corpus.size, first_cpos + per_page)
     cpos_list = [*range(first_cpos, last_cpos)]
-    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
+    lookups = _lookups_by_cpos(cqi_corpus, cpos_list)
     payload = {}
     # the items for the current page
     payload['items'] = [cpos_list]
@@ -220,7 +205,7 @@ def ext_cqp_paginate_subcorpus(
     per_page: int = 20
 ) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
     # Sanity checks
@@ -235,7 +220,7 @@ def ext_cqp_paginate_subcorpus(
         return {'code': 416, 'msg': 'Range Not Satisfiable'}
     offset = (page - 1) * per_page
     cutoff = per_page
-    cqi_results_export = export_subcorpus(
+    cqi_results_export = _export_subcorpus(
         cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
     payload = {}
     # the items for the current page
@@ -267,20 +252,145 @@ def ext_cqp_partial_export_subcorpus(
     context: int = 50
 ) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
+    cqi_subcorpus_partial_export = _partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
     return cqi_subcorpus_partial_export
 
 
-def ext_cqp_export_subcorpus(
-    subcorpus: str,
-    context: int = 50
-) -> dict:
+def ext_cqp_export_subcorpus(subcorpus: str, context: int = 50) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
+    cqi_subcorpus_export = _export_subcorpus(cqi_subcorpus, context=context)
     return cqi_subcorpus_export
+
+
+def _lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
+    lookups = {}
+    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
+    for attr in corpus.positional_attributes.list():
+        cpos_attr_values = attr.values_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
+    for attr in corpus.structural_attributes.list():
+        # We only want to iterate over non subattributes, identifiable by
+        # attr.has_values == False
+        if attr.has_values:
+            continue
+        cpos_attr_ids = attr.ids_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            if cpos_attr_ids[i] == -1:
+                continue
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
+        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
+        if len(occured_attr_ids) == 0:
+            continue
+        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
+        if len(subattrs) == 0:
+            continue
+        lookup_name = f'{attr.name}_lookup'
+        lookups[lookup_name] = {}
+        for attr_id in occured_attr_ids:
+            lookups[lookup_name][attr_id] = {}
+        for subattr in subattrs:
+            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
+            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
+                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
+    return lookups
+
+
+def _partial_export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    match_id_list: list[int],
+    context: int = 25
+) -> dict:
+    if subcorpus.size == 0:
+        return {'matches': []}
+    match_boundaries = []
+    for match_id in match_id_list:
+        if match_id < 0 or match_id >= subcorpus.size:
+            continue
+        match_boundaries.append(
+            (
+                match_id,
+                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
+                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
+            )
+        )
+    cpos_set = set()
+    matches = []
+    for match_boundary in match_boundaries:
+        match_num, match_start, match_end = match_boundary
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = _lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}
+
+
+def _export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    context: int = 25,
+    cutoff: float = float('inf'),
+    offset: int = 0
+) -> dict:
+    if subcorpus.size == 0:
+        return {'matches': []}
+    first_match = max(0, offset)
+    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
+    match_boundaries = zip(
+        range(first_match, last_match + 1),
+        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
+        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
+    )
+    cpos_set = set()
+    matches = []
+    for match_num, match_start, match_end in match_boundaries:
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = _lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}

app/namespaces/cqi_over_sio/utils.py

@@ -0,0 +1,37 @@
+from cqi import CQiClient
+from threading import Lock
+from flask import session
+
+
+class SessionManager:
+    @staticmethod
+    def setup():
+        session['cqi_over_sio'] = {}
+
+    @staticmethod
+    def teardown():
+        session.pop('cqi_over_sio')
+
+    @staticmethod
+    def set_corpus_id(corpus_id: int):
+        session['cqi_over_sio']['corpus_id'] = corpus_id
+
+    @staticmethod
+    def get_corpus_id() -> int:
+        return session['cqi_over_sio']['corpus_id']
+
+    @staticmethod
+    def set_cqi_client(cqi_client: CQiClient):
+        session['cqi_over_sio']['cqi_client'] = cqi_client
+
+    @staticmethod
+    def get_cqi_client() -> CQiClient:
+        return session['cqi_over_sio']['cqi_client']
+
+    @staticmethod
+    def set_cqi_client_lock(cqi_client_lock: Lock):
+        session['cqi_over_sio']['cqi_client_lock'] = cqi_client_lock
+
+    @staticmethod
+    def get_cqi_client_lock() -> Lock:
+        return session['cqi_over_sio']['cqi_client_lock']

app/namespaces/jobs.py

@@ -0,0 +1,109 @@
+from flask import current_app, Flask
+from flask_login import current_user
+from flask_socketio import Namespace
+from app import db, hashids, socketio
+from app.decorators import socketio_admin_required, socketio_login_required
+from app.models import Job, JobStatus
+
+
+def _delete_job(app: Flask, job_id: int):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.delete()
+        db.session.commit()
+
+
+def _restart_job(app, job_id):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.restart()
+        db.session.commit()
+
+
+class UsersNamespace(Namespace):
+    @socketio_login_required
+    def on_delete(self, job_hashid: str) -> dict:
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            job.user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        socketio.start_background_task(
+            _delete_job,
+            current_app._get_current_object(),
+            job_id
+        )
+
+        return {
+            'body': f'Job "{job.title}" marked for deletion',
+            'status': 202,
+            'statusText': 'Accepted'
+        }
+
+    @socketio_admin_required
+    def on_log(self, job_hashid: str):
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+            return {'status': 409, 'statusText': 'Conflict'}
+
+        with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
+            log = log_file.read()
+
+        return {
+            'body': log,
+            'status': 200,
+            'statusText': 'Forbidden'
+        }
+
+    socketio_login_required
+    def on_restart(self, job_hashid: str):
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            job.user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        if job.status == JobStatus.FAILED:
+            return {'status': 409, 'statusText': 'Conflict'}
+
+        socketio.start_background_task(
+            _restart_job,
+            current_app._get_current_object(),
+            job_id
+        )
+
+        return {
+            'body': f'Job "{job.title}" marked for restarting',
+            'status': 202,
+            'statusText': 'Accepted'
+        }

app/namespaces/users.py

@@ -0,0 +1,116 @@
+from flask import current_app, Flask
+from flask_login import current_user
+from flask_socketio import join_room, leave_room, Namespace
+from app import db, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import User
+
+
+def _delete_user(app: Flask, user_id: int):
+    with app.app_context():
+        user = User.query.get(user_id)
+        user.delete()
+        db.session.commit()
+
+
+class UsersNamespace(Namespace):
+    @socketio_login_required
+    def on_get(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        return {
+            'body': user.to_json_serializeable(
+                backrefs=True,
+                relationships=True
+            ),
+            'status': 200,
+            'statusText': 'OK'
+        }
+
+    @socketio_login_required
+    def on_subscribe(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        join_room(f'/users/{user.hashid}')
+
+        return {'status': 200, 'statusText': 'OK'}
+
+    @socketio_login_required
+    def on_unsubscribe(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        leave_room(f'/users/{user.hashid}')
+
+        return {'status': 200, 'statusText': 'OK'}
+
+    @socketio_login_required
+    def on_delete(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        socketio.start_background_task(
+            _delete_user,
+            current_app._get_current_object(),
+            user.id
+        )
+
+        return {
+            'body': f'User "{user.username}" marked for deletion',
+            'status': 202,
+            'statusText': 'Accepted'
+        }

app/static/css/corpus-status-text.css

@@ -2,6 +2,10 @@
   --corpus-status-content: "unprepared";
 }
 
+[data-corpus-status="SUBMITTED"] {
+  --corpus-status-content: "submitted";
+}
+
 [data-corpus-status="QUEUED"] {
   --corpus-status-content: "queued";
 }

app/static/css/height.css

@@ -0,0 +1,47 @@
+.h-10 {
+    height: 10% !important;
+}
+
+.h-20 {
+    height: 20% !important;
+}
+
+.h-25 {
+    height: 25% !important;
+}
+
+.h-30 {
+    height: 30% !important;
+}
+
+.h-40 {
+    height: 40% !important;
+}
+
+.h-50 {
+    height: 50% !important;
+}
+
+.h-60 {
+    height: 60% !important;
+}
+
+.h-70 {
+    height: 70% !important;
+}
+
+.h-75 {
+    height: 75% !important;
+}
+
+.h-80 {
+    height: 80% !important;
+}
+
+.h-90 {
+    height: 90% !important;
+}
+
+.h-100 {
+    height: 100% !important;
+}

app/static/css/materialize.override.css

@@ -1,37 +1,3 @@
-/* #region sidenav-fixed */
-/*
- * The sidenav-fixed class is used which causes the sidenav to be fixed and open
- * on large screens and hides to the regular functionality on smaller screens.
- * In order to prevent the sidenav to overlap the content, the content (header, main and footer)
- * gets an offset equal to the width of the sidenav.
- *
- * Read more: https://materializecss.com/sidenav.html#variations
- */
-body[data-sidenav-fixed="true" i] header,
-body[data-sidenav-fixed="true" i] main,
-body[data-sidenav-fixed="true" i] footer {
-  padding-left: 300px;
-}
-
-@media only screen and (max-width : 992px) {
-  body[data-sidenav-fixed="true" i] header,
-  body[data-sidenav-fixed="true" i] main,
-  body[data-sidenav-fixed="true" i] footer {
-    padding-left: 0;
-  }
-}
-
-body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
-  width: calc(100% - 300px);
-}
-
-@media only screen and (max-width : 992px) {
-  body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
-    width: 100%;
-  }
-}
-/* #endregion sidenav-fixed */
-
 /* #region sticky-footer */
 /*
  * Sticky Footer:
@@ -44,13 +10,13 @@ body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
  *
  * Read more: https://materializecss.com/footer.html#sticky-footer
  */
- body[data-sticky-footer="true" i] {
+ body {
   display: flex;
   min-height: 100vh;
   flex-direction: column;
 }
 
-body[data-sticky-footer="true" i] main {
+main {
   flex: 1 0 auto;
 }
 /* #endregion sticky-footer */

app/static/css/width.css

@@ -0,0 +1,47 @@
+.w-10 {
+    width: 10% !important;
+}
+
+.w-20 {
+    width: 20% !important;
+}
+
+.w-25 {
+    width: 25% !important;
+}
+
+.w-30 {
+    width: 30% !important;
+}
+
+.w-40 {
+    width: 40% !important;
+}
+
+.w-50 {
+    width: 50% !important;
+}
+
+.w-60 {
+    width: 60% !important;
+}
+
+.w-70 {
+    width: 70% !important;
+}
+
+.w-75 {
+    width: 75% !important;
+}
+
+.w-80 {
+    width: 80% !important;
+}
+
+.w-90 {
+    width: 90% !important;
+}
+
+.w-100 {
+    width: 100% !important;
+}

app/static/js/app.js

@@ -1,182 +1,20 @@
 nopaque.App = class App {
   constructor() {
-    this.data = {
-      promises: {getUser: {}, subscribeUser: {}},
-      users: {},
-    };
     this.socket = io({transports: ['websocket'], upgrade: false});
-    this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
-  }
 
-  getUser(userId) {
-    if (userId in this.data.promises.getUser) {
-      return this.data.promises.getUser[userId];
-    }
+    // Endpoints
+    this.users = new nopaque.app.endpoints.Users(this);
 
-    this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('GET /users/<user_id>', userId, (response) => {
-        if (response.status === 200) {
-          this.data.users[userId] = response.body;
-          resolve(this.data.users[userId]);
-        } else {
-          reject(`[${response.status}] ${response.statusText}`);
-        }
-      });
-    });
-
-    return this.data.promises.getUser[userId];
-  }
-
-  subscribeUser(userId) {
-    if (userId in this.data.promises.subscribeUser) {
-      return this.data.promises.subscribeUser[userId];
-    }
-
-    this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
-        if (response.status !== 200) {
-          reject(response);
-          return;
-        }
-        resolve(response);
-      });
-    });
-
-    return this.data.promises.subscribeUser[userId];
-  }
-
-  flash(message, category) {
-    let iconPrefix = '';
-    switch (category) {
-      case 'corpus': {
-        iconPrefix = '<i class="left material-icons">book</i>';
-        break;
-      }
-      case 'error': {
-        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
-        break;
-      }
-      case 'job': {
-        iconPrefix = '<i class="left nopaque-icons">J</i>';
-        break;
-      }
-      case 'settings': {
-        iconPrefix = '<i class="left material-icons">settings</i>';
-        break;
-      }
-      default: {
-        iconPrefix = '<i class="left material-icons">notifications</i>';
-        break;
-      }
-    }
-    let toast = M.toast(
-      {
-        html: `
-          <span>${iconPrefix}${message}</span>
-          <button class="action-button btn-flat toast-action white-text" data-action="close">
-            <i class="material-icons">close</i>
-          </button>
-        `.trim()
-      }
-    );
-    let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
-    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
-  }
-  
-  onPatch(patch) {
-    // Filter Patch to only include operations on users that are initialized
-    let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
-    let filteredPatch = patch.filter(operation => regExp.test(operation.path));
-
-    // Handle job status updates
-    let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
-    let subFilteredPatch = filteredPatch
-      .filter((operation) => {return operation.op === 'replace';})
-      .filter((operation) => {return subRegExp.test(operation.path);});
-    for (let operation of subFilteredPatch) {
-      let [match, userId, jobId] = operation.path.match(subRegExp);
-      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
-    }
-
-    // Apply Patch
-    jsonpatch.applyPatch(this.data, filteredPatch);
+    // Extensions
+    this.toaster = new nopaque.app.extensions.Toaster(this);
+    this.ui = new nopaque.app.extensions.UI(this);
+    this.userHub = new nopaque.app.extensions.UserHub(this);
   }
 
   init() {
-    this.initUi();
-  }
-
-  initUi() {
-    /* Pre-Initialization fixes */
-    // #region
-
-    // Flask-WTF sets the standard HTML maxlength Attribute on input/textarea
-    // elements to specify their maximum length (in characters). Unfortunatly
-    // Materialize won't recognize the maxlength Attribute, instead it uses
-    // the data-length Attribute. It's conversion time :)
-    for (let elem of document.querySelectorAll('input[maxlength], textarea[maxlength]')) {
-      elem.dataset.length = elem.getAttribute('maxlength');
-      elem.removeAttribute('maxlength');
-    }
-
-    // To work around some limitations with the Form setup of Flask-WTF.
-    // HTML option elements with an empty value are considered as placeholder
-    // elements. The user should not be able to actively select these options.
-    // So they get the disabled attribute.
-    for (let optionElement of document.querySelectorAll('option[value=""]')) {
-      optionElement.disabled = true;
-    }
-
-    // TODO: Check why we are doing this.
-    for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) {
-      for (let c of optgroupElement.children) {
-        optgroupElement.parentElement.insertAdjacentElement('afterbegin', c);
-      }
-      optgroupElement.remove();
-    }
-    // #endregion
-
-
-    /* Initialize Materialize Components */
-    // #region
-  
-    // Automatically initialize Materialize Components that do not require
-    // additional configuration.
-    M.AutoInit();
-
-    // CharacterCounters
-    // Materialize didn't include the CharacterCounter plugin within the
-    // AutoInit method (maybe they forgot it?). Anyway... We do it here. :)
-    M.CharacterCounter.init(document.querySelectorAll('input[data-length]:not(.no-autoinit), textarea[data-length]:not(.no-autoinit)'));
-
-    // Header navigation account Dropdown.
-    M.Dropdown.init(
-      document.querySelector('#nav-account-dropdown-trigger'),
-      {
-        alignment: 'right',
-        constrainWidth: false,
-        coverTrigger: false
-      }
-    );
-
-    // Terms of use modal
-    M.Modal.init(
-      document.querySelector('#terms-of-use-modal'),
-      {
-        dismissible: false,
-        onCloseEnd: (modalElement) => {
-          nopaque.requests.users.entity.acceptTermsOfUse();
-        }
-      }
-    );
-    // #endregion
-
-
-    /* Initialize nopaque Components */
-    // #region 
-    nopaque.resource_displays.AutoInit();
-    nopaque.resource_lists.AutoInit();
-    nopaque.forms.AutoInit();
-    // #endregion
+    // Initialize extensions
+    this.toaster.init();
+    this.ui.init();
+    this.userHub.init();
   }
 };

app/static/js/app/endpoints/index.js

@@ -0,0 +1 @@
+nopaque.app.endpoints = {};

app/static/js/app/endpoints/users.js

@@ -0,0 +1,41 @@
+nopaque.app.endpoints.Users = class Users {
+  constructor(app) {
+    this.app = app;
+
+    this.socket = io('/users', {transports: ['websocket'], upgrade: false});
+  }
+
+  async get(userId) {
+    const response = await this.socket.emitWithAck('get', userId);
+
+    if (response.status !== 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+
+    return response.body;
+  }
+
+  async subscribe(userId) {
+    const response = await this.socket.emitWithAck('subscribe', userId);
+
+    if (response.status != 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+
+  async unsubscribe(userId) {
+    const response = await this.socket.emitWithAck('unsubscribe', userId);
+
+    if (response.status != 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+
+  async delete(userId) {
+    const response = await this.socket.emitWithAck('delete', userId);
+
+    if (response.status != 202) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+}