Bump nopaque version

.vscode/settings.json

@@ -1,7 +1,24 @@
 {
     "editor.rulers": [79],
-    "editor.tabSize": 2,
+    "editor.tabSize": 4,
+    "emmet.includeLanguages": {
+        "jinja-html": "html"
+    },
+    "files.associations": {
+        ".flaskenv": "env",
+        "*.env.tpl": "env",
+        "*.txt.j2": "jinja"
+    },
     "files.insertFinalNewline": true,
     "files.trimFinalNewlines": true,
-    "files.trimTrailingWhitespace": true
+    "files.trimTrailingWhitespace": true,
+    "[html]": {
+        "editor.tabSize": 2
+    },
+    "[javascript]": {
+        "editor.tabSize": 2
+    },
+    "[jinja-html]": {
+        "editor.tabSize": 2
+    }
 }

README.md

@@ -35,7 +35,7 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
 # Create data directories
-username@hostname:~$ mkdir volumes/{db,mq}
+username@hostname:~$ mkdir -p volumes/{db,mq}
 username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
 # Fill out the variables within these files.

app/__init__.py

@@ -130,8 +130,11 @@ def create_app(config: Config = Config) -> Flask:
     # endregion Blueprints
 
     # region SocketIO Namespaces
-    from .blueprints.corpora.cqi_over_sio import CQiOverSocketIO
-    socketio.on_namespace(CQiOverSocketIO('/cqi_over_sio'))
+    from .namespaces.cqi_over_sio import CQiOverSocketIONamespace
+    socketio.on_namespace(CQiOverSocketIONamespace('/cqi_over_sio'))
+
+    from .namespaces.users import UsersNamespace
+    socketio.on_namespace(UsersNamespace('/users'))
     # endregion SocketIO Namespaces
 
     # region Database event Listeners

app/blueprints/admin/events.py

@@ -1,49 +0,0 @@
-from flask_login import current_user
-from flask_socketio import disconnect, Namespace
-from app import db, hashids
-from app.decorators import socketio_admin_required
-from app.models import User
-
-
-class AdminNamespace(Namespace):
-    def on_connect(self):
-        # Check if the user is authenticated and is an administrator
-        if not (current_user.is_authenticated and current_user.is_administrator):
-            disconnect()
-
-
-    @socketio_admin_required
-    def on_set_user_confirmed(self, user_hashid: str, confirmed_value: bool):
-        # Decode the user hashid
-        user_id = hashids.decode(user_hashid)
-
-        # Validate user_id
-        if not isinstance(user_id, int):
-            return {
-                'code': 400,
-                'body': 'user_id is invalid'
-            }
-
-        # Validate confirmed_value
-        if not isinstance(confirmed_value, bool):
-            return {
-                'code': 400,
-                'body': 'confirmed_value is invalid'
-            }
-
-        # Load user from database
-        user = User.query.get(user_id)
-        if user is None:
-            return {
-                'code': 404,
-                'body': 'User not found'
-            }
-
-        # Update user confirmed status
-        user.confirmed = confirmed_value
-        db.session.commit()
-
-        return {
-            'code': 200,
-            'body': f'User "{user.username}" is now {"confirmed" if confirmed_value else "unconfirmed"}'
-        }

app/blueprints/auth/__init__.py

@@ -1,5 +1,29 @@
-from flask import Blueprint
+from flask import Blueprint, redirect, request, url_for
+from flask_login import current_user
+from app import db
 
 
 bp = Blueprint('auth', __name__)
+
+
+@bp.before_app_request
+def before_request():
+    if not current_user.is_authenticated:
+        return
+
+    current_user.ping()
+    db.session.commit()
+
+    if (
+        not current_user.confirmed
+        and request.endpoint
+        and request.blueprint != 'auth'
+        and request.endpoint != 'static'
+    ):
+        return redirect(url_for('auth.unconfirmed'))
+
+    if not current_user.terms_of_use_accepted:
+        return redirect(url_for('main.terms_of_use'))
+
+
 from . import routes

app/blueprints/auth/forms.py

@@ -60,7 +60,11 @@ class RegistrationForm(FlaskForm):
 
     def validate_username(self, field):
         if User.query.filter_by(username=field.data).first():
-            raise ValidationError('Username already in use')
+            raise ValidationError('Username already registered')
+
+    def validate_terms_of_use_accepted(self, field):
+        if not field.data:
+            raise ValidationError('Terms of Use not accepted')
 
 
 class LoginForm(FlaskForm):

app/blueprints/auth/routes.py

@@ -12,26 +12,6 @@ from .forms import (
 )
 
 
-@bp.before_app_request
-def before_request():
-    """
-    Checks if a user is unconfirmed when visiting specific sites. Redirects to
-    unconfirmed view if user is unconfirmed.
-    """
-    if not current_user.is_authenticated:
-        return
-
-    current_user.ping()
-    db.session.commit()
-    if (not current_user.confirmed
-            and request.endpoint
-            and request.blueprint != 'auth'
-            and request.endpoint != 'static'):
-        return redirect(url_for('auth.unconfirmed'))
-    if not current_user.terms_of_use_accepted:
-        return redirect(url_for('main.terms_of_use'))
-
-
 @bp.route('/register', methods=['GET', 'POST'])
 def register():
     if current_user.is_authenticated:

app/blueprints/contributions/__init__.py

@@ -15,9 +15,11 @@ def before_request():
     pass
 
 
-from . import (
-    routes,
-    spacy_nlp_pipeline_models,
-    tesseract_ocr_pipeline_models,
-    transkribus_htr_pipeline_models
-)
+from . import routes
+
+
+from .spacy_nlp_pipeline_models import bp as spacy_nlp_pipeline_models_bp
+bp.register_blueprint(spacy_nlp_pipeline_models_bp, url_prefix='/spacy-nlp-pipeline-models')
+
+from .tesseract_ocr_pipeline_models import bp as tesseract_ocr_pipeline_models_bp
+bp.register_blueprint(tesseract_ocr_pipeline_models_bp, url_prefix='/tesseract-ocr-pipeline-models')

app/blueprints/contributions/routes.py

@@ -1,7 +1,7 @@
-from flask import redirect, url_for
+from flask import render_template
 from . import bp
 
 
 @bp.route('')
-def contributions():
-    return redirect(url_for('main.dashboard', _anchor='contributions'))
+def index():
+    return render_template('contributions/index.html.j2', title='Contributions')

app/blueprints/contributions/spacy_nlp_pipeline_models/__init__.py

@@ -1,2 +1,18 @@
-from .. import bp
-from . import json_routes, routes
+from flask import current_app, Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('spacy_nlp_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import routes, json_routes

app/blueprints/contributions/spacy_nlp_pipeline_models/json_routes.py

@@ -1,5 +1,5 @@
 from flask import abort, current_app, request
-from flask_login import current_user
+from flask_login import current_user, login_required
 from threading import Thread
 from app import db
 from app.decorators import content_negotiation, permission_required
@@ -7,7 +7,8 @@ from app.models import SpaCyNLPPipelineModel
 from . import bp
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
+@login_required
 @content_negotiation(produces='application/json')
 def delete_spacy_model(spacy_nlp_pipeline_model_id):
     def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
@@ -15,7 +16,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
             snpm = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
             snpm.delete()
             db.session.commit()
-    
+
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
     if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
@@ -31,7 +32,7 @@ def delete_spacy_model(spacy_nlp_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_spacy_nlp_pipeline_model_is_public(spacy_nlp_pipeline_model_id):

app/blueprints/contributions/spacy_nlp_pipeline_models/routes.py

@@ -1,5 +1,5 @@
 from flask import abort, flash, redirect, render_template, url_for
-from flask_login import current_user
+from flask_login import current_user, login_required
 from app import db
 from app.models import SpaCyNLPPipelineModel
 from . import bp
@@ -9,16 +9,15 @@ from .forms import (
 )
 
 
-@bp.route('/spacy-nlp-pipeline-models')
-def spacy_nlp_pipeline_models():
-    return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2',
-        title='SpaCy NLP Pipeline Models'
-    )
+@bp.route('/')
+@login_required
+def index():
+    return redirect(url_for('contributions.index', _anchor='spacy-nlp-pipeline-models'))
 
 
-@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
-def create_spacy_nlp_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+@login_required
+def create():
     form = CreateSpaCyNLPPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -42,7 +41,7 @@ def create_spacy_nlp_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'SpaCy NLP Pipeline model "{snpm.title}" created')
-        return {}, 201, {'Location': url_for('.spacy_nlp_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/spacy_nlp_pipeline_models/create.html.j2',
         title='Create SpaCy NLP Pipeline Model',
@@ -50,8 +49,9 @@ def create_spacy_nlp_pipeline_model():
     )
 
 
-@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
-def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
+@bp.route('/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
+@login_required
+def entity(spacy_nlp_pipeline_model_id):
     snpm = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
     if not (snpm.user == current_user or current_user.is_administrator):
         abort(403)
@@ -61,9 +61,9 @@ def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
         if db.session.is_modified(snpm):
             flash(f'SpaCy NLP Pipeline model "{snpm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.spacy_nlp_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_model.html.j2',
+        'contributions/spacy_nlp_pipeline_models/entity.html.j2',
         title=f'{snpm.title} {snpm.version}',
         form=form,
         spacy_nlp_pipeline_model=snpm

app/blueprints/contributions/tesseract_ocr_pipeline_models/__init__.py

@@ -1,2 +1,18 @@
-from .. import bp
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('tesseract_ocr_pipeline_models', __name__)
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
 from . import json_routes, routes

app/blueprints/contributions/tesseract_ocr_pipeline_models/json_routes.py

@@ -7,7 +7,7 @@ from app.models import TesseractOCRPipelineModel
 from . import bp
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
 @content_negotiation(produces='application/json')
 def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
@@ -31,7 +31,7 @@ def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
     return response_data, 202
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>/is_public', methods=['PUT'])
 @permission_required('CONTRIBUTE')
 @content_negotiation(consumes='application/json', produces='application/json')
 def update_tesseract_ocr_pipeline_model_is_public(tesseract_ocr_pipeline_model_id):

app/blueprints/contributions/tesseract_ocr_pipeline_models/routes.py

@@ -9,16 +9,13 @@ from .forms import (
 )
 
 
-@bp.route('/tesseract-ocr-pipeline-models')
-def tesseract_ocr_pipeline_models():
-    return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2',
-        title='Tesseract OCR Pipeline Models'
-    )
+@bp.route('/')
+def index():
+    return redirect(url_for('contributions.index', _anchor='tesseract-ocr-pipeline-models'))
 
 
-@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
-def create_tesseract_ocr_pipeline_model():
+@bp.route('/create', methods=['GET', 'POST'])
+def create():
     form = CreateTesseractOCRPipelineModelForm()
     if form.is_submitted():
         if not form.validate():
@@ -41,7 +38,7 @@ def create_tesseract_ocr_pipeline_model():
             abort(500)
         db.session.commit()
         flash(f'Tesseract OCR Pipeline model "{topm.title}" created')
-        return {}, 201, {'Location': url_for('.tesseract_ocr_pipeline_models')}
+        return {}, 201, {'Location': url_for('.index')}
     return render_template(
         'contributions/tesseract_ocr_pipeline_models/create.html.j2',
         title='Create Tesseract OCR Pipeline Model',
@@ -49,8 +46,8 @@ def create_tesseract_ocr_pipeline_model():
     )
 
 
-@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
-def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
+@bp.route('/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
+def entity(tesseract_ocr_pipeline_model_id):
     topm = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
     if not (topm.user == current_user or current_user.is_administrator):
         abort(403)
@@ -60,9 +57,9 @@ def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
         if db.session.is_modified(topm):
             flash(f'Tesseract OCR Pipeline model "{topm.title}" updated')
             db.session.commit()
-        return redirect(url_for('.tesseract_ocr_pipeline_models'))
+        return redirect(url_for('.index'))
     return render_template(
-        'contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_model.html.j2',
+        'contributions/tesseract_ocr_pipeline_models/entity.html.j2',
         title=f'{topm.title} {topm.version}',
         form=form,
         tesseract_ocr_pipeline_model=topm

app/blueprints/contributions/transkribus_htr_pipeline_models/__init__.py

@@ -1,2 +0,0 @@
-from .. import bp
-from . import routes

app/blueprints/contributions/transkribus_htr_pipeline_models/routes.py

@@ -1,7 +0,0 @@
-from flask import abort
-from . import bp
-
-
-@bp.route('/transkribus_htr_pipeline_models')
-def transkribus_htr_pipeline_models():
-    return abort(503)

app/blueprints/corpora/cqi_over_sio/utils.py

@@ -1,130 +0,0 @@
-from cqi.models.corpora import Corpus as CQiCorpus
-from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
-
-
-def lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
-    lookups = {}
-    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
-    for attr in corpus.positional_attributes.list():
-        cpos_attr_values: list[str] = attr.values_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
-    for attr in corpus.structural_attributes.list():
-        # We only want to iterate over non subattributes, identifiable by
-        # attr.has_values == False
-        if attr.has_values:
-            continue
-        cpos_attr_ids: list[int] = attr.ids_by_cpos(cpos_list)
-        for i, cpos in enumerate(cpos_list):
-            if cpos_attr_ids[i] == -1:
-                continue
-            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
-        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
-        if len(occured_attr_ids) == 0:
-            continue
-        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
-        if len(subattrs) == 0:
-            continue
-        lookup_name: str = f'{attr.name}_lookup'
-        lookups[lookup_name] = {}
-        for attr_id in occured_attr_ids:
-            lookups[lookup_name][attr_id] = {}
-        for subattr in subattrs:
-            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
-            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
-                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
-    return lookups
-
-
-def partial_export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    match_id_list: list[int],
-    context: int = 25
-) -> dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    match_boundaries = []
-    for match_id in match_id_list:
-        if match_id < 0 or match_id >= subcorpus.size:
-            continue
-        match_boundaries.append(
-            (
-                match_id,
-                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
-                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
-            )
-        )
-    cpos_set = set()
-    matches = []
-    for match_boundary in match_boundaries:
-        match_num, match_start, match_end = match_boundary
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}
-
-
-def export_subcorpus(
-    subcorpus: CQiSubcorpus,
-    context: int = 25,
-    cutoff: float = float('inf'),
-    offset: int = 0
-) -> dict:
-    if subcorpus.size == 0:
-        return {"matches": []}
-    first_match = max(0, offset)
-    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
-    match_boundaries = zip(
-        range(first_match, last_match + 1),
-        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
-        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
-    )
-    cpos_set = set()
-    matches = []
-    for match_num, match_start, match_end in match_boundaries:
-        c = (match_start, match_end)
-        if match_start == 0 or context == 0:
-            lc = None
-            cpos_list_lbound = match_start
-        else:
-            lc_lbound = max(0, (match_start - context))
-            lc_rbound = match_start - 1
-            lc = (lc_lbound, lc_rbound)
-            cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
-            rc = None
-            cpos_list_rbound = match_end
-        else:
-            rc_lbound = match_end + 1
-            rc_rbound = min(
-                (match_end + context),
-                (subcorpus.collection.corpus.size - 1)
-            )
-            rc = (rc_lbound, rc_rbound)
-            cpos_list_rbound = rc_rbound
-        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
-        matches.append(match)
-        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
-    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
-    return {'matches': matches, **lookups}

app/blueprints/jobs/events.py

@@ -1,138 +0,0 @@
-from flask import current_app
-from flask_login import current_user
-from flask_socketio import Namespace
-from app import db, hashids, socketio
-from app.extensions.flask_socketio import admin_required, login_required
-from app.models import Job, JobStatus
-
-
-class JobsNamespace(Namespace):
-    @login_required
-    def on_delete(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the current user is allowed to delete the job
-        if not (job.user == current_user or current_user.is_administrator):
-            return {
-                'code': 403,
-                'body': 'Forbidden'
-            }
-
-        # TODO: This should be a method in the Job model
-        def _delete_job(app, job_id):
-            with app.app_context():
-                job = Job.query.get(job_id)
-                job.delete()
-                db.session.commit()
-
-        # Delete the job in a background task
-        socketio.start_background_task(
-            target=_delete_job,
-            app=current_app._get_current_object(),
-            job_id=job_id
-        )
-
-        return {
-            'code': 202,
-            'body': f'Job "{job.title}" marked for deletion'
-        }
-
-
-    @admin_required
-    def on_get_log(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the job is already processed
-        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
-            return {
-                'code': 409,
-                'body': 'Job is not done processing'
-            }
-
-        # Read the log file
-        with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
-            job_log = log_file.read()
-
-        return {
-            'code': 200,
-            'body': job_log
-        }
-
-
-    @login_required
-    def on_restart(self, job_hashid: str):
-        # Decode the job hashid
-        job_id = hashids.decode(job_hashid)
-
-        # Validate job_id
-        if not isinstance(job_id, int):
-            return {
-                'code': 400,
-                'body': 'job_id is invalid'
-            }
-
-        # Load job from database
-        job = Job.query.get(job_id)
-        if job is None:
-            return {
-                'code': 404,
-                'body': 'Job not found'
-            }
-
-        # Check if the current user is allowed to restart the job
-        if not (job.user == current_user or current_user.is_administrator):
-            return {
-                'code': 403,
-                'body': 'Forbidden'
-            }
-
-        # TODO: This should be a method in the Job model
-        def _restart_job(app, job_id):
-            with app.app_context():
-                job = Job.query.get(job_id)
-                job.restart()
-                db.session.commit()
-
-        # Restart the job in a background task
-        socketio.start_background_task(
-            target=_restart_job,
-            app=current_app._get_current_object(),
-            job_id=job_id
-        )
-
-        return {
-            'code': 202,
-            'body': f'Job "{job.title}" restarted'
-        }

app/blueprints/main/cli.py

@@ -44,7 +44,7 @@ def deploy():
     TesseractOCRPipelineModel.insert_defaults()
 
     print('Stop running analysis sessions')
-    for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0).all():
+    for corpus in Corpus.query.all():
         corpus.num_analysis_sessions = 0
     db.session.commit()
 

app/blueprints/main/routes.py

@@ -72,16 +72,14 @@ def terms_of_use():
     )
 
 
-@bp.route('/social-area')
+@bp.route('/social')
 @login_required
-def social_area():
-    print('test')
+def social():
     corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
-    print(corpora)
     users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
     return render_template(
-        'main/social_area.html.j2',
-        title='Social Area',
+        'main/social.html.j2',
+        title='Social',
         corpora=corpora,
         users=users
     )

app/blueprints/services/forms.py

@@ -61,7 +61,7 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
         if field.data:
             if not('methods' in service_info and 'binarization' in service_info['methods']):
                 raise ValidationError('Binarization is not available')
-              
+
     def validate_pdf(self, field):
         if field.data.mimetype != 'application/pdf':
             raise ValidationError('PDF files only!')
@@ -146,7 +146,7 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
     encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
     txt = FileField('File', validators=[FileRequired()])
     model = SelectField('Model', validators=[InputRequired()])
-    
+
     def validate_encoding_detection(self, field):
         service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]
         if field.data:
@@ -167,7 +167,6 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
         version = kwargs.pop('version', service_manifest['latest_version'])
         super().__init__(*args, **kwargs)
         service_info = service_manifest['versions'][version]
-        print(service_info)
         if self.encoding_detection.render_kw is None:
             self.encoding_detection.render_kw = {}
         self.encoding_detection.render_kw['disabled'] = True

app/blueprints/users/__init__.py

@@ -15,4 +15,4 @@ def before_request():
     pass
 
 
-from . import cli, events, json_routes, routes, settings
+from . import cli, json_routes, routes, settings

app/blueprints/users/events.py

@@ -1,47 +0,0 @@
-from flask_login import current_user
-from flask_socketio import join_room, leave_room
-from app import hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import User
-
-
-@socketio.on('GET /users/<user_id>')
-@socketio_login_required
-def get_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'status': 404, 'statusText': 'Not found'}
-    if not (user == current_user or current_user.is_administrator):
-        return {'status': 403, 'statusText': 'Forbidden'}
-    return {
-        'body': user.to_json_serializeable(backrefs=True, relationships=True),
-        'status': 200,
-        'statusText': 'OK'
-    }
-
-
-@socketio.on('SUBSCRIBE /users/<user_id>')
-@socketio_login_required
-def subscribe_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'status': 404, 'statusText': 'Not found'}
-    if not (user == current_user or current_user.is_administrator):
-        return {'status': 403, 'statusText': 'Forbidden'}
-    join_room(f'/users/{user.hashid}')
-    return {'status': 200, 'statusText': 'OK'}
-
-
-@socketio.on('UNSUBSCRIBE /users/<user_id>')
-@socketio_login_required
-def unsubscribe_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'status': 404, 'statusText': 'Not found'}
-    if not (user == current_user or current_user.is_administrator):
-        return {'status': 403, 'statusText': 'Forbidden'}
-    leave_room(f'/users/{user.hashid}')
-    return {'status': 200, 'statusText': 'OK'}

app/blueprints/users/nevents.py

@@ -1,114 +0,0 @@
-from flask_login import current_user
-from flask_socketio import join_room
-from app import hashids, socketio
-from app.decorators import socketio_admin_required, socketio_login_required
-from app.models import User
-
-
-@socketio.on('GET /users')
-@socketio_admin_required
-def get_users():
-    users = User.query.filter_by().all()
-    return {
-        'body': [user.to_json_serializable() for user in users],
-        'options': {
-            'status': 200,
-            'statusText': 'OK',
-            'headers': {'Content-Type: application/json'}
-        }
-    }
-
-
-@socketio.on('SUBSCRIBE /users')
-@socketio_admin_required
-def subscribe_users():
-    join_room('/users')
-    return {'options': {'status': 200, 'statusText': 'OK'}}
-
-
-@socketio.on('GET /users/<user_id>')
-@socketio_login_required
-def get_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'options': {'status': 404, 'statusText': 'Not found'}}
-    if not (user == current_user or current_user.is_administrator):
-        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
-    return {
-        'body': user.to_json_serializable(),
-        'options': {
-            'status': 200,
-            'statusText': 'OK',
-            'headers': {'Content-Type: application/json'}
-        }
-    }
-
-
-@socketio.on('SUBSCRIBE /users/<user_id>')
-@socketio_login_required
-def subscribe_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.get(user_id)
-    if user is None:
-        return {'options': {'status': 404, 'statusText': 'Not found'}}
-    if not (user == current_user or current_user.is_administrator):
-        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
-    join_room(f'/users/{user.hashid}')
-    return {'options': {'status': 200, 'statusText': 'OK'}}
-
-
-@socketio.on('GET /public_users')
-@socketio_login_required
-def get_public_users():
-    users = User.query.filter_by(is_public=True).all()
-    return {
-        'body': [
-            user.to_json_serializable(filter_by_privacy_settings=True)
-            for user in users
-        ],
-        'options': {
-            'status': 200,
-            'statusText': 'OK',
-            'headers': {'Content-Type: application/json'}
-        }
-    }
-
-
-@socketio.on('SUBSCRIBE /users')
-@socketio_admin_required
-def subscribe_users():
-    join_room('/public_users')
-    return {'options': {'status': 200, 'statusText': 'OK'}}
-
-
-@socketio.on('GET /public_users/<user_id>')
-@socketio_login_required
-def get_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.filter_by(id=user_id, is_public=True).first()
-    if user is None:
-        return {'options': {'status': 404, 'statusText': 'Not found'}}
-    if not (user == current_user or current_user.is_administrator):
-        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
-    return {
-        'body': user.to_json_serializable(filter_by_privacy_settings=True),
-        'options': {
-            'status': 200,
-            'statusText': 'OK',
-            'headers': {'Content-Type: application/json'}
-        }
-    }
-
-
-@socketio.on('SUBSCRIBE /public_users/<user_id>')
-@socketio_login_required
-def subscribe_user(user_hashid):
-    user_id = hashids.decode(user_hashid)
-    user = User.query.filter_by(id=user_id, is_public=True).first()
-    if user is None:
-        return {'options': {'status': 404, 'statusText': 'Not found'}}
-    if not (user == current_user or current_user.is_administrator):
-        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
-    join_room(f'/public_users/{user.hashid}')
-    return {'options': {'status': 200, 'statusText': 'OK'}}

app/converters/vrt.py

@@ -1,69 +1,25 @@
 from flask import current_app
+from pathlib import Path
 
 
-def normalize_vrt_file(input_file, output_file):
-    def check_pos_attribute_order(vrt_lines):
-        # The following orders are possible:
-        # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
-        # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
-        # since 27.01.2022: 'word,pos,lemma,simple_pos'
-        # This Function tries to find out which order we have by looking at the
-        # number of attributes and the position of the simple_pos attribute
-        SIMPLE_POS_LABELS = [
-            'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ',
-            'DET', 'INTJ', 'NOUN', 'NUM', 'PART',
-            'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',
-            'VERB', 'X'
-        ]
-        for line in vrt_lines:
-            if line.startswith('<'):
-                continue
-            pos_attrs = line.rstrip('\n').split('\t')
-            num_pos_attrs = len(pos_attrs)
-            if num_pos_attrs == 4:
-                if pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos']
-                continue
-            elif num_pos_attrs == 5:
-                if pos_attrs[2] in SIMPLE_POS_LABELS:
-                    return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
-                elif pos_attrs[3] in SIMPLE_POS_LABELS:
-                    return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
-                continue
-        return None
-
-
-    def check_has_ent_as_s_attr(vrt_lines):
-        for line in vrt_lines:
-            if line.startswith('<ent'):
-                return True
-        return False
-
-
-    def pos_attrs_to_string_1(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
-
-
-    def pos_attrs_to_string_2(pos_attrs):
-        return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'
-
+def normalize_vrt_file(input_file: Path, output_file: Path):
     current_app.logger.info(f'Converting {input_file}...')
 
-    with open(input_file) as f:
+    with input_file.open() as f:
         input_vrt_lines = f.readlines()
 
-    pos_attr_order = check_pos_attribute_order(input_vrt_lines)
-    has_ent_as_s_attr = check_has_ent_as_s_attr(input_vrt_lines)
+    pos_attr_order = _check_pos_attribute_order(input_vrt_lines)
+    has_ent_as_s_attr = _check_has_ent_as_s_attr(input_vrt_lines)
 
     current_app.logger.info(f'Detected pos_attr_order: [{",".join(pos_attr_order)}]')
     current_app.logger.info(f'Detected has_ent_as_s_attr: {has_ent_as_s_attr}')
 
     if pos_attr_order == ['word', 'lemma', 'simple_pos', 'pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_1
+        pos_attrs_to_string_function = _pos_attrs_to_string_1
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos', 'ner']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     elif pos_attr_order == ['word', 'pos', 'lemma', 'simple_pos']:
-        pos_attrs_to_string_function = pos_attrs_to_string_2
+        pos_attrs_to_string_function = _pos_attrs_to_string_2
     else:
         raise Exception('Can not handle format')
 
@@ -113,5 +69,49 @@ def normalize_vrt_file(input_file, output_file):
                     current_ent = pos_attrs[4]
         output_vrt += pos_attrs_to_string_function(pos_attrs)
 
-    with open(output_file, 'w') as f:
+    with output_file.open(mode='w') as f:
         f.write(output_vrt)
+
+
+def _check_pos_attribute_order(vrt_lines: list[str]) -> list[str]:
+    # The following orders are possible:
+    # since 26.02.2019: 'word,lemma,simple_pos,pos,ner'
+    # since 26.03.2021: 'word,pos,lemma,simple_pos,ner'
+    # since 27.01.2022: 'word,pos,lemma,simple_pos'
+    # This Function tries to find out which order we have by looking at the
+    # number of attributes and the position of the simple_pos attribute
+    SIMPLE_POS_LABELS = [
+        'ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'DET', 'INTJ', 'NOUN', 'NUM',
+        'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X'
+    ]
+    for line in vrt_lines:
+        if line.startswith('<'):
+            continue
+        pos_attrs = line.rstrip('\n').split('\t')
+        num_pos_attrs = len(pos_attrs)
+        if num_pos_attrs == 4:
+            if pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos']
+            continue
+        elif num_pos_attrs == 5:
+            if pos_attrs[2] in SIMPLE_POS_LABELS:
+                return ['word', 'lemma', 'simple_pos', 'pos', 'ner']
+            elif pos_attrs[3] in SIMPLE_POS_LABELS:
+                return ['word', 'pos', 'lemma', 'simple_pos', 'ner']
+            continue
+    # TODO: raise exception "can't determine attribute order"
+
+
+def _check_has_ent_as_s_attr(vrt_lines: list[str]) -> bool:
+    for line in vrt_lines:
+        if line.startswith('<ent'):
+            return True
+    return False
+
+
+def _pos_attrs_to_string_1(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[3]}\t{pos_attrs[1]}\t{pos_attrs[2]}\n'
+
+
+def _pos_attrs_to_string_2(pos_attrs: list[str]) -> str:
+    return f'{pos_attrs[0]}\t{pos_attrs[1]}\t{pos_attrs[2]}\t{pos_attrs[3]}\n'

app/decorators.py

@@ -26,7 +26,7 @@ def socketio_login_required(f):
     def wrapper(*args, **kwargs):
         if current_user.is_authenticated:
             return f(*args, **kwargs)
-        return {'code': 401, 'body': 'Unauthorized'}
+        return {'status': 401, 'statusText': 'Unauthorized'}
     return wrapper
 
 
@@ -35,7 +35,7 @@ def socketio_permission_required(permission):
         @wraps(f)
         def wrapper(*args, **kwargs):
             if not current_user.can(permission):
-                return {'code': 403, 'body': 'Forbidden'}
+                return {'status': 403, 'statusText': 'Forbidden'}
             return f(*args, **kwargs)
         return wrapper
     return decorator

app/jobs/__init__.py

@@ -1,2 +1,2 @@
-from .handle_corpora import job as handle_corpora
-from .handle_jobs import job as handle_jobs
+from .handle_corpora import handle_corpora
+from .handle_jobs import handle_jobs

app/jobs/handle_corpora.py

@@ -1,12 +1,12 @@
-from app import db, docker_client, scheduler
-from app.models import Corpus, CorpusStatus
 from flask import current_app
 import docker
 import os
 import shutil
+from app import db, docker_client, scheduler
+from app.models import Corpus, CorpusStatus
 
 
-def job():
+def handle_corpora():
     with scheduler.app.app_context():
         _handle_corpora()
 
@@ -21,14 +21,14 @@ def _handle_corpora():
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
         corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
     for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
-        _checkout_analysing_corpus_container(corpus)
+        _checkout_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
         _create_cqpserver_container(corpus)
     for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
         _remove_cqpserver_container(corpus)
     db.session.commit()
 
-def _create_build_corpus_service(corpus):
+def _create_build_corpus_service(corpus: Corpus):
     ''' # Docker service settings # '''
     ''' ## Command ## '''
     command = ['bash', '-c']
@@ -50,12 +50,10 @@ def _create_build_corpus_service(corpus):
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'corpus.build',
-        'corpus_id': str(corpus.id)
+        'nopaque.server_name': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -100,7 +98,7 @@ def _create_build_corpus_service(corpus):
         return
     corpus.status = CorpusStatus.QUEUED
 
-def _checkout_build_corpus_service(corpus):
+def _checkout_build_corpus_service(corpus: Corpus):
     service_name = f'build-corpus_{corpus.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -128,8 +126,7 @@ def _checkout_build_corpus_service(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _create_cqpserver_container(corpus):
-    ''' # Docker container settings # '''
+def _create_cqpserver_container(corpus: Corpus):
     ''' ## Command ## '''
     command = []
     command.append(
@@ -144,9 +141,9 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1887'
     ''' ## Name ## '''
-    name = f'cqpserver_{corpus.id}'
+    name = f'nopaque-cqpserver-{corpus.id}'
     ''' ## Network ## '''
     network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}'
     ''' ## Volumes ## '''
@@ -203,8 +200,8 @@ def _create_cqpserver_container(corpus):
         return
     corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
 
-def _checkout_analysing_corpus_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _checkout_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         docker_client.containers.get(container_name)
     except docker.errors.NotFound as e:
@@ -214,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Get container "{container_name}" failed: {e}')
 
-def _remove_cqpserver_container(corpus):
-    container_name = f'cqpserver_{corpus.id}'
+def _remove_cqpserver_container(corpus: Corpus):
+    container_name = f'nopaque-cqpserver-{corpus.id}'
     try:
         container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:

app/jobs/handle_jobs.py

@@ -1,3 +1,10 @@
+from datetime import datetime
+from flask import current_app
+from werkzeug.utils import secure_filename
+import docker
+import json
+import os
+import shutil
 from app import db, docker_client, hashids, scheduler
 from app.models import (
     Job,
@@ -6,16 +13,9 @@ from app.models import (
     TesseractOCRPipelineModel,
     SpaCyNLPPipelineModel
 )
-from datetime import datetime
-from flask import current_app
-from werkzeug.utils import secure_filename
-import docker
-import json
-import os
-import shutil
 
 
-def job():
+def handle_jobs():
     with scheduler.app.app_context():
         _handle_jobs()
 
@@ -29,7 +29,7 @@ def _handle_jobs():
         _remove_job_service(job)
     db.session.commit()
 
-def _create_job_service(job):
+def _create_job_service(job: Job):
     ''' # Docker service settings # '''
     ''' ## Service specific settings ## '''
     if job.service == 'file-setup-pipeline':
@@ -86,9 +86,7 @@ def _create_job_service(job):
     constraints = ['node.role==worker']
     ''' ## Labels ## '''
     labels = {
-        'origin': current_app.config['SERVER_NAME'],
-        'type': 'job',
-        'job_id': str(job.id)
+        'origin': current_app.config['SERVER_NAME']
     }
     ''' ## Mounts ## '''
     mounts = []
@@ -169,7 +167,7 @@ def _create_job_service(job):
         return
     job.status = JobStatus.QUEUED
 
-def _checkout_job_service(job):
+def _checkout_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)
@@ -218,7 +216,7 @@ def _checkout_job_service(job):
     except docker.errors.DockerException as e:
         current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
 
-def _remove_job_service(job):
+def _remove_job_service(job: Job):
     service_name = f'job_{job.id}'
     try:
         service = docker_client.services.get(service_name)

app/models/corpus.py

@@ -8,7 +8,7 @@ import shutil
 import xml.etree.ElementTree as ET
 from app import db
 from app.converters.vrt import normalize_vrt_file
-from app.extensions.sqlalchemy_extras import IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import IntEnumColumn
 from .corpus_follower_association import CorpusFollowerAssociation
 
 

app/models/default_records/tesseract_ocr_pipeline_model.yml

@@ -10,6 +10,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Amharic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata'
@@ -22,6 +23,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Arabic'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata'
@@ -34,6 +36,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Assamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata'
@@ -46,6 +49,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata'
@@ -58,6 +62,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Azerbaijani - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata'
@@ -70,6 +75,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Belarusian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata'
@@ -82,6 +88,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bengali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata'
@@ -94,6 +101,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tibetan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata'
@@ -106,6 +114,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bosnian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata'
@@ -118,6 +127,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Bulgarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata'
@@ -130,6 +140,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Catalan; Valencian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata'
@@ -142,6 +153,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Cebuano'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata'
@@ -154,6 +166,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Czech'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata'
@@ -166,6 +179,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Chinese - Simplified'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata'
@@ -178,6 +192,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Chinese - Traditional'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata'
@@ -190,6 +205,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Cherokee'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata'
@@ -202,6 +218,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Welsh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata'
@@ -214,6 +231,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Danish'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata'
@@ -226,6 +244,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata'
@@ -238,6 +257,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Dzongkha'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata'
@@ -250,6 +270,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Modern (1453-)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata'
@@ -262,6 +283,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata'
@@ -274,6 +296,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'English, Middle (1100-1500)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata'
@@ -286,6 +309,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Esperanto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata'
@@ -298,6 +322,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Estonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata'
@@ -310,6 +335,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Basque'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata'
@@ -322,6 +348,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Persian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata'
@@ -334,6 +361,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Finnish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata'
@@ -346,6 +374,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'French'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata'
@@ -358,6 +387,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'German Fraktur'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata'
@@ -370,6 +400,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'French, Middle (ca. 1400-1600)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata'
@@ -382,6 +413,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Irish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata'
@@ -394,6 +426,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Galician'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata'
@@ -406,6 +439,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Greek, Ancient (-1453)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata'
@@ -418,6 +452,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Gujarati'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata'
@@ -430,6 +465,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Haitian; Haitian Creole'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata'
@@ -442,6 +478,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hebrew'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata'
@@ -454,6 +491,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hindi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata'
@@ -466,6 +504,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Croatian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata'
@@ -478,6 +517,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Hungarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata'
@@ -490,6 +530,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Inuktitut'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata'
@@ -502,6 +543,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Indonesian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata'
@@ -514,6 +556,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Icelandic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata'
@@ -526,6 +569,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Italian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata'
@@ -538,6 +582,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Italian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata'
@@ -550,6 +595,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Javanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata'
@@ -562,6 +608,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Japanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata'
@@ -574,6 +621,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kannada'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata'
@@ -586,6 +634,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata'
@@ -598,6 +647,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Georgian - Old'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata'
@@ -610,6 +660,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kazakh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata'
@@ -622,6 +673,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Central Khmer'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata'
@@ -634,6 +686,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kirghiz; Kyrgyz'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata'
@@ -646,6 +699,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Korean'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata'
@@ -658,6 +712,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Kurdish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata'
@@ -670,6 +725,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lao'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata'
@@ -682,6 +738,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata'
@@ -694,6 +751,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Latvian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata'
@@ -706,6 +764,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Lithuanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata'
@@ -718,6 +777,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malayalam'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata'
@@ -730,6 +790,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Marathi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata'
@@ -742,6 +803,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Macedonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata'
@@ -754,6 +816,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Maltese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata'
@@ -766,6 +829,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Malay'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata'
@@ -778,6 +842,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Burmese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata'
@@ -790,6 +855,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Nepali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata'
@@ -802,6 +868,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Dutch; Flemish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata'
@@ -814,6 +881,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Norwegian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata'
@@ -826,6 +894,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Oriya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata'
@@ -838,6 +907,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Panjabi; Punjabi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata'
@@ -850,6 +920,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Polish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata'
@@ -862,6 +933,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Portuguese'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata'
@@ -874,6 +946,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Pushto; Pashto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata'
@@ -886,6 +959,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Romanian; Moldavian; Moldovan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata'
@@ -898,6 +972,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Russian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata'
@@ -910,6 +985,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Sanskrit'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata'
@@ -922,6 +998,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Sinhala; Sinhalese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata'
@@ -934,6 +1011,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovak'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata'
@@ -946,6 +1024,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Slovenian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata'
@@ -958,6 +1037,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 - title: 'Spanish; Castilian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata'
@@ -970,6 +1050,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 - title: 'Spanish; Castilian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata'
@@ -982,6 +1063,7 @@
     - '0.1.0'
     - '0.1.1'
     - '0.1.2'
+    - '0.1.3b'
 # - title: 'Albanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata'
@@ -994,6 +1076,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata'
@@ -1006,6 +1089,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Serbian - Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata'
@@ -1018,6 +1102,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swahili'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata'
@@ -1030,6 +1115,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Swedish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata'
@@ -1042,6 +1128,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Syriac'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata'
@@ -1054,6 +1141,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tamil'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata'
@@ -1066,6 +1154,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Telugu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata'
@@ -1078,6 +1167,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tajik'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata'
@@ -1090,6 +1180,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tagalog'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata'
@@ -1102,6 +1193,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Thai'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata'
@@ -1114,6 +1206,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Tigrinya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata'
@@ -1126,6 +1219,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Turkish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata'
@@ -1138,6 +1232,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uighur; Uyghur'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata'
@@ -1150,6 +1245,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Ukrainian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata'
@@ -1162,6 +1258,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Urdu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata'
@@ -1174,6 +1271,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata'
@@ -1186,6 +1284,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Uzbek - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata'
@@ -1198,6 +1297,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Vietnamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata'
@@ -1210,6 +1310,7 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'
 # - title: 'Yiddish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata'
@@ -1222,3 +1323,4 @@
 #     - '0.1.0'
 #     - '0.1.1'
 #     - '0.1.2'
+#     - '0.1.3b'

app/models/event_listeners.py

@@ -42,8 +42,9 @@ def resource_after_delete(mapper, connection, resource):
             'path': resource.jsonpatch_path
         }
     ]
+    namespace = '/users'
     room = f'/users/{resource.user_hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def cfa_after_delete(mapper, connection, cfa):
@@ -54,8 +55,9 @@ def cfa_after_delete(mapper, connection, cfa):
             'path': jsonpatch_path
         }
     ]
+    namespace = '/users'
     room = f'/users/{cfa.corpus.user.hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def resource_after_insert(mapper, connection, resource):
@@ -69,8 +71,9 @@ def resource_after_insert(mapper, connection, resource):
             'value': jsonpatch_value
         }
     ]
+    namespace = '/users'
     room = f'/users/{resource.user_hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def cfa_after_insert(mapper, connection, cfa):
@@ -83,8 +86,9 @@ def cfa_after_insert(mapper, connection, cfa):
             'value': jsonpatch_value
         }
     ]
+    namespace = '/users'
     room = f'/users/{cfa.corpus.user.hashid}'
-    socketio.emit('PATCH', jsonpatch, room=room)
+    socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def resource_after_update(mapper, connection, resource):
@@ -109,8 +113,9 @@ def resource_after_update(mapper, connection, resource):
             }
         )
     if jsonpatch:
+        namespace = '/users'
         room = f'/users/{resource.user_hashid}'
-        socketio.emit('PATCH', jsonpatch, room=room)
+        socketio.emit('patch', jsonpatch, namespace=namespace, room=room)
 
 
 def job_after_update(mapper, connection, job):

app/models/job.py

@@ -6,7 +6,7 @@ from time import sleep
 from pathlib import Path
 import shutil
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn, IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn, IntEnumColumn
 
 
 class JobStatus(IntEnum):

app/models/spacy_nlp_pipeline_model.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import requests
 import yaml
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn
 from .file_mixin import FileMixin
 from .user import User
 
@@ -41,7 +41,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
     @property
     def url(self):
         return url_for(
-            'contributions.spacy_nlp_pipeline_model',
+            'contributions.spacy_nlp_pipeline_models.entity',
             spacy_nlp_pipeline_model_id=self.id
         )
 
@@ -104,7 +104,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
                             f.write(chunk)
                     pbar.close()
         db.session.commit()
-    
+
     def delete(self):
         try:
             self.path.unlink(missing_ok=True)

app/models/tesseract_ocr_pipeline_model.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import requests
 import yaml
 from app import db
-from app.extensions.sqlalchemy_extras import ContainerColumn
+from app.extensions.nopaque_sqlalchemy_extras import ContainerColumn
 from .file_mixin import FileMixin
 from .user import User
 
@@ -40,7 +40,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
     @property
     def url(self):
         return url_for(
-            'contributions.tesseract_ocr_pipeline_model',
+            'contributions.tesseract_ocr_pipeline_models.entity',
             tesseract_ocr_pipeline_model_id=self.id
         )
 

app/models/user.py

@@ -11,7 +11,7 @@ import re
 import secrets
 import shutil
 from app import db, hashids
-from app.extensions.sqlalchemy_extras import IntEnumColumn
+from app.extensions.nopaque_sqlalchemy_extras import IntEnumColumn
 from .corpus import Corpus
 from .corpus_follower_association import CorpusFollowerAssociation
 from .corpus_follower_role import CorpusFollowerRole

app/namespaces/cqi_over_sio/__init__.py

@@ -1,17 +1,16 @@
 from cqi import CQiClient
 from cqi.errors import CQiException
 from cqi.status import CQiStatus
-from docker.models.containers import Container
-from flask import current_app, session
+from flask import current_app
 from flask_login import current_user
 from flask_socketio import Namespace
 from inspect import signature
 from threading import Lock
-from typing import Callable
 from app import db, docker_client, hashids, socketio
 from app.decorators import socketio_login_required
 from app.models import Corpus, CorpusStatus
-from . import extensions
+from . import cqi_extension_functions
+from .utils import SessionManager
 
 
 '''
@@ -38,7 +37,7 @@ Basic concept:
 '''
 
 
-CQI_API_FUNCTION_NAMES: list[str] = [
+CQI_API_FUNCTION_NAMES = [
     'ask_feature_cl_2_3',
     'ask_feature_cqi_1_0',
     'ask_feature_cqp_2_3',
@@ -86,68 +85,90 @@ CQI_API_FUNCTION_NAMES: list[str] = [
 ]
 
 
-class CQiOverSocketIO(Namespace):
+CQI_EXTENSION_FUNCTION_NAMES = [
+    'ext_corpus_update_db',
+    'ext_corpus_static_data',
+    'ext_corpus_paginate_corpus',
+    'ext_cqp_paginate_subcorpus',
+    'ext_cqp_partial_export_subcorpus',
+    'ext_cqp_export_subcorpus',
+]
+
+
+class CQiOverSocketIONamespace(Namespace):
     @socketio_login_required
     def on_connect(self):
         pass
 
     @socketio_login_required
-    def on_init(self, db_corpus_hashid: str):
-        db_corpus_id: int = hashids.decode(db_corpus_hashid)
-        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
-        if db_corpus is None:
+    def on_init(self, corpus_hashid: str) -> dict:
+        corpus_id = hashids.decode(corpus_hashid)
+
+        if not isinstance(corpus_id, int):
+            return {'code': 400, 'msg': 'Bad Request'}
+
+        corpus = Corpus.query.get(corpus_id)
+
+        if corpus is None:
             return {'code': 404, 'msg': 'Not Found'}
-        if not (db_corpus.user == current_user
-                or current_user.is_following_corpus(db_corpus)
-                or current_user.is_administrator):
+
+        if not (
+            corpus.user == current_user
+            or current_user.is_following_corpus(corpus)
+            or current_user.is_administrator
+        ):
             return {'code': 403, 'msg': 'Forbidden'}
-        if db_corpus.status not in [
+
+        if corpus.status not in [
             CorpusStatus.BUILT,
             CorpusStatus.STARTING_ANALYSIS_SESSION,
             CorpusStatus.RUNNING_ANALYSIS_SESSION,
             CorpusStatus.CANCELING_ANALYSIS_SESSION
         ]:
             return {'code': 424, 'msg': 'Failed Dependency'}
-        if db_corpus.num_analysis_sessions is None:
-            db_corpus.num_analysis_sessions = 0
-            db.session.commit()
-        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+
+        corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
         db.session.commit()
-        retry_counter: int = 20
-        while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+        retry_counter = 20
+        while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
             if retry_counter == 0:
-                db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+                corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
                 db.session.commit()
                 return {'code': 408, 'msg': 'Request Timeout'}
             socketio.sleep(3)
             retry_counter -= 1
-            db.session.refresh(db_corpus)
-        # cqi_client: CQiClient = CQiClient(f'cqpserver_{db_corpus_id}')
-        cqpserver_container_name: str = f'cqpserver_{db_corpus_id}'
-        cqpserver_container: Container = docker_client.containers.get(cqpserver_container_name)
-        cqpserver_host: str = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
-        cqi_client: CQiClient = CQiClient(cqpserver_host)
-        session['cqi_over_sio'] = {
-            'cqi_client': cqi_client,
-            'cqi_client_lock': Lock(),
-            'db_corpus_id': db_corpus_id
-        }
+            db.session.refresh(corpus)
+
+        cqpserver_container_name = f'nopaque-cqpserver-{corpus_id}'
+        cqpserver_container = docker_client.containers.get(cqpserver_container_name)
+        cqpserver_ip_address = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
+        cqi_client = CQiClient(cqpserver_ip_address)
+        cqi_client_lock = Lock()
+
+        SessionManager.setup()
+        SessionManager.set_corpus_id(corpus_id)
+        SessionManager.set_cqi_client(cqi_client)
+        SessionManager.set_cqi_client_lock(cqi_client_lock)
+
         return {'code': 200, 'msg': 'OK'}
 
     @socketio_login_required
-    def on_exec(self, fn_name: str, fn_args: dict = {}):
+    def on_exec(self, fn_name: str, fn_args: dict = {}) -> dict:
         try:
-            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+            cqi_client = SessionManager.get_cqi_client()
+            cqi_client_lock = SessionManager.get_cqi_client_lock()
         except KeyError:
             return {'code': 424, 'msg': 'Failed Dependency'}
+
         if fn_name in CQI_API_FUNCTION_NAMES:
-            fn: Callable = getattr(cqi_client.api, fn_name)
-        elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
-            fn: Callable = getattr(extensions, fn_name)
+            fn = getattr(cqi_client.api, fn_name)
+        elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
+            fn = getattr(cqi_extension_functions, fn_name)
         else:
             return {'code': 400, 'msg': 'Bad Request'}
+
         for param in signature(fn).parameters.values():
+            # Check if the parameter is optional or required
             if param.default is param.empty:
                 if param.name not in fn_args:
                     return {'code': 400, 'msg': 'Bad Request'}
@@ -156,6 +177,7 @@ class CQiOverSocketIO(Namespace):
                     continue
             if type(fn_args[param.name]) is not param.annotation:
                 return {'code': 400, 'msg': 'Bad Request'}
+
         cqi_client_lock.acquire()
         try:
             fn_return_value = fn(**fn_args)
@@ -173,6 +195,7 @@ class CQiOverSocketIO(Namespace):
             }
         finally:
             cqi_client_lock.release()
+
         if isinstance(fn_return_value, CQiStatus):
             payload = {
                 'code': fn_return_value.code,
@@ -180,27 +203,31 @@ class CQiOverSocketIO(Namespace):
             }
         else:
             payload = fn_return_value
+
         return {'code': 200, 'msg': 'OK', 'payload': payload}
 
     def on_disconnect(self):
         try:
-            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
-            db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+            corpus_id = SessionManager.get_corpus_id()
+            cqi_client = SessionManager.get_cqi_client()
+            cqi_client_lock = SessionManager.get_cqi_client_lock()
+            SessionManager.teardown()
         except KeyError:
             return
+
         cqi_client_lock.acquire()
-        try:
-            session.pop('cqi_over_sio')
-        except KeyError:
-            pass
+
         try:
             cqi_client.api.ctrl_bye()
         except (BrokenPipeError, CQiException):
             pass
+
         cqi_client_lock.release()
-        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
-        if db_corpus is None:
+
+        corpus = Corpus.query.get(corpus_id)
+
+        if corpus is None:
             return
-        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+
+        corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
         db.session.commit()

app/namespaces/cqi_over_sio/cqi_extension_functions.py

@@ -1,54 +1,39 @@
 from collections import Counter
-from cqi import CQiClient
 from cqi.models.corpora import Corpus as CQiCorpus
 from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
-from cqi.models.attributes import (
-    PositionalAttribute as CQiPositionalAttribute,
-    StructuralAttribute as CQiStructuralAttribute
-)
 from cqi.status import StatusOk as CQiStatusOk
-from flask import session
+from flask import current_app
 import gzip
 import json
 import math
 from app import db
 from app.models import Corpus
-from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
-
-
-CQI_EXTENSION_FUNCTION_NAMES: list[str] = [
-    'ext_corpus_update_db',
-    'ext_corpus_static_data',
-    'ext_corpus_paginate_corpus',
-    'ext_cqp_paginate_subcorpus',
-    'ext_cqp_partial_export_subcorpus',
-    'ext_cqp_export_subcorpus',
-]
+from .utils import SessionManager
 
 
 def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
-    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
-    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    corpus_id = SessionManager.get_corpus_id()
+    cqi_client = SessionManager.get_cqi_client()
+    db_corpus = Corpus.query.get(corpus_id)
+    cqi_corpus = cqi_client.corpora.get(corpus)
     db_corpus.num_tokens = cqi_corpus.size
     db.session.commit()
     return CQiStatusOk()
 
 
 def ext_corpus_static_data(corpus: str) -> dict:
-    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
-    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+    corpus_id = SessionManager.get_corpus_id()
+    db_corpus = Corpus.query.get(corpus_id)
 
     static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
     if static_data_file_path.exists():
         with static_data_file_path.open('rb') as f:
             return f.read()
 
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
-    cqi_p_attrs: list[CQiPositionalAttribute] = cqi_corpus.positional_attributes.list()
-    cqi_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list()
+    cqi_client = SessionManager.get_cqi_client()
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    cqi_p_attrs = cqi_corpus.positional_attributes.list()
+    cqi_s_attrs = cqi_corpus.structural_attributes.list()
 
     static_data = {
         'corpus': {
@@ -61,21 +46,21 @@ def ext_corpus_static_data(corpus: str) -> dict:
     }
 
     for p_attr in cqi_p_attrs:
-        print(f'corpus.freqs.{p_attr.name}')
+        current_app.logger.info(f'corpus.freqs.{p_attr.name}')
         static_data['corpus']['freqs'][p_attr.name] = []
-        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        p_attr_id_list = list(range(p_attr.lexicon_size))
         static_data['corpus']['freqs'][p_attr.name].extend(p_attr.freqs_by_ids(p_attr_id_list))
         del p_attr_id_list
 
-        print(f'p_attrs.{p_attr.name}')
+        current_app.logger.info(f'p_attrs.{p_attr.name}')
         static_data['p_attrs'][p_attr.name] = []
-        cpos_list: list[int] = list(range(cqi_corpus.size))
+        cpos_list = list(range(cqi_corpus.size))
         static_data['p_attrs'][p_attr.name].extend(p_attr.ids_by_cpos(cpos_list))
         del cpos_list
 
-        print(f'values.p_attrs.{p_attr.name}')
+        current_app.logger.info(f'values.p_attrs.{p_attr.name}')
         static_data['values']['p_attrs'][p_attr.name] = []
-        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        p_attr_id_list = list(range(p_attr.lexicon_size))
         static_data['values']['p_attrs'][p_attr.name].extend(p_attr.values_by_ids(p_attr_id_list))
         del p_attr_id_list
 
@@ -91,9 +76,9 @@ def ext_corpus_static_data(corpus: str) -> dict:
             # Note: Needs more testing, don't use it in production       #
             ##############################################################
             cqi_corpus.query('Last', f'<{s_attr.name}> []* </{s_attr.name}>;')
-            cqi_subcorpus: CQiSubcorpus = cqi_corpus.subcorpora.get('Last')
-            first_match: int = 0
-            last_match: int = cqi_subcorpus.size - 1
+            cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
+            first_match = 0
+            last_match = cqi_subcorpus.size - 1
             match_boundaries = zip(
                 range(first_match, last_match + 1),
                 cqi_subcorpus.dump(
@@ -111,7 +96,7 @@ def ext_corpus_static_data(corpus: str) -> dict:
             del cqi_subcorpus, first_match, last_match
             for id, lbound, rbound in match_boundaries:
                 static_data['s_attrs'][s_attr.name]['lexicon'].append({})
-                print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+                current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
                 static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
             del match_boundaries
 
@@ -123,33 +108,33 @@ def ext_corpus_static_data(corpus: str) -> dict:
             # This is a very slow operation, thats why we only use it for
             # the text attribute
             lbound, rbound = s_attr.cpos_by_id(id)
-            print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+            current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
             static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
             static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
-            cpos_list: list[int] = list(range(lbound, rbound + 1))
+            cpos_list = list(range(lbound, rbound + 1))
             for p_attr in cqi_p_attrs:
-                p_attr_ids: list[int] = []
+                p_attr_ids = []
                 p_attr_ids.extend(p_attr.ids_by_cpos(cpos_list))
-                print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
+                current_app.logger.info(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
                 static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids))
                 del p_attr_ids
             del cpos_list
 
-        sub_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
-        print(f's_attrs.{s_attr.name}.values')
+        sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
+        current_app.logger.info(f's_attrs.{s_attr.name}.values')
         static_data['s_attrs'][s_attr.name]['values'] = [
             sub_s_attr.name[(len(s_attr.name) + 1):]
             for sub_s_attr in sub_s_attrs
         ]
-        s_attr_id_list: list[int] = list(range(s_attr.size))
-        sub_s_attr_values: list[str] = []
+        s_attr_id_list = list(range(s_attr.size))
+        sub_s_attr_values = []
         for sub_s_attr in sub_s_attrs:
             tmp = []
             tmp.extend(sub_s_attr.values_by_ids(s_attr_id_list))
             sub_s_attr_values.append(tmp)
             del tmp
         del s_attr_id_list
-        print(f'values.s_attrs.{s_attr.name}')
+        current_app.logger.info(f'values.s_attrs.{s_attr.name}')
         static_data['values']['s_attrs'][s_attr.name] = [
             {
                 s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id]
@@ -159,11 +144,11 @@ def ext_corpus_static_data(corpus: str) -> dict:
             } for s_attr_id in range(0, s_attr.size)
         ]
         del sub_s_attr_values
-    print('Saving static data to file')
+    current_app.logger.info('Saving static data to file')
     with gzip.open(static_data_file_path, 'wt') as f:
         json.dump(static_data, f)
     del static_data
-    print('Sending static data to client')
+    current_app.logger.info('Sending static data to client')
     with open(static_data_file_path, 'rb') as f:
         return f.read()
 
@@ -173,7 +158,7 @@ def ext_corpus_paginate_corpus(
     page: int = 1,
     per_page: int = 20
 ) -> dict:
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus)
     # Sanity checks
     if (
@@ -188,7 +173,7 @@ def ext_corpus_paginate_corpus(
     first_cpos = (page - 1) * per_page
     last_cpos = min(cqi_corpus.size, first_cpos + per_page)
     cpos_list = [*range(first_cpos, last_cpos)]
-    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
+    lookups = _lookups_by_cpos(cqi_corpus, cpos_list)
     payload = {}
     # the items for the current page
     payload['items'] = [cpos_list]
@@ -220,7 +205,7 @@ def ext_cqp_paginate_subcorpus(
     per_page: int = 20
 ) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
     # Sanity checks
@@ -235,7 +220,7 @@ def ext_cqp_paginate_subcorpus(
         return {'code': 416, 'msg': 'Range Not Satisfiable'}
     offset = (page - 1) * per_page
     cutoff = per_page
-    cqi_results_export = export_subcorpus(
+    cqi_results_export = _export_subcorpus(
         cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
     payload = {}
     # the items for the current page
@@ -267,20 +252,145 @@ def ext_cqp_partial_export_subcorpus(
     context: int = 50
 ) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
+    cqi_subcorpus_partial_export = _partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
     return cqi_subcorpus_partial_export
 
 
-def ext_cqp_export_subcorpus(
-    subcorpus: str,
-    context: int = 50
-) -> dict:
+def ext_cqp_export_subcorpus(subcorpus: str, context: int = 50) -> dict:
     corpus_name, subcorpus_name = subcorpus.split(':', 1)
-    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_client = SessionManager.get_cqi_client()
     cqi_corpus = cqi_client.corpora.get(corpus_name)
     cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
+    cqi_subcorpus_export = _export_subcorpus(cqi_subcorpus, context=context)
     return cqi_subcorpus_export
+
+
+def _lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
+    lookups = {}
+    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
+    for attr in corpus.positional_attributes.list():
+        cpos_attr_values = attr.values_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
+    for attr in corpus.structural_attributes.list():
+        # We only want to iterate over non subattributes, identifiable by
+        # attr.has_values == False
+        if attr.has_values:
+            continue
+        cpos_attr_ids = attr.ids_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            if cpos_attr_ids[i] == -1:
+                continue
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
+        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
+        if len(occured_attr_ids) == 0:
+            continue
+        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
+        if len(subattrs) == 0:
+            continue
+        lookup_name = f'{attr.name}_lookup'
+        lookups[lookup_name] = {}
+        for attr_id in occured_attr_ids:
+            lookups[lookup_name][attr_id] = {}
+        for subattr in subattrs:
+            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
+            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
+                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
+    return lookups
+
+
+def _partial_export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    match_id_list: list[int],
+    context: int = 25
+) -> dict:
+    if subcorpus.size == 0:
+        return {'matches': []}
+    match_boundaries = []
+    for match_id in match_id_list:
+        if match_id < 0 or match_id >= subcorpus.size:
+            continue
+        match_boundaries.append(
+            (
+                match_id,
+                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
+                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
+            )
+        )
+    cpos_set = set()
+    matches = []
+    for match_boundary in match_boundaries:
+        match_num, match_start, match_end = match_boundary
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = _lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}
+
+
+def _export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    context: int = 25,
+    cutoff: float = float('inf'),
+    offset: int = 0
+) -> dict:
+    if subcorpus.size == 0:
+        return {'matches': []}
+    first_match = max(0, offset)
+    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
+    match_boundaries = zip(
+        range(first_match, last_match + 1),
+        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
+        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
+    )
+    cpos_set = set()
+    matches = []
+    for match_num, match_start, match_end in match_boundaries:
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = _lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}

app/namespaces/cqi_over_sio/utils.py

@@ -0,0 +1,37 @@
+from cqi import CQiClient
+from threading import Lock
+from flask import session
+
+
+class SessionManager:
+    @staticmethod
+    def setup():
+        session['cqi_over_sio'] = {}
+
+    @staticmethod
+    def teardown():
+        session.pop('cqi_over_sio')
+
+    @staticmethod
+    def set_corpus_id(corpus_id: int):
+        session['cqi_over_sio']['corpus_id'] = corpus_id
+
+    @staticmethod
+    def get_corpus_id() -> int:
+        return session['cqi_over_sio']['corpus_id']
+
+    @staticmethod
+    def set_cqi_client(cqi_client: CQiClient):
+        session['cqi_over_sio']['cqi_client'] = cqi_client
+
+    @staticmethod
+    def get_cqi_client() -> CQiClient:
+        return session['cqi_over_sio']['cqi_client']
+
+    @staticmethod
+    def set_cqi_client_lock(cqi_client_lock: Lock):
+        session['cqi_over_sio']['cqi_client_lock'] = cqi_client_lock
+
+    @staticmethod
+    def get_cqi_client_lock() -> Lock:
+        return session['cqi_over_sio']['cqi_client_lock']

app/namespaces/jobs.py

@@ -0,0 +1,109 @@
+from flask import current_app, Flask
+from flask_login import current_user
+from flask_socketio import Namespace
+from app import db, hashids, socketio
+from app.decorators import socketio_admin_required, socketio_login_required
+from app.models import Job, JobStatus
+
+
+def _delete_job(app: Flask, job_id: int):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.delete()
+        db.session.commit()
+
+
+def _restart_job(app, job_id):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        job.restart()
+        db.session.commit()
+
+
+class UsersNamespace(Namespace):
+    @socketio_login_required
+    def on_delete(self, job_hashid: str) -> dict:
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            job.user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        socketio.start_background_task(
+            _delete_job,
+            current_app._get_current_object(),
+            job_id
+        )
+
+        return {
+            'body': f'Job "{job.title}" marked for deletion',
+            'status': 202,
+            'statusText': 'Accepted'
+        }
+
+    @socketio_admin_required
+    def on_log(self, job_hashid: str):
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+            return {'status': 409, 'statusText': 'Conflict'}
+
+        with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
+            log = log_file.read()
+
+        return {
+            'body': log,
+            'status': 200,
+            'statusText': 'Forbidden'
+        }
+
+    socketio_login_required
+    def on_restart(self, job_hashid: str):
+        job_id = hashids.decode(job_hashid)
+
+        if not isinstance(job_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        job = Job.query.get(job_id)
+
+        if job is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            job.user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        if job.status == JobStatus.FAILED:
+            return {'status': 409, 'statusText': 'Conflict'}
+
+        socketio.start_background_task(
+            _restart_job,
+            current_app._get_current_object(),
+            job_id
+        )
+
+        return {
+            'body': f'Job "{job.title}" marked for restarting',
+            'status': 202,
+            'statusText': 'Accepted'
+        }

app/namespaces/users.py

@@ -0,0 +1,116 @@
+from flask import current_app, Flask
+from flask_login import current_user
+from flask_socketio import join_room, leave_room, Namespace
+from app import db, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import User
+
+
+def _delete_user(app: Flask, user_id: int):
+    with app.app_context():
+        user = User.query.get(user_id)
+        user.delete()
+        db.session.commit()
+
+
+class UsersNamespace(Namespace):
+    @socketio_login_required
+    def on_get(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        return {
+            'body': user.to_json_serializeable(
+                backrefs=True,
+                relationships=True
+            ),
+            'status': 200,
+            'statusText': 'OK'
+        }
+
+    @socketio_login_required
+    def on_subscribe(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        join_room(f'/users/{user.hashid}')
+
+        return {'status': 200, 'statusText': 'OK'}
+
+    @socketio_login_required
+    def on_unsubscribe(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        leave_room(f'/users/{user.hashid}')
+
+        return {'status': 200, 'statusText': 'OK'}
+
+    @socketio_login_required
+    def on_delete(self, user_hashid: str) -> dict:
+        user_id = hashids.decode(user_hashid)
+
+        if not isinstance(user_id, int):
+            return {'status': 400, 'statusText': 'Bad Request'}
+
+        user = User.query.get(user_id)
+
+        if user is None:
+            return {'status': 404, 'statusText': 'Not found'}
+
+        if not (
+            user == current_user
+            or current_user.is_administrator
+        ):
+            return {'status': 403, 'statusText': 'Forbidden'}
+
+        socketio.start_background_task(
+            _delete_user,
+            current_app._get_current_object(),
+            user.id
+        )
+
+        return {
+            'body': f'User "{user.username}" marked for deletion',
+            'status': 202,
+            'statusText': 'Accepted'
+        }

app/static/css/corpus-status-text.css

@@ -2,6 +2,10 @@
   --corpus-status-content: "unprepared";
 }
 
+[data-corpus-status="SUBMITTED"] {
+  --corpus-status-content: "submitted";
+}
+
 [data-corpus-status="QUEUED"] {
   --corpus-status-content: "queued";
 }

app/static/css/height.css

@@ -0,0 +1,47 @@
+.h-10 {
+    height: 10% !important;
+}
+
+.h-20 {
+    height: 20% !important;
+}
+
+.h-25 {
+    height: 25% !important;
+}
+
+.h-30 {
+    height: 30% !important;
+}
+
+.h-40 {
+    height: 40% !important;
+}
+
+.h-50 {
+    height: 50% !important;
+}
+
+.h-60 {
+    height: 60% !important;
+}
+
+.h-70 {
+    height: 70% !important;
+}
+
+.h-75 {
+    height: 75% !important;
+}
+
+.h-80 {
+    height: 80% !important;
+}
+
+.h-90 {
+    height: 90% !important;
+}
+
+.h-100 {
+    height: 100% !important;
+}

app/static/css/materialize.override.css

@@ -1,37 +1,3 @@
-/* #region sidenav-fixed */
-/*
- * The sidenav-fixed class is used which causes the sidenav to be fixed and open
- * on large screens and hides to the regular functionality on smaller screens.
- * In order to prevent the sidenav to overlap the content, the content (header, main and footer)
- * gets an offset equal to the width of the sidenav.
- *
- * Read more: https://materializecss.com/sidenav.html#variations
- */
-body[data-sidenav-fixed="true" i] header,
-body[data-sidenav-fixed="true" i] main,
-body[data-sidenav-fixed="true" i] footer {
-  padding-left: 300px;
-}
-
-@media only screen and (max-width : 992px) {
-  body[data-sidenav-fixed="true" i] header,
-  body[data-sidenav-fixed="true" i] main,
-  body[data-sidenav-fixed="true" i] footer {
-    padding-left: 0;
-  }
-}
-
-body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
-  width: calc(100% - 300px);
-}
-
-@media only screen and (max-width : 992px) {
-  body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
-    width: 100%;
-  }
-}
-/* #endregion sidenav-fixed */
-
 /* #region sticky-footer */
 /*
  * Sticky Footer:
@@ -44,13 +10,13 @@ body[data-sidenav-fixed="true" i] .navbar-fixed > nav {
  *
  * Read more: https://materializecss.com/footer.html#sticky-footer
  */
- body[data-sticky-footer="true" i] {
+ body {
   display: flex;
   min-height: 100vh;
   flex-direction: column;
 }
 
-body[data-sticky-footer="true" i] main {
+main {
   flex: 1 0 auto;
 }
 /* #endregion sticky-footer */

app/static/css/width.css

@@ -0,0 +1,47 @@
+.w-10 {
+    width: 10% !important;
+}
+
+.w-20 {
+    width: 20% !important;
+}
+
+.w-25 {
+    width: 25% !important;
+}
+
+.w-30 {
+    width: 30% !important;
+}
+
+.w-40 {
+    width: 40% !important;
+}
+
+.w-50 {
+    width: 50% !important;
+}
+
+.w-60 {
+    width: 60% !important;
+}
+
+.w-70 {
+    width: 70% !important;
+}
+
+.w-75 {
+    width: 75% !important;
+}
+
+.w-80 {
+    width: 80% !important;
+}
+
+.w-90 {
+    width: 90% !important;
+}
+
+.w-100 {
+    width: 100% !important;
+}

app/static/js/app.js

@@ -1,182 +1,20 @@
 nopaque.App = class App {
   constructor() {
-    this.data = {
-      promises: {getUser: {}, subscribeUser: {}},
-      users: {},
-    };
     this.socket = io({transports: ['websocket'], upgrade: false});
-    this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
-  }
 
-  getUser(userId) {
-    if (userId in this.data.promises.getUser) {
-      return this.data.promises.getUser[userId];
-    }
+    // Endpoints
+    this.users = new nopaque.app.endpoints.Users(this);
 
-    this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('GET /users/<user_id>', userId, (response) => {
-        if (response.status === 200) {
-          this.data.users[userId] = response.body;
-          resolve(this.data.users[userId]);
-        } else {
-          reject(`[${response.status}] ${response.statusText}`);
-        }
-      });
-    });
-
-    return this.data.promises.getUser[userId];
-  }
-
-  subscribeUser(userId) {
-    if (userId in this.data.promises.subscribeUser) {
-      return this.data.promises.subscribeUser[userId];
-    }
-
-    this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
-        if (response.status !== 200) {
-          reject(response);
-          return;
-        }
-        resolve(response);
-      });
-    });
-
-    return this.data.promises.subscribeUser[userId];
-  }
-
-  flash(message, category) {
-    let iconPrefix = '';
-    switch (category) {
-      case 'corpus': {
-        iconPrefix = '<i class="left material-icons">book</i>';
-        break;
-      }
-      case 'error': {
-        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
-        break;
-      }
-      case 'job': {
-        iconPrefix = '<i class="left nopaque-icons">J</i>';
-        break;
-      }
-      case 'settings': {
-        iconPrefix = '<i class="left material-icons">settings</i>';
-        break;
-      }
-      default: {
-        iconPrefix = '<i class="left material-icons">notifications</i>';
-        break;
-      }
-    }
-    let toast = M.toast(
-      {
-        html: `
-          <span>${iconPrefix}${message}</span>
-          <button class="action-button btn-flat toast-action white-text" data-action="close">
-            <i class="material-icons">close</i>
-          </button>
-        `.trim()
-      }
-    );
-    let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
-    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
-  }
-  
-  onPatch(patch) {
-    // Filter Patch to only include operations on users that are initialized
-    let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
-    let filteredPatch = patch.filter(operation => regExp.test(operation.path));
-
-    // Handle job status updates
-    let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
-    let subFilteredPatch = filteredPatch
-      .filter((operation) => {return operation.op === 'replace';})
-      .filter((operation) => {return subRegExp.test(operation.path);});
-    for (let operation of subFilteredPatch) {
-      let [match, userId, jobId] = operation.path.match(subRegExp);
-      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
-    }
-
-    // Apply Patch
-    jsonpatch.applyPatch(this.data, filteredPatch);
+    // Extensions
+    this.toaster = new nopaque.app.extensions.Toaster(this);
+    this.ui = new nopaque.app.extensions.UI(this);
+    this.userHub = new nopaque.app.extensions.UserHub(this);
   }
 
   init() {
-    this.initUi();
-  }
-
-  initUi() {
-    /* Pre-Initialization fixes */
-    // #region
-
-    // Flask-WTF sets the standard HTML maxlength Attribute on input/textarea
-    // elements to specify their maximum length (in characters). Unfortunatly
-    // Materialize won't recognize the maxlength Attribute, instead it uses
-    // the data-length Attribute. It's conversion time :)
-    for (let elem of document.querySelectorAll('input[maxlength], textarea[maxlength]')) {
-      elem.dataset.length = elem.getAttribute('maxlength');
-      elem.removeAttribute('maxlength');
-    }
-
-    // To work around some limitations with the Form setup of Flask-WTF.
-    // HTML option elements with an empty value are considered as placeholder
-    // elements. The user should not be able to actively select these options.
-    // So they get the disabled attribute.
-    for (let optionElement of document.querySelectorAll('option[value=""]')) {
-      optionElement.disabled = true;
-    }
-
-    // TODO: Check why we are doing this.
-    for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) {
-      for (let c of optgroupElement.children) {
-        optgroupElement.parentElement.insertAdjacentElement('afterbegin', c);
-      }
-      optgroupElement.remove();
-    }
-    // #endregion
-
-
-    /* Initialize Materialize Components */
-    // #region
-  
-    // Automatically initialize Materialize Components that do not require
-    // additional configuration.
-    M.AutoInit();
-
-    // CharacterCounters
-    // Materialize didn't include the CharacterCounter plugin within the
-    // AutoInit method (maybe they forgot it?). Anyway... We do it here. :)
-    M.CharacterCounter.init(document.querySelectorAll('input[data-length]:not(.no-autoinit), textarea[data-length]:not(.no-autoinit)'));
-
-    // Header navigation account Dropdown.
-    M.Dropdown.init(
-      document.querySelector('#nav-account-dropdown-trigger'),
-      {
-        alignment: 'right',
-        constrainWidth: false,
-        coverTrigger: false
-      }
-    );
-
-    // Terms of use modal
-    M.Modal.init(
-      document.querySelector('#terms-of-use-modal'),
-      {
-        dismissible: false,
-        onCloseEnd: (modalElement) => {
-          nopaque.requests.users.entity.acceptTermsOfUse();
-        }
-      }
-    );
-    // #endregion
-
-
-    /* Initialize nopaque Components */
-    // #region 
-    nopaque.resource_displays.AutoInit();
-    nopaque.resource_lists.AutoInit();
-    nopaque.forms.AutoInit();
-    // #endregion
+    // Initialize extensions
+    this.toaster.init();
+    this.ui.init();
+    this.userHub.init();
   }
 };

app/static/js/app/endpoints/index.js

@@ -0,0 +1 @@
+nopaque.app.endpoints = {};

app/static/js/app/endpoints/users.js

@@ -0,0 +1,41 @@
+nopaque.app.endpoints.Users = class Users {
+  constructor(app) {
+    this.app = app;
+
+    this.socket = io('/users', {transports: ['websocket'], upgrade: false});
+  }
+
+  async get(userId) {
+    const response = await this.socket.emitWithAck('get', userId);
+
+    if (response.status !== 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+
+    return response.body;
+  }
+
+  async subscribe(userId) {
+    const response = await this.socket.emitWithAck('subscribe', userId);
+
+    if (response.status != 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+
+  async unsubscribe(userId) {
+    const response = await this.socket.emitWithAck('unsubscribe', userId);
+
+    if (response.status != 200) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+
+  async delete(userId) {
+    const response = await this.socket.emitWithAck('delete', userId);
+
+    if (response.status != 202) {
+      throw new Error(`[${response.status}] ${response.statusText}`);
+    }
+  }
+}

app/static/js/app/extensions/index.js

@@ -0,0 +1 @@
+nopaque.app.extensions = {};

app/static/js/app/extensions/toaster.js

@@ -0,0 +1,56 @@
+nopaque.app.extensions.Toaster = class Toaster {
+  constructor(app) {
+    this.app = app;
+  }
+
+  init() {
+    this.app.userHub.addEventListener('patch', (event) => {this.#onPatch(event.detail);});
+  }
+
+  async #onPatch(patch) {
+    // Handle corpus updates
+    const corpusRegExp = new RegExp(`^/users/([A-Za-z0-9]+)/corpora/([A-Za-z0-9]+)`);
+    const corpusPatch = patch.filter((operation) => {return corpusRegExp.test(operation.path);});
+
+    this.#onCorpusPatch(corpusPatch);
+
+    // Handle job updates
+    const jobRegExp = new RegExp(`^/users/([A-Za-z0-9]+)/jobs/([A-Za-z0-9]+)`);
+    const jobPatch = patch.filter((operation) => {return jobRegExp.test(operation.path);});
+
+    this.#onJobPatch(jobPatch);
+  }
+
+  async #onCorpusPatch(patch) {
+    return;
+    // Handle corpus status updates
+    const corpusStatusRegExp = new RegExp(`^/users/([A-Za-z0-9]+)/corpora/([A-Za-z0-9]+)/status$`);
+    const corpusStatusPatch = patch
+      .filter((operation) => {return corpusStatusRegExp.test(operation.path);})
+      .filter((operation) => {return operation.op === 'replace';});
+
+    for (let operation of corpusStatusPatch) {
+      const [match, userId, corpusId] = operation.path.match(corpusStatusRegExp);
+      const user = await this.app.userHub.get(userId);
+      const corpus = user.corpora[corpusId];
+
+      this.app.ui.flash(`[<a href="/corpora/${corpusId}">${corpus.title}</a>] New status: <span class="corpus-status-text" data-corpus-status="${operation.value}"></span>`, 'corpus');
+    }
+  }
+
+  async #onJobPatch(patch) {
+    // Handle job status updates
+    const jobStatusRegExp = new RegExp(`^/users/([A-Za-z0-9]+)/jobs/([A-Za-z0-9]+)/status$`);
+    const jobStatusPatch = patch
+      .filter((operation) => {return jobStatusRegExp.test(operation.path);})
+      .filter((operation) => {return operation.op === 'replace';});
+
+    for (let operation of jobStatusPatch) {
+      const [match, userId, jobId] = operation.path.match(jobStatusRegExp);
+      const user = await this.app.userHub.get(userId);
+      const job = user.jobs[jobId];
+
+      this.app.ui.flash(`[<a href="/jobs/${jobId}">${job.title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
+    }
+  }
+}

app/static/js/app/extensions/ui.js

@@ -0,0 +1,126 @@
+nopaque.app.extensions.UI = class UI {
+  constructor(app) {
+    this.app = app;
+  }
+
+  init() {
+    /* Pre-Initialization fixes */
+    // #region
+
+    // Flask-WTF sets the standard HTML maxlength Attribute on input/textarea
+    // elements to specify their maximum length (in characters). Unfortunatly
+    // Materialize won't recognize the maxlength Attribute, instead it uses
+    // the data-length Attribute. It's conversion time :)
+    for (let elem of document.querySelectorAll('input[maxlength], textarea[maxlength]')) {
+      elem.dataset.length = elem.getAttribute('maxlength');
+      elem.removeAttribute('maxlength');
+    }
+
+    // To work around some limitations with the Form setup of Flask-WTF.
+    // HTML option elements with an empty value are considered as placeholder
+    // elements. The user should not be able to actively select these options.
+    // So they get the disabled attribute.
+    for (let optionElement of document.querySelectorAll('option[value=""]')) {
+      optionElement.disabled = true;
+    }
+
+    // TODO: Check why we are doing this.
+    for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) {
+      for (let c of optgroupElement.children) {
+        optgroupElement.parentElement.insertAdjacentElement('afterbegin', c);
+      }
+      optgroupElement.remove();
+    }
+    // #endregion
+
+
+    /* Initialize Materialize Components */
+    // #region
+
+    // Automatically initialize Materialize Components that do not require
+    // additional configuration.
+    M.AutoInit();
+
+    // CharacterCounters
+    // Materialize didn't include the CharacterCounter plugin within the
+    // AutoInit method (maybe they forgot it?). Anyway... We do it here. :)
+    M.CharacterCounter.init(document.querySelectorAll('input[data-length]:not(.no-autoinit), textarea[data-length]:not(.no-autoinit)'));
+
+    // Header navigation processes and services Dropdown.
+    M.Dropdown.init(
+      document.querySelector('#navbar-data-processing-and-analysis-dropdown-trigger'),
+      {
+        constrainWidth: false,
+        container: document.querySelector('#dropdowns'),
+        coverTrigger: false
+      }
+    );
+
+    // Header navigation account Dropdown.
+    M.Dropdown.init(
+      document.querySelector('#navbar-account-dropdown-trigger'),
+      {
+        alignment: 'right',
+        constrainWidth: false,
+        container: document.querySelector('#dropdowns'),
+        coverTrigger: false
+      }
+    );
+
+    // Terms of use modal
+    M.Modal.init(
+      document.querySelector('#terms-of-use-modal'),
+      {
+        dismissible: false,
+        onCloseEnd: (modalElement) => {
+          nopaque.requests.users.entity.acceptTermsOfUse();
+        }
+      }
+    );
+    // #endregion
+
+
+    /* Initialize nopaque Components */
+    // #region
+    nopaque.resource_displays.AutoInit();
+    nopaque.resource_lists.AutoInit();
+    nopaque.forms.AutoInit();
+    // #endregion
+  }
+
+  flash(message, category) {
+    let iconPrefix;
+
+    switch (category) {
+      case 'corpus': {
+        iconPrefix = '<i class="material-icons left">book</i>';
+        break;
+      }
+      case 'job': {
+        iconPrefix = '<i class="nopaque-icons left">J</i>';
+        break;
+      }
+      case 'error': {
+        iconPrefix = '<i class="material-icons left error-color-text">error</i>';
+        break;
+      }
+      default: {
+        iconPrefix = '<i class="material-icons left">notifications</i>';
+        break;
+      }
+    }
+
+    let toast = M.toast(
+      {
+        html: `
+          <span>${iconPrefix}${message}</span>
+          <button class="btn-flat toast-action white-text" data-toast-action="dismiss">
+            <i class="material-icons">close</i>
+          </button>
+        `.trim()
+      }
+    );
+    let dismissToastElement = toast.el.querySelector('.toast-action[data-toast-action="dismiss"]');
+    dismissToastElement.addEventListener('click', () => {toast.dismiss();});
+  }
+}

app/static/js/app/extensions/user-hub.js

@@ -0,0 +1,68 @@
+nopaque.app.extensions.UserHub = class UserHub extends EventTarget {
+  #data;
+
+  constructor(app) {
+    super();
+
+    this.app = app;
+
+    this.#data = {
+      users: {},
+      promises: {}
+    };
+  }
+
+  init() {
+    this.app.users.socket.on('patch', (patch) => {this.#onPatch(patch)});
+  }
+
+  add(userId) {
+    if (!(userId in this.#data.promises)) {
+      this.#data.promises[userId] = this.#add(userId);
+    }
+
+    return this.#data.promises[userId];
+  }
+
+  async #add(userId) {
+    await this.app.users.subscribe(userId);
+    this.#data.users[userId] = await this.app.users.get(userId);
+  }
+
+  async get(userId) {
+    await this.add(userId);
+    return this.#data.users[userId];
+  }
+
+  #onPatch(patch) {
+    // Filter patch to only include operations on users that are initialized
+    const filterRegExp = new RegExp(`^/users/(${Object.keys(this.#data.users).join('|')})`);
+    const filteredPatch = patch.filter(operation => filterRegExp.test(operation.path));
+
+    // Apply patch
+    jsonpatch.applyPatch(this.#data, filteredPatch);
+
+    // Notify event listeners
+    const patchEventa = new CustomEvent('patch', {detail: filteredPatch});
+    this.dispatchEvent(patchEventa);
+
+    // Notify event listeners. Event type: "patch *"
+    const patchEvent = new CustomEvent('patch *', {detail: filteredPatch});
+    this.dispatchEvent(patchEvent);
+
+    // Group patches by user id: {<user-id>: [op, ...], ...}
+    const patches = {};
+    const matchRegExp = new RegExp(`^/users/([A-Za-z0-9]+)`);
+    for (let operation of filteredPatch) {
+      const [match, userId] = operation.path.match(matchRegExp);
+      if (!(userId in patches)) {patches[userId] = [];}
+      patches[userId].push(operation);
+    }
+
+    // Notify event listeners. Event type: "patch <user-id>"
+    for (let [userId, patch] of Object.entries(patches)) {
+      const userPatchEvent = new CustomEvent(`patch ${userId}`, {detail: patch});
+      this.dispatchEvent(userPatchEvent);
+    }
+  }
+}

app/static/js/app/index.js

@@ -0,0 +1 @@
+nopaque.app = {};

app/static/js/corpus-analysis/concordance-extension.js

@@ -66,7 +66,7 @@ nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
       errorString += `${error.constructor.name}`;
       this.elements.error.innerText = errorString;
       this.elements.error.classList.remove('hide');
-      app.flash(errorString, 'error');
+      app.ui.flash(errorString, 'error');
       this.elements.progress.classList.add('hide');
     }
     this.app.enableActionElements();
@@ -239,7 +239,7 @@ nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
           if (subcorpus.selectedItems.size === 0) {
             this.elements.progress.classList.add('hide');
             this.app.enableActionElements();
-            app.flash('No matches selected', 'error');
+            app.ui.flash('No matches selected', 'error');
             return;
           }
           promise = subcorpus.o.partialExport([...subcorpus.selectedItems], 50);
@@ -298,7 +298,7 @@ nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
       let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
       subcorpus.o.drop().then(
         (cQiStatus) => {
-          app.flash(`${subcorpus.o.name} deleted`, 'corpus');
+          app.ui.flash(`${subcorpus.o.name} deleted`, 'corpus');
           delete this.data.subcorpora[subcorpus.o.name];
           this.settings.selectedSubcorpus = undefined;
           for (let subcorpusName in this.data.subcorpora) {
@@ -320,7 +320,7 @@ nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
         },
         (cqiError) => {
           let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
-          app.flash(errorString, 'error');
+          app.ui.flash(errorString, 'error');
         }
       );
     });

app/static/js/corpus-analysis/reader-extension.js

@@ -46,7 +46,7 @@ nopaque.corpus_analysis.ReaderExtension = class ReaderExtension {
       if ('description' in error) {errorString += `: ${error.description}`;}
       this.elements.error.innerText = errorString;
       this.elements.error.classList.remove('hide');
-      app.flash(errorString, 'error');
+      app.ui.flash(errorString, 'error');
       this.elements.progress.classList.add('hide');
     }
     this.app.enableActionElements();
@@ -205,7 +205,7 @@ nopaque.corpus_analysis.ReaderExtension = class ReaderExtension {
       `
     );
     this.elements.corpusPagination.appendChild(pageElement);
-    
+
     for (let paginateTriggerElement of this.elements.corpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
       paginateTriggerElement.addEventListener('click', (event) => {
         event.preventDefault();

app/static/js/forms/base-form.js

@@ -101,7 +101,7 @@ nopaque.forms.BaseForm = class BaseForm {
         }
       }
       if (request.status === 500) {
-        app.flash('Internal Server Error', 'error');
+        app.ui.flash('Internal Server Error', 'error');
       }
       modal.close();
     });

app/static/js/requests/index.js

@@ -18,23 +18,23 @@ nopaque.requests.JSONfetch = (input, init={}) => {
           }
           if (response.status === 204) {
             return;
-          } 
+          }
           response.json()
             .then(
               (json) => {
                 let message = json.message;
                 let category = json.category || 'message';
                 if (message) {
-                  app.flash(message, category);
+                  app.ui.flash(message, category);
                 }
               },
               (error) => {
-                app.flash(`[${response.status}]: ${response.statusText}`, 'error');
+                app.ui.flash(`[${response.status}]: ${response.statusText}`, 'error');
               }
             );
         },
         (response) => {
-          app.flash('Something went wrong', 'error');
+          app.ui.flash('Something went wrong', 'error');
           reject(response);
         }
       );

app/static/js/resource-displays/corpus-display.js

@@ -52,22 +52,23 @@ nopaque.resource_displays.CorpusDisplay = class CorpusDisplay extends nopaque.re
     }
   }
 
-  setTitle(title) {
-    this.setElements(this.displayElement.querySelectorAll('.corpus-title'), title);
+  async setTitle(title) {
+    const corpusTitleElements = this.displayElement.querySelectorAll('.corpus-title');
+    this.setElements(corpusTitleElements, title);
   }
 
   setNumTokens(numTokens) {
-    this.setElements(
-      this.displayElement.querySelectorAll('.corpus-token-ratio'),
-      `${numTokens}/${app.data.users[this.userId].corpora[this.corpusId].max_num_tokens}`
-    );
+    const corpusTokenRatioElements = this.displayElement.querySelectorAll('.corpus-token-ratio');
+    const maxNumTokens = 2147483647;
+
+    this.setElements(corpusTokenRatioElements, `${numTokens}/${maxNumTokens}`);
   }
 
   setDescription(description) {
     this.setElements(this.displayElement.querySelectorAll('.corpus-description'), description);
   }
 
-  setStatus(status) {
+  async setStatus(status) {
     let elements = this.displayElement.querySelectorAll('.action-button[data-action="analyze"]');
     for (let element of elements) {
       if (['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
@@ -77,8 +78,10 @@ nopaque.resource_displays.CorpusDisplay = class CorpusDisplay extends nopaque.re
       }
     }
     elements = this.displayElement.querySelectorAll('.action-button[data-action="build-request"]');
+    const user = await app.userHub.get(this.userId);
+    const corpusFiles = user.corpora[this.corpusId].files;
     for (let element of elements) {
-      if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(app.data.users[this.userId].corpora[this.corpusId].files).length > 0) {
+      if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(corpusFiles.length > 0)) {
         element.classList.remove('disabled');
       } else {
         element.classList.add('disabled');

app/static/js/resource-displays/resource-display.js

@@ -5,19 +5,14 @@ nopaque.resource_displays.ResourceDisplay = class ResourceDisplay {
     this.displayElement = displayElement;
     this.userId = this.displayElement.dataset.userId;
     this.isInitialized = false;
-    if (this.userId) {
-      app.subscribeUser(this.userId)
-        .then((response) => {
-          app.socket.on('PATCH', (patch) => {
-            if (this.isInitialized) {this.onPatch(patch);}
-          });
-        });
-      app.getUser(this.userId)
-        .then((user) => {
-          this.init(user);
-          this.isInitialized = true;
-        });
-    }
+    if (this.userId === undefined) {return;}
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
+    });
+    app.userHub.get(this.userId).then((user) => {
+      this.init(user);
+      this.isInitialized = true;
+    });
   }
 
   init(user) {throw 'Not implemented';}

app/static/js/resource-lists/corpus-file-list.js

@@ -14,12 +14,11 @@ nopaque.resource_lists.CorpusFileList = class CorpusFileList extends nopaque.res
     this.hasPermissionView =  listContainerElement.dataset?.hasPermissionView == 'true' || false;
     this.hasPermissionManageFiles =  listContainerElement.dataset?.hasPermissionManageFiles == 'true' || false;
     if (this.userId === undefined || this.corpusId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
+      // TODO: Make this better understandable
       this.add(Object.values(user.corpora[this.corpusId].files || user.followed_corpora[this.corpusId].files));
       this.isInitialized = true;
     });
@@ -271,7 +270,7 @@ nopaque.resource_lists.CorpusFileList = class CorpusFileList extends nopaque.res
         this.selectedItemIds.clear();
         this.renderingItemSelection();
         break;
-      }  
+      }
       default: {
         break;
       }

app/static/js/resource-lists/corpus-follower-list.js

@@ -12,15 +12,16 @@ nopaque.resource_lists.CorpusFollowerList = class CorpusFollowerList extends nop
     this.userId = listContainerElement.dataset.userId;
     this.corpusId = listContainerElement.dataset.corpusId;
     if (this.userId === undefined || this.corpusId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
+      // TODO: Check if the following is better
       // let corpusFollowerAssociations = Object.values(user.corpora[this.corpusId].corpus_follower_associations);
       // let filteredList = corpusFollowerAssociations.filter(association => association.follower.id != currentUserId);
       // this.add(filteredList);
+
+      // TODO: Make this better understandable
       this.add(Object.values(user.corpora[this.corpusId].corpus_follower_associations));
       this.isInitialized = true;
     });
@@ -128,7 +129,7 @@ nopaque.resource_lists.CorpusFollowerList = class CorpusFollowerList extends nop
       }
     }
   }
-  
+
   onClick(event) {
     if (event.target.closest('.disable-on-click') !== null) {return;}
     let listItemElement = event.target.closest('.list-item[data-id]');

app/static/js/resource-lists/corpus-list.js

@@ -11,12 +11,10 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
     this.selectedItemIds = new Set();
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
       this.add(this.aggregateData(user));
       this.isInitialized = true;
     });
@@ -69,6 +67,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
               <span class="disable-on-click"></span>
             </label>
           </td>
+          <td><a class="btn-floating service-color darken" data-service="corpus-analysis"><i class="material-icons">book</i></a></td>
           <td>
             <b class="title"></b><br>
             <i class="description"></i>
@@ -80,7 +79,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
           <td>${values['current-user-is-following'] ? '<span><i class="left material-icons">visibility</i>Following</span>' : ''}</td>
           <td class="right-align">
             <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete-request"><i class="material-icons">delete</i></a>
-            <a class="list-action-trigger btn-floating darken waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
+            <a class="list-action-trigger btn-floating waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
           </td>
         </tr>
       `.trim();
@@ -119,6 +118,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
                 <span></span>
               </label>
             </th>
+            <th></th>
             <th>Title and Description</th>
             <th>Owner</th>
             <th>Status</th>
@@ -254,7 +254,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
           let listItem = this.listjs.get('id', selectedItemId)[0].elm;
           let values = this.listjs.get('id', listItem.dataset.id)[0].values();
           let itemElement = nopaque.Utils.HTMLToElement(`<li> - ${values.title}</li>`);
-          // if (!values['is-owner']) { 
+          // if (!values['is-owner']) {
           //   itemUnfollowList.appendChild(itemElement);
           // } else {
           itemDeletionList.appendChild(itemElement);
@@ -286,7 +286,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
           });
           this.selectedItemIds.clear();
           this.renderingItemSelection();
-   
+
         });
         modal.open();
         break;

app/static/js/resource-lists/job-input-list.js

@@ -8,8 +8,10 @@ nopaque.resource_lists.JobInputList = class JobInputList extends nopaque.resourc
     this.userId = listContainerElement.dataset.userId;
     this.jobId = listContainerElement.dataset.jobId;
     if (this.userId === undefined || this.jobId === undefined) {return;}
-    app.subscribeUser(this.userId);
-    app.getUser(this.userId).then((user) => {
+    // app.userHub.addEventListener('patch', (event) => {
+    //   if (this.isInitialized) {this.onPatch(event.detail);}
+    // });
+    app.userHub.get(this.userId).then((user) => {
       this.add(Object.values(user.jobs[this.jobId].inputs));
       this.isInitialized = true;
     });

app/static/js/resource-lists/job-list.js

@@ -12,12 +12,10 @@ nopaque.resource_lists.JobList = class JobList extends nopaque.resource_lists.Re
     this.selectedItemIds = new Set();
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
       this.add(Object.values(user.jobs));
       this.isInitialized = true;
     });
@@ -25,19 +23,19 @@ nopaque.resource_lists.JobList = class JobList extends nopaque.resource_lists.Re
 
   get item() {
     return `
-      <tr class="list-item clickable hoverable">
+      <tr class="list-item clickable hoverable service-color lighten">
         <td>
           <label class="list-action-trigger" data-list-action="select">
             <input class="select-checkbox" type="checkbox">
             <span class="disable-on-click"></span>
           </label>
         </td>
-        <td><a class="btn-floating service-color darken" data-service="inherit"><i class="nopaque-icons service-icons" data-service="inherit"></i></a></td>
+        <td><a class="btn-floating service-color darken"><i class="nopaque-icons service-icons"></i></a></td>
         <td><b class="title"></b><br><i class="description"></i></td>
         <td><span class="status badge new job-status-color job-status-text" data-badge-caption=""></span></td>
         <td class="right-align">
           <a class="list-action-trigger btn-floating red waves-effect waves-light" data-list-action="delete-request"><i class="material-icons">delete</i></a>
-          <a class="list-action-trigger btn-floating darken waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
+          <a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view"><i class="material-icons">send</i></a>
         </td>
       </tr>
     `.trim();

app/static/js/resource-lists/job-result-list.js

@@ -8,12 +8,10 @@ nopaque.resource_lists.JobResultList = class JobResultList extends nopaque.resou
     this.userId = listContainerElement.dataset.userId;
     this.jobId = listContainerElement.dataset.jobId;
     if (this.userId === undefined || this.jobId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
       this.add(Object.values(user.jobs[this.jobId].results));
       this.isInitialized = true;
     });

app/static/js/resource-lists/spacy-nlp-pipeline-model-list.js

@@ -8,12 +8,10 @@ nopaque.resource_lists.SpaCyNLPPipelineModelList = class SpaCyNLPPipelineModelLi
     this.isInitialized = false;
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
       this.add(Object.values(user.spacy_nlp_pipeline_models));
       this.isInitialized = true;
     });

app/static/js/resource-lists/tesseract-ocr-pipeline-model-list.js

@@ -8,21 +8,11 @@ nopaque.resource_lists.TesseractOCRPipelineModelList = class TesseractOCRPipelin
     this.isInitialized = false;
     this.userId = listContainerElement.dataset.userId;
     if (this.userId === undefined) {return;}
-    app.subscribeUser(this.userId).then((response) => {
-      app.socket.on('PATCH', (patch) => {
-        if (this.isInitialized) {this.onPatch(patch);}
-      });
+    app.userHub.addEventListener('patch', (event) => {
+      if (this.isInitialized) {this.onPatch(event.detail);}
     });
-    app.getUser(this.userId).then((user) => {
+    app.userHub.get(this.userId).then((user) => {
       this.add(Object.values(user.tesseract_ocr_pipeline_models));
-      for (let uncheckedCheckbox of this.listjs.list.querySelectorAll('input[data-checked="True"]')) {
-        uncheckedCheckbox.setAttribute('checked', '');
-      }
-      if (user.role.name !== ('Administrator' || 'Contributor')) {
-        for (let switchElement of this.listjs.list.querySelectorAll('.is_public')) {
-          switchElement.setAttribute('disabled', '');
-        }
-      }
       this.isInitialized = true;
     });
   }

app/templates/_base/_modals/terms_of_use.html.j2

@@ -1,32 +0,0 @@
-<div id="terms-of-use-modal" class="modal modal-fixed-footer">
-  <div class="modal-content">
-    <div class="container">
-      <div class="row">
-        <div class="col s12">
-          <h1 id="title">Terms of use</h1>
-        </div>
-        <div class="col s12">
-          <div class="switch">
-            <label>
-              DE
-              <input type="checkbox" id="terms-of-use-modal-switch">
-              <span class="lever"></span>
-              EN
-            </label>
-          </div>
-          <br>
-        </div>
-        <div class="terms-of-use-modal-content hide">
-          {% include "main/terms_of_use_en.html.j2" %}
-        </div>
-        <div class="terms-of-use-modal-content">
-          {% include "main/terms_of_use_de.html.j2" %}
-        </div>
-      </div>
-    </div>
-  </div>
-  <div class="modal-footer">
-    <span style="margin-right:20px;">I have taken note of the new GTC and agree to their validity in the context of my further use.</span>
-    <a href="#!" class="modal-close waves-effect waves-green btn">Yes</a>
-  </div>
-</div>

app/templates/_base/dropdowns.html.j2

@@ -1,6 +1,60 @@
 {% if current_user.is_authenticated %}
-<ul class="dropdown-content" id="nav-account-dropdown-content">
-  <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>Settings</a></li>
-  <li><a href="{{ url_for('auth.logout') }}"><i class="material-icons left">logout</i>Log out</a></li>
+<ul class="dropdown-content" id="navbar-data-processing-and-analysis-dropdown-content">
+  <li {% if request.path == url_for('services.file_setup_pipeline') %}class="active"{% endif %}>
+    <a href="{{ url_for('services.file_setup_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="file-setup-pipeline"></i>
+      File Setup Pipeline
+    </a>
+  </li>
+  <li {% if request.path == url_for('services.tesseract_ocr_pipeline') %}class="active"{% endif %}>
+    <a href="{{ url_for('services.tesseract_ocr_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="tesseract-ocr-pipeline"></i>
+      Tesseract OCR Pipeline
+    </a>
+  </li>
+  {% if config.NOPAQUE_TRANSKRIBUS_ENABLED %}
+  <li {% if request.path == url_for('services.transkribus_htr_pipeline') %}class="active"{% endif %}>
+    <a href="{{ url_for('services.transkribus_htr_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="transkribus-htr-pipeline"></i>
+      Transkribus HTR Pipeline
+    </a>
+  </li>
+  {% endif %}
+  <li {% if request.path == url_for('services.spacy_nlp_pipeline') %}class="active"{% endif %}>
+    <a href="{{ url_for('services.spacy_nlp_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="spacy-nlp-pipeline"></i>
+      SpaCy NLP Pipeline
+    </a>
+  </li>
+  <li class="divider" tabindex="-1"></li>
+  <li {% if request.path == url_for('services.corpus_analysis') %}class="active"{% endif %}>
+    <a href="{{ url_for('services.corpus_analysis') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="corpus-analysis"></i>
+      Corpus Analyis
+    </a>
+  </li>
+</ul>
+{% endif %}
+
+{% if current_user.is_authenticated %}
+<ul class="dropdown-content" id="navbar-account-dropdown-content">
+  <li {% if request.path == url_for('users.user', user_id=current_user.id) %}class="active"{% endif %}>
+    <a href="{{ url_for('users.user', user_id=current_user.id) }}">
+      <i class="material-icons">person</i>
+      Your profile
+    </a>
+  </li>
+  <li {% if request.path == url_for('settings.settings') %}class="active"{% endif %}>
+    <a href="{{ url_for('settings.settings') }}">
+      <i class="material-icons">settings</i>
+      Settings
+    </a>
+  </li>
+  <li>
+    <a href="{{ url_for('auth.logout') }}">
+      <i class="material-icons">logout</i>
+      Log out
+    </a>
+  </li>
 </ul>
 {% endif %}

app/templates/_base/footer.html.j2

@@ -1,40 +1,45 @@
 <div class="container">
   <div class="row">
-    <div class="col s6 m3">
-      <a href="https://www.dfg.de/">
-        <img class="responsive-img" src="{{ url_for('static', filename='images/logo_-_dfg.gif') }}">
-      </a>
-    </div>
-    <div class="col s6 m3 offset-m1 center-align">
-      <a href="https://www.uni-bielefeld.de/sfb1288/">
-        <img class="responsive-img" src="{{ url_for('static', filename='images/logo_-_sfb_1288.png') }}">
-      </a>
-    </div>
-    <div class="col s12 m3 offset-m1">
+    <div class="col s12 l3">
       <h5 class="white-text">Legal Notice</h5>
       <ul>
         <li><a class="grey-text text-lighten-3" href="https://www.uni-bielefeld.de/(en)/impressum/">Legal Notice</a></li>
         <li><a class="grey-text text-lighten-3" href="{{ url_for('main.privacy_policy') }}">Privacy statement (GDPR)</a></li>
         <li><a class="grey-text text-lighten-3" href="{{ url_for('main.terms_of_use') }}">Terms of use</a></li>
-        <li></li>
       </ul>
     </div>
-  </div>
-</div>
-<div class="footer-copyright">
-  <div class="container">
-    <div class="row" style="margin-bottom: 0;">
-      <div class="col s12 m3">
-        <span>© 2020 Bielefeld University</span>
-      </div>
-      <div class="col s12 m2">
-        <span class="right"><b>Version {{ config.NOPAQUE_VERSION }}</b></span>
-      </div>
-      <div class="col s12 m7 right-align">
-        <a class="btn-small waves-effect waves-light" href="{{ url_for('main.faq') }}"><i class="left material-icons">info_outline</i>Frequently Asked Questions</a>
-        <a class="btn-small waves-effect waves-light" href="mailto:{{ config.NOPAQUE_SERVICE_DESK }}"><i class="left material-icons">mail</i>Report an issue</a>
-        <a class="btn-small waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque" target="_blank"><i class="left material-icons">code</i>GitLab</a>
-      </div>
+
+    <div class="col s12 l3">
+      <h5 class="white-text">More Resources</h5>
+      <ul>
+        <li><a class="grey-text text-lighten-3" href="{{ url_for('main.faq') }}">Frequently asked questions</a></li>
+        <li><a class="grey-text text-lighten-3" href="mailto:{{ config.NOPAQUE_SERVICE_DESK }}">Report an issue</a></li>
+        <li><a class="grey-text text-lighten-3" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque">GitLab (source code)</a></li>
+      </ul>
+    </div>
+
+    <div class="col s12 l4">
+      <h5 class="white-text">Who made this?</h5>
+      <p class="grey-text text-lighten-4">
+        This software is developed by the SFB 1288 INF project at Bielefeld University.
+        Thanks to all the people who made nopaque possible.
+        <span class="red-text">&hearts;</span>
+      </p>
+    </div>
+
+    <div class="col s12 l2">
+      <br class="hide-on-med-and-down">
+      <br class="hide-on-med-and-down">
+      <a href="https://www.dfg.de/">
+        <img class="responsive-img" src="{{ url_for('static', filename='images/logo_-_dfg.gif') }}">
+      </a>
     </div>
   </div>
 </div>
+
+<div class="footer-copyright">
+  <div class="container">
+    © 2024 Bielefeld University
+    <a class="grey-text text-lighten-4 right" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/-/releases/{{ config.NOPAQUE_VERSION }}">Version {{ config.NOPAQUE_VERSION }}</a>
+  </div>
+</div>

app/templates/_base/icons.html.j2

@@ -0,0 +1 @@
+<link href="{{ url_for('static', filename='images/nopaque_-_favicon.png') }}" rel="icon">

app/templates/_base/modals.html.j2

@@ -1,3 +1,34 @@
 {% if current_user.is_authenticated and not current_user.terms_of_use_accepted %}
-{% include "_base/_modals/terms_of_use.html.j2" %}
+<div id="terms-of-use-modal" class="modal modal-fixed-footer">
+  <div class="modal-content">
+    <div class="container">
+      <div class="row">
+        <div class="col s12">
+          <h1 id="title">Terms of use</h1>
+        </div>
+        <div class="col s12">
+          <div class="switch">
+            <label>
+              DE
+              <input type="checkbox" id="terms-of-use-modal-switch">
+              <span class="lever"></span>
+              EN
+            </label>
+          </div>
+          <br>
+        </div>
+        <div class="terms-of-use-modal-content hide">
+          {% include "main/terms_of_use_en.html.j2" %}
+        </div>
+        <div class="terms-of-use-modal-content">
+          {% include "main/terms_of_use_de.html.j2" %}
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <span style="margin-right:20px;">I have taken note of the new GTC and agree to their validity in the context of my further use.</span>
+    <a href="#!" class="modal-close waves-effect waves-green btn">Yes</a>
+  </div>
+</div>
 {% endif %}

app/templates/_base/navbar.html.j2

@@ -0,0 +1,105 @@
+<div class="navbar-fixed">
+  <nav>
+    <div class="nav-wrapper">
+      {# menu icon #}
+      {# small/medium devices #}
+      <a href="#!" class="sidenav-trigger" data-target="sidenav">
+        <i class="material-icons">menu</i>
+      </a>
+
+      {% if current_user.is_authenticated %}
+      {# nopaque logo #}
+      {# large devices #}
+      <a href="{{ url_for('main.index') }}" class="brand-logo hide-on-med-and-down h-100">
+        <img src="{{ url_for('static', filename='images/nopaque_-_logo.png') }}" alt="" class="mx-3 py-3 h-100">
+      </a>
+
+      {# left aligned navigation items #}
+      {# large devices #}
+      <ul class="hide-on-med-and-down" style="margin-left: calc(57px + 1.5rem);">
+        {# dashboard #}
+        <li {% if request.path == url_for('main.dashboard') %}class="active"{% endif %}>
+          <a href="{{ url_for('main.dashboard') }}">
+            <i class="material-icons left">dashboard</i>
+            Dashboard
+          </a>
+        </li>
+
+        {# data processing & analysis #}
+        <li>
+          <a href="#!" class="dropdown-trigger no-autoinit" data-target="navbar-data-processing-and-analysis-dropdown-content" id="navbar-data-processing-and-analysis-dropdown-trigger">
+            <i class="material-icons left">miscellaneous_services</i>
+            Data Processing & Analysis
+          </a>
+        </li>
+
+        {# contributions #}
+        <li {% if request.path == url_for('contributions.index') %}class="active"{% endif %}>
+          <a href="{{ url_for('contributions.index') }}">
+            <i class="material-icons left">new_label</i>
+            Contributions
+          </a>
+        </li>
+
+        {# social #}
+        <li {% if request.path == url_for('main.social') %}class="active"{% endif %}>
+          <a href="{{ url_for('main.social') }}">
+            <i class="material-icons left">groups</i>
+            Social
+          </a>
+        </li>
+      </ul>
+      {% else %}
+      {# nopaque logo+wordmark+slogan #}
+      {# large devices #}
+      <a href="{{ url_for('main.index') }}" class="brand-logo hide-on-med-and-down h-100">
+        <img src="{{ url_for('static', filename='images/nopaque_-_logo+wordmark+slogan.png') }}" alt="" class="mx-3 py-3 h-100">
+      </a>
+      {% endif %}
+
+      {# nopaque logo+wordmark #}
+      {# small/medium devices #}
+      <a href="{{ url_for('main.index') }}" class="brand-logo center hide-on-large-only h-100">
+        <img src="{{ url_for('static', filename='images/nopaque_-_logo+wordmark.png') }}" alt="" class="py-3 h-100">
+      </a>
+
+      {# right aligned navigation items #}
+      {# large devices #}
+      <ul class="right hide-on-med-and-down h-100">
+        {# manual #}
+        <li class="tooltipped {% if request.path == url_for('main.manual') %}active{% endif %}" data-position="bottom" data-tooltip="Manual">
+          <a href="{{ url_for('main.manual') }}">
+            <i class="material-icons">help_outline</i>
+          </a>
+        </li>
+
+        {# news #}
+        <li class="tooltipped {% if request.path == url_for('main.news') %}active{% endif %}" data-position="bottom" data-tooltip="News">
+          <a href="{{ url_for('main.news') }}">
+            <i class="material-icons">newspaper</i>
+          </a>
+        </li>
+
+        {% if current_user.is_authenticated %}
+        {# avatar #}
+        <li class="h-100">
+          <a href="#!" class="dropdown-trigger no-autoinit h-100" data-target="navbar-account-dropdown-content" id="navbar-account-dropdown-trigger">
+            <span class="mr-3">{{ current_user.username }}</span>
+            <img src="{{ url_for('users.user_avatar', user_id=current_user.id) }}" alt="" class="circle py-3 h-100 right">
+          </a>
+        </li>
+        {% else %}
+        {# log in #}
+        <li {% if request.path == url_for('auth.login') %}class="active"{% endif %}>
+          <a href="{{ url_for('auth.login') }}">Log in</a>
+        </li>
+
+        {# register #}
+        <li {% if request.path == url_for('auth.register') %}class="active"{% endif %}>
+          <a href="{{ url_for('auth.register') }}" class="btn waves-effect waves-light primary-color lighten">Register</a>
+        </li>
+        {% endif %}
+      </ul>
+    </div>
+  </nav>
+</div>

app/templates/_base/navbar_primary_content.html.j2

@@ -1,38 +0,0 @@
-{% if current_user.is_authenticated %}
-<!-- menu icon -->
-<!-- small/medium devices -->
-<a href="#!" class="sidenav-trigger" data-target="sidenav"><i class="material-icons">menu</i></a>
-{% endif %}
-
-<!-- nopaque logo+wordmark -->
-<!-- small/medium devices -->
-<a href="{{ url_for('main.index') }}" class="brand-logo center hide-on-large-only" style="height: 100%;">
-  <img src="{{ url_for('static', filename='images/nopaque_-_logo+wordmark.png') }}" alt="" class="py-3" style="height: 100%;">
-</a>
-<!-- large devices -->
-<a href="{{ url_for('main.index') }}" class="brand-logo hide-on-med-and-down ml-2" style="height: 100%;">
-  <img src="{{ url_for('static', filename='images/nopaque_-_logo+wordmark.png') }}" alt="" class="py-3" style="height: 100%;">
-</a>
-
-<!-- right aligned navigation links -->
-<!-- large devices -->
-<ul class="right hide-on-med-and-down" style="height: 64px;">
-  {% if current_user.is_authenticated %}
-  <!-- avatar, username and email -->
-  <li style="height: 100%;">
-    <a href="#!" class="dropdown-trigger no-autoinit" data-target="nav-account-dropdown-content" id="nav-account-dropdown-trigger" style="height: 100%;">
-      <img src="{{ url_for('users.user_avatar', user_id=current_user.id) }}" alt="" class="left circle py-3" style="height: 100%;">
-      <span class="ml-2">{{ current_user.username }} ({{ current_user.email }})</span>
-    </a>
-  </li>
-  {% else %}
-  <!-- log in -->
-  <li {% if request.path == url_for('auth.login') %}class="active"{% endif %}>
-    <a href="{{ url_for('auth.login') }}"><i class="material-icons left">login</i>Log in</a>
-  </li>
-  <!-- register -->
-  <li {% if request.path == url_for('auth.register') %}class="active"{% endif %}>
-    <a href="{{ url_for('auth.register') }}"><i class="material-icons left">assignment</i>Register</a>
-  </li>
-  {% endif %}
-</ul>

app/templates/_base/scripts.html.j2

@@ -5,69 +5,76 @@
 <script src="{{ url_for('static', filename='external/plotly.js/js/plotly.min.js') }}"></script>
 <script src="{{ url_for('static', filename='external/socket.io/js/socket.io.min.js') }}"></script>
 {% assets
-  filters='rjsmin',
-  output='gen/nopaque.%(version)s.js',
-  'js/index.js',
-  'js/app.js',
-  'js/utils.js',
+    filters='rjsmin',
+    output='gen/nopaque.%(version)s.js',
+    'js/index.js',
+    'js/app.js',
+    'js/app/index.js',
+    'js/app/endpoints/index.js',
+    'js/app/endpoints/users.js',
+    'js/app/extensions/index.js',
+    'js/app/extensions/toaster.js',
+    'js/app/extensions/ui.js',
+    'js/app/extensions/user-hub.js',
+    'js/utils.js',
 
-  'js/forms/index.js',
-  'js/forms/base-form.js',
-  'js/forms/create-contribution-form.js',
-  'js/forms/create-corpus-file-form.js',
-  'js/forms/create-job-form.js',
+    'js/forms/index.js',
+    'js/forms/base-form.js',
+    'js/forms/create-contribution-form.js',
+    'js/forms/create-corpus-file-form.js',
+    'js/forms/create-job-form.js',
 
-  'js/resource-displays/index.js',
-  'js/resource-displays/resource-display.js',
-  'js/resource-displays/corpus-display.js',
-  'js/resource-displays/job-display.js',
+    'js/resource-displays/index.js',
+    'js/resource-displays/resource-display.js',
+    'js/resource-displays/corpus-display.js',
+    'js/resource-displays/job-display.js',
 
-  'js/resource-lists/index.js',
-  'js/resource-lists/resource-list.js',
-  'js/resource-lists/admin-user-list.js',
-  'js/resource-lists/corpus-file-list.js',
-  'js/resource-lists/corpus-follower-list.js',
-  'js/resource-lists/corpus-list.js',
-  'js/resource-lists/corpus-text-info-list.js',
-  'js/resource-lists/corpus-token-list.js',
-  'js/resource-lists/detailed-public-corpus-list.js',
-  'js/resource-lists/job-input-list.js',
-  'js/resource-lists/job-list.js',
-  'js/resource-lists/job-result-list.js',
-  'js/resource-lists/public-corpus-list.js',
-  'js/resource-lists/public-user-list.js',
-  'js/resource-lists/spacy-nlp-pipeline-model-list.js',
-  'js/resource-lists/tesseract-ocr-pipeline-model-list.js',
+    'js/resource-lists/index.js',
+    'js/resource-lists/resource-list.js',
+    'js/resource-lists/admin-user-list.js',
+    'js/resource-lists/corpus-file-list.js',
+    'js/resource-lists/corpus-follower-list.js',
+    'js/resource-lists/corpus-list.js',
+    'js/resource-lists/corpus-text-info-list.js',
+    'js/resource-lists/corpus-token-list.js',
+    'js/resource-lists/detailed-public-corpus-list.js',
+    'js/resource-lists/job-input-list.js',
+    'js/resource-lists/job-list.js',
+    'js/resource-lists/job-result-list.js',
+    'js/resource-lists/public-corpus-list.js',
+    'js/resource-lists/public-user-list.js',
+    'js/resource-lists/spacy-nlp-pipeline-model-list.js',
+    'js/resource-lists/tesseract-ocr-pipeline-model-list.js',
 
-  'js/requests/index.js',
-  'js/requests/admin.js',
-  'js/requests/contributions.js',
-  'js/requests/corpora.js',
-  'js/requests/jobs.js',
-  'js/requests/users.js',
+    'js/requests/index.js',
+    'js/requests/admin.js',
+    'js/requests/contributions.js',
+    'js/requests/corpora.js',
+    'js/requests/jobs.js',
+    'js/requests/users.js',
 
-  'js/corpus-analysis/index.js',
-  'js/corpus-analysis/cqi/index.js',
-  'js/corpus-analysis/cqi/constants.js',
-  'js/corpus-analysis/cqi/errors.js',
-  'js/corpus-analysis/cqi/status.js',
-  'js/corpus-analysis/cqi/api/index.js',
-  'js/corpus-analysis/cqi/api/client.js',
-  'js/corpus-analysis/cqi/models/index.js',
-  'js/corpus-analysis/cqi/models/resource.js',
-  'js/corpus-analysis/cqi/models/attributes.js',
-  'js/corpus-analysis/cqi/models/subcorpora.js',
-  'js/corpus-analysis/cqi/models/corpora.js',
-  'js/corpus-analysis/cqi/client.js',
-  'js/corpus-analysis/query-builder/index.js',
-  'js/corpus-analysis/query-builder/element-references.js',
-  'js/corpus-analysis/query-builder/query-builder.js',
-  'js/corpus-analysis/query-builder/structural-attribute-builder-functions.js',
-  'js/corpus-analysis/query-builder/token-attribute-builder-functions.js',
-  'js/corpus-analysis/app.js',
-  'js/corpus-analysis/concordance-extension.js',
-  'js/corpus-analysis/reader-extension.js',
-  'js/corpus-analysis/static-visualization-extension.js'
+    'js/corpus-analysis/index.js',
+    'js/corpus-analysis/cqi/index.js',
+    'js/corpus-analysis/cqi/constants.js',
+    'js/corpus-analysis/cqi/errors.js',
+    'js/corpus-analysis/cqi/status.js',
+    'js/corpus-analysis/cqi/api/index.js',
+    'js/corpus-analysis/cqi/api/client.js',
+    'js/corpus-analysis/cqi/models/index.js',
+    'js/corpus-analysis/cqi/models/resource.js',
+    'js/corpus-analysis/cqi/models/attributes.js',
+    'js/corpus-analysis/cqi/models/subcorpora.js',
+    'js/corpus-analysis/cqi/models/corpora.js',
+    'js/corpus-analysis/cqi/client.js',
+    'js/corpus-analysis/query-builder/index.js',
+    'js/corpus-analysis/query-builder/element-references.js',
+    'js/corpus-analysis/query-builder/query-builder.js',
+    'js/corpus-analysis/query-builder/structural-attribute-builder-functions.js',
+    'js/corpus-analysis/query-builder/token-attribute-builder-functions.js',
+    'js/corpus-analysis/app.js',
+    'js/corpus-analysis/concordance-extension.js',
+    'js/corpus-analysis/reader-extension.js',
+    'js/corpus-analysis/static-visualization-extension.js'
 -%}
 <script src="{{ ASSET_URL }}"></script>
 {% endassets -%}
@@ -77,26 +84,22 @@
   const app = new nopaque.App();
   app.init();
 
-  {% if current_user.is_authenticated -%}
-  // TODO: Set this as a property of the app object
+  {% if current_user.is_authenticated %}
   const currentUserId = {{ current_user.hashid|tojson }};
 
-  // Subscribe to the current user's data events
-  app.subscribeUser(currentUserId)
+  app.userHub.add(currentUserId)
     .catch((error) => {throw JSON.stringify(error);});
 
-  // Get the current user's data
-  app.getUser(currentUserId, true, true)
-    .catch((error) => {throw JSON.stringify(error);});
-
-  {% if not current_user.terms_of_use_accepted -%}
+  {% if not current_user.terms_of_use_accepted %}
   M.Modal.getInstance(document.querySelector('#terms-of-use-modal')).open();
-  {% endif -%}
-  {% endif -%}
+  {% endif %}
+  {% else %}
+  const currentUserId = null;
+  {% endif %}
 
   // Display flashed messages
   for (let [category, message] of {{ get_flashed_messages(with_categories=True)|tojson }}) {
-    app.flash(message, message);
+    app.ui.flash(message, message);
   }
 </script>
 
@@ -104,7 +107,7 @@
   let languageModalSwitch = document.querySelector('#terms-of-use-modal-switch');
   let termsOfUseModalContent = document.querySelectorAll('.terms-of-use-modal-content');
   if (languageModalSwitch) {
-    languageModalSwitch.addEventListener('change', function() {
+    languageModalSwitch.addEventListener('change', () => {
       termsOfUseModalContent.forEach(content => {
         content.classList.toggle('hide');
       });

app/templates/_base/sidenav.html.j2

@@ -1,7 +1,7 @@
-<ul class="sidenav sidenav-fixed" id="sidenav">
+<ul class="sidenav" id="sidenav">
+  {% if current_user.is_authenticated %}
   {# user view #}
-  {# shown for small/medium devices #}
-  <li class="hide-on-large-only">
+  <li>
     <div class="user-view">
       <div class="background primary-color"></div>
       <a><img class="circle" src="{{ url_for('users.user_avatar', user_id=current_user.id) }}" alt=""></a>
@@ -9,86 +9,115 @@
       <a><span class="white-text email">{{ current_user.email }}</span></a>
     </div>
   </li>
+  {% endif %}
 
-  {# general items #}
-  {% if current_user.can('USE_API') %}
-  <li>
-    <a class="waves-effect" href="{{ url_for('apifairy.docs') }}"><i class="material-icons">api</i>API</a>
+  {% if current_user.is_authenticated %}
+  {# dashboard #}
+  <li {% if request.path == url_for('main.dashboard') %}class="active"{% endif %}>
+    <a class="waves-effect" href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a>
+  </li>
+
+  {# contributions #}
+  <li {% if request.path == url_for('contributions.index') %}class="active"{% endif %}>
+    <a href="{{ url_for('contributions.index') }}">
+      <i class="material-icons left">new_label</i>
+      Contributions
+    </a>
+  </li>
+
+  {# social #}
+  <li {% if request.path == url_for('main.social') %}class="active"{% endif %}>
+    <a class="waves-effect" href="{{ url_for('main.social') }}"><i class="material-icons">groups</i>Social</a>
   </li>
   {% endif %}
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.manual') }}"><i class="material-icons">school</i>Manual</a>
-  </li>
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.news') }}"><i class="material-icons">email</i>News</a>
+
+  {# news #}
+  <li {% if request.path == url_for('main.news') %}class="active"{% endif %}>
+    <a class="waves-effect" href="{{ url_for('main.news') }}"><i class="material-icons">newspaper</i>News</a>
   </li>
 
-  {# dashboard items #}
-  <li><div class="divider"></div></li>
-  <li><a class="subheader">Dashboard</a></li>
-  <li {% if request.path == url_for('corpora.corpora') %}class="active"{% endif %}>
-    <a class="waves-effect" href="{{ url_for('corpora.corpora') }}"><i class="nopaque-icons">I</i>My Corpora</a>
-  </li>
-  <li {% if request.path == url_for('jobs.jobs') %}class="active"{% endif %}>
-    <a class="waves-effect" href="{{ url_for('jobs.jobs') }}"><i class="nopaque-icons">J</i>My Jobs</a>
-  </li>
-  <li>
-    <a class="waves-effect" href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>My Contributions</a>
+  {# manual #}
+  <li {% if request.path == url_for('main.manual') %}class="active"{% endif %}>
+    <a class="waves-effect" href="{{ url_for('main.manual') }}"><i class="material-icons">help_outline</i>Manual</a>
   </li>
 
-  {# processes & services items #}
+  {% if current_user.is_authenticated %}
+  {# data processing & analysis section #}
   <li><div class="divider"></div></li>
-  <li><a class="subheader">Processes & Services</a></li>
+  <li><a class="subheader">Data Processing & Analysis</a></li>
+
+  {# file setup pipeline #}
   <li class="service-color service-color-border border-darken" data-service="file-setup-pipeline" style="border-left: 10px solid;">
     <a class="waves-effect" href="{{ url_for('services.file_setup_pipeline') }}"><i class="nopaque-icons service-icons" data-service="file-setup-pipeline"></i>File setup</a>
   </li>
+
+  {# tesseract ocr pipeline #}
   <li class="service-color service-color-border border-darken mt-1" data-service="tesseract-ocr-pipeline" style="border-left: 10px solid;">
     <a class="waves-effect" href="{{ url_for('services.tesseract_ocr_pipeline') }}"><i class="nopaque-icons service-icons" data-service="tesseract-ocr-pipeline"></i>OCR</a>
   </li>
+
   {% if config.NOPAQUE_TRANSKRIBUS_ENABLED %}
+  {# transkribus htr pipeline #}
   <li class="service-color service-color-border border-darken mt-1" data-service="transkribus-htr-pipeline" style="border-left: 10px solid;">
     <a class="waves-effect" href="{{ url_for('services.transkribus_htr_pipeline') }}"><i class="nopaque-icons service-icons" data-service="transkribus-htr-pipeline"></i>HTR</a>
   </li>
   {% endif %}
+
+  {# spacy nlp pipeline #}
   <li class="service-color service-color-border border-darken mt-1" data-service="spacy-nlp-pipeline" style="border-left: 10px solid;">
     <a class="waves-effect" href="{{ url_for('services.spacy_nlp_pipeline') }}"><i class="nopaque-icons service-icons" data-service="spacy-nlp-pipeline"></i>NLP</a>
   </li>
+
+  {# corpus analysis #}
   <li class="service-color service-color-border border-darken mt-1" data-service="corpus-analysis" style="border-left: 10px solid;">
     <a class="waves-effect" href="{{ url_for('services.corpus_analysis') }}"><i class="nopaque-icons service-icons" data-service="corpus-analysis"></i>Corpus Analysis</a>
   </li>
+  {% endif %}
 
-  {# social items #}
+  {# account section #}
   <li><div class="divider"></div></li>
-  <li><a class="subheader">Social</a></li>
-  <li>
+  <li><a class="subheader">Account</a></li>
+
+  {% if current_user.is_authenticated %}
+  {# my profile #}
+  <li {% if request.path == url_for('users.user', user_id=current_user.id) %}class="active"{% endif %}>
     <a href="{{ url_for('users.user', user_id=current_user.id) }}"><i class="material-icons">person</i>My Profile</a>
   </li>
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-users') }}"><i class="material-icons">group</i>Public Users</a>
-  </li>
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-corpora') }}"><i class="nopaque-icons">I</i>Public Corpora</a>
+
+  {# settings #}
+  <li {% if request.path == url_for('settings.settings') %}class="active"{% endif %}>
+    <a class="waves-effect" href="{{ url_for('settings.settings') }}"><i class="material-icons">settings</i>Settings</a>
   </li>
 
-  {# administration items #}
-  {% if current_user.can('ADMINISTRATE') %}
+  {# log out #}
+  <li>
+    <a class="waves-effect" href="{{ url_for('auth.logout') }}"><i class="material-icons">logout</i>Log out</a>
+  </li>
+  {% else %}
+  {# log in #}
+  <li {% if request.path == url_for('auth.login') %}class="active"{% endif %}>
+    <a href="{{ url_for('auth.login') }}">Log in</a>
+  </li>
+
+  {# register #}
+  <li {% if request.path == url_for('auth.register') %}class="active"{% endif %}>
+    <a href="{{ url_for('auth.register') }}" class="btn waves-effect waves-light">Register</a>
+  </li>
+  {% endif %}
+
+  {% if current_user.is_authenticated and current_user.can('ADMINISTRATE') %}
+  {# administration section #}
   <li><div class="divider"></div></li>
   <li><a class="subheader">Administration</a></li>
+
+  {# corpora #}
   <li>
     <a class="waves-effect" href="{{ url_for('admin.corpora') }}"><i class="nopaque-icons">I</i>Corpora</a>
   </li>
+
+  {# users #}
   <li>
     <a class="waves-effect" href="{{ url_for('admin.users') }}"><i class="material-icons">manage_accounts</i>Users</a>
   </li>
   {% endif %}
-
-  {# account items #}
-  <li class="hide-on-large-only"><div class="divider"></div></li>
-  <li class="hide-on-large-only"><a class="subheader">Account</a></li>
-  <li class="hide-on-large-only">
-    <a class="waves-effect" href="{{ url_for('settings.settings') }}"><i class="material-icons">settings</i>Settings</a>
-  </li>
-  <li class="hide-on-large-only">
-    <a class="waves-effect" href="{{ url_for('auth.logout') }}"><i class="material-icons">logout</i>Log out</a>
-  </li>
 </ul>

app/templates/_base/stylesheets.html.j2

@@ -1,21 +1,23 @@
 <link href="{{ url_for('static', filename='external/material-design-icons/css/material-icons.css') }}" rel="stylesheet">
 {% assets
-  output='gen/nopaque.%(version)s.css',
-  'css/materialize.css',
-  'css/materialize.override.css',
-  'css/nopaque-icons.css',
-  'css/theme-colors.css',
-  'css/corpus-status-colors.css',
-  'css/corpus-status-text.css',
-  'css/job-status-colors.css',
-  'css/job-status-text.css',
-  'css/service-colors.css',
-  'css/pagination.css',
-  'css/service-icons.css',
-  'css/s-attr-colors.css',
-  'css/spacing.css',
-  'css/status-spinner.css',
-  'css/utils.css'
--%}
+    output='gen/nopaque.%(version)s.css',
+    'css/materialize.css',
+    'css/materialize.override.css',
+    'css/nopaque-icons.css',
+    'css/theme-colors.css',
+    'css/corpus-status-colors.css',
+    'css/corpus-status-text.css',
+    'css/height.css',
+    'css/job-status-colors.css',
+    'css/job-status-text.css',
+    'css/service-colors.css',
+    'css/pagination.css',
+    'css/service-icons.css',
+    'css/s-attr-colors.css',
+    'css/spacing.css',
+    'css/status-spinner.css',
+    'css/utils.css',
+    'css/width.css'
+%}
 <link href="{{ ASSET_URL }}" rel="stylesheet">
-{% endassets -%}
+{% endassets %}

app/templates/auth/login.html.j2

@@ -5,7 +5,7 @@
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12 m8 offset-m2">
+    <div class="col s12 l8 offset-l2">
       <h1 id="title">{{ title }}</h1>
       <p>Want to boost your research and get going? Nopaque is free and no download is needed. <a href="{{ url_for('.register') }}">Register now</a>!</p>
 
@@ -15,14 +15,14 @@
           {{ wtf.render_field(form.user, material_icon='person') }}
           {{ wtf.render_field(form.password, material_icon='vpn_key') }}
           <div class="row">
-            <div class="col s6 left-align">
-              <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
-            </div>
-            <div class="col s6 right-align">
+            <div class="col s12 l6">
               {{ wtf.render_field(form.remember_me) }}
             </div>
+            <div class="col s12 l6 right-align">
+              <a class="mr-3" href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
+              {{ wtf.render_field(form.submit, material_icon='send') }}
+            </div>
           </div>
-          {{ wtf.render_field(form.submit, material_icon='send', class_='width-100') }}
         </div>
       </form>
     </div>

app/templates/auth/register.html.j2

@@ -5,7 +5,7 @@
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12 m8 offset-m2">
+    <div class="col s12 l8 offset-l2">
       <h1 id="title">{{ title }}</h1>
       <p>
         Simply enter a username and password to receive your registration email.
@@ -22,14 +22,18 @@
           {{ wtf.render_field(form.username, material_icon='person') }}
           {{ wtf.render_field(form.password, material_icon='vpn_key') }}
           {{ wtf.render_field(form.password_2, material_icon='vpn_key') }}
-          {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
-          <br>
-          {{ wtf.render_field(form.terms_of_use_accepted, type='checkbox')}}
-          <p></p>
-          <br>
-          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+          {{ wtf.render_field(form.email, material_icon='email', type='email') }}
+          <div class="row">
+            <div class="col s12 l6">
+              {{ wtf.render_field(form.terms_of_use_accepted)}}
+            </div>
+            <div class="col s12 l6 right-align">
+              {{ wtf.render_field(form.submit, material_icon='send') }}
+            </div>
+          </div>
         </div>
       </form>
+    </div>
   </div>
 </div>
 {% endblock page_content %}

app/templates/auth/reset_password.html.j2

@@ -13,7 +13,9 @@
           {{ form.hidden_tag() }}
           {{ wtf.render_field(form.password) }}
           {{ wtf.render_field(form.password_2) }}
-          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+          <div class="right-align">
+            {{ wtf.render_field(form.submit, material_icon='send') }}
+          </div>
         </div>
       </form>
     </div>

app/templates/auth/reset_password_request.html.j2

@@ -4,15 +4,17 @@
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12 m8 offset-m2">
+    <div class="col s12 l8 offset-l2">
       <h1 id="title">{{ title }}</h1>
       <p>After entering your email address you will receive instructions on how to reset your password.</p>
 
       <form method="POST">
         <div class="card-panel">
           {{ form.hidden_tag() }}
-          {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
-          {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+          {{ wtf.render_field(form.email, material_icon='email', type='email') }}
+          <div class="right-align">
+            {{ wtf.render_field(form.submit, material_icon='send') }}
+          </div>
         </div>
       </form>
     </div>

app/templates/base.html.j2

@@ -1,103 +1,72 @@
 {% if title is not defined %}
-{% set title = 'nopaque' %}
-{% endif %}
-
-{% if sidenav_fixed is not defined %}
-{% set sidenav_fixed = current_user.is_authenticated %}
-{% endif %}
-
-{% if sticky_footer is not defined %}
-{% set sticky_footer = true %}
-{% endif %}
-
-{% if navbar_fixed is not defined %}
-{% set navbar_fixed = true %}
-{% endif %}
-
-{% if navbar_extended is not defined %}
-{% set navbar_extended = false %}
+  {% set title = 'nopaque' %}
 {% endif %}
 
 {% block doc %}
 <!DOCTYPE html>
-<html {% block html_attribs %}lang="en"{% endblock html_attribs %}>
+<html {% block html_attributes %}lang="en"{% endblock html_attributes %}>
   {% block html %}
-  <head>
+  <head {% block head_attributes %}{% endblock head_attributes %}>
     {% block head %}
     {% block metas %}
-    {% include "_base/metas.html.j2" %}
+    {% include '_base/metas.html.j2' %}
     {% endblock metas %}
 
     <title {% block title_attribs %}{% endblock title_attribs %}>
-      {%- block title %}{{ title }}{% endblock title -%}
+      {% block title %}
+      {{ title }}
+      {% endblock title %}
     </title>
 
-    <link href="{{ url_for('static', filename='images/nopaque_-_favicon.png') }}" rel="icon">
+    {% block icons %}
+    {% include '_base/icons.html.j2' %}
+    {% endblock icons %}
 
-    {% block stylesheets %}
-    {% include "_base/stylesheets.html.j2" %}
-    {% endblock stylesheets %}
+    {% block styles %}
+    {% include '_base/stylesheets.html.j2' %}
+    {% endblock styles %}
     {% endblock head %}
   </head>
-  <body {% block body_attribs %}data-sidenav-fixed="{{ sidenav_fixed }}" data-sticky-footer="{{ sticky_footer }}"{% endblock body_attribs %}>
+  <body {% block body_attributes %}{% endblock body_attributes %}>
     {% block body %}
-    <header {% block header_attribs %}{% endblock header_attribs %}>
+    <header {% block header_attributes %}{% endblock header_attributes %}>
       {% block header %}
-      {% if navbar_fixed %}
-      <div class="navbar-fixed">
-      {% endif %}
-        <nav {% block navbar_attribs %}{% if navbar_extended %}class="nav-extended"{% endif %}{% endblock navbar_attribs %}>
-        {% block navbar %}
-          <div {% block navbar_primary_content_attribs %}class="nav-wrapper"{% endblock navbar_primary_content_attribs %}>
-            {% block navbar_primary_content %}
-            {% include "_base/navbar_primary_content.html.j2" %}
-            {% endblock navbar_primary_content %}
-          </div>
-          {% if navbar_extended %}
-          <div {% block navbar_secondary_content_attribs %}class="nav-content"{% endblock navbar_secondary_content_attribs %}>
-            {% block navbar_secondary_content %}
-            {% endblock navbar_secondary_content %}
-          </div>
-          {% endif %}
-        {% endblock navbar %}
-        </nav>
-      {% if navbar_fixed %}
-      </div>
-      {% endif %}
+      {% block navbar %}
+      {% include '_base/navbar.html.j2' %}
+      {% endblock navbar %}
+
       {% block sidenav %}
-      {% if current_user.is_authenticated %}
-      {% include "_base/sidenav.html.j2" %}
-      {% endif %}
+      {% include '_base/sidenav.html.j2' %}
       {% endblock sidenav %}
       {% endblock header %}
     </header>
 
-    <main {% block main_attribs %}{% endblock main_attribs %}>
+    <main {% block main_attributes %}{% endblock main_attributes %}>
       {% block main %}
       {% block page_content %}{% endblock page_content %}
       {% endblock main %}
-
-      <div id="dropdowns">
-        {% block dropdowns %}
-        {% include "_base/dropdowns.html.j2" %}
-        {% endblock dropdowns %}
-      </div>
-
-      <div id="modals">
-        {% block modals %}
-        {% include "_base/modals.html.j2" %}
-        {% endblock modals %}
-      </div>
     </main>
 
-    <footer {% block footer_attribs %}class="page-footer"{% endblock footer_attribs %}>
+    <footer {% block footer_attributes %}class="page-footer"{% endblock footer_attributes %}>
       {% block footer %}
-      {% include "_base/footer.html.j2" %}
+      {% include '_base/footer.html.j2' %}
       {% endblock footer %}
     </footer>
 
+    <div {% block dropdowns_attributes %}id="dropdowns"{% endblock dropdowns_attributes %}>
+      {% block dropdowns %}
+      {% include '_base/dropdowns.html.j2' %}
+      {% endblock dropdowns %}
+    </div>
+
+    <div {% block modals_attributes %}id="modals"{% endblock modals_attributes %}>
+      {% block modals %}
+      {% include '_base/modals.html.j2' %}
+      {% endblock modals %}
+    </div>
+
     {% block scripts %}
-    {% include "_base/scripts.html.j2" %}
+    {% include '_base/scripts.html.j2' %}
     {% endblock scripts %}
     {% endblock body %}
   </body>

app/templates/contributions/index.html.j2

@@ -0,0 +1,48 @@
+{% extends "base.html.j2" %}
+
+{% block page_content %}
+<div class="container">
+  <div class="row">
+    <div class="col s12">
+      <h1 id="title">{{ title }}</h1>
+      <p>
+        Upload your own language models into nopaque. This is useful for
+        working with different languages that are not available as standard in
+        nopaque or if a you want to work with a language model that you have
+        developed by yourself. Uploaded models can be found in the model list
+        of the corresponding service and can be used immediately.
+      </p>
+    </div>
+
+    <div class="col s12 l4">
+      <h4>Tesseract OCR Pipeline Models</h4>
+    </div>
+
+    <div class="col s12 l8">
+      <div class="card">
+        <div class="card-content">
+          <div class="tesseract-ocr-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
+        </div>
+        <div class="card-action right-align">
+          <a href="{{ url_for('.tesseract_ocr_pipeline_models.create') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
+        </div>
+      </div>
+    </div>
+
+    <div class="col s12 l4">
+      <h4>SpaCy NLP Pipeline Models</h4>
+    </div>
+
+    <div class="col s12 l8">
+      <div class="card">
+        <div class="card-content">
+          <div class="spacy-nlp-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
+        </div>
+        <div class="card-action right-align">
+          <a href="{{ url_for('.spacy_nlp_pipeline_models.create') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+{% endblock page_content %}

app/templates/contributions/spacy_nlp_pipeline_models/spacy_nlp_pipeline_models.html.j2

@@ -1,24 +0,0 @@
-{% extends "base.html.j2" %}
-{% import "wtf.html.j2" as wtf %}
-
-{% block page_content %}
-<div class="container">
-  <div class="row">
-    <div class="col s12">
-      <h1 id="title">{{ title }}</h1>
-      <p>Here you can see and edit the models that you have created. You can also create new models.</p>
-    </div>
-
-    <div class="col s12">
-      <div class="card">
-        <div class="card-content">
-          <div class="spacy-nlp-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
-        </div>
-        <div class="card-action right-align">
-          <a href="{{ url_for('.create_spacy_nlp_pipeline_model') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/contributions/tesseract_ocr_pipeline_models/tesseract_ocr_pipeline_models.html.j2

@@ -1,24 +0,0 @@
-{% extends "base.html.j2" %}
-{% import "wtf.html.j2" as wtf %}
-
-{% block page_content %}
-<div class="container">
-  <div class="row">
-    <div class="col s12">
-      <h1 id="title">{{ title }}</h1>
-      <p>Here you can see and edit the models that you have created. You can also create new models.</p>
-    </div>
-
-    <div class="col s12">
-      <div class="card">
-        <div class="card-content">
-          <div class="tesseract-ocr-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
-        </div>
-        <div class="card-action right-align">
-          <a href="{{ url_for('.create_tesseract_ocr_pipeline_model') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-{% endblock page_content %}

app/templates/corpora/_analysis/query_builder/_expert_mode.html.j2

@@ -5,9 +5,9 @@
       <i class="material-icons prefix">search</i>
       <input class="validate corpus-analysis-action" id="corpus-analysis-concordance-form-query" name="query" type="text" required pattern=".*\S+.*" placeholder="Type in your query via CQL"></input>
       <span class="error-color-text helper-text hide" id="corpus-analysis-concordance-error"></span>
-      <a class="modal-trigger" data-manual-modal-chapter="manual-modal-cqp-query-language" href="#manual-modal" style="margin-left: 40px;"><i class="material-icons" style="font-size: inherit;">help</i> Corpus Query Language tutorial</a>
+      <a href="{{ url_for('main.manual', _anchor='manual-cqp-query-language') }}" target="manual" style="margin-left: 40px;"><i class="material-icons" style="font-size: inherit;">help</i> Corpus Query Language tutorial</a>
       <span> | </span>
-      <a class="modal-trigger" data-manual-modal-chapter="manual-modal-tagsets" href="#manual-modal"><i class="material-icons" style="font-size: inherit;">info</i> Tagsets</a>
+      <a href="{{ url_for('main.manual', _anchor='manual-tagsets') }}" target="manual"><i class="material-icons" style="font-size: inherit;">info</i> Tagsets</a>
     </div>
     <div class="input-field col s12 m3">
       <i class="material-icons prefix">arrow_forward</i>
@@ -17,7 +17,7 @@
     <div class="col s12 m3 l3 right-align">
       <p class="hide-on-small-only">&nbsp;</p>
       <button class="btn waves-effect waves-light corpus-analysis-action" id="corpus-analysis-concordance-form-submit" type="submit" name="submit">
-        Send 
+        Send
         <i class="material-icons right">send</i>
       </button>
     </div>

app/templates/corpora/_analysis/query_builder/_query_builder.html.j2

@@ -22,9 +22,9 @@
   <div class="row">
     <div class="col s12">
       <span class="error-color-text helper-text hide" id="corpus-analysis-concordance-error"></span>
-      <a class="modal-trigger" data-manual-modal-chapter="manual-modal-cqp-query-language" href="#manual-modal"><i class="material-icons" style="font-size: inherit;">help</i> Corpus Query Language tutorial</a>
+      <a href="{{ url_for('main.manual', _anchor='manual-cqp-query-language') }}" target="manual"><i class="material-icons" style="font-size: inherit;">help</i> Corpus Query Language tutorial</a>
       <span> | </span>
-      <a class="modal-trigger" data-manual-modal-chapter="manual-modal-tagsets" href="#manual-modal"><i class="material-icons" style="font-size: inherit;">info</i> Tagsets</a>
+      <a href="{{ url_for('main.manual', _anchor='manual-tagsets') }}" target="manual"><i class="material-icons" style="font-size: inherit;">info</i> Tagsets</a>
     </div>
   </div>
   <div class="row">
@@ -40,14 +40,14 @@
       <a class="btn-small waves-effect waves-light tooltipped modal-trigger" href="#corpus-analysis-concordance-positional-attr-modal" data-position="bottom" data-tooltip="Search for any token, for example a word, a lemma or a part-of-speech tag">Add new token to your query</a>
       <a class="btn-small waves-effect waves-light tooltipped modal-trigger" href="#corpus-analysis-concordance-structural-attr-modal" data-position="bottom" data-tooltip="Structure your query with structural attributes, for example sentences, entities or annotate the text">Add structural attributes to your query</a>
       <a class="btn-small waves-effect waves-light tooltipped dropdown-trigger disabled" data-target="corpus-analysis-concordance-token-incidence-modifiers-dropdown" data-toggle-area="token-incidence-modifiers" data-position="top" data-tooltip="Incidence Modifiers are special characters or patterns, <br>which determine how often a character represented previously should occur.">incidence modifiers</a>
-      <a class="modal-trigger" data-manual-modal-chapter="manual-modal-query-builder" href="#manual-modal"><i class="material-icons left" style="color:black">help_outline</i></a>
+      <a href="{{ url_for('main.manual', _anchor='manual-query-builder') }}" target="manual"><i class="material-icons left" style="color:black">help_outline</i></a>
     </div>
   </div>
   <div class="row">
     <div class="col s12 right-align">
       <p class="hide-on-small-only">&nbsp;</p>
       <button class="btn waves-effect waves-light corpus-analysis-action" type="submit" name="submit">
-        Send 
+        Send
         <i class="material-icons right">send</i>
       </button>
     </div>
@@ -67,7 +67,7 @@
 <div id="corpus-analysis-concordance-structural-attr-modal" class="modal">
   <div class="modal-content">
     <div class="attr-modal-header">
-      <h5>Which structural attribute do you want to add to your query?<a class="modal-trigger" data-manual-modal-chapter="manual-modal-query-builder" data-manual-modal-chapter-anchor="add-structural-attribute-tutorial" href="#manual-modal"><i class="material-icons left" id="corpus-analysis-concordance-add-structural-attribute-tutorial-info-icon">help_outline</i></a></h5>
+      <h5>Which structural attribute do you want to add to your query?<a href="{{ url_for('main.manual', _anchor='add-structural-attribute-tutorial') }}" target="manual"><i class="material-icons left" id="corpus-analysis-concordance-add-structural-attribute-tutorial-info-icon">help_outline</i></a></h5>
     </div>
     <p></p>
     <br>
@@ -128,7 +128,7 @@
     <div class="row attr-modal-header">
       <p></p>
       <div class="col s12">
-        <h5>Which kind of token are you looking for? <a class="modal-trigger" data-manual-modal-chapter="manual-modal-query-builder" data-manual-modal-chapter-anchor="add-new-token-tutorial" href="#manual-modal"><i class="material-icons left" id="corpus-analysis-concordance-token-tutorial-info-icon">help_outline</i></a></h5>
+        <h5>Which kind of token are you looking for? <a href="{{ url_for('main.manual', _anchor='add-new-token-tutorial') }}" target="manual"><i class="material-icons left" id="corpus-analysis-concordance-token-tutorial-info-icon">help_outline</i></a></h5>
       </div>
       <div class="input-field col s3" style="margin-left:42px;">
         <select id="corpus-analysis-concordance-positional-attr-selection">
@@ -234,7 +234,7 @@
           <label>Part-of-speech tags</label>
         </div>
       </template>
-      
+
       <template class="token-builder-section" data-token-builder-section="german-pos">
         <div class= "input-field col s4" data-kind-of-token="german-pos">
           <select name="germanpos">
@@ -337,7 +337,7 @@
     </div>
     <div id="corpus-analysis-concordance-token-edit-options" data-toggle-area="input-field-options">
       <div class="row">
-        <h6>Options to edit your token: <a class="modal-trigger" data-manual-modal-chapter="manual-modal-query-builder" href="#manual-modal" data-manual-modal-chapter-anchor="edit-options-tutorial"><i class="material-icons left" id="corpus-analysis-concordance-edit-options-tutorial-info-icon">help_outline</i></a></h6>
+        <h6>Options to edit your token: <a href="{{ url_for('main.manual', _anchor='edit-options-tutorial') }}" target="manual"><i class="material-icons left" id="corpus-analysis-concordance-edit-options-tutorial-info-icon">help_outline</i></a></h6>
       </div>
       <p></p>
       <div class="row">
@@ -359,10 +359,10 @@
 
       </div>
     </div>
-    
+
     {{ exactly_n_modal_content("character") }}
     {{ exactly_nm_modal_content("character") }}
-      
+
   </div>
 </div>
 {% endmacro %}

app/templates/corpora/analysis.html.j2

@@ -14,8 +14,11 @@
 {% endblock stylesheets %}
 
 
-{% block navbar_secondary_content %}
-<ul class="tabs tabs-transparent no-autoinit" id="corpus-analysis-extension-tabs">
+{% block main_attributes %}class="service-color lighten" data-service="corpus-analysis" id="corpus-analysis-container"{% endblock main_attributes %}
+
+
+{% block page_content %}
+<ul class="tabs no-autoinit" id="corpus-analysis-extension-tabs">
   <li class="tab">
     <a class="active" href="#corpus-analysis-home-container"><i class="nopaque-icons service-icons left" data-service="corpus-analysis" style="line-height: inherit;"></i>Corpus analysis</a>
   </li>
@@ -26,13 +29,7 @@
     <a href="#corpus-analysis-reader-container"><i class="material-icons left" style="line-height: inherit;">{{ reader_extension.icon }}</i>{{ reader_extension.name }}</a>
   </li>
 </ul>
-{% endblock navbar_secondary_content %}
 
-
-{% block main_attribs %} class="service-color lighten" data-service="corpus-analysis" id="corpus-analysis-container" style="margin-top: 48px;"{% endblock main_attribs %}
-
-
-{% block page_content %}
 <div id="corpus-analysis-home-container">
   <h1>{{ title }}</h1>
 
@@ -72,7 +69,9 @@
 {{ super() }}
 <div class="modal no-autoinit" id="corpus-analysis-init-modal">
   <div class="modal-content">
-    <h4>We are preparing your analysis session</h4>
+    <div class="card-panel primary-color white-text" data-service="corpus-analysis">
+      <h4 class="m-3"><i class="material-icons left" style="font-size: inherit; line-height: inherit;">hourglass_empty</i>We are preparing your analysis session</h4>
+    </div>
     <p>
       Our server works as hard as it can to prepare your analysis session. Please be patient and give it some time.<br>
       If initialization takes longer than usual or an error occurs, <a onclick="window.location.reload()" href="#">reload the page</a>.

app/templates/corpora/corpus.html.j2

@@ -8,7 +8,7 @@
 {% endblock stylesheets %}
 
 
-{% block main_attribs %} class="service-color lighten" data-service="corpus-analysis"{% endblock main_attribs %}
+{% block main_attributes %} class="service-color lighten" data-service="corpus-analysis"{% endblock main_attributes %}
 
 {% block page_content %}
 <div class="container">
@@ -46,7 +46,7 @@
         </div>
       </div>
     </div>
-    
+
     {% if cfr.has_permission('VIEW') %}
     <div class="col s12 l5">
       <div class="card">
@@ -151,9 +151,9 @@
   <div class="modal-content">
     <h4>Invite a nopaque user by username</h4>
     <p>
-      Add other nopaque users as followers to your corpus. You can also add multiple 
-      users at the same time. Added users get the role of "viewer" 
-      by default, so they are only allowed to analyze files within nopaque, but not 
+      Add other nopaque users as followers to your corpus. You can also add multiple
+      users at the same time. Added users get the role of "viewer"
+      by default, so they are only allowed to analyze files within nopaque, but not
       to download or edit them. You can customize the roles later below.
     </p>
     <p><b>Please make sure that the invited users are legally allowed to view the included corpus files.</b></p>
@@ -176,8 +176,8 @@
   <div class="modal-content">
     <h4>Create a link to share your corpus</h4>
     <p>
-      With the link other users follow your corpus directly, if it has not expired. 
-      You can set different roles via the link, you can also edit them later in the menu below. 
+      With the link other users follow your corpus directly, if it has not expired.
+      You can set different roles via the link, you can also edit them later in the menu below.
       It is recommended not to set the expiration date of the link too far.
     </p>
     <p><b>Please make sure that the invited users are legally allowed to view the included corpus files.</b></p>
@@ -364,10 +364,10 @@ shareLinkModalCreateButtonElement.addEventListener('click', (event) => {
 shareLinkModalOutputCopyButtonElement.addEventListener('click', (event) => {
   navigator.clipboard.writeText(shareLinkModalOutputFieldElement.value)
     .then(
-      () => {app.flash('Copied!');},
-      () => {app.flash('Could not copy to clipboard. Please copy manually.', 'error');}
+      () => {app.ui.flash('Copied!');},
+      () => {app.ui.flash('Could not copy to clipboard. Please copy manually.', 'error');}
     );
-  
+
 });
 // #endregion Share link
 {% endif %}

app/templates/corpora/create.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
 {% import "wtf.html.j2" as wtf %}
 
-{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %}
+{% block main_attributes %}class="service-color lighten" data-service="corpus-analysis"{% endblock main_attributes %}
 
 {% block page_content %}
 <div class="container">

app/templates/corpora/import.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
 {% import "wtf.html.j2" as wtf %}
 
-{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %}
+{% block main_attributes %}class="service-color lighten" data-service="corpus-analysis"{% endblock main_attributes %}
 
 {% block page_content %}
 <div class="container">

app/templates/corpora/public_corpus.html.j2

@@ -1,7 +1,7 @@
 {% extends "base.html.j2" %}
 {% import "wtf.html.j2" as wtf %}
 
-{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %}
+{% block main_attributes %}class="service-color lighten" data-service="corpus-analysis"{% endblock main_attributes %}
 
 {% block page_content %}
 <div class="container">
@@ -85,7 +85,7 @@
         </div>
       </div>
     </div>
-    {% endif %} 
+    {% endif %}
 
     <div class="col s12">
       <div class="card">
@@ -161,9 +161,9 @@
   <div class="modal-content">
     <h4>Invite a nopaque user by username</h4>
     <p>
-      Add other nopaque users as followers to your corpus. You can also add multiple 
-      users at the same time. Added users get the role of "viewer" 
-      by default, so they are only allowed to analyze files within nopaque, but not 
+      Add other nopaque users as followers to your corpus. You can also add multiple
+      users at the same time. Added users get the role of "viewer"
+      by default, so they are only allowed to analyze files within nopaque, but not
       to download or edit them. You can customize the roles later below.
     </p>
     <p><b>Please make sure that the invited users are legally allowed to view the included corpus files.</b></p>
@@ -186,8 +186,8 @@
   <div class="modal-content">
     <h4>Create a link to share your corpus</h4>
     <p>
-      With the link other users follow your corpus directly, if it has not expired. 
-      You can set different roles via the link, you can also edit them later in the menu below. 
+      With the link other users follow your corpus directly, if it has not expired.
+      You can set different roles via the link, you can also edit them later in the menu below.
       It is recommended not to set the expiration date of the link too far.
     </p>
     <p><b>Please make sure that the invited users are legally allowed to view the included corpus files.</b></p>
@@ -396,10 +396,10 @@ shareLinkModalCreateButtonElement.addEventListener('click', (event) => {
 shareLinkModalOutputCopyButtonElement.addEventListener('click', (event) => {
   navigator.clipboard.writeText(shareLinkModalOutputFieldElement.value)
     .then(
-      () => {app.flash('Copied!');},
-      () => {app.flash('Could not copy to clipboard. Please copy manually.', 'error');}
+      () => {app.ui.flash('Copied!');},
+      () => {app.ui.flash('Could not copy to clipboard. Please copy manually.', 'error');}
     );
-  
+
 });
 // #endregion Share link
 {% endif %}

app/templates/jobs/job.html.j2

@@ -1,6 +1,6 @@
 {% extends "base.html.j2" %}
 
-{% block main_attribs %} class="service-scheme" data-service="{{ job.service }}"{% endblock main_attribs %}
+{% block main_attributes %} class="service-color lighten" data-service="{{ job.service }}"{% endblock main_attributes %}
 
 {% block page_content %}
 <div class="container">

app/templates/main/dashboard.html.j2

@@ -41,97 +41,58 @@
           <div class="job-list" data-user-id="{{ current_user.hashid }}"></div>
         </div>
         <div class="card-action right-align">
-          <p><a class="btn modal-trigger waves-effect waves-light" data-target="create-job-modal">Create job<i class="material-icons right">add</i></a></p>
+          <p><a data-target="dashboard-create-job-dropdown-content" class="btn waves-effect waves-light dropdown-trigger no-autoinit" id="dashboard-create-job-dropdown-trigger">Create job<i class="material-icons right">add</i></a></p>
         </div>
       </div>
     </div>
-
-    <div class="col s12" id="contributions">
-      <h2>My Contributions</h2>
-    </div>
-
-    <div class="col s4">
-      <div class="card extension-selector hoverable service-color" data-service="tesseract-ocr-pipeline">
-        <a href="{{ url_for('contributions.tesseract_ocr_pipeline_models') }}" style="position: absolute; width: 100%; height: 100%;"></a>
-        <div class="card-content">
-          <span class="card-title">Tesseract OCR Pipeline Models</span>
-          <p>Here you can see and edit the models that you have created. You can also create new models.</p>
-        </div>
-      </div>
-    </div>
-
-    <div class="col s4">
-      <div class="card extension-selector hoverable service-color" data-service="spacy-nlp-pipeline">
-        <a href="{{ url_for('contributions.spacy_nlp_pipeline_models') }}" style="position: absolute; width: 100%; height: 100%;"></a>
-        <div class="card-content">
-          <span class="card-title">SpaCy NLP Pipeline Models</span>
-          <p>Here you can see and edit the models that you have created. You can also create new models.</p>
-        </div>
-      </div>
-    </div>
-
-    {% if config.NOPAQUE_TRANSKRIBUS_ENABLED %}
-    <div class="col s4">
-      <div class="card extension-selector hoverable service-color" data-service="transkribus-htr-pipeline">
-      <a href="{{ url_for('contributions.transkribus_htr_pipeline_models') }}" style="position: absolute; width: 100%; height: 100%;"></a>
-        <div class="card-content">
-          <span class="card-title">Transkribus HTR Pipeline Models</span>
-          <p>Here you can see and edit the models that you have created. You can also create new models.</p>
-        </div>
-      </div>
-    </div>
-    {% endif %}
   </div>
 </div>
 {% endblock page_content %}
 
-{% block modals %}
+
+{% block dropdowns %}
 {{ super() }}
-<div id="create-job-modal" class="modal">
-  <div class="modal-content">
-    <h4>Select a service</h4>
-    <p>&nbsp;</p>
-    <div class="row">
-      <div class="col s12 m4">
-          <div class="card-panel center-align hoverable">
-            <br>
-            <a href="{{ url_for('services.file_setup_pipeline') }}" class="btn-floating btn-large waves-effect waves-light" style="transform: scale(2);">
-              <i class="nopaque-icons service-color darken service-icons" data-service="file-setup-pipeline"></i>
-            </a>
-            <br><br>
-            <p class="service-color-text darken" data-service="file-setup-pipeline"><b>File setup</b></p>
-            <p class="light">Digital copies of text based research data (books, letters, etc.) often comprise various files and formats. nopaque converts and merges those files to facilitate further processing.</p>
-            <a href="{{ url_for('services.file_setup_pipeline') }}" class="waves-effect waves-light btn service-color darken" data-service="file-setup-pipeline">Create Job</a>
-          </div>
-      </div>
-      <div class="col s12 m4">
-          <div class="card-panel center-align hoverable">
-            <br>
-            <a href="{{ url_for('services.tesseract_ocr_pipeline') }}" class="btn-floating btn-large waves-effect waves-light" style="transform: scale(2);">
-              <i class="nopaque-icons service-color darken service-icons" data-service="tesseract-ocr-pipeline" style="font-size: 2.5rem;"></i>
-            </a>
-            <br><br>
-            <p class="service-color-text darken" data-service="tesseract-ocr-pipeline"><b>Optical Character Recognition</b></p>
-            <p class="light">nopaque converts your image data – like photos or scans – into text data through a process called OCR. This step enables you to proceed with further computational analysis of your documents.</p>
-            <a href="{{ url_for('services.tesseract_ocr_pipeline') }}" class="waves-effect waves-light btn service-color darken" data-service="tesseract-ocr-pipeline">Create Job</a>
-          </div>
-      </div>
-      <div class="col s12 m4">
-          <div class="card-panel center-align hoverable">
-            <br>
-            <a href="{{ url_for('services.spacy_nlp_pipeline') }}" class="btn-floating btn-large waves-effect waves-light" style="transform: scale(2);">
-              <i class="nopaque-icons service-color darken service-icons" data-service="spacy-nlp-pipeline" style="font-size: 2.5rem;"></i>
-            </a>
-            <br><br>
-            <p class="service-color-text darken" data-service="spacy-nlp-pipeline"><b>Natural Language Processing</b></p>
-            <p class="light">By means of computational linguistic data processing (tokenization, lemmatization, part-of-speech tagging and named-entity recognition) nopaque extracts additional information from your text.</p>
-            <a href="{{ url_for('services.spacy_nlp_pipeline') }}" class="waves-effect waves-light btn service-color darken" data-service="spacy-nlp-pipeline">Create Job</a>
-          </div>
-      </div>
-    </div>
-  </div>
-  <div class="modal-footer">
-    <a class="btn-flat modal-close waves-effect waves-light">Close</a>
-  </div>
-</div>
-{% endblock modals %}
+<ul class="dropdown-content" id="dashboard-create-job-dropdown-content">
+  <li>
+    <a href="{{ url_for('services.file_setup_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="file-setup-pipeline"></i>
+      File Setup Pipeline
+    </a>
+  </li>
+  <li>
+    <a href="{{ url_for('services.tesseract_ocr_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="tesseract-ocr-pipeline"></i>
+      Tesseract OCR Pipeline
+    </a>
+  </li>
+  {% if config.NOPAQUE_TRANSKRIBUS_ENABLED %}
+  <li>
+    <a href="{{ url_for('services.transkribus_htr_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="transkribus-htr-pipeline"></i>
+      Transkribus HTR Pipeline
+    </a>
+  </li>
+  {% endif %}
+  <li>
+    <a href="{{ url_for('services.spacy_nlp_pipeline') }}">
+      <i class="nopaque-icons service-icons service-color-text text-darken" data-service="spacy-nlp-pipeline"></i>
+      SpaCy NLP Pipeline
+    </a>
+  </li>
+</ul>
+{% endblock dropdowns %}
+
+
+{% block scripts %}
+{{ super() }}
+<script>
+  M.Dropdown.init(
+    document.querySelector('#dashboard-create-job-dropdown-trigger'),
+    {
+      constrainWidth: false,
+      container: document.querySelector('#dropdowns'),
+      coverTrigger: false
+    }
+  );
+</script>
+{% endblock scripts %}