Merge branch 'access-pipeline' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into access-pipeline

.dockerignore

@@ -8,6 +8,6 @@
 !.flaskenv
 !boot.sh
 !config.py
-!docker-entrypoint.sh
+!docker-nopaque-entrypoint.sh
 !nopaque.py
 !requirements.txt

.gitlab-ci.yml

@@ -1,3 +1,37 @@
+include:
+  - template: Security/Container-Scanning.gitlab-ci.yml
+
+##############################################################################
+# Pipeline stages in order of execution                                      #
+##############################################################################
+stages:
+  - build
+  - publish
+  - sca
+
+##############################################################################
+# Pipeline behavior                                                          #
+##############################################################################
+workflow:
+  rules:
+    # Run the pipeline on commits to the default branch
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      variables:
+        # Set the Docker image tag to `latest`
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest
+      when: always
+    # Run the pipeline on tag creation
+    - if: $CI_COMMIT_TAG
+      variables:
+        # Set the Docker image tag to the Git tag name
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+      when: always
+    # Don't run the pipeline on all other occasions
+    - when: never
+
+##############################################################################
+# Default values for pipeline jobs                                           #
+##############################################################################
 default:
   image: docker:24.0.6
   services:
@@ -5,38 +39,46 @@ default:
   tags:
     - docker
 
+##############################################################################
+# CI/CD variables for all jobs in the pipeline                               #
+##############################################################################
 variables:
   DOCKER_TLS_CERTDIR: /certs
+  DOCKER_BUILD_PATH: .
+  DOCKERFILE: Dockerfile
 
-build_image:
+##############################################################################
+# Pipeline jobs                                                              #
+##############################################################################
+build:
   stage: build
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: on_success
-      variables:
-        IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
-    - if: $CI_COMMIT_TAG
-      when: "on_success"
-      variables:
-        IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-    - when: never
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - docker build -t $IMAGE_TAG .
-    - docker push $IMAGE_TAG
+    - docker build --tag $DOCKER_IMAGE --file $DOCKERFILE $DOCKER_BUILD_PATH
+    - docker save $DOCKER_IMAGE > docker_image.tar
+  artifacts:
+    paths:
+      - docker_image.tar
 
-include:
-  - template: Security/Container-Scanning.gitlab-ci.yml
+publish:
+  stage: publish
+  before_script:
+    - docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - docker load --input docker_image.tar
+    - docker push $DOCKER_IMAGE
+  after_script:
+    - docker logout $CI_REGISTRY
 
 container_scanning:
+  stage: sca
   rules:
+    # Run the job on commits to the default branch
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: on_success
-      variables:
-        CS_IMAGE: $CI_REGISTRY_IMAGE:latest
+      when: always
+    # Run the job on tag creation
     - if: $CI_COMMIT_TAG
-      when: on_success
-      variables:
-        CS_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}
+      when: always
+    # Don't run the job on all other occasions
     - when: never
+  variables:
+    CS_IMAGE: $DOCKER_IMAGE

.vscode/extensions.json

@@ -1,7 +1,8 @@
 {
     "recommendations": [
-        "samuelcolvin.jinjahtml",
+        "irongeek.vscode-env",
         "ms-azuretools.vscode-docker",
-        "ms-python.python"
+        "ms-python.python",
+        "samuelcolvin.jinjahtml"
     ]
 }

.vscode/settings.json

@@ -1,13 +1,9 @@
 {
     "editor.rulers": [79],
     "files.insertFinalNewline": true,
-    "python.terminal.activateEnvironment": false,
     "[css]": {
         "editor.tabSize": 2
     },
-    "[scss]": {
-        "editor.tabSize": 2
-    },
     "[html]": {
         "editor.tabSize": 2
     },
@@ -17,7 +13,7 @@
     "[jinja-html]": {
         "editor.tabSize": 2
     },
-    "[jinja-js]": {
+    "[scss]": {
         "editor.tabSize": 2
     }
 }

Dockerfile

@@ -4,11 +4,13 @@ FROM python:3.10.13-slim-bookworm
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
 
 
+# Set environment variables
 ENV LANG="C.UTF-8"
 ENV PYTHONDONTWRITEBYTECODE="1"
 ENV PYTHONUNBUFFERED="1"
 
 
+# Install system dependencies
 RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
@@ -17,37 +19,42 @@ RUN apt-get update \
  && rm --recursive /var/lib/apt/lists/*
 
 
-COPY docker-entrypoint.sh /usr/local/bin/
-
-
+# Create a non-root user
 RUN useradd --create-home --no-log-init nopaque \
  && groupadd docker \
  && usermod --append --groups docker nopaque
 
-
 USER nopaque
 WORKDIR /home/nopaque
 
 
+# Create a Python virtual environment
 ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv"
 RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}"
 ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}"
 
 
+# Install Python dependencies
+COPY --chown=nopaque:nopaque requirements.txt requirements.txt
+RUN python3 -m pip install --requirement requirements.txt \
+ && rm requirements.txt
+
+
+# Install the application
+COPY docker-nopaque-entrypoint.sh /usr/local/bin/
+
 COPY --chown=nopaque:nopaque app app
 COPY --chown=nopaque:nopaque migrations migrations
 COPY --chown=nopaque:nopaque tests tests
 COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py requirements.txt ./
 
-
-RUN python3 -m pip install --requirement requirements.txt \
- && mkdir logs
-
-
-USER root
+RUN mkdir logs
 
 
 EXPOSE 5000
 
 
-ENTRYPOINT ["docker-entrypoint.sh"]
+USER root
+
+
+ENTRYPOINT ["docker-nopaque-entrypoint.sh"]

README.md

@@ -1,5 +1,8 @@
 # nopaque
 
+![release badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/-/badges/release.svg)
+![pipeline badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/badges/master/pipeline.svg?ignore_skipped=true)
+
 nopaque bundles various tools and services that provide humanities scholars with DH methods and thus can support their various individual research processes. Using nopaque, researchers can subject digitized sources to Optical Character Recognition (OCR). The resulting text files can then be used as a data basis for Natural Language Processing (NLP). The texts are automatically subjected to various linguistic annotations. The data processed via NLP can then be summarized in the web application as corpora and analyzed by means of an information retrieval system through complex search queries. The range of functions of the web application will be successively extended according to the needs of the researchers.
 
 ## Prerequisites and requirements

app/__init__.py

@@ -57,6 +57,9 @@ def create_app(config: Config = Config) -> Flask:
     scheduler.init_app(app)
     socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
 
+    from .models.event_listeners import register_event_listeners
+    register_event_listeners()
+
     from .admin import bp as admin_blueprint
     default_breadcrumb_root(admin_blueprint, '.admin')
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
@@ -99,7 +102,7 @@ def create_app(config: Config = Config) -> Flask:
 
     from .users import bp as users_blueprint
     default_breadcrumb_root(users_blueprint, '.users')
-    app.register_blueprint(users_blueprint, url_prefix='/users')
+    app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
 
     from .workshops import bp as workshops_blueprint
     app.register_blueprint(workshops_blueprint, url_prefix='/workshops')

app/contributions/spacy_nlp_pipeline_models/forms.py

@@ -16,8 +16,8 @@ class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
     )
 
     def validate_spacy_model_file(self, field):
-        if not field.data.filename.lower().endswith('.tar.gz'):
-            raise ValidationError('.tar.gz files only!')
+        if not field.data.filename.lower().endswith(('.tar.gz', ('.whl'))):
+            raise ValidationError('.tar.gz or .whl files only!')
 
     def __init__(self, *args, **kwargs):
         if 'prefix' not in kwargs:

app/converters/sandpaper.py

@@ -2,32 +2,34 @@ from flask import current_app
 from app import db
 from app.models import User, Corpus, CorpusFile
 from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
 import json
-import os
 import shutil
 
 
 class SandpaperConverter:
-    def __init__(self, json_db_file, data_dir):
+    def __init__(self, json_db_file: Path, data_dir: Path):
         self.json_db_file = json_db_file
         self.data_dir = data_dir
 
     def run(self):
-        with open(self.json_db_file, 'r') as f:
-            json_db = json.loads(f.read())
+        with self.json_db_file.open('r') as f:
+            json_db: List[Dict] = json.load(f)
 
         for json_user in json_db:
             if not json_user['confirmed']:
                 current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
                 continue
-            user_dir = os.path.join(self.data_dir, str(json_user['id']))
+            user_dir = self.data_dir / f'{json_user["id"]}'
             self.convert_user(json_user, user_dir)
             db.session.commit()
 
 
-    def convert_user(self, json_user, user_dir):
+    def convert_user(self, json_user: Dict, user_dir: Path):
         current_app.logger.info(f'Create User {json_user["username"]}...')
-        user = User(
+        try:
+            user = User.create(
                 confirmed=json_user['confirmed'],
                 email=json_user['email'],
                 last_seen=datetime.fromtimestamp(json_user['last_seen']),
@@ -35,47 +37,34 @@ class SandpaperConverter:
                 password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
                 username=json_user['username']
             )
-        db.session.add(user)
-        db.session.flush(objects=[user])
-        db.session.refresh(user)
-        try:
-            user.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus in json_user['corpora'].values():
             if not json_corpus['files'].values():
                 current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
                 continue
-            corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
+            corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
             self.convert_corpus(json_corpus, user, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus(self, json_corpus, user, corpus_dir):
+    def convert_corpus(self, json_corpus: Dict, user: User, corpus_dir: Path):
         current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
-        corpus = Corpus(
+        try:
+            corpus = Corpus.create(
                 user=user,
                 creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
                 description=json_corpus['description'],
                 title=json_corpus['title']
             )
-        db.session.add(corpus)
-        db.session.flush(objects=[corpus])
-        db.session.refresh(corpus)
-        try:
-            corpus.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus_file in json_corpus['files'].values():
             self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
+    def convert_corpus_file(self, json_corpus_file: Dict, corpus: Corpus, corpus_dir: Path):
         current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
         corpus_file = CorpusFile(
             corpus=corpus,
@@ -99,13 +88,13 @@ class SandpaperConverter:
         db.session.refresh(corpus_file)
         try:
             shutil.copy2(
-                os.path.join(corpus_dir, json_corpus_file['filename']),
+                corpus_dir / json_corpus_file['filename'],
                 corpus_file.path
             )
         except:
             current_app.logger.warning(
                 'Can not convert corpus file: '
-                f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
+                f'{corpus_dir / json_corpus_file["filename"]}'
                 ' -> '
                 f'{corpus_file.path}'
             )

app/corpora/cli.py

@@ -1,7 +1,7 @@
-from app.models import Corpus, CorpusStatus
-import os
+from flask import current_app
 import shutil
 from app import db
+from app.models import Corpus, CorpusStatus
 from . import bp
 
 
@@ -18,10 +18,17 @@ def reset():
     ]
     for corpus in [x for x in Corpus.query.all() if x.status in status]:
         print(f'Resetting corpus {corpus}')
-        shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
-        os.mkdir(os.path.join(corpus.path, 'cwb'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'data'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'registry'))
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
         corpus.status = CorpusStatus.UNPREPARED
         corpus.num_analysis_sessions = 0
     db.session.commit()

app/corpora/cqi_over_sio/extensions.py

@@ -12,7 +12,6 @@ from typing import Dict, List
 import gzip
 import json
 import math
-import os
 from app import db
 from app.models import Corpus
 from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
@@ -42,9 +41,9 @@ def ext_corpus_static_data(corpus: str) -> Dict:
     db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
     db_corpus: Corpus = Corpus.query.get(db_corpus_id)
 
-    static_data_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
-    if os.path.exists(static_data_file_path):
-        with open(static_data_file_path, 'rb') as f:
+    static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
+    if static_data_file_path.exists():
+        with static_data_file_path.open('rb') as f:
             return f.read()
 
     cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']

app/corpora/files/routes.py

@@ -7,7 +7,6 @@ from flask import (
     url_for
 )
 from flask_breadcrumbs import register_breadcrumb
-import os
 from app import db
 from app.models import Corpus, CorpusFile, CorpusStatus
 from ..decorators import corpus_follower_permission_required
@@ -92,8 +91,8 @@ def corpus_file(corpus_id, corpus_file_id):
 def download_corpus_file(corpus_id, corpus_file_id):
     corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
     return send_from_directory(
-        os.path.dirname(corpus_file.path),
-        os.path.basename(corpus_file.path),
+        corpus_file.path.parent,
+        corpus_file.path.name,
         as_attachment=True,
         attachment_filename=corpus_file.filename,
         mimetype=corpus_file.mimetype

app/corpora/followers/json_routes.py

@@ -12,65 +12,65 @@ from ..decorators import corpus_follower_permission_required
 from . import bp
 
 
-# @bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
-# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
-# @content_negotiation(consumes='application/json', produces='application/json')
-# def create_corpus_followers(corpus_id):
-#     usernames = request.json
-#     if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
-#         abort(400)
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     for username in usernames:
-#         user = User.query.filter_by(username=username, is_public=True).first_or_404()
-#         user.follow_corpus(corpus)
-#     db.session.commit()
-#     response_data = {
-#         'message': f'Users are now following "{corpus.title}"',
-#         'category': 'corpus'
-#     }
-#     return response_data, 200
+@bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def create_corpus_followers(corpus_id):
+    usernames = request.json
+    if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    for username in usernames:
+        user = User.query.filter_by(username=username, is_public=True).first_or_404()
+        user.follow_corpus(corpus)
+    db.session.commit()
+    response_data = {
+        'message': f'Users are now following "{corpus.title}"',
+        'category': 'corpus'
+    }
+    return response_data, 200
 
 
-# @bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
-# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
-# @content_negotiation(consumes='application/json', produces='application/json')
-# def update_corpus_follower_role(corpus_id, follower_id):
-#     role_name = request.json
-#     if not isinstance(role_name, str):
-#         abort(400)
-#     cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
-#     if cfr is None:
-#         abort(400)
-#     cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
-#     cfa.role = cfr
-#     db.session.commit()
-#     response_data = {
-#         'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
-#         'category': 'corpus'
-#     }
-#     return response_data, 200
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_follower_role(corpus_id, follower_id):
+    role_name = request.json
+    if not isinstance(role_name, str):
+        abort(400)
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    cfa.role = cfr
+    db.session.commit()
+    response_data = {
+        'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
+        'category': 'corpus'
+    }
+    return response_data, 200
 
 
-# @bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
-# def delete_corpus_follower(corpus_id, follower_id):
-#     cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
-#     if not (
-#         current_user.id == follower_id
-#         or current_user == cfa.corpus.user 
-#         or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
-#         or current_user.is_administrator()):
-#         abort(403)
-#     if current_user.id == follower_id:
-#         flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
-#         response = make_response()
-#         response.status_code = 204
-#     else:
-#         response_data = {
-#             'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
-#             'category': 'corpus'
-#         }
-#         response = jsonify(response_data)
-#         response.status_code = 200
-#     cfa.follower.unfollow_corpus(cfa.corpus)
-#     db.session.commit()
-#     return response
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
+def delete_corpus_follower(corpus_id, follower_id):
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    if not (
+        current_user.id == follower_id
+        or current_user == cfa.corpus.user 
+        or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
+        or current_user.is_administrator()):
+        abort(403)
+    if current_user.id == follower_id:
+        flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
+        response = make_response()
+        response.status_code = 204
+    else:
+        response_data = {
+            'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
+            'category': 'corpus'
+        }
+        response = jsonify(response_data)
+        response.status_code = 200
+    cfa.follower.unfollow_corpus(cfa.corpus)
+    db.session.commit()
+    return response

app/corpora/json_routes.py

@@ -61,7 +61,7 @@ def build_corpus(corpus_id):
 @bp.route('/stopwords')
 @content_negotiation(produces='application/json')
 def get_stopwords():
-    nltk.download('stopwords')
+    nltk.download('stopwords', quiet=True)
     languages = ["german", "english", "catalan", "greek", "spanish", "french", "italian", "russian", "chinese"]
     stopwords = {}
     for language in languages:
@@ -71,55 +71,55 @@ def get_stopwords():
     response_data = stopwords
     return response_data, 202
 
-# @bp.route('/<hashid:corpus_id>/generate-share-link', methods=['POST'])
-# @corpus_follower_permission_required('MANAGE_FOLLOWERS')
-# @content_negotiation(consumes='application/json', produces='application/json')
-# def generate_corpus_share_link(corpus_id):
-#     data = request.json
-#     if not isinstance(data, dict):
-#         abort(400)
-#     expiration = data.get('expiration')
-#     if not isinstance(expiration, str):
-#         abort(400)
-#     role_name = data.get('role')
-#     if not isinstance(role_name, str):
-#         abort(400)
-#     expiration_date = datetime.strptime(expiration, '%b %d, %Y')
-#     cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
-#     if cfr is None:
-#         abort(400)
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date)
-#     corpus_share_link = url_for(
-#         'corpora.follow_corpus',
-#         corpus_id=corpus_id,
-#         token=token,
-#         _external=True
-#     )
-#     response_data = {
-#         'message': 'Corpus share link generated',
-#         'category': 'corpus',
-#         'corpusShareLink': corpus_share_link
-#     }
-#     return response_data, 200
+@bp.route('/<hashid:corpus_id>/generate-share-link', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def generate_corpus_share_link(corpus_id):
+    data = request.json
+    if not isinstance(data, dict):
+        abort(400)
+    expiration = data.get('expiration')
+    if not isinstance(expiration, str):
+        abort(400)
+    role_name = data.get('role')
+    if not isinstance(role_name, str):
+        abort(400)
+    expiration_date = datetime.strptime(expiration, '%b %d, %Y')
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date)
+    corpus_share_link = url_for(
+        'corpora.follow_corpus',
+        corpus_id=corpus_id,
+        token=token,
+        _external=True
+    )
+    response_data = {
+        'message': 'Corpus share link generated',
+        'category': 'corpus',
+        'corpusShareLink': corpus_share_link
+    }
+    return response_data, 200
     
 
 
-# @bp.route('/<hashid:corpus_id>/is_public', methods=['PUT'])
-# @corpus_owner_or_admin_required
-# @content_negotiation(consumes='application/json', produces='application/json')
-# def update_corpus_is_public(corpus_id):
-#     is_public = request.json
-#     if not isinstance(is_public, bool):
-#         abort(400)
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     corpus.is_public = is_public
-#     db.session.commit()
-#     response_data = {
-#         'message': (
-#             f'Corpus "{corpus.title}" is now'
-#             f' {"public" if is_public else "private"}'
-#         ),
-#         'category': 'corpus'
-#     }
-#     return response_data, 200
+@bp.route('/<hashid:corpus_id>/is_public', methods=['PUT'])
+@corpus_owner_or_admin_required
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_is_public(corpus_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    corpus.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Corpus "{corpus.title}" is now'
+            f' {"public" if is_public else "private"}'
+        ),
+        'category': 'corpus'
+    }
+    return response_data, 200

app/corpora/routes.py

@@ -68,20 +68,19 @@ def corpus(corpus_id):
             corpus=corpus,
             cfr=cfr,
             cfrs=cfrs,
-            users = users
+            users=users
         )
     if (current_user.is_following_corpus(corpus) or corpus.is_public):
-        abort(404)
-        # cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all()
-        # return render_template(
-        #     'corpora/public_corpus.html.j2',
-        #     title=corpus.title,
-        #     corpus=corpus,
-        #     cfrs=cfrs,
-        #     cfr=cfr,
-        #     cfas=cfas,
-        #     users = users
-        # )
+        cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all()
+        return render_template(
+            'corpora/public_corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfrs=cfrs,
+            cfr=cfr,
+            cfas=cfas,
+            users=users
+        )
     abort(403)
 
 
@@ -98,14 +97,14 @@ def analysis(corpus_id):
     )
 
 
-# @bp.route('/<hashid:corpus_id>/follow/<token>')
-# def follow_corpus(corpus_id, token):
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     if current_user.follow_corpus_by_token(token):
-#         db.session.commit()
-#         flash(f'You are following "{corpus.title}" now', category='corpus')
-#         return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
-#     abort(403)
+@bp.route('/<hashid:corpus_id>/follow/<token>')
+def follow_corpus(corpus_id, token):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if current_user.follow_corpus_by_token(token):
+        db.session.commit()
+        flash(f'You are following "{corpus.title}" now', category='corpus')
+        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    abort(403)
 
 
 @bp.route('/import', methods=['GET', 'POST'])

app/daemon/corpus_utils.py

@@ -45,7 +45,7 @@ def _create_build_corpus_service(corpus):
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1853'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
     ''' ## Labels ## '''
     labels = {
         'origin': current_app.config['SERVER_NAME'],
@@ -139,7 +139,7 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1853'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
     ''' ## Name ## '''
     name = f'cqpserver_{corpus.id}'
     ''' ## Network ## '''

app/ext/flask_sqlalchemy/__init__.py

@@ -0,0 +1,2 @@
+from .container_column import ContainerColumn
+from .int_enum_column import IntEnumColumn

app/ext/flask_sqlalchemy/container_column.py

@@ -0,0 +1,21 @@
+import json
+from app import db
+
+
+class ContainerColumn(db.TypeDecorator):
+    impl = db.String
+
+    def __init__(self, container_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.container_type = container_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.container_type):
+            return json.dumps(value)
+        elif isinstance(value, str) and isinstance(json.loads(value), self.container_type):
+            return value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return json.loads(value)

app/ext/flask_sqlalchemy/int_enum_column.py

@@ -0,0 +1,22 @@
+from app import db
+
+
+class IntEnumColumn(db.TypeDecorator):
+    impl = db.Integer
+
+    def __init__(self, enum_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_type = enum_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.enum_type) and isinstance(value.value, int):
+            return value.value
+        elif isinstance(value, int):
+            return self.enum_type(value).value
+        elif isinstance(value, str):
+            return self.enum_type[value].value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return self.enum_type(value)

app/jobs/json_routes.py

@@ -1,7 +1,6 @@
 from flask import abort, current_app
 from flask_login import current_user
 from threading import Thread
-import os
 from app import db
 from app.decorators import admin_required, content_negotiation
 from app.models import Job, JobStatus
@@ -39,7 +38,7 @@ def job_log(job_id):
     if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
         response = {'errors': {'message': 'Job status is not completed or failed'}}
         return response, 409
-    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+    with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
         log = log_file.read()
     response_data = {
         'jobLog': log

app/jobs/routes.py

@@ -7,7 +7,6 @@ from flask import (
 )
 from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
-import os
 from app.models import Job, JobInput, JobResult
 from . import bp
 from .utils import job_dynamic_list_constructor as job_dlc
@@ -38,8 +37,8 @@ def download_job_input(job_id, job_input_id):
     if not (job_input.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_input.path),
-        os.path.basename(job_input.path),
+        job_input.path.parent,
+        job_input.path.name,
         as_attachment=True,
         attachment_filename=job_input.filename,
         mimetype=job_input.mimetype
@@ -52,8 +51,8 @@ def download_job_result(job_id, job_result_id):
     if not (job_result.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_result.path),
-        os.path.basename(job_result.path),
+        job_result.path.parent,
+        job_result.path.name,
         as_attachment=True,
         attachment_filename=job_result.filename,
         mimetype=job_result.mimetype

app/main/cli.py

@@ -1,6 +1,7 @@
 from flask import current_app
 from flask_migrate import upgrade
-import os
+from pathlib import Path
+from typing import List
 from app.models import (
     CorpusFollowerRole,
     Role,
@@ -17,16 +18,15 @@ def deploy():
     # Make default directories
     print('Make default directories')
     base_dir = current_app.config['NOPAQUE_DATA_DIR']
-    default_dirs = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
+    default_dirs: List[Path] = [
+        base_dir / 'tmp',
+        base_dir / 'users'
     ]
-    for dir in default_dirs:
-        if os.path.exists(dir):
-            if not os.path.isdir(dir):
-                raise NotADirectoryError(f'{dir} is not a directory')
-        else:
-            os.mkdir(dir)
+    for default_dir in default_dirs:
+        if not default_dir.exists():
+            default_dir.mkdir()
+        if not default_dir.is_dir():
+            raise NotADirectoryError(f'{default_dir} is not a directory')
 
     # migrate database to latest revision
     print('Migrate database to latest revision')

app/main/routes.py

@@ -45,12 +45,6 @@ def dashboard():
     )
 
 
-# @bp.route('/user_manual')
-# @register_breadcrumb(bp, '.user_manual', '<i class="material-icons left">help</i>User manual')
-# def user_manual():
-#     return render_template('main/user_manual.html.j2', title='User manual')
-
-
 @bp.route('/news')
 @register_breadcrumb(bp, '.news', '<i class="material-icons left">email</i>News')
 def news():
@@ -78,15 +72,17 @@ def terms_of_use():
     )
 
 
-# @bp.route('/social-area')
-# @register_breadcrumb(bp, '.social_area', '<i class="material-icons left">group</i>Social Area')
-# @login_required
-# def social_area():
-#     corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
-#     users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
-#     return render_template(
-#         'main/social_area.html.j2',
-#         title='Social Area',
-#         corpora=corpora,
-#         users=users
-#     )
+@bp.route('/social-area')
+@register_breadcrumb(bp, '.social_area', '<i class="material-icons left">group</i>Social Area')
+@login_required
+def social_area():
+    print('test')
+    corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
+    print(corpora)
+    users = User.query.filter(User.is_public == True, User.id != current_user.id).all()
+    return render_template(
+        'main/social_area.html.j2',
+        title='Social Area',
+        corpora=corpora,
+        users=users
+    )

app/models/__init__.py

@@ -0,0 +1,19 @@
+from .avatar import *
+from .corpus_file import *
+from .corpus_follower_association import *
+from .corpus_follower_role import *
+from .corpus import *
+from .job_input import *
+from .job_result import *
+from .job import *
+from .role import *
+from .spacy_nlp_pipeline_model import *
+from .tesseract_ocr_pipeline_model import *
+from .token import *
+from .user import *
+from app import login
+
+
+@login.user_loader
+def load_user(user_id):
+    return User.query.get(int(user_id))

app/models/avatar.py

@@ -0,0 +1,40 @@
+from flask import current_app
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class Avatar(HashidMixin, FileMixin, db.Model):
+    __tablename__ = 'avatars'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Relationships
+    user = db.relationship('User', back_populates='avatar')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'avatar'
+        # return os.path.join(self.user.path, 'avatar')
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus.py

@@ -0,0 +1,200 @@
+from datetime import datetime
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from sqlalchemy.ext.associationproxy import association_proxy
+from typing import Union
+from pathlib import Path
+import shutil
+import xml.etree.ElementTree as ET
+from app import db
+from app.converters.vrt import normalize_vrt_file
+from app.ext.flask_sqlalchemy import IntEnumColumn
+from .corpus_follower_association import CorpusFollowerAssociation
+
+
+class CorpusStatus(IntEnum):
+    UNPREPARED = 1
+    SUBMITTED = 2
+    QUEUED = 3
+    BUILDING = 4
+    BUILT = 5
+    FAILED = 6
+    STARTING_ANALYSIS_SESSION = 7
+    RUNNING_ANALYSIS_SESSION = 8
+    CANCELING_ANALYSIS_SESSION = 9
+
+    @staticmethod
+    def get(corpus_status: Union['CorpusStatus', int, str]) -> 'CorpusStatus':
+        if isinstance(corpus_status, CorpusStatus):
+            return corpus_status
+        if isinstance(corpus_status, int):
+            return CorpusStatus(corpus_status)
+        if isinstance(corpus_status, str):
+            return CorpusStatus[corpus_status]
+        raise TypeError('corpus_status must be CorpusStatus, int, or str')
+
+
+class Corpus(HashidMixin, db.Model):
+    '''
+    Class to define a corpus.
+    '''
+    __tablename__ = 'corpora'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
+    description = db.Column(db.String(255))
+    status = db.Column(
+        IntEnumColumn(CorpusStatus),
+        default=CorpusStatus.UNPREPARED
+    )
+    title = db.Column(db.String(32))
+    num_analysis_sessions = db.Column(db.Integer, default=0)
+    num_tokens = db.Column(db.Integer, default=0)
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    files = db.relationship(
+        'CorpusFile',
+        back_populates='corpus',
+        lazy='dynamic',
+        cascade='all, delete-orphan'
+    )
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='corpus',
+        cascade='all, delete-orphan'
+    )
+    followers = association_proxy(
+        'corpus_follower_associations',
+        'follower',
+        creator=lambda u: CorpusFollowerAssociation(follower=u)
+    )
+    user = db.relationship('User', back_populates='corpora')
+    # "static" attributes
+    max_num_tokens = 2_147_483_647
+
+    def __repr__(self):
+        return f'<Corpus {self.title}>'
+
+    @property
+    def analysis_url(self):
+        return url_for('corpora.analysis', corpus_id=self.id)
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/corpora/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'corpora' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for('corpora.corpus', corpus_id=self.id)
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def create(**kwargs):
+        corpus = Corpus(**kwargs)
+        db.session.add(corpus)
+        db.session.flush(objects=[corpus])
+        db.session.refresh(corpus)
+        corpus_files_dir = corpus.path / 'files'
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            corpus.path.mkdir()
+            corpus_files_dir.mkdir()
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return corpus
+
+    def build(self):
+        corpus_cwb_dir = self.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus_cwb_dir, ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            self.status = CorpusStatus.FAILED
+            raise
+        corpus_element = ET.fromstring('<corpus>\n</corpus>')
+        for corpus_file in self.files:
+            normalized_vrt_path = corpus_cwb_dir / f'{corpus_file.id}.norm.vrt'
+            try:
+                normalize_vrt_file(corpus_file.path, normalized_vrt_path)
+            except:
+                self.status = CorpusStatus.FAILED
+                return
+            element_tree = ET.parse(normalized_vrt_path)
+            text_element = element_tree.getroot()
+            text_element.set('author', corpus_file.author)
+            text_element.set('title', corpus_file.title)
+            text_element.set(
+                'publishing_year',
+                f'{corpus_file.publishing_year}'
+            )
+            text_element.set('address', corpus_file.address or 'NULL')
+            text_element.set('booktitle', corpus_file.booktitle or 'NULL')
+            text_element.set('chapter', corpus_file.chapter or 'NULL')
+            text_element.set('editor', corpus_file.editor or 'NULL')
+            text_element.set('institution', corpus_file.institution or 'NULL')
+            text_element.set('journal', corpus_file.journal or 'NULL')
+            text_element.set('pages', f'{corpus_file.pages}' or 'NULL')
+            text_element.set('publisher', corpus_file.publisher or 'NULL')
+            text_element.set('school', corpus_file.school or 'NULL')
+            text_element.tail = '\n'
+            # corpus_element.insert(1, text_element)
+            corpus_element.append(text_element)
+        ET.ElementTree(corpus_element).write(
+            corpus_cwb_dir / 'corpus.vrt',
+            encoding='utf-8'
+        )
+        self.status = CorpusStatus.SUBMITTED
+
+    def delete(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'description': self.description,
+            'max_num_tokens': self.max_num_tokens,
+            'num_analysis_sessions': self.num_analysis_sessions,
+            'num_tokens': self.num_tokens,
+            'status': self.status.name,
+            'title': self.title,
+            'is_public': self.is_public
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
+            json_serializeable['files'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.files
+            }
+        return json_serializeable

app/models/corpus_file.py

@@ -0,0 +1,102 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .corpus import CorpusStatus
+from .file_mixin import FileMixin
+
+
+class CorpusFile(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'corpus_files'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    # Fields
+    author = db.Column(db.String(255))
+    description = db.Column(db.String(255))
+    publishing_year = db.Column(db.Integer)
+    title = db.Column(db.String(255))
+    address = db.Column(db.String(255))
+    booktitle = db.Column(db.String(255))
+    chapter = db.Column(db.String(255))
+    editor = db.Column(db.String(255))
+    institution = db.Column(db.String(255))
+    journal = db.Column(db.String(255))
+    pages = db.Column(db.String(255))
+    publisher = db.Column(db.String(255))
+    school = db.Column(db.String(255))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='files'
+    )
+
+    @property
+    def download_url(self):
+        return url_for(
+            'corpora.download_corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.corpus.jsonpatch_path}/files/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.corpus.path / 'files' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'corpora.corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.corpus.user.hashid
+
+    @property
+    def user_id(self):
+        return self.corpus.user_id
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+        self.corpus.status = CorpusStatus.UNPREPARED
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'address': self.address,
+            'author': self.author,
+            'description': self.description,
+            'booktitle': self.booktitle,
+            'chapter': self.chapter,
+            'editor': self.editor,
+            'institution': self.institution,
+            'journal': self.journal,
+            'pages': self.pages,
+            'publisher': self.publisher,
+            'publishing_year': self.publishing_year,
+            'school': self.school,
+            'title': self.title,
+            **self.file_mixin_to_json_serializeable(
+                backrefs=backrefs,
+                relationships=relationships
+            )
+        }
+        if backrefs:
+            json_serializeable['corpus'] = \
+                self.corpus.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_association.py

@@ -0,0 +1,47 @@
+from flask_hashids import HashidMixin
+from app import db
+from .corpus_follower_role import CorpusFollowerRole
+
+
+class CorpusFollowerAssociation(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_associations'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    follower_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    role_id = db.Column(db.Integer, db.ForeignKey('corpus_follower_roles.id'))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='corpus_follower_associations'
+    )
+    follower = db.relationship(
+        'User',
+        back_populates='corpus_follower_associations'
+    )
+    role = db.relationship(
+        'CorpusFollowerRole',
+        back_populates='corpus_follower_associations'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = CorpusFollowerRole.query.filter_by(default=True).first()
+        super().__init__(**kwargs)
+
+    def __repr__(self):
+        return f'<CorpusFollowerAssociation {self.follower.__repr__()} ~ {self.role.__repr__()} ~ {self.corpus.__repr__()}>'
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'corpus': self.corpus.to_json_serializeable(backrefs=True),
+            'follower': self.follower.to_json_serializeable(),
+            'role': self.role.to_json_serializeable()
+        }
+        if backrefs:
+            pass
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_role.py

@@ -0,0 +1,107 @@
+from flask_hashids import HashidMixin
+from enum import IntEnum
+from typing import Union
+from app import db
+
+
+class CorpusFollowerPermission(IntEnum):
+    VIEW = 1
+    MANAGE_FILES = 2
+    MANAGE_FOLLOWERS = 4
+    MANAGE_CORPUS = 8
+
+    @staticmethod
+    def get(corpus_follower_permission: Union['CorpusFollowerPermission', int, str]) -> 'CorpusFollowerPermission':
+        if isinstance(corpus_follower_permission, CorpusFollowerPermission):
+            return corpus_follower_permission
+        if isinstance(corpus_follower_permission, int):
+            return CorpusFollowerPermission(corpus_follower_permission)
+        if isinstance(corpus_follower_permission, str):
+            return CorpusFollowerPermission[corpus_follower_permission]
+        raise TypeError('corpus_follower_permission must be CorpusFollowerPermission, int, or str')
+
+
+class CorpusFollowerRole(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='role'
+    )
+
+    def __repr__(self):
+        return f'<CorpusFollowerRole {self.name}>'
+
+    def has_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        return self.permissions & perm.value == perm.value
+    
+    def add_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if not self.has_permission(perm):
+            self.permissions += perm.value
+    
+    def remove_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if self.has_permission(perm):
+            self.permissions -= perm.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name
+                for x in CorpusFollowerPermission
+                if self.has_permission(x)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['corpus_follower_association'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpus_follower_association
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'Anonymous': [],
+            'Viewer': [
+                CorpusFollowerPermission.VIEW
+            ],
+            'Contributor': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES
+            ],
+            'Administrator': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES,
+                CorpusFollowerPermission.MANAGE_FOLLOWERS,
+                CorpusFollowerPermission.MANAGE_CORPUS
+
+            ]
+        }
+        default_role_name = 'Viewer'
+        for role_name, permissions in roles.items():
+            role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+            if role is None:
+                role = CorpusFollowerRole(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()

app/models/default_records/tesseract_ocr_pipeline_model.yml

@@ -9,6 +9,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Amharic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata'
@@ -20,6 +21,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Arabic'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata'
@@ -31,6 +33,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Assamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata'
@@ -42,6 +45,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Azerbaijani'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata'
@@ -53,6 +57,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Azerbaijani - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata'
@@ -64,6 +69,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Belarusian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata'
@@ -75,6 +81,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bengali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata'
@@ -86,6 +93,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tibetan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata'
@@ -97,6 +105,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bosnian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata'
@@ -108,6 +117,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bulgarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata'
@@ -119,6 +129,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Catalan; Valencian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata'
@@ -130,6 +141,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Cebuano'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata'
@@ -141,6 +153,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Czech'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata'
@@ -152,6 +165,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Chinese - Simplified'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata'
@@ -163,6 +177,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Chinese - Traditional'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata'
@@ -174,6 +189,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Cherokee'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata'
@@ -185,6 +201,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Welsh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata'
@@ -196,6 +213,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Danish'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata'
@@ -207,6 +225,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'German'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata'
@@ -218,6 +237,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Dzongkha'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata'
@@ -229,6 +249,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Greek, Modern (1453-)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata'
@@ -240,6 +261,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'English'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata'
@@ -251,6 +273,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'English, Middle (1100-1500)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata'
@@ -262,6 +285,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Esperanto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata'
@@ -273,6 +297,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Estonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata'
@@ -284,6 +309,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Basque'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata'
@@ -295,6 +321,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Persian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata'
@@ -306,6 +333,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Finnish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata'
@@ -317,6 +345,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'French'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata'
@@ -328,6 +357,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'German Fraktur'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata'
@@ -339,6 +369,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'French, Middle (ca. 1400-1600)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata'
@@ -350,6 +381,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Irish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata'
@@ -361,6 +393,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Galician'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata'
@@ -372,6 +405,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Greek, Ancient (-1453)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata'
@@ -383,6 +417,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Gujarati'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata'
@@ -394,6 +429,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Haitian; Haitian Creole'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata'
@@ -405,6 +441,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hebrew'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata'
@@ -416,6 +453,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hindi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata'
@@ -427,6 +465,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Croatian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata'
@@ -438,6 +477,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hungarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata'
@@ -449,6 +489,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Inuktitut'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata'
@@ -460,6 +501,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Indonesian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata'
@@ -471,6 +513,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Icelandic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata'
@@ -482,6 +525,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Italian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata'
@@ -493,6 +537,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'Italian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata'
@@ -504,6 +549,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Javanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata'
@@ -515,6 +561,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Japanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata'
@@ -526,6 +573,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kannada'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata'
@@ -537,6 +585,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Georgian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata'
@@ -548,6 +597,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Georgian - Old'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata'
@@ -559,6 +609,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kazakh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata'
@@ -570,6 +621,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Central Khmer'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata'
@@ -581,6 +633,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kirghiz; Kyrgyz'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata'
@@ -592,6 +645,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Korean'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata'
@@ -603,6 +657,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kurdish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata'
@@ -614,6 +669,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Lao'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata'
@@ -625,6 +681,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata'
@@ -636,6 +693,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Latvian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata'
@@ -647,6 +705,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Lithuanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata'
@@ -658,6 +717,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Malayalam'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata'
@@ -669,6 +729,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Marathi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata'
@@ -680,6 +741,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Macedonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata'
@@ -691,6 +753,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Maltese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata'
@@ -702,6 +765,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Malay'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata'
@@ -713,6 +777,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Burmese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata'
@@ -724,6 +789,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Nepali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata'
@@ -735,6 +801,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Dutch; Flemish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata'
@@ -746,6 +813,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Norwegian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata'
@@ -757,6 +825,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Oriya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata'
@@ -768,6 +837,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Panjabi; Punjabi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata'
@@ -779,6 +849,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Polish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata'
@@ -790,6 +861,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Portuguese'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata'
@@ -801,6 +873,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Pushto; Pashto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata'
@@ -812,6 +885,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Romanian; Moldavian; Moldovan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata'
@@ -823,6 +897,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Russian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata'
@@ -834,6 +909,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Sanskrit'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata'
@@ -845,6 +921,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Sinhala; Sinhalese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata'
@@ -856,6 +933,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Slovak'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata'
@@ -867,6 +945,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Slovenian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata'
@@ -878,6 +957,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Spanish; Castilian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata'
@@ -889,6 +969,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'Spanish; Castilian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata'
@@ -900,6 +981,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Albanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata'
@@ -911,6 +993,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Serbian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata'
@@ -922,6 +1005,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Serbian - Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata'
@@ -933,6 +1017,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Swahili'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata'
@@ -944,6 +1029,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Swedish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata'
@@ -955,6 +1041,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Syriac'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata'
@@ -966,6 +1053,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tamil'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata'
@@ -977,6 +1065,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Telugu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata'
@@ -988,6 +1077,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tajik'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata'
@@ -999,6 +1089,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tagalog'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata'
@@ -1010,6 +1101,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Thai'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata'
@@ -1021,6 +1113,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tigrinya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata'
@@ -1032,6 +1125,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Turkish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata'
@@ -1043,6 +1137,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uighur; Uyghur'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata'
@@ -1054,6 +1149,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Ukrainian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata'
@@ -1065,6 +1161,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Urdu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata'
@@ -1076,6 +1173,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uzbek'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata'
@@ -1087,6 +1185,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uzbek - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata'
@@ -1098,6 +1197,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Vietnamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata'
@@ -1109,6 +1209,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Yiddish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata'
@@ -1120,3 +1221,4 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'

app/models/event_listeners.py

@@ -0,0 +1,133 @@
+from datetime import datetime
+from enum import Enum
+from app import db, mail, socketio
+from app.email import create_message
+from .corpus_file import CorpusFile
+from .corpus_follower_association import CorpusFollowerAssociation
+from .corpus import Corpus
+from .job_input import JobInput
+from .job_result import JobResult
+from .job import Job, JobStatus
+from .spacy_nlp_pipeline_model import SpaCyNLPPipelineModel
+from .tesseract_ocr_pipeline_model import TesseractOCRPipelineModel
+from .user import UserSettingJobStatusMailNotificationLevel
+
+
+def register_event_listeners():
+    resources = [
+        Corpus,
+        CorpusFile,
+        Job,
+        JobInput,
+        JobResult,
+        SpaCyNLPPipelineModel,
+        TesseractOCRPipelineModel
+    ]
+
+    for resource in resources:
+        db.event.listen(resource, 'after_delete', resource_after_delete)
+        db.event.listen(resource, 'after_insert', resource_after_insert)
+        db.event.listen(resource, 'after_update', resource_after_update)
+
+    db.event.listen(CorpusFollowerAssociation, 'after_delete', cfa_after_delete)
+    db.event.listen(CorpusFollowerAssociation, 'after_insert', cfa_after_insert)
+
+    db.event.listen(Job, 'after_update', job_after_update)
+
+
+def resource_after_delete(mapper, connection, resource):
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': resource.jsonpatch_path
+        }
+    ]
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def cfa_after_delete(mapper, connection, cfa):
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': jsonpatch_path
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def resource_after_insert(mapper, connection, resource):
+    jsonpatch_value = resource.to_json_serializeable()
+    for attr in mapper.relationships:
+        jsonpatch_value[attr.key] = {}
+    jsonpatch = [
+        {
+            'op': 'add',
+            'path': resource.jsonpatch_path,
+            'value': jsonpatch_value
+        }
+    ]
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def cfa_after_insert(mapper, connection, cfa):
+    jsonpatch_value = cfa.to_json_serializeable()
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'add',
+            'path': jsonpatch_path,
+            'value': jsonpatch_value
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def resource_after_update(mapper, connection, resource):
+    jsonpatch = []
+    for attr in db.inspect(resource).attrs:
+        if attr.key in mapper.relationships:
+            continue
+        if not attr.load_history().has_changes():
+            continue
+        jsonpatch_path = f'{resource.jsonpatch_path}/{attr.key}'
+        if isinstance(attr.value, datetime):
+            jsonpatch_value = f'{attr.value.isoformat()}Z'
+        elif isinstance(attr.value, Enum):
+            jsonpatch_value = attr.value.name
+        else:
+            jsonpatch_value = attr.value
+        jsonpatch.append(
+            {
+                'op': 'replace',
+                'path': jsonpatch_path,
+                'value': jsonpatch_value
+            }
+        )
+    if jsonpatch:
+        room = f'/users/{resource.user_hashid}'
+        socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def job_after_update(mapper, connection, job):
+    for attr in db.inspect(job).attrs:
+        if attr.key != 'status':
+            continue
+        if not attr.load_history().has_changes():
+            return
+        if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:
+            return
+        if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:
+            if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+                return
+        msg = create_message(
+            job.user.email,
+            f'Status update for your Job "{job.title}"',
+            'tasks/email/notification',
+            job=job
+        )
+        mail.send(msg)

app/models/file_mixin.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+from flask import current_app
+from werkzeug.utils import secure_filename
+from app import db
+
+
+class FileMixin:
+    '''
+    Mixin for db.Model classes. All file related models should use this.
+    '''
+    creation_date = db.Column(db.DateTime, default=datetime.utcnow)
+    filename = db.Column(db.String(255))
+    mimetype = db.Column(db.String(255))
+
+    def file_mixin_to_json_serializeable(self, backrefs=False, relationships=False):
+        return {
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'filename': self.filename,
+            'mimetype': self.mimetype
+        }
+    
+    @classmethod
+    def create(cls, file_storage, **kwargs):
+        filename = kwargs.pop('filename', file_storage.filename)
+        mimetype = kwargs.pop('mimetype', file_storage.mimetype)
+        obj = cls(
+            filename=secure_filename(filename),
+            mimetype=mimetype,
+            **kwargs
+        )
+        db.session.add(obj)
+        db.session.flush(objects=[obj])
+        db.session.refresh(obj)
+        try:
+            file_storage.save(obj.path)
+        except (AttributeError, OSError) as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise e
+        return obj

app/models/job.py

@@ -0,0 +1,172 @@
+from datetime import datetime
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from time import sleep
+from typing import Union
+from pathlib import Path
+import shutil
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn, IntEnumColumn
+
+
+class JobStatus(IntEnum):
+    INITIALIZING = 1
+    SUBMITTED = 2
+    QUEUED = 3
+    RUNNING = 4
+    CANCELING = 5
+    CANCELED = 6
+    COMPLETED = 7
+    FAILED = 8
+
+    @staticmethod
+    def get(job_status: Union['JobStatus', int, str]) -> 'JobStatus':
+        if isinstance(job_status, JobStatus):
+            return job_status
+        if isinstance(job_status, int):
+            return JobStatus(job_status)
+        if isinstance(job_status, str):
+            return JobStatus[job_status]
+        raise TypeError('job_status must be JobStatus, int, or str')
+
+
+class Job(HashidMixin, db.Model):
+    '''
+    Class to define Jobs.
+    '''
+    __tablename__ = 'jobs'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    creation_date = \
+        db.Column(db.DateTime(), default=datetime.utcnow)
+    description = db.Column(db.String(255))
+    end_date = db.Column(db.DateTime())
+    service = db.Column(db.String(64))
+    service_args = db.Column(ContainerColumn(dict, 255))
+    service_version = db.Column(db.String(16))
+    status = db.Column(
+        IntEnumColumn(JobStatus),
+        default=JobStatus.INITIALIZING
+    )
+    title = db.Column(db.String(32))
+    # Relationships
+    inputs = db.relationship(
+        'JobInput',
+        back_populates='job',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    results = db.relationship(
+        'JobResult',
+        back_populates='job',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    user = db.relationship(
+        'User',
+        back_populates='jobs'
+    )
+
+    def __repr__(self):
+        return f'<Job {self.title}>'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/jobs/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'jobs' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for('jobs.job', job_id=self.id)
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def create(**kwargs):
+        job = Job(**kwargs)
+        db.session.add(job)
+        db.session.flush(objects=[job])
+        db.session.refresh(job)
+        job_inputs_dir = job.path / 'inputs'
+        job_pipeline_data_dir = job.path / 'pipeline_data'
+        job_results_dir = job.path / 'results'
+        try:
+            job.path.mkdir()
+            job_inputs_dir.mkdir()
+            job_pipeline_data_dir.mkdir()
+            job_results_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return job
+
+    def delete(self):
+        ''' Delete the job and its inputs and results from the database. '''
+        if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:  # noqa
+            self.status = JobStatus.CANCELING
+            db.session.commit()
+            while self.status != JobStatus.CANCELED:
+                # In case the daemon handled a job in any way
+                if self.status != JobStatus.CANCELING:
+                    self.status = JobStatus.CANCELING
+                    db.session.commit()
+                sleep(1)
+                db.session.refresh(self)
+        try:
+            shutil.rmtree(self.path)
+        except OSError as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise e
+        db.session.delete(self)
+
+    def restart(self):
+        ''' Restart a job - only if the status is failed '''
+        if self.status != JobStatus.FAILED:
+            raise Exception('Job status is not "failed"')
+        shutil.rmtree(self.path / 'results', ignore_errors=True)
+        shutil.rmtree(self.path / 'pyflow.data', ignore_errors=True)
+        for result in self.results:
+            db.session.delete(result)
+        self.end_date = None
+        self.status = JobStatus.SUBMITTED
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'description': self.description,
+            'end_date': (
+                None if self.end_date is None
+                else f'{self.end_date.isoformat()}Z'
+            ),
+            'service': self.service,
+            'service_args': self.service_args,
+            'service_version': self.service_version,
+            'status': self.status.name,
+            'title': self.title
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['inputs'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.inputs
+            }
+            json_serializeable['results'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.results
+            }
+        return json_serializeable

app/models/job_input.py

@@ -0,0 +1,65 @@
+from flask import url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class JobInput(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'job_inputs'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
+    # Relationships
+    job = db.relationship(
+        'Job',
+        back_populates='inputs'
+    )
+
+    def __repr__(self):
+        return f'<JobInput {self.filename}>'
+
+    @property
+    def content_url(self):
+        return url_for(
+            'jobs.download_job_input',
+            job_id=self.job.id,
+            job_input_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.job.jsonpatch_path}/inputs/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.job.path / 'inputs' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'jobs.job',
+            job_id=self.job_id,
+            _anchor=f'job-{self.job.hashid}-input-{self.hashid}'
+        )
+
+    @property
+    def user_hashid(self):
+        return self.job.user.hashid
+
+    @property
+    def user_id(self):
+        return self.job.user.id
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['job'] = \
+                self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/job_result.py

@@ -0,0 +1,71 @@
+from flask import url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class JobResult(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'job_results'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
+    # Fields
+    description = db.Column(db.String(255))
+    # Relationships
+    job = db.relationship(
+        'Job',
+        back_populates='results'
+    )
+
+    def __repr__(self):
+        return f'<JobResult {self.filename}>'
+
+    @property
+    def download_url(self):
+        return url_for(
+            'jobs.download_job_result',
+            job_id=self.job_id,
+            job_result_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.job.jsonpatch_path}/results/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.job.path / 'results' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'jobs.job',
+            job_id=self.job_id,
+            _anchor=f'job-{self.job.hashid}-result-{self.hashid}'
+        )
+
+    @property
+    def user_hashid(self):
+        return self.job.user.hashid
+
+    @property
+    def user_id(self):
+        return self.job.user.id
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'description': self.description,
+            **self.file_mixin_to_json_serializeable(
+                backrefs=backrefs,
+                relationships=relationships
+            )
+        }
+        if backrefs:
+            json_serializeable['job'] = \
+                self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/role.py

@@ -0,0 +1,100 @@
+from enum import IntEnum
+from flask_hashids import HashidMixin
+from typing import Union
+from app import db
+
+
+class Permission(IntEnum):
+    '''
+    Defines User permissions as integers by the power of 2. User permission
+    can be evaluated using the bitwise operator &.
+    '''
+    ADMINISTRATE = 1
+    CONTRIBUTE = 2
+    USE_API = 4
+
+    @staticmethod
+    def get(permission: Union['Permission', int, str]) -> 'Permission':
+        if isinstance(permission, Permission):
+            return permission
+        if isinstance(permission, int):
+            return Permission(permission)
+        if isinstance(permission, str):
+            return Permission[permission]
+        raise TypeError('permission must be Permission, int, or str')
+
+
+class Role(HashidMixin, db.Model):
+    __tablename__ = 'roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    users = db.relationship('User', back_populates='role', lazy='dynamic')
+
+    def __repr__(self):
+        return f'<Role {self.name}>'
+
+    def has_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        return self.permissions & p.value == p.value
+    
+    def add_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if not self.has_permission(p):
+            self.permissions += p.value
+    
+    def remove_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if self.has_permission(p):
+            self.permissions -= p.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name for x in Permission
+                if self.has_permission(x.value)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['users'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.users
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'User': [],
+            'API user': [Permission.USE_API],
+            'Contributor': [Permission.CONTRIBUTE],
+            'Administrator': [
+                Permission.ADMINISTRATE,
+                Permission.CONTRIBUTE,
+                Permission.USE_API
+            ],
+            'System user': []
+        }
+        default_role_name = 'User'
+        for role_name, permissions in roles.items():
+            role = Role.query.filter_by(name=role_name).first()
+            if role is None:
+                role = Role(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()

app/models/spacy_nlp_pipeline_model.py

@@ -0,0 +1,136 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from tqdm import tqdm
+from pathlib import Path
+import requests
+import yaml
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn
+from .file_mixin import FileMixin
+from .user import User
+
+
+class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'spacy_nlp_pipeline_models'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    title = db.Column(db.String(64))
+    description = db.Column(db.String(255))
+    version = db.Column(db.String(16))
+    compatible_service_versions = db.Column(ContainerColumn(list, 255))
+    publisher = db.Column(db.String(128))
+    publisher_url = db.Column(db.String(512))
+    publishing_url = db.Column(db.String(512))
+    publishing_year = db.Column(db.Integer)
+    pipeline_name = db.Column(db.String(64))
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    user = db.relationship('User', back_populates='spacy_nlp_pipeline_models')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'spacy_nlp_pipeline_models' / f'{self.id}'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}'
+
+    @property
+    def url(self):
+        return url_for(
+            'contributions.spacy_nlp_pipeline_model',
+            spacy_nlp_pipeline_model_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def insert_defaults(force_download=False):
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        default_records_file = Path(__file__).parent / 'default_records' / 'spacy_nlp_pipeline_model.yml'
+        with default_records_file.open('r') as f:
+            default_records = yaml.safe_load(f)
+        for m in default_records:
+            model = SpaCyNLPPipelineModel.query.filter_by(title=m['title'], version=m['version']).first()  # noqa
+            if model is not None:
+                model.compatible_service_versions = m['compatible_service_versions']
+                model.description = m['description']
+                model.filename = m['url'].split('/')[-1]
+                model.publisher = m['publisher']
+                model.publisher_url = m['publisher_url']
+                model.publishing_url = m['publishing_url']
+                model.publishing_year = m['publishing_year']
+                model.is_public = True
+                model.title = m['title']
+                model.version = m['version']
+                model.pipeline_name = m['pipeline_name']
+            else:
+                model = SpaCyNLPPipelineModel(
+                    compatible_service_versions=m['compatible_service_versions'],
+                    description=m['description'],
+                    filename=m['url'].split('/')[-1],
+                    publisher=m['publisher'],
+                    publisher_url=m['publisher_url'],
+                    publishing_url=m['publishing_url'],
+                    publishing_year=m['publishing_year'],
+                    is_public=True,
+                    title=m['title'],
+                    user=nopaque_user,
+                    version=m['version'],
+                    pipeline_name=m['pipeline_name']
+                )
+                db.session.add(model)
+                db.session.flush(objects=[model])
+                db.session.refresh(model)
+            if not model.path.exists() or force_download:
+                r = requests.get(m['url'], stream=True)
+                pbar = tqdm(
+                    desc=f'{model.title} ({model.filename})',
+                    unit="B",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                    total=int(r.headers['Content-Length'])
+                )
+                pbar.clear()
+                with open(model.path, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=1024):
+                        if chunk:  # filter out keep-alive new chunks
+                            pbar.update(len(chunk))
+                            f.write(chunk)
+                    pbar.close()
+        db.session.commit()
+    
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'compatible_service_versions': self.compatible_service_versions,
+            'description': self.description,
+            'publisher': self.publisher,
+            'publisher_url': self.publisher_url,
+            'publishing_url': self.publishing_url,
+            'publishing_year': self.publishing_year,
+            'pipeline_name': self.pipeline_name,
+            'is_public': self.is_public,
+            'title': self.title,
+            'version': self.version,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/tesseract_ocr_pipeline_model.py

@@ -0,0 +1,132 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from tqdm import tqdm
+from pathlib import Path
+import requests
+import yaml
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn
+from .file_mixin import FileMixin
+from .user import User
+
+
+class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'tesseract_ocr_pipeline_models'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    title = db.Column(db.String(64))
+    description = db.Column(db.String(255))
+    version = db.Column(db.String(16))
+    compatible_service_versions = db.Column(ContainerColumn(list, 255))
+    publisher = db.Column(db.String(128))
+    publisher_url = db.Column(db.String(512))
+    publishing_url = db.Column(db.String(512))
+    publishing_year = db.Column(db.Integer)
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    user = db.relationship('User', back_populates='tesseract_ocr_pipeline_models')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'tesseract_ocr_pipeline_models' / f'{self.id}'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/tesseract_ocr_pipeline_models/{self.hashid}'
+
+    @property
+    def url(self):
+        return url_for(
+            'contributions.tesseract_ocr_pipeline_model',
+            tesseract_ocr_pipeline_model_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def insert_defaults(force_download=False):
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        default_records_file = Path(__file__).parent / 'default_records' / 'tesseract_ocr_pipeline_model.yml'
+        with default_records_file.open('r') as f:
+            default_records = yaml.safe_load(f)
+        for m in default_records:
+            model = TesseractOCRPipelineModel.query.filter_by(title=m['title'], version=m['version']).first()  # noqa
+            if model is not None:
+                model.compatible_service_versions = m['compatible_service_versions']
+                model.description = m['description']
+                model.filename = f'{model.id}.traineddata'
+                model.publisher = m['publisher']
+                model.publisher_url = m['publisher_url']
+                model.publishing_url = m['publishing_url']
+                model.publishing_year = m['publishing_year']
+                model.is_public = True
+                model.title = m['title']
+                model.version = m['version']
+            else:
+                model = TesseractOCRPipelineModel(
+                    compatible_service_versions=m['compatible_service_versions'],
+                    description=m['description'],
+                    publisher=m['publisher'],
+                    publisher_url=m['publisher_url'],
+                    publishing_url=m['publishing_url'],
+                    publishing_year=m['publishing_year'],
+                    is_public=True,
+                    title=m['title'],
+                    user=nopaque_user,
+                    version=m['version']
+                )
+                db.session.add(model)
+                db.session.flush(objects=[model])
+                db.session.refresh(model)
+                model.filename = f'{model.id}.traineddata'
+            if not model.path.exists() or force_download:
+                r = requests.get(m['url'], stream=True)
+                pbar = tqdm(
+                    desc=f'{model.title} ({model.filename})',
+                    unit="B",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                    total=int(r.headers['Content-Length'])
+                )
+                pbar.clear()
+                with open(model.path, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=1024):
+                        if chunk:  # filter out keep-alive new chunks
+                            pbar.update(len(chunk))
+                            f.write(chunk)
+                    pbar.close()
+        db.session.commit()
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'compatible_service_versions': self.compatible_service_versions,
+            'description': self.description,
+            'publisher': self.publisher,
+            'publisher_url': self.publisher_url,
+            'publishing_url': self.publishing_url,
+            'publishing_year': self.publishing_year,
+            'is_public': self.is_public,
+            'title': self.title,
+            'version': self.version,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/token.py

@@ -0,0 +1,48 @@
+from datetime import datetime, timedelta
+from app import db
+
+
+class Token(db.Model):
+    __tablename__ = 'tokens'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    access_token = db.Column(db.String(64), index=True)
+    access_expiration = db.Column(db.DateTime)
+    refresh_token = db.Column(db.String(64), index=True)
+    refresh_expiration = db.Column(db.DateTime)
+    # Relationships
+    user = db.relationship('User', back_populates='tokens')
+
+    def expire(self):
+        self.access_expiration = datetime.utcnow()
+        self.refresh_expiration = datetime.utcnow()
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'access_token': self.access_token,
+            'access_expiration': (
+                None if self.access_expiration is None
+                else f'{self.access_expiration.isoformat()}Z'
+            ),
+            'refresh_token': self.refresh_token,
+            'refresh_expiration': (
+                None if self.refresh_expiration is None
+                else f'{self.refresh_expiration.isoformat()}Z'
+            )
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable
+
+    @staticmethod
+    def clean():
+        """Remove any tokens that have been expired for more than a day."""
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        Token.query.filter(Token.refresh_expiration < yesterday).delete()

app/models/user.py

@@ -0,0 +1,452 @@
+from datetime import datetime, timedelta
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from flask_login import UserMixin
+from sqlalchemy.ext.associationproxy import association_proxy
+from pathlib import Path
+from typing import Union
+from werkzeug.security import generate_password_hash, check_password_hash
+import jwt
+import re
+import secrets
+import shutil
+from app import db, hashids
+from app.ext.flask_sqlalchemy import IntEnumColumn
+from .corpus import Corpus
+from .corpus_follower_association import CorpusFollowerAssociation
+from .corpus_follower_role import CorpusFollowerRole
+from .role import Permission, Role
+from .token import Token
+
+
+class ProfilePrivacySettings(IntEnum):
+    SHOW_EMAIL = 1
+    SHOW_LAST_SEEN = 2
+    SHOW_MEMBER_SINCE = 4
+
+    @staticmethod
+    def get(profile_privacy_setting: Union['ProfilePrivacySettings', int, str]) -> 'ProfilePrivacySettings':
+        if isinstance(profile_privacy_setting, ProfilePrivacySettings):
+            return profile_privacy_setting
+        if isinstance(profile_privacy_setting, int):
+            return ProfilePrivacySettings(profile_privacy_setting)
+        if isinstance(profile_privacy_setting, str):
+            return ProfilePrivacySettings[profile_privacy_setting]
+        raise TypeError('profile_privacy_setting must be ProfilePrivacySettings, int, or str')
+
+
+class UserSettingJobStatusMailNotificationLevel(IntEnum):
+    NONE = 1
+    END = 2
+    ALL = 3
+
+
+class User(HashidMixin, UserMixin, db.Model):
+    __tablename__ = 'users'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
+    # Fields
+    email = db.Column(db.String(254), index=True, unique=True)
+    username = db.Column(db.String(64), index=True, unique=True)
+    username_pattern = re.compile(r'^[A-Za-zÄÖÜäöüß0-9_.]*$')
+    password_hash = db.Column(db.String(128))
+    confirmed = db.Column(db.Boolean, default=False)
+    terms_of_use_accepted = db.Column(db.Boolean, default=False)
+    member_since = db.Column(db.DateTime(), default=datetime.utcnow)
+    setting_job_status_mail_notification_level = db.Column(
+        IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
+        default=UserSettingJobStatusMailNotificationLevel.END
+    )
+    last_seen = db.Column(db.DateTime())
+    full_name = db.Column(db.String(64))
+    about_me = db.Column(db.String(256))
+    location = db.Column(db.String(64))
+    website = db.Column(db.String(128))
+    organization = db.Column(db.String(128))
+    is_public = db.Column(db.Boolean, default=False)
+    profile_privacy_settings = db.Column(db.Integer(), default=0)
+    # Relationships
+    avatar = db.relationship(
+        'Avatar',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        uselist=False
+    )
+    corpora = db.relationship(
+        'Corpus',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='follower',
+        cascade='all, delete-orphan'
+    )
+    followed_corpora = association_proxy(
+        'corpus_follower_associations',
+        'corpus',
+        creator=lambda c: CorpusFollowerAssociation(corpus=c)
+    )
+    jobs = db.relationship(
+        'Job',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    role = db.relationship(
+        'Role',
+        back_populates='users'
+    )
+    spacy_nlp_pipeline_models = db.relationship(
+        'SpaCyNLPPipelineModel',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    tesseract_ocr_pipeline_models = db.relationship(
+        'TesseractOCRPipelineModel',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    tokens = db.relationship(
+        'Token',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = (
+                Role.query.filter_by(name='Administrator').first()
+                if kwargs['email'] == current_app.config['NOPAQUE_ADMIN']
+                else Role.query.filter_by(default=True).first()
+            )
+        super().__init__(**kwargs)
+
+    def __repr__(self):
+        return f'<User {self.username}>'
+
+    @property
+    def jsonpatch_path(self):
+        return f'/users/{self.hashid}'
+
+    @property
+    def password(self):
+        raise AttributeError('password is not a readable attribute')
+
+    @password.setter
+    def password(self, password):
+        self.password_hash = generate_password_hash(password)
+
+    @property
+    def path(self) -> Path:
+        return current_app.config.get('NOPAQUE_DATA_DIR') / 'users' / f'{self.id}'
+
+    @staticmethod
+    def create(**kwargs):
+        user = User(**kwargs)
+        db.session.add(user)
+        db.session.flush(objects=[user])
+        db.session.refresh(user)
+        user_spacy_nlp_pipeline_models_dir = user.path / 'spacy_nlp_pipeline_models'
+        user_tesseract_ocr_pipeline_models_dir = user.path / 'tesseract_ocr_pipeline_models'
+        user_corpora_dir = user.path / 'corpora'
+        user_jobs_dir = user.path / 'jobs'
+        try:
+            user.path.mkdir()
+            user_spacy_nlp_pipeline_models_dir.mkdir()
+            user_tesseract_ocr_pipeline_models_dir.mkdir()
+            user_corpora_dir.mkdir()
+            user_jobs_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return user
+
+    @staticmethod
+    def insert_defaults():
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        system_user_role = Role.query.filter_by(name='System user').first()
+        if nopaque_user is None:
+            nopaque_user = User.create(
+                username='nopaque',
+                role=system_user_role
+            )
+            db.session.add(nopaque_user)
+        elif nopaque_user.role != system_user_role:
+            nopaque_user.role = system_user_role
+        db.session.commit()
+
+    @staticmethod
+    def reset_password(token, new_password):
+        try:
+            payload = jwt.decode(
+                token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'User.reset_password':
+            return False
+        user_hashid = payload.get('sub')
+        user_id = hashids.decode(user_hashid)
+        user = User.query.get(user_id)
+        if user is None:
+            return False
+        user.password = new_password
+        db.session.add(user)
+        return True
+
+    @staticmethod
+    def verify_access_token(access_token, refresh_token=None):
+        token = Token.query.filter(Token.access_token == access_token).first()
+        if token is not None:
+            if token.access_expiration > datetime.utcnow():
+                token.user.ping()
+                db.session.commit()
+                if token.user.role.name != 'System user':
+                    return token.user
+
+    @staticmethod
+    def verify_refresh_token(refresh_token, access_token):
+        token = Token.query.filter((Token.refresh_token == refresh_token) & (Token.access_token == access_token)).first()
+        if token is not None:
+            if token.refresh_expiration > datetime.utcnow():
+                return token
+            # someone tried to refresh with an expired token
+            # revoke all tokens from this user as a precaution
+            token.user.revoke_auth_tokens()
+            db.session.commit()
+
+    def can(self, permission):
+        return self.role is not None and self.role.has_permission(permission)
+
+    def confirm(self, confirmation_token):
+        try:
+            payload = jwt.decode(
+                confirmation_token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'user.confirm':
+            return False
+        if payload.get('sub') != self.hashid:
+            return False
+        self.confirmed = True
+        db.session.add(self)
+        return True
+
+    def delete(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        db.session.delete(self)
+
+    def generate_auth_token(self):
+        return Token(
+            access_token=secrets.token_urlsafe(),
+            access_expiration=datetime.utcnow() + timedelta(minutes=15),
+            refresh_token=secrets.token_urlsafe(),
+            refresh_expiration=datetime.utcnow() + timedelta(days=7),
+            user=self
+        )
+
+    def generate_confirm_token(self, expiration=3600):
+        now = datetime.utcnow()
+        payload = {
+            'exp': now + timedelta(seconds=expiration),
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'user.confirm',
+            'sub': self.hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+
+    def generate_reset_password_token(self, expiration=3600):
+        now = datetime.utcnow()
+        payload = {
+            'exp': now + timedelta(seconds=expiration),
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'User.reset_password',
+            'sub': self.hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+
+    def is_administrator(self):
+        return self.can(Permission.ADMINISTRATE)
+
+    def ping(self):
+        self.last_seen = datetime.utcnow()
+
+    def revoke_auth_tokens(self):
+        for token in self.tokens:
+            db.session.delete(token)
+
+    def verify_password(self, password):
+        if self.role.name == 'System user':
+            return False
+        return check_password_hash(self.password_hash, password)
+
+    #region Profile Privacy settings
+    def has_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        return self.profile_privacy_settings & s.value == s.value
+    
+    def add_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        if not self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings += s.value
+
+    def remove_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        if self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings -= s.value
+
+    def reset_profile_privacy_settings(self):
+        self.profile_privacy_settings = 0
+    #endregion Profile Privacy settings
+
+    def follow_corpus(self, corpus, role=None):
+        if role is None:
+            cfr = CorpusFollowerRole.query.filter_by(default=True).first()
+        else:
+            cfr = role
+        if self.is_following_corpus(corpus):
+            cfa = CorpusFollowerAssociation.query.filter_by(corpus=corpus, follower=self).first()
+            if cfa.role != cfr:
+                cfa.role = cfr
+        else:
+            cfa = CorpusFollowerAssociation(corpus=corpus, role=cfr, follower=self)
+            db.session.add(cfa)
+
+    def unfollow_corpus(self, corpus):
+        if not self.is_following_corpus(corpus):
+            return
+        self.followed_corpora.remove(corpus)
+
+    def is_following_corpus(self, corpus):
+        return corpus in self.followed_corpora
+    
+    def generate_follow_corpus_token(self, corpus_hashid, role_name, expiration=7):
+        now = datetime.utcnow()
+        payload = {
+            'exp': expiration,
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'User.follow_corpus',
+            'role_name': role_name,
+            'sub': corpus_hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+    
+    def follow_corpus_by_token(self, token):
+        try:
+            payload = jwt.decode(
+                token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'role_name', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'User.follow_corpus':
+            return False
+        corpus_hashid = payload.get('sub')
+        corpus_id = hashids.decode(corpus_hashid)
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if corpus is None:
+            return False
+        role_name = payload.get('role_name')
+        role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+        if role is None:
+            return False
+        self.follow_corpus(corpus, role)
+        # db.session.add(self)
+        return True
+
+    def to_json_serializeable(self, backrefs=False, relationships=False, filter_by_privacy_settings=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'confirmed': self.confirmed,
+            'avatar': url_for('users.user_avatar', user_id=self.id),
+            'email': self.email,
+            'last_seen': (
+                None if self.last_seen is None
+                else f'{self.last_seen.isoformat()}Z'
+            ),
+            'member_since': f'{self.member_since.isoformat()}Z',
+            'username': self.username,
+            'full_name': self.full_name,
+            'about_me': self.about_me,
+            'website': self.website,
+            'location': self.location,
+            'organization': self.organization,
+            'job_status_mail_notification_level': \
+                    self.setting_job_status_mail_notification_level.name,
+            'profile_privacy_settings': {
+                'is_public': self.is_public,
+                'show_email': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL),
+                'show_last_seen': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN),
+                'show_member_since': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE)
+            }
+        }
+        if backrefs:
+            json_serializeable['role'] = \
+                self.role.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
+            json_serializeable['corpora'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpora
+            }
+            json_serializeable['jobs'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.jobs
+            }
+            json_serializeable['tesseract_ocr_pipeline_models'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.tesseract_ocr_pipeline_models
+            }
+            json_serializeable['spacy_nlp_pipeline_models'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.spacy_nlp_pipeline_models
+            }
+
+        if filter_by_privacy_settings:
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL):
+                json_serializeable.pop('email')
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN):
+                json_serializeable.pop('last_seen')
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE):
+                json_serializeable.pop('member_since')
+        return json_serializeable

app/services/__init__.py

@@ -1,12 +1,11 @@
 from flask import Blueprint
 from flask_login import login_required
-import os
+from pathlib import Path
 import yaml
 
 
-services_file = \
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'services.yml')
-with open(services_file, 'r') as f:
+services_file = Path(__file__).parent / 'services.yml'
+with services_file.open('r') as f:
     SERVICES = yaml.safe_load(f)
 
 bp = Blueprint('services', __name__)

app/services/routes.py

@@ -6,6 +6,7 @@ from app import db, hashids
 from app.models import (
     Job,
     JobInput,
+    JobResult,
     JobStatus,
     TesseractOCRPipelineModel,
     SpaCyNLPPipelineModel
@@ -74,6 +75,8 @@ def tesseract_ocr_pipeline():
     version = request.args.get('version', service_manifest['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
+    job_results = JobResult.query.all()
+    choosable_job_ids = [job_result.job.hashid for job_result in job_results if job_result.job.service == "file-setup-pipeline" and job_result.filename.endswith('.pdf')]
     form = CreateTesseractOCRPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
@@ -111,6 +114,7 @@ def tesseract_ocr_pipeline():
     return render_template(
         'services/tesseract_ocr_pipeline.html.j2',
         title=service_manifest['name'],
+        choosable_job_ids=choosable_job_ids,
         form=form,
         tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models,
         user_tesseract_ocr_pipeline_models_count=user_tesseract_ocr_pipeline_models_count

app/services/services.yml

@@ -10,7 +10,7 @@ file-setup-pipeline:
 tesseract-ocr-pipeline:
   name: 'Tesseract OCR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.1'
+  latest_version: '0.1.2'
   versions:
     0.1.0:
       methods:
@@ -23,6 +23,12 @@ tesseract-ocr-pipeline:
         - 'ocropus_nlbin_threshold'
       publishing_year: 2022
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
+    0.1.2:
+      methods:
+        - 'binarization'
+        - 'ocropus_nlbin_threshold'
+      publishing_year: 2023
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.2'
 transkribus-htr-pipeline:
   name: 'Transkribus HTR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
@@ -41,7 +47,7 @@ transkribus-htr-pipeline:
 spacy-nlp-pipeline:
   name: 'SpaCy NLP Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.2'
+  latest_version: '0.1.1'
   versions:
     0.1.0:
       methods:
@@ -56,5 +62,5 @@ spacy-nlp-pipeline:
     0.1.2:
       methods:
         - 'encoding_detection'
-      publishing_year: 2022
+      publishing_year: 2024
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2'

app/static/js/App.js

@@ -1,104 +0,0 @@
-class App {
-  constructor() {
-    this.data = {
-      promises: {getUser: {}, subscribeUser: {}},
-      users: {},
-    };
-    this.socket = io({transports: ['websocket'], upgrade: false});
-    this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
-  }
-
-  getUser(userId) {
-    if (userId in this.data.promises.getUser) {
-      return this.data.promises.getUser[userId];
-    }
-
-    this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('GET /users/<user_id>', userId, (response) => {
-        if (response.status === 200) {
-          this.data.users[userId] = response.body;
-          resolve(this.data.users[userId]);
-        } else {
-          reject(`[${response.status}] ${response.statusText}`);
-        }
-      });
-    });
-
-    return this.data.promises.getUser[userId];
-  }
-
-  subscribeUser(userId) {
-    if (userId in this.data.promises.subscribeUser) {
-      return this.data.promises.subscribeUser[userId];
-    }
-
-    this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
-      this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
-        if (response.status !== 200) {
-          reject(response);
-          return;
-        }
-        resolve(response);
-      });
-    });
-
-    return this.data.promises.subscribeUser[userId];
-  }
-
-  flash(message, category) {
-    let iconPrefix = '';
-    switch (category) {
-      case 'corpus': {
-        iconPrefix = '<i class="left material-icons">book</i>';
-        break;
-      }
-      case 'error': {
-        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
-        break;
-      }
-      case 'job': {
-        iconPrefix = '<i class="left nopaque-icons">J</i>';
-        break;
-      }
-      case 'settings': {
-        iconPrefix = '<i class="left material-icons">settings</i>';
-        break;
-      }
-      default: {
-        iconPrefix = '<i class="left material-icons">notifications</i>';
-        break;
-      }
-    }
-    let toast = M.toast(
-      {
-        html: `
-          <span>${iconPrefix}${message}</span>
-          <button class="action-button btn-flat toast-action white-text" data-action="close">
-            <i class="material-icons">close</i>
-          </button>
-        `.trim()
-      }
-    );
-    let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
-    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
-  }
-  
-  onPatch(patch) {
-    // Filter Patch to only include operations on users that are initialized
-    let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
-    let filteredPatch = patch.filter(operation => regExp.test(operation.path));
-
-    // Handle job status updates
-    let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
-    let subFilteredPatch = filteredPatch
-      .filter((operation) => {return operation.op === 'replace';})
-      .filter((operation) => {return subRegExp.test(operation.path);});
-    for (let operation of subFilteredPatch) {
-      let [match, userId, jobId] = operation.path.match(subRegExp);
-      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-status="${operation.value}"></span>`, 'job');
-    }
-
-    // Apply Patch
-    jsonpatch.applyPatch(this.data, filteredPatch);
-  }
-}

app/static/js/CorpusAnalysis/QueryBuilder.js

@@ -1,41 +0,0 @@
-class ConcordanceQueryBuilder {
-
-  constructor() {
-    
-    this.elements = new ElementReferencesQueryBuilder();
-    this.generalFunctions = new GeneralFunctionsQueryBuilder(this.elements);
-    this.tokenAttributeBuilderFunctions = new TokenAttributeBuilderFunctionsQueryBuilder(this.elements);
-    this.structuralAttributeBuilderFunctions = new StructuralAttributeBuilderFunctionsQueryBuilder(this.elements);
-
-    // Eventlisteners for the incidence modifiers. There are two different types of incidence modifiers: token and character incidence modifiers.
-    document.querySelectorAll('.incidence-modifier-selection').forEach(button => {
-      let dropdownId = button.parentNode.parentNode.id;
-      if (dropdownId === 'corpus-analysis-concordance-token-incidence-modifiers-dropdown') {
-        button.addEventListener('click', () => this.generalFunctions.tokenIncidenceModifierHandler(button.dataset.token, button.innerHTML));
-      } else if (dropdownId === 'corpus-analysis-concordance-character-incidence-modifiers-dropdown') {
-        button.addEventListener('click', () => this.tokenAttributeBuilderFunctions.characterIncidenceModifierHandler(button));
-      }
-    });
-    
-    // Eventlisteners for the submit of n- and m-values of the incidence modifier modal for "exactly n" or "between n and m".
-    document.querySelectorAll('.n-m-submit-button').forEach(button => {
-      let modalId = button.dataset.modalId;
-      if (modalId === 'corpus-analysis-concordance-exactly-n-token-modal' || modalId === 'corpus-analysis-concordance-between-nm-token-modal') {
-        button.addEventListener('click', () => this.generalFunctions.tokenNMSubmitHandler(modalId));
-      } else if (modalId === 'corpus-analysis-concordance-exactly-n-character-modal' || modalId === 'corpus-analysis-concordance-between-nm-character-modal') {
-        button.addEventListener('click', () => this.tokenAttributeBuilderFunctions.characterNMSubmitHandler(modalId));
-      }
-    });
-
-    document.querySelector('#corpus-analysis-concordance-text-annotation-submit').addEventListener('click', () => this.structuralAttributeBuilderFunctions.textAnnotationSubmitHandler());
-
-    this.elements.positionalAttrModal = M.Modal.init(
-      document.querySelector('#corpus-analysis-concordance-positional-attr-modal'), 
-      {
-        onOpenStart: () => {
-          this.tokenAttributeBuilderFunctions.optionToggleHandler();
-        }
-      }
-    );
-  }
-}

app/static/js/CorpusAnalysis/QueryBuilder/GeneralFunctionsQueryBuilder.js

@@ -1,364 +0,0 @@
-class GeneralFunctionsQueryBuilder {
-  constructor(elements) {
-    this.elements = elements;
-  }
-
-  toggleClass(elements, className, action){
-    elements.forEach(element => {
-      document.querySelector('[data-toggle-area="' + element + '"]').classList[action](className);
-    });
-  }
-
-  resetQueryInputField() {
-    this.elements.queryInputField.innerHTML = '';
-    this.addPlaceholder();
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  updateChipList() {
-    this.elements.queryChipElements = this.elements.queryInputField.querySelectorAll('.query-component');
-  }
-
-  removePlaceholder() {
-    let placeholder = this.elements.queryInputField.querySelector('#corpus-analysis-concordance-query-builder-input-field-placeholder');  
-    if (placeholder) {
-      this.elements.queryInputField.innerHTML = '';
-    }
-  }
-
-  addPlaceholder() {
-    let placeholder = Utils.HTMLToElement('<span id="corpus-analysis-concordance-query-builder-input-field-placeholder">Click on a button to add a query component</span>');
-    this.elements.queryInputField.appendChild(placeholder);
-  }
-  
-  resetMaterializeSelection(selectionElements, value = "default") {
-    selectionElements.forEach(selectionElement => {
-      selectionElement.querySelector(`option[value=${value}]`).selected = true;
-      let instance = M.FormSelect.getInstance(selectionElement);
-      instance.destroy();
-      M.FormSelect.init(selectionElement);
-    })
-  }
-
-
-  queryChipFactory(dataType, prettyQueryText, queryText, index = null, isClosingTag = false) {
-    // Creates a new query chip element, adds Eventlisteners for selection, deletion and drag and drop and appends it to the query input field.
-    
-    queryText = Utils.escape(queryText);
-    prettyQueryText = Utils.escape(prettyQueryText);
-    let queryChipElement = Utils.HTMLToElement(
-      `
-        <span class="chip query-component" data-type="${dataType}" data-query="${queryText}" draggable="true" data-closing-tag="${isClosingTag}">
-          ${prettyQueryText}
-          ${isClosingTag ? '<i class="material-icons" style="padding-top:5px; font-size:20px; cursor:pointer;">lock_open</i>' : '<i class="material-icons close">close</i>'}
-        </span>
-      `
-    );
-    this.actionListeners(queryChipElement, isClosingTag);
-    queryChipElement.addEventListener('dragstart', this.handleDragStart.bind(this, queryChipElement));
-    queryChipElement.addEventListener('dragend', this.handleDragEnd);
-
-    // Ensures that metadata is always at the end of the query and if an index is given, inserts the query chip at the given index and if there is a closing tag, inserts the query chip before the closing tag.
-    this.removePlaceholder();
-    let lastChild = this.elements.queryInputField.lastChild;
-    let isLastChildTextAnnotation = lastChild && lastChild.dataset.type === 'text-annotation';
-    if (!index) {
-      let closingTagElement = this.elements.queryInputField.querySelector('[data-closing-tag="true"]');
-      if (closingTagElement) {
-        index = Array.from(this.elements.queryInputField.children).indexOf(closingTagElement);
-      }
-    }
-    if (index || isLastChildTextAnnotation) {
-      let insertingElement = isLastChildTextAnnotation ? lastChild : this.elements.queryChipElements[index];
-      this.elements.queryInputField.insertBefore(queryChipElement, insertingElement);
-    } else {
-      this.elements.queryInputField.appendChild(queryChipElement);
-    }
-
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  actionListeners(queryChipElement, isClosingTag) {
-    let notQuantifiableDataTypes = ['start-sentence', 'end-sentence', 'start-entity', 'start-empty-entity', 'end-entity', 'token-incidence-modifier'];
-    queryChipElement.addEventListener('click', (event) => {
-      if (event.target.classList.contains('chip')) {
-        if (!notQuantifiableDataTypes.includes(queryChipElement.dataset.type)) {
-          this.selectChipElement(queryChipElement);
-        }
-      }
-    });
-    queryChipElement.querySelector('i').addEventListener('click', () => {
-      if (isClosingTag) {
-        this.lockClosingChipElement(queryChipElement);
-      } else {
-        this.deleteChipElement(queryChipElement);
-      }
-    });
-  }
-
-  lockClosingChipElement(queryChipElement) {
-    let chipIndex = Array.from(this.elements.queryInputField.children).indexOf(queryChipElement);
-    this.queryChipFactory(queryChipElement.dataset.type, queryChipElement.firstChild.textContent, queryChipElement.dataset.query, chipIndex+1);
-    this.deleteChipElement(queryChipElement);
-    this.updateChipList();
-  }
-
-  handleDragStart(queryChipElement, event) {
-    // is called when a query chip is dragged. It creates a dropzone (in form of a chip) for the dragged chip and adds it to the query input field.
-    let queryChips = this.elements.queryInputField.querySelectorAll('.query-component');
-    setTimeout(() => {
-      let targetChipElement = Utils.HTMLToElement('<span class="chip drop-target">Drop here</span>');
-      for (let element of queryChips) {
-        if (element === queryChipElement.nextSibling) {continue;}
-        let targetChipClone = targetChipElement.cloneNode(true);
-        if (element === queryChipElement && queryChips[queryChips.length - 1] !== element) {
-          queryChips[queryChips.length - 1].insertAdjacentElement('afterend', targetChipClone);
-        } else {
-          element.insertAdjacentElement('beforebegin', targetChipClone);
-        }
-        this.addDragDropListeners(targetChipClone, queryChipElement);
-      }
-    }, 0);
-  }
-
-  handleDragEnd(event) {
-    document.querySelectorAll('.drop-target').forEach(target => target.remove());
-  }
-
-  addDragDropListeners(targetChipClone, queryChipElement) {
-    targetChipClone.addEventListener('dragover', (event) => {
-      event.preventDefault();
-    });
-    targetChipClone.addEventListener('dragenter', (event) => {
-      event.preventDefault();
-      event.target.style.borderStyle = 'solid dotted';
-    });
-    targetChipClone.addEventListener('dragleave', (event) => {
-      event.preventDefault();
-      event.target.style.borderStyle = 'hidden';
-    });
-    targetChipClone.addEventListener('drop', (event) => {
-      let dropzone = event.target;
-      dropzone.parentElement.replaceChild(queryChipElement, dropzone);
-      this.updateChipList();
-      this.queryPreviewBuilder();
-    });
-  }
-
-  queryPreviewBuilder() {
-    // Builds the query preview in the form of pure CQL and displays it in the query preview field.
-    let queryPreview = document.querySelector('#corpus-analysis-concordance-query-preview');
-    let queryInputFieldContent = [];
-    this.elements.queryChipElements.forEach(element => {
-      let queryElement = element.dataset.query;
-      if (queryElement !== undefined) {
-        queryElement = Utils.escape(queryElement);
-      }
-      queryInputFieldContent.push(queryElement);
-    });
-  
-    let queryString = queryInputFieldContent.join(' ');
-    let replacements = {
-      ' +': '+',
-      ' *': '*',
-      ' ?': '?',
-      ' {': '{'
-    };
-
-    for (let key in replacements) {
-      queryString = queryString.replace(key, replacements[key]);
-    }
-    queryString += ';';
-  
-    queryPreview.innerHTML = queryString;
-    queryPreview.parentNode.classList.toggle('hide', queryString === ';');
-  }
-
-  deleteChipElement(attr) {
-    if (attr.dataset.type === "start-sentence") {
-      this.elements.sentenceElement.innerHTML = 'Sentence';
-    } else if (attr.dataset.type === "start-entity" || attr.dataset.type === "start-empty-entity") {
-      this.elements.entityElement.innerHTML = 'Entity';
-    }
-    this.elements.queryInputField.removeChild(attr);
-    if (this.elements.queryInputField.children.length === 0) {
-      this.addPlaceholder();
-    }
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  selectChipElement(attr) {
-    document.querySelectorAll('.chip.teal').forEach(element => {
-        if (element !== attr) {
-          element.classList.remove('teal', 'lighten-2');
-          this.toggleClass(['token-incidence-modifiers'], 'disabled', 'add');
-        }
-    });
-
-    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'toggle');
-    attr.classList.toggle('teal');
-    attr.classList.toggle('lighten-5');
-  }
-
-  tokenIncidenceModifierHandler(incidenceModifier, incidenceModifierPretty) {
-    // Adds a token incidence modifier to the query input field.
-    let selectedChip = this.elements.queryInputField.querySelector('.chip.teal');
-    let selectedChipIndex = Array.from(this.elements.queryInputField.children).indexOf(selectedChip);
-    this.queryChipFactory('token-incidence-modifier', incidenceModifierPretty, incidenceModifier, selectedChipIndex+1);
-    this.selectChipElement(selectedChip);
-  }
-
-  tokenNMSubmitHandler(modalId) {
-    // Adds a token incidence modifier (exactly n or between n and m) to the query input field.
-    let modal = document.querySelector(`#${modalId}`);
-    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
-    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
-    input_m = input_m !== undefined ? input_m.value : '';
-    let input = `{${input_n}${input_m !== '' ? ',' : ''}${input_m}}`;
-    let pretty_input = `between ${input_n} and ${input_m} (${input})`;
-    if (input_m === '') {
-     pretty_input = `exactly ${input_n} (${input})`;
-    }
-
-    let instance = M.Modal.getInstance(modal);
-    instance.close();
-
-    this.tokenIncidenceModifierHandler(input, pretty_input);
-  }
-
-  switchToExpertModeParser() {
-    let expertModeInputField = document.querySelector('#corpus-analysis-concordance-form-query');
-    expertModeInputField.value = '';
-    let queryBuilderInputFieldValue = Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
-    if (queryBuilderInputFieldValue !== "" && queryBuilderInputFieldValue !== ";") {
-      expertModeInputField.value = queryBuilderInputFieldValue;
-    }
-  }
-
-  switchToQueryBuilderParser() {
-    this.resetQueryInputField();
-    let expertModeInputFieldValue = document.querySelector('#corpus-analysis-concordance-form-query').value;
-    let chipElements = this.parseTextToChip(expertModeInputFieldValue);
-    for (let chipElement of chipElements) {
-      this.queryChipFactory(chipElement['type'], chipElement['pretty'], chipElement['query']);
-    }
-  }
-  
-
-  parseTextToChip(query) {
-    const parsingElementDict = {
-      '<s>': {
-        pretty: 'Sentence Start',
-        type: 'start-sentence'
-      },
-      '<\/s>': {
-        pretty: 'Sentence End',
-        type: 'end-sentence'
-      },
-      '<ent>': {
-        pretty: 'Entity Start',
-        type: 'start-empty-entity'
-      },
-      '<ent_type="([A-Z]+)">': {
-        pretty: '',
-        type: 'start-entity'
-      },
-      '<\\\/ent(_type)?>': {
-        pretty: 'Entity End',
-        type: 'end-entity'
-      },
-      ':: ?match\\.text_[A-Za-z]+="[^"]+"': {
-        pretty: '',
-        type: 'text-annotation'
-      },
-      '\\[(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?)*\\]': {
-        pretty: '',
-        type: 'token'
-      },
-      '\\[\\]': {
-        pretty: 'Empty Token',
-        type: 'token'
-      },
-      '(?<!\\[) ?\\+ ?(?![^\\]]\\])': {
-        pretty: ' one or more (+)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\* ?(?![^\\]]\\])': {
-        pretty: 'zero or more (*)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\? ?(?![^\\]]\\])': {
-        pretty: 'zero or one (?)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])': {
-        pretty: '',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])': {
-        pretty: '',
-        type: 'token-incidence-modifier'
-      }
-    }
-  
-    let chipElements = [];
-    let regexPattern = Object.keys(parsingElementDict).map(pattern => `(${pattern})`).join('|');
-    const regex = new RegExp(regexPattern, 'gi');
-    let match;
-  
-    while ((match = regex.exec(query)) !== null) {
-      // This is necessary to avoid infinite loops with zero-width matches
-      if (match.index === regex.lastIndex) {
-        regex.lastIndex++;
-      }
-      let stringElement = match[0];
-      for (let [pattern, chipElement] of Object.entries(parsingElementDict)) {
-        const parsingRegex = new RegExp(pattern, 'gi');
-        if (parsingRegex.exec(stringElement)) {
-          // Creating the pretty text for the chip element
-          let prettyText;
-          switch (pattern) {
-            case '<ent_type="([A-Z]+)">':
-              prettyText = `Entity Type=${stringElement.replace(/<ent_type="|">/g, '')}`;
-              break;
-            case ':: ?match\\.text_[A-Za-z]+="[^"]+"':
-              prettyText = stringElement.replace(/:: ?match\.text_|"|"/g, '');
-              break;
-            case '\\[(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?)*\\]':
-              let doubleQuotes = /(word|lemma|pos|simple_pos)="[^"]+"/gi;
-              let singleQuotes = /(word|lemma|pos|simple_pos)='[^']+'/gi;
-              if (doubleQuotes.exec(stringElement)) {
-                prettyText = stringElement.replace(/^\[|\]$|"/g, '');
-              } else if (singleQuotes.exec(stringElement)) {
-                prettyText = stringElement.replace(/^\[|\]$|'/g, '');
-              }
-              prettyText = prettyText.replace(/\&/g, ' and ').replace(/\|/g, ' or ');
-              break;
-            case '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])':
-              prettyText = `exactly ${stringElement.replace(/{|}/g, '')} (${stringElement})`;
-              break;
-            case '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])':
-              prettyText = `between${stringElement.replace(/{|}/g, ' ').replace(',', ' and ')}(${stringElement})`;
-              break;
-            default:
-              prettyText = chipElement.pretty;
-              break;
-          }
-          chipElements.push({
-            type: chipElement.type,
-            pretty: prettyText,
-            query: stringElement
-          });
-          break;
-        }
-      }
-    }
-  
-    return chipElements;
-  }
-
-}
-

app/static/js/CorpusAnalysis/QueryBuilder/StructuralAttributeBuilderFunctionsQueryBuilder.js

@@ -1,75 +0,0 @@
-class StructuralAttributeBuilderFunctionsQueryBuilder extends GeneralFunctionsQueryBuilder {
-  constructor(elements) {
-    super(elements);
-    this.elements = elements;
-
-    document.querySelectorAll('[data-structural-attr-modal-action-button]').forEach(button => {
-      button.addEventListener('click', () => {
-        this.actionButtonInStrucAttrModalHandler(button.dataset.structuralAttrModalActionButton);
-      });
-    });
-    document.querySelector('.ent-type-selection-action[data-ent-type="any"]').addEventListener('click', () => {
-      this.queryChipFactory('start-empty-entity', 'Entity Start', '<ent>');
-      this.queryChipFactory('end-entity', 'Entity End', '</ent>', null, true);
-      this.resetAndCloseStructuralAttrModal();
-    });
-    document.querySelector('.ent-type-selection-action[data-ent-type="english"]').addEventListener('change', (event) => {
-      this.queryChipFactory('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`);
-      this.queryChipFactory('end-entity', 'Entity End', '</ent_type>', null, true);
-      this.resetAndCloseStructuralAttrModal();
-    });
-  }
-
-  actionButtonInStrucAttrModalHandler(action) {
-    switch (action) {
-      case 'sentence':
-        this.queryChipFactory('start-sentence', 'Sentence Start', '<s>');
-        this.queryChipFactory('end-sentence', 'Sentence End', '</s>', null, true);
-        this.resetAndCloseStructuralAttrModal();
-        break;
-      case 'entity':
-        this.toggleClass(['entity-builder'], 'hide', 'toggle');
-        this.toggleClass(['text-annotation-builder'], 'hide', 'add');
-        break;
-      case 'meta-data':
-        this.toggleClass(['text-annotation-builder'], 'hide', 'toggle');
-        this.toggleClass(['entity-builder'], 'hide', 'add');
-        break;
-      default:
-        break;
-    }
-  }
-
-  textAnnotationSubmitHandler() {
-    let noValueMetadataMessage = document.querySelector('#corpus-analysis-concordance-no-value-metadata-message');
-    let textAnnotationSubmit = document.querySelector('#corpus-analysis-concordance-text-annotation-submit');
-    let textAnnotationInput = document.querySelector('#corpus-analysis-concordance-text-annotation-input');
-    let textAnnotationOptions = document.querySelector('#corpus-analysis-concordance-text-annotation-options');
-
-    if (textAnnotationInput.value === '') {
-      textAnnotationSubmit.classList.add('red');
-      noValueMetadataMessage.classList.remove('hide');
-      setTimeout(() => {
-        textAnnotationSubmit.classList.remove('red');
-      }, 500);
-      setTimeout(() => {
-        noValueMetadataMessage.classList.add('hide');
-      }, 3000);
-    } else {
-      let queryText = `:: match.text_${textAnnotationOptions.value}="${textAnnotationInput.value}"`;
-      this.queryChipFactory('text-annotation', `${textAnnotationOptions.value}=${textAnnotationInput.value}`, queryText);
-      this.resetAndCloseStructuralAttrModal();
-    }
-  }
-
-  resetAndCloseStructuralAttrModal() {
-    let textAnnotatinInput = document.querySelector('#corpus-analysis-concordance-text-annotation-input');
-    textAnnotatinInput.value = '';
-    this.resetMaterializeSelection([this.elements.englishEntTypeSelection, this.elements.germanEntTypeSelection]);
-    this.resetMaterializeSelection([this.elements.textAnnotationSelection], 'address');
-
-    this.toggleClass(['entity-builder', 'text-annotation-builder'], 'hide', 'add');
-    this.elements.structuralAttrModal.close();
-  }
-
-}

app/static/js/CorpusAnalysis/QueryBuilder/TokenAttributeBuilderFunctionsQueryBuilder.js

@@ -1,334 +0,0 @@
-class TokenAttributeBuilderFunctionsQueryBuilder extends GeneralFunctionsQueryBuilder {
-  constructor(elements) {
-    super(elements);
-    this.elements = elements;
-
-    this.elements.positionalAttrSelection.addEventListener('change', (event) => {
-      this.toggleClass(['word', 'lemma', 'english-pos', 'german-pos', 'simple-pos'], 'hide', 'add');
-      if (event.target.value !== 'empty-token') {
-        this.toggleClass([event.target.value], 'hide', 'remove');
-        this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-        this.resetMaterializeSelection([this.elements.englishPosSelection, this.elements.germanPosSelection, this.elements.simplePosSelection]);
-      }
-      if (event.target.value === 'word' || event.target.value === 'lemma') {
-        this.toggleClass(['input-field-options'], 'hide', 'remove');
-      } else if (event.target.value === 'empty-token'){
-        this.addTokenToQuery();
-      } else {
-        this.toggleClass(['input-field-options'], 'hide', 'add');
-      }
-    });
-
-    // Options for positional attribute selection
-    document.querySelectorAll('.positional-attr-options-action-button[data-options-action]').forEach(button => {
-      button.addEventListener('click', () => {this.actionButtonInOptionSectionHandler(button.dataset.optionsAction);});
-    });
-
-    // Eventlistener for kind of token 
-    this.elements.tokenSubmitButton.addEventListener('click', () => {this.addTokenToQuery();});
-    this.elements.wordInput.addEventListener('input', () => {this.optionToggleHandler();});
-    this.elements.lemmaInput.addEventListener('input', () => {this.optionToggleHandler();});
-    this.elements.englishPosSelection.addEventListener('change', () => {this.optionToggleHandler();});
-    this.elements.germanPosSelection.addEventListener('change', () => {this.optionToggleHandler();});
-    this.elements.simplePosSelection.addEventListener('change', () => {this.optionToggleHandler();});
-  }
-
-  tokenInputCheck() {
-    let input;
-
-    if (!document.querySelector('[data-toggle-area="word"]').classList.contains('hide')) {
-      input = this.elements.wordInput;
-    } else if (!document.querySelector('[data-toggle-area="lemma"]').classList.contains('hide')){
-      input = this.elements.lemmaInput;
-    } else if (!document.querySelector('[data-toggle-area="english-pos"]').classList.contains('hide')){
-      input = this.elements.englishPosSelection;
-    } else if (!document.querySelector('[data-toggle-area="german-pos"]').classList.contains('hide')){
-      input = this.elements.germanPosSelection;
-    } else if (!document.querySelector('[data-toggle-area="simple-pos"]').classList.contains('hide')){
-      input = this.elements.simplePosSelection;
-    }
-
-    return input;
-  }
-
-  optionToggleHandler() {
-    let input;
-    input = this.tokenInputCheck();
-
-    if (input.value === '' || input.value === 'default') {
-      this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-    } else {
-      this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'remove');
-    }
-  }
-
-  disableTokenSubmit() {
-    this.elements.isTokenQueryInvalid = true;
-    this.elements.tokenSubmitButton.classList.add('red');
-    this.elements.noValueMessage.classList.remove('hide');
-    setTimeout(() => {
-      this.elements.tokenSubmitButton.classList.remove('red');
-    }, 500);
-    setTimeout(() => {
-      this.elements.noValueMessage.classList.add('hide');
-    }, 3000);
-  }
-
-  tokenChipFactory(prettyQueryText, tokenText) {
-    tokenText = encodeURI(tokenText);
-    let builderElement;
-    let queryChipElement;
-    builderElement = document.createElement('div');
-    builderElement.innerHTML = `
-    <div class='chip col s2 l2' style='margin-top:20px;' data-tokentext='${tokenText}'>
-      ${prettyQueryText}
-      <i class='material-icons close'>close</i>
-    </div>`;
-    queryChipElement = builderElement.firstElementChild;
-    this.elements.tokenQuery.appendChild(queryChipElement);
-  }
-
-  addTokenToQuery() {
-    let c = this.elements.ignoreCaseCheckbox.checked ? ' %c' : '';
-    let tokenQueryPrettyText = '';
-    let tokenQueryCQLText = '';
-    this.elements.isTokenQueryInvalid = false;
-
-    this.elements.tokenQuery.childNodes.forEach(element => {
-      tokenQueryPrettyText += ' ' + element.firstChild.data + ' ';
-      tokenQueryCQLText += decodeURI(element.dataset.tokentext);
-    });
-
-    switch (this.elements.positionalAttrSelection.value) {
-      case 'word':
-        if (this.elements.wordInput.value === '') {
-          this.disableTokenSubmit();
-        } else {
-          tokenQueryPrettyText += `word=${this.elements.wordInput.value}${c}`;
-          tokenQueryCQLText += `word="${this.elements.wordInput.value}"${c}`;
-          this.elements.wordInput.value = '';
-        }
-        break;
-      case 'lemma':
-        if (this.elements.lemmaInput.value === '') {
-          this.disableTokenSubmit();
-        } else {
-          tokenQueryPrettyText += `lemma=${this.elements.lemmaInput.value}${c}`;
-          tokenQueryCQLText += `lemma="${this.elements.lemmaInput.value}"${c}`;
-          this.elements.lemmaInput.value = '';
-        }
-        break;
-      case 'english-pos':
-        if (this.elements.englishPosSelection.value === 'default') {
-          this.disableTokenSubmit();
-        } else {
-          tokenQueryPrettyText += `pos=${this.elements.englishPosSelection.value}`;
-          tokenQueryCQLText += `pos="${this.elements.englishPosSelection.value}"`;
-          this.elements.englishPosSelection.value = '';
-        }
-        break;
-      case 'german-pos':
-        if (this.elements.germanPosSelection.value === 'default') {
-          this.disableTokenSubmit();
-        } else {
-          tokenQueryPrettyText += `pos=${this.elements.germanPosSelection.value}`;
-          tokenQueryCQLText += `pos="${this.elements.germanPosSelection.value}"`;
-          this.elements.germanPosSelection.value = '';
-        }
-        break;
-      case 'simple-pos':
-        if (this.elements.simplePosSelection.value === 'default') {
-          this.disableTokenSubmit();
-        } else {
-          tokenQueryPrettyText += `simple_pos=${this.elements.simplePosSelection.value}`;
-          tokenQueryCQLText += `simple_pos="${this.elements.simplePosSelection.value}"`;
-          this.elements.simplePosSelection.value = '';
-        }
-        break;
-      case 'empty-token':
-        tokenQueryPrettyText += 'empty token';
-      default:
-        break;
-    }
-
-    // isTokenQueryInvalid looks if a valid value is passed. If the input fields/dropdowns are empty (isTokenQueryInvalid === true), no token is added.
-    if (this.elements.isTokenQueryInvalid === false) {
-      tokenQueryCQLText = '[' + tokenQueryCQLText + ']';
-      this.queryChipFactory('token', tokenQueryPrettyText, tokenQueryCQLText);
-      this.resetPositionalAttrModal();
-      this.elements.positionalAttrModal.close();
-    }
-  }
-
-  actionButtonInOptionSectionHandler(elem) {
-    let input = this.tokenInputCheck();
-    switch (elem) {
-      case 'option-group':
-        input.value  += '(option1|option2)';
-        let firstIndex = input.value.indexOf('option1');
-        let lastIndex = firstIndex + 'option1'.length;
-        input.focus();
-        input.setSelectionRange(firstIndex, lastIndex);
-        break;
-      case 'wildcard-char':
-        input.value += '.';
-        break;
-      case 'and':
-        this.conditionHandler('and', " & ");
-        break;
-      case 'or':
-        this.conditionHandler('or', " | ");
-        break;
-      default:
-        break;
-    }
-    this.optionToggleHandler();
-  }
-
-  characterIncidenceModifierHandler(elem) {
-    // For word and lemma, the incidence modifiers are inserted in the input field. For the others, one or two chips are created which contain the respective value of the token and the incidence modifier.
-    switch (this.elements.positionalAttrSelection.value) {
-      case 'empty-token':
-        this.tokenChipFactory(elem.innerText, elem.dataset.token);
-        break;
-      case 'english-pos':
-        this.tokenChipFactory(`pos=${this.elements.englishPosSelection.value}`, `pos="${this.elements.englishPosSelection.value}"`);
-        this.tokenChipFactory(elem.innerText, elem.dataset.token);
-        break;
-      case 'german-pos':
-        this.tokenChipFactory(`pos=${this.elements.germanPosSelection.value}`, `pos="${this.elements.germanPosSelection.value}"`);
-        this.tokenChipFactory(elem.innerText, elem.dataset.token);
-        break;
-      case 'simple-pos':
-        this.tokenChipFactory(`simple_pos=${this.elements.simplePosSelection.value}`, `simple_pos="${this.elements.simplePosSelection.value}"`);
-        this.tokenChipFactory(elem.innerText, elem.dataset.token);
-        break;     
-      default:
-        let input = this.tokenInputCheck();
-        input.value += elem.dataset.token;
-        break;
-    }
-
-    if (this.elements.positionalAttrSelection.value !== "word" && this.elements.positionalAttrSelection.value !== "lemma") {
-      this.toggleClass([this.elements.positionalAttrSelection.value], "hide", "add");
-    }
-  }
-
-  characterNMSubmitHandler(modalId) {
-    let modal = document.querySelector(`#${modalId}`);
-    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
-    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
-    input_m = input_m !== undefined ? ',' + input_m.value : '';
-    let input = `${input_n}${input_m}`;
-
-    let instance = M.Modal.getInstance(modal);
-    instance.close();
-    
-    switch (this.elements.positionalAttrSelection.value) {
-      case 'word':
-        this.elements.wordInput.value += '{' + input + '}';
-        break;
-      case 'lemma':
-        this.elements.lemmaInput.value += '{' + input + '}';
-        break;
-      default:
-        break;
-    }
-  }
-
-  conditionHandler(conditionText, conditionQueryContent) {
-    let tokenQueryPrettyText;
-    let tokenQueryCQLText;
-    let c = this.elements.ignoreCaseCheckbox.checked ? ' %c' : '';
-
-    switch (this.elements.positionalAttrSelection.value) {
-      case 'word':
-        tokenQueryPrettyText = `word=${this.elements.wordInput.value}${c}`;
-        tokenQueryCQLText = `word="${this.elements.wordInput.value}"${c}`;
-        this.elements.wordInput.value = '';
-        break;
-      case 'lemma':
-        tokenQueryPrettyText = `lemma=${this.elements.lemmaInput.value}${c}`;
-        tokenQueryCQLText = `lemma="${this.elements.lemmaInput.value}"${c}`;
-        this.elements.lemmaInput.value = '';
-        break;
-      case 'english-pos':
-        tokenQueryPrettyText = `pos=${this.elements.englishPosSelection.value}`;
-        tokenQueryCQLText = `pos="${this.elements.englishPosSelection.value}"`;
-        this.elements.englishPosSelection.value = '';
-        break;
-      case 'german-pos':
-        tokenQueryPrettyText = `pos=${this.elements.germanPosSelection.value}`;
-        tokenQueryCQLText = `pos="${this.elements.germanPosSelection.value}"`;
-        this.elements.germanPosSelection.value = '';
-        break;
-      case 'simple-pos':
-        tokenQueryPrettyText = `simple_pos=${this.elements.simplePosSelection.value}`;
-        tokenQueryCQLText = `simple_pos="${this.elements.simplePosSelection.value}"`;
-        this.elements.simplePosSelection.value = '';
-        break;
-      default:
-        break;
-    }
-
-    // Deleting the options which do not make sense in the context of the condition like "word" AND "word". Also sets selection default.
-    let selectionDefault = "word";
-    let optionDeleteList = ['empty-token'];
-    if (conditionText === 'and') {
-      if (this.elements.positionalAttrSelection.value === 'word' || this.elements.positionalAttrSelection.value === 'lemma') {
-        selectionDefault = "english-pos";
-        optionDeleteList.push('word', 'lemma');
-      } else if (this.elements.positionalAttrSelection.value === 'english-pos' || this.elements.positionalAttrSelection.value === 'german-pos') {
-        optionDeleteList.push('english-pos', 'german-pos');
-      } else {
-        optionDeleteList.push('simple-pos');
-      }
-    }
-
-    this.resetMaterializeSelection([this.elements.englishPosSelection, this.elements.germanPosSelection, this.elements.simplePosSelection]);
-
-    this.tokenChipFactory(tokenQueryPrettyText, tokenQueryCQLText);
-    this.tokenChipFactory(conditionText, conditionQueryContent);
-    this.setTokenSelection(selectionDefault, optionDeleteList);
-  }
-
-  setTokenSelection(selection, optionDeleteList) {
-    optionDeleteList.forEach(option => {
-      this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`).remove();
-    });
-
-    this.resetMaterializeSelection([this.elements.positionalAttrSelection], selection);
-
-    this.toggleClass(['word', 'lemma', 'english-pos', 'german-pos', 'simple-pos'], 'hide', 'add');
-    this.toggleClass([selection], 'hide', 'remove');
-    this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-    if (selection === "word" || selection === "lemma") {
-      this.toggleClass(['input-field-options'], 'hide', 'remove');
-    } else {
-      this.toggleClass(['input-field-options'], 'hide', 'add');
-    }
-  }
-
-  
-  resetPositionalAttrModal() {
-    let originalSelectionList = 
-      `
-        <option value="word" selected>word</option>
-        <option value="lemma" >lemma</option>
-        <option value="english-pos">english pos</option>
-        <option value="german-pos">german pos</option>
-        <option value="simple-pos">simple_pos</option>
-        <option value="empty-token">empty token</option>
-      `;
-    this.elements.positionalAttrSelection.innerHTML = originalSelectionList;
-    this.elements.tokenQuery.innerHTML = '';
-    this.toggleClass(['word', 'lemma', 'english-pos', 'german-pos', 'simple-pos'], 'hide', 'add');
-    this.toggleClass(['word', 'input-field-options'], 'hide', 'remove');
-    this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-
-    document.querySelector('#corpus-analysis-concordance-positional-attr-selection option[value="word"]').selected = true;
-
-    this.resetMaterializeSelection([this.elements.englishPosSelection, this.elements.germanPosSelection, this.elements.simplePosSelection]);
-    this.resetMaterializeSelection([this.elements.positionalAttrSelection], "word");
-  }
-}

app/static/js/app.js

@@ -0,0 +1,204 @@
+nopaque.App = class App {
+  constructor() {
+    this.data = {
+      promises: {getUser: {}, subscribeUser: {}},
+      users: {},
+    };
+    this.socket = io({transports: ['websocket'], upgrade: false});
+    this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
+  }
+
+  getUser(userId) {
+    if (userId in this.data.promises.getUser) {
+      return this.data.promises.getUser[userId];
+    }
+
+    this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
+      this.socket.emit('GET /users/<user_id>', userId, (response) => {
+        if (response.status === 200) {
+          this.data.users[userId] = response.body;
+          resolve(this.data.users[userId]);
+        } else {
+          reject(`[${response.status}] ${response.statusText}`);
+        }
+      });
+    });
+
+    return this.data.promises.getUser[userId];
+  }
+
+  subscribeUser(userId) {
+    if (userId in this.data.promises.subscribeUser) {
+      return this.data.promises.subscribeUser[userId];
+    }
+
+    this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
+      this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
+        if (response.status !== 200) {
+          reject(response);
+          return;
+        }
+        resolve(response);
+      });
+    });
+
+    return this.data.promises.subscribeUser[userId];
+  }
+
+  flash(message, category) {
+    let iconPrefix = '';
+    switch (category) {
+      case 'corpus': {
+        iconPrefix = '<i class="left material-icons">book</i>';
+        break;
+      }
+      case 'error': {
+        iconPrefix = '<i class="error-color-text left material-icons">error</i>';
+        break;
+      }
+      case 'job': {
+        iconPrefix = '<i class="left nopaque-icons">J</i>';
+        break;
+      }
+      case 'settings': {
+        iconPrefix = '<i class="left material-icons">settings</i>';
+        break;
+      }
+      default: {
+        iconPrefix = '<i class="left material-icons">notifications</i>';
+        break;
+      }
+    }
+    let toast = M.toast(
+      {
+        html: `
+          <span>${iconPrefix}${message}</span>
+          <button class="action-button btn-flat toast-action white-text" data-action="close">
+            <i class="material-icons">close</i>
+          </button>
+        `.trim()
+      }
+    );
+    let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
+    toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
+  }
+  
+  onPatch(patch) {
+    // Filter Patch to only include operations on users that are initialized
+    let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
+    let filteredPatch = patch.filter(operation => regExp.test(operation.path));
+
+    // Handle job status updates
+    let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
+    let subFilteredPatch = filteredPatch
+      .filter((operation) => {return operation.op === 'replace';})
+      .filter((operation) => {return subRegExp.test(operation.path);});
+    for (let operation of subFilteredPatch) {
+      let [match, userId, jobId] = operation.path.match(subRegExp);
+      this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-status="${operation.value}"></span>`, 'job');
+    }
+
+    // Apply Patch
+    jsonpatch.applyPatch(this.data, filteredPatch);
+  }
+
+  init() {
+    this.initUi();
+  }
+
+  initUi() {
+    /* Pre-Initialization fixes */
+    // #region
+
+    // Flask-WTF sets the standard HTML maxlength Attribute on input/textarea
+    // elements to specify their maximum length (in characters). Unfortunatly
+    // Materialize won't recognize the maxlength Attribute, instead it uses
+    // the data-length Attribute. It's conversion time :)
+    for (let elem of document.querySelectorAll('input[maxlength], textarea[maxlength]')) {
+      elem.dataset.length = elem.getAttribute('maxlength');
+      elem.removeAttribute('maxlength');
+    }
+
+    // To work around some limitations with the Form setup of Flask-WTF.
+    // HTML option elements with an empty value are considered as placeholder
+    // elements. The user should not be able to actively select these options.
+    // So they get the disabled attribute.
+    for (let optionElement of document.querySelectorAll('option[value=""]')) {
+      optionElement.disabled = true;
+    }
+
+    // TODO: Check why we are doing this.
+    for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) {
+      for (let c of optgroupElement.children) {
+        optgroupElement.parentElement.insertAdjacentElement('afterbegin', c);
+      }
+      optgroupElement.remove();
+    }
+    // #endregion
+
+
+    /* Initialize Materialize Components */
+    // #region
+  
+    // Automatically initialize Materialize Components that do not require
+    // additional configuration.
+    M.AutoInit();
+
+    // CharacterCounters
+    // Materialize didn't include the CharacterCounter plugin within the
+    // AutoInit method (maybe they forgot it?). Anyway... We do it here. :)
+    M.CharacterCounter.init(document.querySelectorAll('input[data-length]:not(.no-autoinit), textarea[data-length]:not(.no-autoinit)'));
+
+    // Header navigation "more" Dropdown.
+    M.Dropdown.init(
+      document.querySelector('#nav-more-dropdown-trigger'),
+      {
+        alignment: 'right',
+        constrainWidth: false,
+        coverTrigger: false
+      }
+    );
+
+    // Manual modal
+    M.Modal.init(
+      document.querySelector('#manual-modal'),
+      {
+        onOpenStart: (modalElement, modalTriggerElement) => {
+          if ('manualModalChapter' in modalTriggerElement.dataset) {
+            let manualModalTocElement = document.querySelector('#manual-modal-toc');
+            let manualModalToc = M.Tabs.getInstance(manualModalTocElement);
+            manualModalToc.select(modalTriggerElement.dataset.manualModalChapter);
+            // TODO: Make this work.
+            // if ('manualModalChapterAnchor' in modalTriggerElement.dataset) {
+            //   let manualModalChapterAnchor = document.querySelector(`#${modalTriggerElement.dataset.manualModalChapterAnchor}`);
+            //   let xCoord = manualModalChapterAnchor.getBoundingClientRect().left;
+            //   let yCoord = manualModalChapterAnchor.getBoundingClientRect().top;
+            //   let modalContentElement = modalElement.querySelector('.modal-content');
+            //   modalContentElement.scroll(xCoord, yCoord);
+            // }
+          }
+        }
+      }
+    );
+
+    // Terms of use modal
+    M.Modal.init(
+      document.querySelector('#terms-of-use-modal'),
+      {
+        dismissible: false,
+        onCloseEnd: (modalElement) => {
+          nopaque.requests.users.entity.acceptTermsOfUse();
+        }
+      }
+    );
+    // #endregion
+
+
+    /* Initialize nopaque Components */
+    // #region 
+    nopaque.resource_displays.AutoInit();
+    nopaque.resource_lists.AutoInit();
+    nopaque.forms.AutoInit();
+    // #endregion
+  }
+};

app/static/js/corpus-analysis/app.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisApp {
+nopaque.corpus_analysis.App = class App {
   constructor(corpusId) {
     this.corpusId = corpusId;
 
@@ -30,7 +30,7 @@ class CorpusAnalysisApp {
       // Setup CQi over SocketIO connection and gather data from the CQPServer
       const statusTextElement = this.elements.initModal.querySelector('.status-text');
       statusTextElement.innerText = 'Creating CQi over SocketIO client...';
-      const cqiClient = new cqi.CQiClient('/cqi_over_sio');
+      const cqiClient = new nopaque.corpus_analysis.cqi.Client('/cqi_over_sio');
       statusTextElement.innerText += ' Done';
       statusTextElement.innerHTML = 'Waiting for the CQP server...';
       const response = await cqiClient.api.socket.emitWithAck('init', this.corpusId);

app/static/js/corpus-analysis/concordance-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisConcordance {
+nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
   name = 'Concordance';
 
   constructor(app) {
@@ -33,7 +33,7 @@ class CorpusAnalysisConcordance {
 
   async submitForm(queryModeId) {
     this.app.disableActionElements();
-    let queryBuilderQuery = Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
+    let queryBuilderQuery = nopaque.Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
     let expertModeQuery = this.elements.expertModeForm.query.value.trim();
     let query = queryModeId === 'corpus-analysis-concordance-expert-mode-form' ? expertModeQuery : queryBuilderQuery;
     let form = queryModeId === 'corpus-analysis-concordance-expert-mode-form' ? this.elements.expertModeForm : this.elements.queryBuilderForm;
@@ -171,11 +171,11 @@ class CorpusAnalysisConcordance {
     this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', (event) => {
       event.preventDefault();
       let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
-      let modalElementId = Utils.generateElementId('export-subcorpus-modal-');
-      let exportFormatSelectElementId = Utils.generateElementId('export-format-select-');
-      let exportSelectedMatchesOnlyCheckboxElementId = Utils.generateElementId('export-selected-matches-only-checkbox-');
-      let exportFileNameInputElementId = Utils.generateElementId('export-file-name-input-');
-      let modalElement = Utils.HTMLToElement(
+      let modalElementId = nopaque.Utils.generateElementId('export-subcorpus-modal-');
+      let exportFormatSelectElementId = nopaque.Utils.generateElementId('export-format-select-');
+      let exportSelectedMatchesOnlyCheckboxElementId = nopaque.Utils.generateElementId('export-selected-matches-only-checkbox-');
+      let exportFileNameInputElementId = nopaque.Utils.generateElementId('export-file-name-input-');
+      let modalElement = nopaque.Utils.HTMLToElement(
         `
           <div class="modal" id="${modalElementId}">
             <div class="modal-content">

app/static/js/corpus-analysis/cqi/api/client.js

@@ -1,4 +1,4 @@
-cqi.api.APIClient = class APIClient {
+nopaque.corpus_analysis.cqi.api.Client = class Client {
   /**
    * @param {string} host
    * @param {number} [timeout=60] timeout
@@ -30,10 +30,10 @@ cqi.api.APIClient = class APIClient {
     } else if (response.code === 500) {
       throw new Error(`[${response.code}] ${response.msg}`);
     } else if (response.code === 502) {
-      if (response.payload.code in cqi.errors.lookup) {
-        throw new cqi.errors.lookup[response.payload.code]();
+      if (response.payload.code in nopaque.corpus_analysis.cqi.errors.lookup) {
+        throw new nopaque.corpus_analysis.cqi.errors.lookup[response.payload.code]();
       } else {
-        throw new cqi.errors.CQiError();
+        throw new nopaque.corpus_analysis.cqi.errors.CQiError();
       }
     }
   }
@@ -41,22 +41,22 @@ cqi.api.APIClient = class APIClient {
   /**
    * @param {string} username
    * @param {string} password
-   * @returns {Promise<cqi.status.StatusConnectOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusConnectOk>}
    */
   async ctrl_connect(username, password) {
     const fn_name = 'ctrl_connect';
     const fn_args = {username: username, password: password};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusByeOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusByeOk>}
    */
   async ctrl_bye() {
     const fn_name = 'ctrl_bye';
     let payload = await this.#request(fn_name);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -68,12 +68,12 @@ cqi.api.APIClient = class APIClient {
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusPingOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusPingOk>}
    */
   async ctrl_ping() {
     const fn_name = 'ctrl_ping';
     let payload = await this.#request(fn_name);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -208,13 +208,13 @@ cqi.api.APIClient = class APIClient {
    * try to unload a corpus and all its attributes from memory
    * 
    * @param {string} corpus
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async corpus_drop_corpus(corpus) {
     const fn_name = 'corpus_drop_corpus';
     const fn_args = {corpus: corpus};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -250,13 +250,13 @@ cqi.api.APIClient = class APIClient {
    * unload attribute from memory
    * 
    * @param {string} attribute
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async cl_drop_attribute(attribute) {
     const fn_name = 'cl_drop_attribute';
     const fn_args = {attribute: attribute};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -482,13 +482,13 @@ cqi.api.APIClient = class APIClient {
    * @param {string} mother_corpus
    * @param {string} subcorpus_name
    * @param {string} query
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async cqp_query(mother_corpus, subcorpus_name, query) {
     const fn_name = 'cqp_query';
     const fn_args = {mother_corpus: mother_corpus, subcorpus_name: subcorpus_name, query: query};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -524,7 +524,7 @@ cqi.api.APIClient = class APIClient {
 
   /**
    * Dump the values of <field> for match ranges <first> .. <last>
-   * in <subcorpus>. <field> is one of the CQI_CONST_FIELD_* constants.
+   * in <subcorpus>. <field> is one of the nopaque.corpus_analysis.cqi.constants.FIELD_* constants.
    * 
    * @param {string} subcorpus
    * @param {number} field
@@ -542,13 +542,13 @@ cqi.api.APIClient = class APIClient {
    * delete a subcorpus from memory
    * 
    * @param {string} subcorpus
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async cqp_drop_subcorpus(subcorpus) {
     const fn_name = 'cqp_drop_subcorpus';
     const fn_args = {subcorpus: subcorpus};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**
@@ -561,9 +561,9 @@ cqi.api.APIClient = class APIClient {
    *
    * returns <n> (id, frequency) pairs flattened into a list of size 2*<n>
    * field is one of
-   * - CQI_CONST_FIELD_MATCH
-   * - CQI_CONST_FIELD_TARGET
-   * - CQI_CONST_FIELD_KEYWORD
+   * - nopaque.corpus_analysis.cqi.constants.FIELD_MATCH
+   * - nopaque.corpus_analysis.cqi.constants.FIELD_TARGET
+   * - nopaque.corpus_analysis.cqi.constants.FIELD_KEYWORD
    *
    * NB: pairs are sorted by frequency desc.
    * 
@@ -610,13 +610,13 @@ cqi.api.APIClient = class APIClient {
 
   /**
    * @param {string} corpus
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async ext_corpus_update_db(corpus) {
     const fn_name = 'ext_corpus_update_db';
     const fn_args = {corpus: corpus};
     let payload = await this.#request(fn_name, fn_args);
-    return new cqi.status.lookup[payload.code]();
+    return new nopaque.corpus_analysis.cqi.status.lookup[payload.code]();
   }
 
   /**

app/static/js/corpus-analysis/cqi/api/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis.cqi.api = {};

app/static/js/corpus-analysis/cqi/client.js

@@ -1,23 +1,23 @@
-cqi.CQiClient = class CQiClient {
+nopaque.corpus_analysis.cqi.Client = class Client {
   /**
    * @param {string} host
    * @param {number} [timeout=60] timeout
    * @param {string} [version=0.1] version
    */
   constructor(host, timeout = 60, version = '0.1') {
-     /** @type {cqi.api.APIClient} */
-    this.api = new cqi.api.APIClient(host, timeout, version);
+     /** @type {nopaque.corpus_analysis.cqi.api.Client} */
+    this.api = new nopaque.corpus_analysis.cqi.api.Client(host, timeout, version);
   }
 
   /**
-   * @returns {cqi.models.corpora.CorpusCollection}
+   * @returns {nopaque.corpus_analysis.cqi.models.corpora.CorpusCollection}
    */
   get corpora() {
-    return new cqi.models.corpora.CorpusCollection(this);
+    return new nopaque.corpus_analysis.cqi.models.corpora.CorpusCollection(this);
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusByeOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusByeOk>}
    */
   async bye() {
     return await this.api.ctrl_bye();
@@ -26,14 +26,14 @@ cqi.CQiClient = class CQiClient {
   /**
    * @param {string} username
    * @param {string} password
-   * @returns {Promise<cqi.status.StatusConnectOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusConnectOk>}
    */
   async connect(username, password) {
     return await this.api.ctrl_connect(username, password);
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusPingOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusPingOk>}
    */
   async ping() {
     return await this.api.ctrl_ping();
@@ -49,7 +49,7 @@ cqi.CQiClient = class CQiClient {
   /**
    * Alias for "bye" method
    * 
-   * @returns {Promise<cqi.status.StatusByeOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusByeOk>}
    */
   async disconnect() {
     return await this.api.ctrl_bye();

app/static/js/corpus-analysis/cqi/constants.js

@@ -0,0 +1,43 @@
+nopaque.corpus_analysis.cqi.constants = {};
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_KEYWORD = 9;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_MATCH = 16;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_MATCHEND = 17;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET = 0;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_0 = 0;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_1 = 1;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_2 = 2;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_3 = 3;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_4 = 4;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_5 = 5;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_6 = 6;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_7 = 7;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_8 = 8;
+
+/** @type {number} */
+nopaque.corpus_analysis.cqi.constants.FIELD_TARGET_9 = 9;

app/static/js/corpus-analysis/cqi/errors.js

@@ -0,0 +1,185 @@
+nopaque.corpus_analysis.cqi.errors = {};
+
+
+/**
+ * A base class from which all other errors inherit.
+ * If you want to catch all errors that the CQi package might throw,
+ * catch this base error.
+ */
+nopaque.corpus_analysis.cqi.errors.CQiError = class CQiError extends Error {
+  constructor(message) {
+    super(message);
+    this.code = undefined;
+    this.description = undefined;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.Error = class Error extends nopaque.corpus_analysis.cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 2;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.ErrorGeneralError = class ErrorGeneralError extends nopaque.corpus_analysis.cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 513;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.ErrorConnectRefused = class ErrorConnectRefused extends nopaque.corpus_analysis.cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 514;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.ErrorUserAbort = class ErrorUserAbort extends nopaque.corpus_analysis.cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 515;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.ErrorSyntaxError = class ErrorSyntaxError extends nopaque.corpus_analysis.cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 516;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLError = class Error extends nopaque.corpus_analysis.cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 4;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorNoSuchAttribute = class CLErrorNoSuchAttribute extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1025;
+    this.description = "CQi server couldn't open attribute";
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorWrongAttributeType = class CLErrorWrongAttributeType extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1026;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorOutOfRange = class CLErrorOutOfRange extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1027;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorRegex = class CLErrorRegex extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1028;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorCorpusAccess = class CLErrorCorpusAccess extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1029;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorOutOfMemory = class CLErrorOutOfMemory extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1030;
+    this.description = 'CQi server has run out of memory; try discarding some other corpora and/or subcorpora';
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CLErrorInternal = class CLErrorInternal extends nopaque.corpus_analysis.cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1031;
+    this.description = "The classical 'please contact technical support' error";
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CQPError = class Error extends nopaque.corpus_analysis.cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 5;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CQPErrorGeneral = class CQPErrorGeneral extends nopaque.corpus_analysis.cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1281;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CQPErrorNoSuchCorpus = class CQPErrorNoSuchCorpus extends nopaque.corpus_analysis.cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1282;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CQPErrorInvalidField = class CQPErrorInvalidField extends nopaque.corpus_analysis.cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1283;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.CQPErrorOutOfRange = class CQPErrorOutOfRange extends nopaque.corpus_analysis.cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1284;
+    this.description = 'A number is out of range';
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.errors.lookup = {
+  2: nopaque.corpus_analysis.cqi.errors.Error,
+  513: nopaque.corpus_analysis.cqi.errors.ErrorGeneralError,
+  514: nopaque.corpus_analysis.cqi.errors.ErrorConnectRefused,
+  515: nopaque.corpus_analysis.cqi.errors.ErrorUserAbort,
+  516: nopaque.corpus_analysis.cqi.errors.ErrorSyntaxError,
+  4: nopaque.corpus_analysis.cqi.errors.CLError,
+  1025: nopaque.corpus_analysis.cqi.errors.CLErrorNoSuchAttribute,
+  1026: nopaque.corpus_analysis.cqi.errors.CLErrorWrongAttributeType,
+  1027: nopaque.corpus_analysis.cqi.errors.CLErrorOutOfRange,
+  1028: nopaque.corpus_analysis.cqi.errors.CLErrorRegex,
+  1029: nopaque.corpus_analysis.cqi.errors.CLErrorCorpusAccess,
+  1030: nopaque.corpus_analysis.cqi.errors.CLErrorOutOfMemory,
+  1031: nopaque.corpus_analysis.cqi.errors.CLErrorInternal,
+  5: nopaque.corpus_analysis.cqi.errors.CQPError,
+  1281: nopaque.corpus_analysis.cqi.errors.CQPErrorGeneral,
+  1282: nopaque.corpus_analysis.cqi.errors.CQPErrorNoSuchCorpus,
+  1283: nopaque.corpus_analysis.cqi.errors.CQPErrorInvalidField,
+  1284: nopaque.corpus_analysis.cqi.errors.CQPErrorOutOfRange
+};

app/static/js/corpus-analysis/cqi/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis.cqi = {};

app/static/js/corpus-analysis/cqi/models/attributes.js

@@ -1,7 +1,7 @@
-cqi.models.attributes = {};
+nopaque.corpus_analysis.cqi.models.attributes = {};
 
 
-cqi.models.attributes.Attribute = class Attribute extends cqi.models.resource.Model {
+nopaque.corpus_analysis.cqi.models.attributes.Attribute = class Attribute extends nopaque.corpus_analysis.cqi.models.resource.Model {
   /**
    * @returns {string}
    */
@@ -24,7 +24,7 @@ cqi.models.attributes.Attribute = class Attribute extends cqi.models.resource.Mo
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async drop() {
     return await this.client.api.cl_drop_attribute(this.apiName);
@@ -32,17 +32,17 @@ cqi.models.attributes.Attribute = class Attribute extends cqi.models.resource.Mo
 };
 
 
-cqi.models.attributes.AttributeCollection = class AttributeCollection extends cqi.models.resource.Collection {
-   /** @type{typeof cqi.models.attributes.Attribute} */
-  static model = cqi.models.attributes.Attribute;
+nopaque.corpus_analysis.cqi.models.attributes.AttributeCollection = class AttributeCollection extends nopaque.corpus_analysis.cqi.models.resource.Collection {
+   /** @type{typeof nopaque.corpus_analysis.cqi.models.attributes.Attribute} */
+  static model = nopaque.corpus_analysis.cqi.models.attributes.Attribute;
 
   /**
-   * @param {cqi.CQiClient} client
-   * @param {cqi.models.corpora.Corpus} corpus
+   * @param {nopaque.corpus_analysis.cqi.Client} client
+   * @param {nopaque.corpus_analysis.cqi.models.corpora.Corpus} corpus
    */
   constructor(client, corpus) {
     super(client);
-     /** @type {cqi.models.corpora.Corpus} */
+     /** @type {nopaque.corpus_analysis.cqi.models.corpora.Corpus} */
     this.corpus = corpus;
   }
 
@@ -62,7 +62,7 @@ cqi.models.attributes.AttributeCollection = class AttributeCollection extends cq
 
   /**
    * @param {string} attributeName
-   * @returns {Promise<cqi.models.attributes.Attribute>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.attributes.Attribute>}
    */
   async get(attributeName) {
     return this.prepareModel(await this._get(attributeName));
@@ -70,7 +70,7 @@ cqi.models.attributes.AttributeCollection = class AttributeCollection extends cq
 };
 
 
-cqi.models.attributes.AlignmentAttribute = class AlignmentAttribute extends cqi.models.attributes.Attribute {
+nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttribute = class AlignmentAttribute extends nopaque.corpus_analysis.cqi.models.attributes.Attribute {
   /**
    * @param {number} id 
    * @returns {Promise<[number, number, number, number]>}
@@ -89,17 +89,17 @@ cqi.models.attributes.AlignmentAttribute = class AlignmentAttribute extends cqi.
 };
 
 
-cqi.models.attributes.AlignmentAttributeCollection = class AlignmentAttributeCollection extends cqi.models.attributes.AttributeCollection {
-   /** @type{typeof cqi.models.attributes.AlignmentAttribute} */
-  static model = cqi.models.attributes.AlignmentAttribute;
+nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttributeCollection = class AlignmentAttributeCollection extends nopaque.corpus_analysis.cqi.models.attributes.AttributeCollection {
+   /** @type{typeof nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttribute} */
+  static model = nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttribute;
 
   /**
-   * @returns {Promise<cqi.models.attributes.AlignmentAttribute[]>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttribute[]>}
    */
   async list() {
      /** @type {string[]} */
      let alignmentAttributeNames = await this.client.api.corpus_alignment_attributes(this.corpus.apiName);
-     /** @type {cqi.models.attributes.AlignmentAttribute[]} */
+     /** @type {nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttribute[]} */
     let alignmentAttributes = [];
     for (let alignmentAttributeName of alignmentAttributeNames) {
       alignmentAttributes.push(await this.get(alignmentAttributeName));
@@ -109,7 +109,7 @@ cqi.models.attributes.AlignmentAttributeCollection = class AlignmentAttributeCol
 };
 
 
-cqi.models.attributes.PositionalAttribute = class PositionalAttribute extends cqi.models.attributes.Attribute {
+nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute = class PositionalAttribute extends nopaque.corpus_analysis.cqi.models.attributes.Attribute {
   /**
    * @returns {number}
    */
@@ -183,9 +183,9 @@ cqi.models.attributes.PositionalAttribute = class PositionalAttribute extends cq
 };
 
 
-cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeCollection extends cqi.models.attributes.AttributeCollection {
-   /** @type{typeof cqi.models.attributes.PositionalAttribute} */
-  static model = cqi.models.attributes.PositionalAttribute;
+nopaque.corpus_analysis.cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeCollection extends nopaque.corpus_analysis.cqi.models.attributes.AttributeCollection {
+   /** @type{typeof nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute} */
+  static model = nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute;
 
   /**
    * @param {string} positionalAttributeName
@@ -198,7 +198,7 @@ cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeC
   }
 
   /**
-   * @returns {Promise<cqi.models.attributes.PositionalAttribute[]>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute[]>}
    */
   async list() {
     let positionalAttributeNames = await this.client.api.corpus_positional_attributes(this.corpus.apiName);
@@ -211,7 +211,7 @@ cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeC
 };
 
 
-cqi.models.attributes.StructuralAttribute = class StructuralAttribute extends cqi.models.attributes.Attribute {
+nopaque.corpus_analysis.cqi.models.attributes.StructuralAttribute = class StructuralAttribute extends nopaque.corpus_analysis.cqi.models.attributes.Attribute {
   /**
    * @returns {boolean}
    */
@@ -261,9 +261,9 @@ cqi.models.attributes.StructuralAttribute = class StructuralAttribute extends cq
 };
 
 
-cqi.models.attributes.StructuralAttributeCollection = class StructuralAttributeCollection extends cqi.models.attributes.AttributeCollection {
-   /** @type{typeof cqi.models.attributes.StructuralAttribute} */
-  static model = cqi.models.attributes.StructuralAttribute;
+nopaque.corpus_analysis.cqi.models.attributes.StructuralAttributeCollection = class StructuralAttributeCollection extends nopaque.corpus_analysis.cqi.models.attributes.AttributeCollection {
+   /** @type{typeof nopaque.corpus_analysis.cqi.models.attributes.StructuralAttribute} */
+  static model = nopaque.corpus_analysis.cqi.models.attributes.StructuralAttribute;
 
   /**
    * @param {string} structuralAttributeName
@@ -276,7 +276,7 @@ cqi.models.attributes.StructuralAttributeCollection = class StructuralAttributeC
   }
 
   /**
-   * @returns {Promise<cqi.models.attributes.StructuralAttribute[]>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.attributes.StructuralAttribute[]>}
    */
   async list() {
     let structuralAttributeNames = await this.client.api.corpus_structural_attributes(this.corpus.apiName);

app/static/js/corpus-analysis/cqi/models/corpora.js

@@ -1,7 +1,7 @@
-cqi.models.corpora = {};
+nopaque.corpus_analysis.cqi.models.corpora = {};
 
 
-cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
+nopaque.corpus_analysis.cqi.models.corpora.Corpus = class Corpus extends nopaque.corpus_analysis.cqi.models.resource.Model {
   /**
    * @returns {string}
    */
@@ -38,35 +38,35 @@ cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
   }
 
   /**
-   * @returns {cqi.models.attributes.AlignmentAttributeCollection}
+   * @returns {nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttributeCollection}
    */
   get alignmentAttributes() {
-    return new cqi.models.attributes.AlignmentAttributeCollection(this.client, this);
+    return new nopaque.corpus_analysis.cqi.models.attributes.AlignmentAttributeCollection(this.client, this);
   }
 
   /**
-   * @returns {cqi.models.attributes.PositionalAttributeCollection}
+   * @returns {nopaque.corpus_analysis.cqi.models.attributes.PositionalAttributeCollection}
    */
   get positionalAttributes() {
-    return new cqi.models.attributes.PositionalAttributeCollection(this.client, this);
+    return new nopaque.corpus_analysis.cqi.models.attributes.PositionalAttributeCollection(this.client, this);
   }
 
   /**
-   * @returns {cqi.models.attributes.StructuralAttributeCollection}
+   * @returns {nopaque.corpus_analysis.cqi.models.attributes.StructuralAttributeCollection}
    */
   get structuralAttributes() {
-    return new cqi.models.attributes.StructuralAttributeCollection(this.client, this);
+    return new nopaque.corpus_analysis.cqi.models.attributes.StructuralAttributeCollection(this.client, this);
   }
 
   /**
-   * @returns {cqi.models.subcorpora.SubcorpusCollection}
+   * @returns {nopaque.corpus_analysis.cqi.models.subcorpora.SubcorpusCollection}
    */
   get subcorpora() {
-    return new cqi.models.subcorpora.SubcorpusCollection(this.client, this);
+    return new nopaque.corpus_analysis.cqi.models.subcorpora.SubcorpusCollection(this.client, this);
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async drop() {
     return await this.client.api.corpus_drop_corpus(this.apiName);
@@ -75,7 +75,7 @@ cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
   /**
    * @param {string} subcorpusName
    * @param {string} query
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async query(subcorpusName, query) {
     return await this.client.api.cqp_query(this.apiName, subcorpusName, query);
@@ -96,7 +96,7 @@ cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
   }
 
   /**
-   * @returns {cqi.status.StatusOk}
+   * @returns {nopaque.corpus_analysis.cqi.status.StatusOk}
    */
   async updateDb() {
     return await this.client.api.ext_corpus_update_db(this.apiName);
@@ -113,9 +113,9 @@ cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
 };
 
 
-cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.resource.Collection {
-   /** @type {typeof cqi.models.corpora.Corpus} */
-  static model = cqi.models.corpora.Corpus;
+nopaque.corpus_analysis.cqi.models.corpora.CorpusCollection = class CorpusCollection extends nopaque.corpus_analysis.cqi.models.resource.Collection {
+   /** @type {typeof nopaque.corpus_analysis.cqi.models.corpora.Corpus} */
+  static model = nopaque.corpus_analysis.cqi.models.corpora.Corpus;
 
   /**
    * @param {string} corpusName
@@ -144,19 +144,19 @@ cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.
 
   /**
    * @param {string} corpusName
-   * @returns {Promise<cqi.models.corpora.Corpus>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.corpora.Corpus>}
    */
   async get(corpusName) {
     return this.prepareModel(await this._get(corpusName));
   }
 
   /**
-   * @returns {Promise<cqi.models.corpora.Corpus[]>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.corpora.Corpus[]>}
    */
   async list() {
      /** @type {string[]} */
     let corpusNames = await this.client.api.corpus_list_corpora();
-     /** @type {cqi.models.corpora.Corpus[]} */
+     /** @type {nopaque.corpus_analysis.cqi.models.corpora.Corpus[]} */
     let corpora = [];
     for (let corpusName of corpusNames) {
       corpora.push(await this.get(corpusName));

app/static/js/corpus-analysis/cqi/models/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis.cqi.models = {};

app/static/js/corpus-analysis/cqi/models/resource.js

@@ -1,26 +1,26 @@
-cqi.models.resource = {};
+nopaque.corpus_analysis.cqi.models.resource = {};
 
 
 /**
  * A base class for representing a single object on the server.
  */
-cqi.models.resource.Model = class Model {
+nopaque.corpus_analysis.cqi.models.resource.Model = class Model {
   /**
    * @param {object} attrs
-   * @param {cqi.CQiClient} client
-   * @param {cqi.models.resource.Collection} collection
+   * @param {nopaque.corpus_analysis.cqi.CQiClient} client
+   * @param {nopaque.corpus_analysis.cqi.models.resource.Collection} collection
    */
   constructor(attrs, client, collection) {
      /**
       * A client pointing at the server that this object is on.
       *
-      * @type {cqi.CQiClient}
+      * @type {nopaque.corpus_analysis.cqi.CQiClient}
       */
     this.client = client;
      /**
       * The collection that this model is part of.
       *
-      * @type {cqi.models.resource.Collection}
+      * @type {nopaque.corpus_analysis.cqi.models.resource.Collection}
       */
     this.collection = collection;
      /**
@@ -50,22 +50,22 @@ cqi.models.resource.Model = class Model {
 /**
  * A base class for representing all objects of a particular type on the server.
  */
-cqi.models.resource.Collection = class Collection {
+nopaque.corpus_analysis.cqi.models.resource.Collection = class Collection {
    /** 
     * The type of object this collection represents, set by subclasses
     * 
-    * @type {typeof cqi.models.resource.Model}
+    * @type {typeof nopaque.corpus_analysis.cqi.models.resource.Model}
     */
   static model;
 
   /**
-   * @param {cqi.CQiClient} client
+   * @param {nopaque.corpus_analysis.cqi.CQiClient} client
    */
   constructor(client) {
      /**
       * A client pointing at the server that this object is on.
       *
-      * @type {cqi.CQiClient}
+      * @type {nopaque.corpus_analysis.cqi.CQiClient}
       */
      this.client = client;
   }
@@ -82,7 +82,7 @@ cqi.models.resource.Collection = class Collection {
    * Create a model from a set of attributes.
    * 
    * @param {object} attrs
-   * @returns {cqi.models.resource.Model}
+   * @returns {nopaque.corpus_analysis.cqi.models.resource.Model}
    */
   prepareModel(attrs) {
     return new this.constructor.model(attrs, this.client, this);

app/static/js/corpus-analysis/cqi/models/subcorpora.js

@@ -1,7 +1,7 @@
-cqi.models.subcorpora = {};
+nopaque.corpus_analysis.cqi.models.subcorpora = {};
 
 
-cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Model {
+nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus = class Subcorpus extends nopaque.corpus_analysis.cqi.models.resource.Model {
   /**
    * @returns {string}
    */
@@ -31,7 +31,7 @@ cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Mo
   }
 
   /**
-   * @returns {Promise<cqi.status.StatusOk>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.status.StatusOk>}
    */
   async drop() {
     return await this.client.api.cqp_drop_subcorpus(this.apiName);
@@ -55,7 +55,7 @@ cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Mo
   /**
    * @param {number} cutoff
    * @param {number} field
-   * @param {cqi.models.attributes.PositionalAttribute} attribute
+   * @param {nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute} attribute
    * @returns {Promise<number[]>}
    */
   async fdist1(cutoff, field, attribute) {
@@ -70,9 +70,9 @@ cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Mo
   /**
    * @param {number} cutoff
    * @param {number} field1
-   * @param {cqi.models.attributes.PositionalAttribute} attribute1
+   * @param {nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute} attribute1
    * @param {number} field2
-   * @param {cqi.models.attributes.PositionalAttribute} attribute2
+   * @param {nopaque.corpus_analysis.cqi.models.attributes.PositionalAttribute} attribute2
    * @returns {Promise<number[]>}
    */ 
   async fdist2(cutoff, field1, attribute1, field2, attribute2) {
@@ -122,17 +122,17 @@ cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Mo
 };
 
 
-cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends cqi.models.resource.Collection {
-   /** @type {typeof cqi.models.subcorpora.Subcorpus} */
-  static model = cqi.models.subcorpora.Subcorpus;
+nopaque.corpus_analysis.cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends nopaque.corpus_analysis.cqi.models.resource.Collection {
+   /** @type {typeof nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus} */
+  static model = nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus;
 
   /**
-   * @param {cqi.CQiClient} client
-   * @param {cqi.models.corpora.Corpus} corpus
+   * @param {nopaque.corpus_analysis.cqi.CQiClient} client
+   * @param {nopaque.corpus_analysis.cqi.models.corpora.Corpus} corpus
    */
   constructor(client, corpus) {
     super(client);
-     /** @type {cqi.models.corpora.Corpus} */
+     /** @type {nopaque.corpus_analysis.cqi.models.corpora.Corpus} */
     this.corpus = corpus;
   }
 
@@ -145,17 +145,17 @@ cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends cq
     let apiName = `${this.corpus.apiName}:${subcorpusName}`;
      /** @type {object} */
     let fields = {};
-    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCH)) {
-      fields.match = cqi.CONST_FIELD_MATCH;
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, nopaque.corpus_analysis.cqi.constants.FIELD_MATCH)) {
+      fields.match = nopaque.corpus_analysis.cqi.constants.FIELD_MATCH;
     }
-    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCHEND)) {
-      fields.matchend = cqi.CONST_FIELD_MATCHEND
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, nopaque.corpus_analysis.cqi.constants.FIELD_MATCHEND)) {
+      fields.matchend = nopaque.corpus_analysis.cqi.constants.FIELD_MATCHEND
     }
-    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_TARGET)) {
-      fields.target = cqi.CONST_FIELD_TARGET
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, nopaque.corpus_analysis.cqi.constants.FIELD_TARGET)) {
+      fields.target = nopaque.corpus_analysis.cqi.constants.FIELD_TARGET
     }
-    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_KEYWORD)) {
-      fields.keyword = cqi.CONST_FIELD_KEYWORD
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, nopaque.corpus_analysis.cqi.constants.FIELD_KEYWORD)) {
+      fields.keyword = nopaque.corpus_analysis.cqi.constants.FIELD_KEYWORD
     }
     return {
       api_name: apiName,
@@ -167,19 +167,19 @@ cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends cq
 
   /**
    * @param {string} subcorpusName
-   * @returns {Promise<cqi.models.subcorpora.Subcorpus>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus>}
    */
   async get(subcorpusName) {
     return this.prepareModel(await this._get(subcorpusName));
   }
 
   /**
-   * @returns {Promise<cqi.models.subcorpora.Subcorpus[]>}
+   * @returns {Promise<nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus[]>}
    */
   async list() {
      /** @type {string[]} */
     let subcorpusNames = await this.client.api.cqp_list_subcorpora(this.corpus.apiName);
-     /** @type {cqi.models.subcorpora.Subcorpus[]} */
+     /** @type {nopaque.corpus_analysis.cqi.models.subcorpora.Subcorpus[]} */
     let subcorpora = [];
     for (let subcorpusName of subcorpusNames) {
       subcorpora.push(await this.get(subcorpusName));

app/static/js/corpus-analysis/cqi/status.js

@@ -0,0 +1,51 @@
+nopaque.corpus_analysis.cqi.status = {};
+
+
+/**
+ * A base class from which all other status inherit.
+ */
+nopaque.corpus_analysis.cqi.status.CQiStatus = class CQiStatus {
+  constructor() {
+    this.code = undefined;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.status.StatusOk = class StatusOk extends nopaque.corpus_analysis.cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 257;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.status.StatusConnectOk = class StatusConnectOk extends nopaque.corpus_analysis.cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 258;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.status.StatusByeOk = class StatusByeOk extends nopaque.corpus_analysis.cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 259;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.status.StatusPingOk = class StatusPingOk extends nopaque.corpus_analysis.cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 260;
+  }
+};
+
+
+nopaque.corpus_analysis.cqi.status.lookup = {
+  257: nopaque.corpus_analysis.cqi.status.StatusOk,
+  258: nopaque.corpus_analysis.cqi.status.StatusConnectOk,
+  259: nopaque.corpus_analysis.cqi.status.StatusByeOk,
+  260: nopaque.corpus_analysis.cqi.status.StatusPingOk
+};

app/static/js/corpus-analysis/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis = {};

app/static/js/corpus-analysis/query-builder/element-references.js

@@ -1,32 +1,28 @@
-class ElementReferencesQueryBuilder {
+nopaque.corpus_analysis.query_builder.ElementReferences = class ElementReferences {
   constructor() {
     // General Elements
     this.queryInputField = document.querySelector('#corpus-analysis-concordance-query-builder-input-field');
     this.queryChipElements = [];
+    this.queryElementTarget = document.querySelector('.query-element-target')
+    this.editingModusOn = false;
+    this.editedQueryChipElementIndex = undefined;
+    this.deleteQueryButton = document.querySelector('#corpus-analysis-concordance-delete-query-button');
 
     // Structural Attribute Builder Elements
     this.structuralAttrModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-structural-attr-modal'));
-    this.sentenceElement = document.querySelector('[data-structural-attr-modal-action-button="sentence"]');
-    this.entityElement = document.querySelector('[data-structural-attr-modal-action-button="entity"]');
-    this.textAnnotationElement = document.querySelector('[data-structural-attr-modal-action-button="text-annotation"]');
     this.englishEntTypeSelection = document.querySelector('#corpus-analysis-concordance-english-ent-type-selection');
     this.germanEntTypeSelection = document.querySelector('#corpus-analysis-concordance-german-ent-type-selection');
-    this.textAnnotationSelection = document.querySelector('#corpus-analysis-concordance-text-annotation-options');
     
     // Token Attribute Builder Elements
     this.positionalAttrModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-positional-attr-modal'));
     this.positionalAttrSelection = document.querySelector('#corpus-analysis-concordance-positional-attr-selection');
+    this.tokenBuilderContent = document.querySelector('#corpus-analysis-concordance-token-builder-content');
     this.tokenQuery = document.querySelector('#corpus-analysis-concordance-token-query');
+    this.tokenQueryTemplate = document.querySelector('#corpus-analysis-concordance-token-query-template');
     this.tokenSubmitButton = document.querySelector('#corpus-analysis-concordance-token-submit');
     this.noValueMessage = document.querySelector('#corpus-analysis-concordance-no-value-message');
     this.isTokenQueryInvalid = false;
 
-    this.wordInput = document.querySelector('#corpus-analysis-concordance-word-input');
-    this.lemmaInput = document.querySelector('#corpus-analysis-concordance-lemma-input');
-    this.englishPosSelection = document.querySelector('#corpus-analysis-concordance-english-pos-selection');
-    this.germanPosSelection = document.querySelector('#corpus-analysis-concordance-german-pos-selection');
-    this.simplePosSelection = document.querySelector('#corpus-analysis-concordance-simple-pos-selection');
-
     this.ignoreCaseCheckbox = document.querySelector('#corpus-analysis-concordance-ignore-case-checkbox');
   }
-}
+};

app/static/js/corpus-analysis/query-builder/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis.query_builder = {};

app/static/js/corpus-analysis/query-builder/query-builder.js

@@ -0,0 +1,500 @@
+nopaque.corpus_analysis.query_builder.QueryBuilder = class QueryBuilder {
+  constructor() {
+    this.elements = new nopaque.corpus_analysis.query_builder.ElementReferences();
+
+    this.addEventListenersToQueryElementTarget();
+    this.addEventListenersToIncidenceModifier();
+    this.addEventListenersToNAndMInputSubmit();
+
+    this.elements.deleteQueryButton.addEventListener('click', () => {this.resetQueryInputField()});
+    this.expertModeQueryBuilderSwitchHandler();
+
+    this.extensions = {
+      structuralAttributeBuilderFunctions: new nopaque.corpus_analysis.query_builder.StructuralAttributeBuilderFunctions(this),
+      tokenAttributeBuilderFunctions: new nopaque.corpus_analysis.query_builder.TokenAttributeBuilderFunctions(this),
+    };
+
+    this.dropdown = M.Dropdown.init(
+      document.querySelector('.dropdown-trigger[data-toggle-area="token-incidence-modifiers"]'), 
+      {
+        onCloseStart: () => {
+          this.unselectChipElement(this.elements.queryInputField.querySelector('.chip.teal'));
+        }
+      }
+    )
+  }
+
+  addEventListenersToQueryElementTarget() {
+    this.elements.queryElementTarget.addEventListener('click', () => {
+      this.elements.positionalAttrModal.open();
+    });
+    this.elements.queryElementTarget.addEventListener('dragstart', this.handleDragStart.bind(this, this.elements.queryElementTarget));
+    this.elements.queryElementTarget.addEventListener('dragend', this.handleDragEnd);
+  }
+
+  addEventListenersToIncidenceModifier() {
+    // Eventlisteners for the incidence modifiers. There are two different types of incidence modifiers: token and character incidence modifiers.
+    document.querySelectorAll('.incidence-modifier-selection').forEach(button => {
+      let dropdownId = button.parentNode.parentNode.id;
+      if (dropdownId === 'corpus-analysis-concordance-token-incidence-modifiers-dropdown') {
+        button.addEventListener('click', () => this.tokenIncidenceModifierHandler(button.dataset.token, button.innerHTML));
+      } else if (dropdownId === 'corpus-analysis-concordance-character-incidence-modifiers-dropdown') {
+        button.addEventListener('click', () => this.extensions.tokenAttributeBuilderFunctions.characterIncidenceModifierHandler(button));
+      }
+    });
+  }
+
+  addEventListenersToNAndMInputSubmit() {
+    // Eventlisteners for the submit of n- and m-values of the incidence modifier modal for "exactly n" or "between n and m".
+    document.querySelectorAll('.n-m-submit-button').forEach(button => {
+      let modalId = button.dataset.modalId;
+      if (modalId === 'corpus-analysis-concordance-exactly-n-token-modal' || modalId === 'corpus-analysis-concordance-between-nm-token-modal') {
+        button.addEventListener('click', () => this.tokenNMSubmitHandler(modalId));
+      } else if (modalId === 'corpus-analysis-concordance-exactly-n-character-modal' || modalId === 'corpus-analysis-concordance-between-nm-character-modal') {
+        button.addEventListener('click', () => this.extensions.tokenAttributeBuilderFunctions.characterNMSubmitHandler(modalId));
+      }
+    });
+  }
+
+  toggleClass(elements, className, action) {
+    elements.forEach(element => {
+      document.querySelector(`[data-toggle-area="${element}"]`).classList[action](className);
+    });
+  }
+
+  resetQueryInputField() {
+    this.elements.queryInputField.innerHTML = '';
+    this.addQueryElementTarget();
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  addQueryElementTarget() {
+    let queryElementTarget = nopaque.Utils.HTMLToElement(
+      `
+      <a class="query-element-target btn-floating btn-small blue-grey lighten-4 waves-effect waves-light tooltipped" style="margin-bottom:10px; margin-right:5px;" draggable="true" data-position="bottom" data-tooltip="Add an Element to your query">
+        <i class="material-icons">add</i>
+      </a>
+      `
+    );
+    this.elements.queryInputField.appendChild(queryElementTarget);
+    this.elements.queryElementTarget = queryElementTarget;
+    this.addEventListenersToQueryElementTarget();
+  }
+
+  updateChipList() {
+    this.elements.queryChipElements = this.elements.queryInputField.querySelectorAll('.query-component');
+  }
+  
+  resetMaterializeSelection(selectionElements, value = "default") {
+    selectionElements.forEach(selectionElement => {
+      if (selectionElement.querySelector(`option[value=${value}]`) !== null) {
+        selectionElement.querySelector(`option[value=${value}]`).selected = true;
+      }
+      let instance = M.FormSelect.getInstance(selectionElement);
+      instance.destroy();
+      M.FormSelect.init(selectionElement);
+    })
+  }
+
+  submitQueryChipElement(dataType=undefined, prettyQueryText=undefined, queryText=undefined, index=null, isClosingTag=false, isEditable=false) {
+    if (this.elements.editingModusOn) {
+      let editedQueryChipElement = this.elements.queryChipElements[this.elements.editedQueryChipElementIndex];
+      editedQueryChipElement.dataset.type = dataType;
+      editedQueryChipElement.dataset.query = queryText;
+      editedQueryChipElement.firstChild.textContent = prettyQueryText;
+      this.updateChipList();
+      this.queryPreviewBuilder();
+    } else {
+      this.queryChipFactory(dataType, prettyQueryText, queryText, index, isClosingTag, isEditable);
+    }
+  }
+
+  queryChipFactory(dataType, prettyQueryText, queryText, index=null, isClosingTag=false, isEditable=false) {
+    // Creates a new query chip element, adds Eventlisteners for selection, deletion and drag and drop and appends it to the query input field.
+    queryText = nopaque.Utils.escape(queryText);
+    prettyQueryText = nopaque.Utils.escape(prettyQueryText);
+    let queryChipElement = nopaque.Utils.HTMLToElement(
+      `
+        <span class="chip query-component" data-type="${dataType}" data-query="${queryText}" draggable="true"">
+          ${prettyQueryText}${isEditable ? '<i class="material-icons chip-action-button" data-chip-action="edit" style="padding-left:5px; font-size:18px; cursor:pointer;">edit</i>': ''}
+          ${isClosingTag ? '' : '<i class="material-icons close chip-action-button" data-chip-action="delete">close</i>'}
+        </span>
+      `
+    );
+    this.addActionListeners(queryChipElement);
+    queryChipElement.addEventListener('dragstart', this.handleDragStart.bind(this, queryChipElement));
+    queryChipElement.addEventListener('dragend', this.handleDragEnd);
+    // If an index is given, inserts the query chip after the given index (only relevant for Incidence Modifier) and if there is a closing tag, inserts the query chip before the closing tag.
+    if (index !== null) {
+      this.updateChipList();
+      this.elements.queryChipElements[index].after(queryChipElement);
+    } else {
+      this.elements.queryInputField.insertBefore(queryChipElement, this.elements.queryElementTarget);
+    }
+    if (isClosingTag) {
+      this.moveQueryElementTarget(queryChipElement);
+    }
+
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  moveQueryElementTarget(element) {
+    this.elements.queryInputField.insertBefore(this.elements.queryElementTarget, element);
+  }
+
+  addActionListeners(queryChipElement) {
+    let notQuantifiableDataTypes = ['start-sentence', 'end-sentence', 'start-entity', 'start-empty-entity', 'end-entity', 'token-incidence-modifier'];
+    queryChipElement.addEventListener('click', (event) => {
+      if (event.target.classList.contains('chip')) {
+        if (!notQuantifiableDataTypes.includes(queryChipElement.dataset.type)) {
+          this.selectChipElement(queryChipElement);
+        }
+      }
+    });
+    let chipActionButtons = queryChipElement.querySelectorAll('.chip-action-button');
+    chipActionButtons.forEach(button => {
+      button.addEventListener('click', (event) => {
+      if (event.target.dataset.chipAction === 'delete') {
+        this.deleteChipElement(queryChipElement);
+      } else if (event.target.dataset.chipAction === 'edit') {
+        this.editChipElement(queryChipElement);
+      }
+      });
+    });
+  }
+
+  editChipElement(queryChipElement) {
+    this.elements.editingModusOn = true;
+    this.elements.editedQueryChipElementIndex = Array.from(this.elements.queryInputField.children).indexOf(queryChipElement);
+    switch (queryChipElement.dataset.type) {
+      case 'start-entity':
+        this.extensions.structuralAttributeBuilderFunctions.editStartEntityChipElement(queryChipElement);
+        break;
+      case 'token':
+        let queryElementsContent = this.extensions.tokenAttributeBuilderFunctions.prepareTokenQueryElementsContent(queryChipElement);
+        this.extensions.tokenAttributeBuilderFunctions.editTokenChipElement(queryElementsContent);
+        break;
+      default:
+        break;
+    }
+  }
+
+  deleteChipElement(attr) {
+    let elementIndex = Array.from(this.elements.queryInputField.children).indexOf(attr);
+    switch (attr.dataset.type) {
+      case 'start-sentence':
+        this.deleteClosingTagHandler(elementIndex, 'end-sentence');
+        break;
+      case 'start-empty-entity':
+      case 'start-entity':
+        this.deleteClosingTagHandler(elementIndex, 'end-entity');
+        break;
+      case 'token':
+        let nextElement = Array.from(this.elements.queryInputField.children)[elementIndex+1];
+        if (nextElement !== undefined && nextElement.dataset.type === 'token-incidence-modifier') {
+          this.deleteChipElement(nextElement);
+        }
+      default:
+        break;
+    }
+    this.elements.queryInputField.removeChild(attr);
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  deleteClosingTagHandler(elementIndex, closingTagType) {
+    let closingTags = this.elements.queryInputField.querySelectorAll(`[data-type="${closingTagType}"]`);
+    for (let i = 0; i < closingTags.length; i++) {
+      let closingTag = closingTags[i];
+    
+      if (Array.from(this.elements.queryInputField.children).indexOf(closingTag) > elementIndex) {
+        this.deleteChipElement(closingTag);
+        break;
+      }
+    }
+  }
+
+  handleDragStart(queryChipElement) {
+    // is called when a query chip is dragged. It creates a dropzone (in form of a chip) for the dragged chip and adds it to the query input field.
+    let queryChips = this.elements.queryInputField.querySelectorAll('.query-component');
+    if (queryChipElement.dataset.type === 'token-incidence-modifier') {
+      queryChips = this.elements.queryInputField.querySelectorAll('.query-component[data-type="token"]');
+    }
+    setTimeout(() => {
+      let targetChipElement = nopaque.Utils.HTMLToElement('<span class="chip drop-target">Drop here</span>');
+      for (let element of queryChips) {
+        if (element === this.elements.queryInputField.querySelectorAll('.query-component')[0]) {
+          let secondTargetChipClone = targetChipElement.cloneNode(true);
+          element.insertAdjacentElement('beforebegin', secondTargetChipClone);
+          this.addDragDropListeners(secondTargetChipClone, queryChipElement);
+        }
+        if (element === queryChipElement || element.nextSibling === queryChipElement) {continue;}
+
+        let targetChipClone = targetChipElement.cloneNode(true);
+        element.insertAdjacentElement('afterend', targetChipClone);
+        //TODO: Change to two different functions for drag and drop
+        this.addDragDropListeners(targetChipClone, queryChipElement);
+      }
+    }, 0);
+  }
+
+  handleDragEnd(event) {
+    // is called when a query chip is dropped. It removes the dropzones and initializes the tooltips if the dragged element is the query element target.
+    if (event.target.classList.contains('query-element-target')) {
+      M.Tooltip.init(event.target);
+    }
+    document.querySelectorAll('.drop-target').forEach(target => target.remove());
+  }
+
+  addDragDropListeners(targetChipClone, queryChipElement) {
+    targetChipClone.addEventListener('dragover', (event) => {
+      event.preventDefault();
+    });
+    targetChipClone.addEventListener('dragenter', (event) => {
+      event.preventDefault();
+      event.target.style.borderStyle = 'solid dotted';
+    });
+    targetChipClone.addEventListener('dragleave', (event) => {
+      event.preventDefault();
+      event.target.style.borderStyle = 'hidden';
+    });
+    targetChipClone.addEventListener('drop', (event) => {
+      let dropzone = event.target;
+      dropzone.parentElement.replaceChild(queryChipElement, dropzone);
+      this.updateChipList();
+      this.queryPreviewBuilder();
+    });
+  }
+
+  queryPreviewBuilder() {
+    // Builds the query preview in the form of pure CQL and displays it in the query preview field.
+    let queryPreview = document.querySelector('#corpus-analysis-concordance-query-preview');
+    let queryInputFieldContent = [];
+    this.elements.queryChipElements.forEach(element => {
+      let queryElement = element.dataset.query;
+      if (queryElement !== undefined) {
+        queryElement = nopaque.Utils.escape(queryElement);
+      }
+      queryInputFieldContent.push(queryElement);
+    });
+  
+    let queryString = queryInputFieldContent.join(' ');
+    let replacements = {
+      ' +': '+',
+      ' *': '*',
+      ' ?': '?',
+      ' {': '{'
+    };
+
+    for (let key in replacements) {
+      queryString = queryString.replace(key, replacements[key]);
+    }
+    queryString += ';';
+  
+    queryPreview.innerHTML = queryString;
+    queryPreview.parentNode.classList.toggle('hide', queryString === ';');
+  }
+
+  selectChipElement(attr) {
+    if (attr.classList.contains('teal')) {
+      return;
+    }
+
+    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'toggle');
+    attr.classList.toggle('teal');
+    attr.classList.toggle('lighten-5');
+
+    M.Dropdown.getInstance(document.querySelector('.dropdown-trigger[data-toggle-area="token-incidence-modifiers"]')).open();
+
+  }
+
+  unselectChipElement(attr) {
+    let nModalInstance = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-exactly-n-token-modal'));
+    let nmModalInstance = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-between-nm-token-modal'));
+    if (nModalInstance.isOpen || nmModalInstance.isOpen) {
+      return;
+    }
+    attr.classList.remove('teal', 'lighten-5');
+    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'add');
+  }
+
+  tokenIncidenceModifierHandler(incidenceModifier, incidenceModifierPretty, nOrNM = false) {
+    // Adds a token incidence modifier to the query input field.
+    let selectedChip = this.elements.queryInputField.querySelector('.chip.teal');
+    let selectedChipIndex = Array.from(this.elements.queryChipElements).indexOf(selectedChip);
+    if (nOrNM) {
+      this.unselectChipElement(selectedChip);
+    }
+    this.submitQueryChipElement('token-incidence-modifier', incidenceModifierPretty, incidenceModifier, selectedChipIndex);
+  }
+
+  tokenNMSubmitHandler(modalId) {
+    // Adds a token incidence modifier (exactly n or between n and m) to the query input field.
+    let modal = document.querySelector(`#${modalId}`);
+    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
+    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
+    input_m = input_m !== undefined ? input_m.value : '';
+    let input = `{${input_n}${input_m !== '' ? ',' : ''}${input_m}}`;
+    let pretty_input = `between ${input_n} and ${input_m} (${input})`;
+    if (input_m === '') {
+     pretty_input = `exactly ${input_n} (${input})`;
+    }
+
+    let instance = M.Modal.getInstance(modal);
+    instance.close();
+
+    this.tokenIncidenceModifierHandler(input, pretty_input, true);
+  }
+
+  expertModeQueryBuilderSwitchHandler() {
+    let queryBuilderDisplay = document.querySelector("#corpus-analysis-concordance-query-builder-display");
+    let expertModeDisplay = document.querySelector("#corpus-analysis-concordance-expert-mode-display");
+    let expertModeSwitch = document.querySelector("#corpus-analysis-concordance-expert-mode-switch");
+    let submitModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-switch-to-query-builder-submit-modal'));
+    
+    let confirmSwitchToQueryBuilderButton = document.querySelector('.switch-action[data-switch-action="confirm"]');
+    confirmSwitchToQueryBuilderButton.addEventListener("click", () => {
+      queryBuilderDisplay.classList.remove("hide");
+      expertModeDisplay.classList.add("hide");
+      this.switchToQueryBuilderParser();
+    });
+
+    expertModeSwitch.addEventListener("change", () => {
+      const isChecked = expertModeSwitch.checked;
+      if (isChecked) {
+        queryBuilderDisplay.classList.add("hide");
+        expertModeDisplay.classList.remove("hide");
+        this.switchToExpertModeParser();
+      } else {
+        submitModal.open();
+      }
+    });
+  }
+
+  switchToExpertModeParser() {
+    let expertModeInputField = document.querySelector('#corpus-analysis-concordance-form-query');
+    expertModeInputField.value = '';
+    let queryBuilderInputFieldValue = nopaque.Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
+    if (queryBuilderInputFieldValue !== "" && queryBuilderInputFieldValue !== ";") {
+      expertModeInputField.value = queryBuilderInputFieldValue;
+    }
+  }
+
+  switchToQueryBuilderParser() {
+    this.resetQueryInputField();
+    let expertModeInputFieldValue = document.querySelector('#corpus-analysis-concordance-form-query').value;
+    let chipElements = this.parseTextToChip(expertModeInputFieldValue);
+    let editableElements = ['start-entity', 'token'];
+    for (let chipElement of chipElements) {
+      let isEditable = editableElements.includes(chipElement['type']);
+      if (chipElement['query'] === '[]'){
+        isEditable = false;
+      }
+      this.submitQueryChipElement(chipElement['type'], chipElement['pretty'], chipElement['query'], null, false, isEditable);
+    }
+  }
+
+  parseTextToChip(query) {
+    const parsingElementDict = {
+      '<s>': {
+        pretty: 'Sentence Start',
+        type: 'start-sentence'
+      },
+      '<\/s>': {
+        pretty: 'Sentence End',
+        type: 'end-sentence'
+      },
+      '<ent>': {
+        pretty: 'Entity Start',
+        type: 'start-empty-entity'
+      },
+      '<ent_type="([A-Z]+)">': {
+        pretty: '',
+        type: 'start-entity'
+      },
+      '<\\\/ent(_type)?>': {
+        pretty: 'Entity End',
+        type: 'end-entity'
+      },
+      '\\[(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?)*\\]': {
+        pretty: '',
+        type: 'token'
+      },
+      '\\[\\]': {
+        pretty: 'Empty Token',
+        type: 'token'
+      },
+      '(?<!\\[) ?\\+ ?(?![^\\]]\\])': {
+        pretty: ' one or more (+)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\* ?(?![^\\]]\\])': {
+        pretty: 'zero or more (*)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\? ?(?![^\\]]\\])': {
+        pretty: 'zero or one (?)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])': {
+        pretty: '',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])': {
+        pretty: '',
+        type: 'token-incidence-modifier'
+      }
+    }
+  
+    let chipElements = [];
+    let regexPattern = Object.keys(parsingElementDict).map(pattern => `(${pattern})`).join('|');
+    const regex = new RegExp(regexPattern, 'gi');
+    let match;
+  
+    while ((match = regex.exec(query)) !== null) {
+      // this is necessary to avoid infinite loops with zero-width matches
+      if (match.index === regex.lastIndex) {
+        regex.lastIndex++;
+      }
+      let stringElement = match[0];
+      for (let [pattern, chipElement] of Object.entries(parsingElementDict)) {
+        const parsingRegex = new RegExp(pattern, 'gi');
+        if (parsingRegex.exec(stringElement)) {
+          // Creating the pretty text for the chip element
+          let prettyText;
+          switch (pattern) {
+            case '<ent_type="([A-Z]+)">':
+              prettyText = `Entity Type=${stringElement.replace(/<ent_type="|">/g, '')}`;
+              break;
+            case ':: ?match\\.text_[A-Za-z]+="[^"]+"':
+              prettyText = stringElement.replace(/:: ?match\.text_|"|"/g, '');
+              break;
+            case '\\[(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?)*\\]':
+              prettyText = stringElement.replace(/^\[|\]$|(?<!\\)"/g, '');
+              prettyText = prettyText.replace(/\&/g, ' and ').replace(/\|/g, ' or ');
+              break;
+            case '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])':
+              prettyText = `exactly ${stringElement.replace(/{|}/g, '')} (${stringElement})`;
+              break;
+            case '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])':
+              prettyText = `between${stringElement.replace(/{|}/g, ' ').replace(',', ' and ')}(${stringElement})`;
+              break;
+            default:
+              prettyText = chipElement.pretty;
+              break;
+          }
+          chipElements.push({
+            type: chipElement.type,
+            pretty: prettyText,
+            query: stringElement
+          });
+          break;
+        }
+      }
+    }
+  
+    return chipElements;
+  }
+};

app/static/js/corpus-analysis/query-builder/structural-attribute-builder-functions.js

@@ -0,0 +1,82 @@
+nopaque.corpus_analysis.query_builder.StructuralAttributeBuilderFunctions = class StructuralAttributeBuilderFunctions {
+  constructor(app) {
+    this.app = app;
+    this.elements = app.elements;
+
+    this.structuralAttrModalEventlisteners();
+
+    this.elements.structuralAttrModal = M.Modal.init(
+      document.querySelector('#corpus-analysis-concordance-structural-attr-modal'), 
+      {
+        onCloseStart: () => {
+          this.resetStructuralAttrModal();
+        }
+      }
+    );
+  }
+
+  structuralAttrModalEventlisteners() {
+    document.querySelectorAll('[data-structural-attr-modal-action-button]').forEach(button => {
+      button.addEventListener('click', () => {
+        this.actionButtonInStrucAttrModalHandler(button.dataset.structuralAttrModalActionButton);
+      });
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="any"]').addEventListener('click', () => {
+      this.app.submitQueryChipElement('start-empty-entity', 'Entity Start', '<ent>');
+      this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent>', null, true);
+      this.elements.structuralAttrModal.close();
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="english"]').addEventListener('change', (event) => {
+      this.app.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true);
+      if (!this.elements.editingModusOn) {
+        this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
+      }
+      this.elements.structuralAttrModal.close();
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="german"]').addEventListener('change', (event) => {
+      this.app.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true); 
+      if (!this.elements.editingModusOn) {
+        this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
+      }
+      this.elements.structuralAttrModal.close();
+    });
+  }
+
+  resetStructuralAttrModal() {
+    this.app.resetMaterializeSelection([this.elements.englishEntTypeSelection, this.elements.germanEntTypeSelection]);
+    this.app.toggleClass(['entity-builder'], 'hide', 'add');
+    this.toggleEditingAreaStructuralAttrModal('remove');
+    this.elements.editingModusOn = false;
+    this.elements.editedQueryChipElementIndex = undefined;
+  }
+
+  actionButtonInStrucAttrModalHandler(action) {
+    switch (action) {
+      case 'sentence':
+        this.app.submitQueryChipElement('start-sentence', 'Sentence Start', '<s>');
+        this.app.submitQueryChipElement('end-sentence', 'Sentence End', '</s>', null, true);
+        this.elements.structuralAttrModal.close();
+        break;
+      case 'entity':
+        this.app.toggleClass(['entity-builder'], 'hide', 'toggle');
+        break;
+      default:
+        break;
+    }
+  }
+
+  toggleEditingAreaStructuralAttrModal(action) {
+    // If the user edits a query chip element, the corresponding editing area is displayed and the other areas are hidden or disabled.
+    this.app.toggleClass(['sentence-button', 'entity-button', 'any-type-entity-button'], 'disabled', action);
+  }
+
+  editStartEntityChipElement(queryChipElement) {
+    this.elements.structuralAttrModal.open();
+    this.app.toggleClass(['entity-builder'], 'hide', 'remove');
+    this.toggleEditingAreaStructuralAttrModal('add');
+    let entType = queryChipElement.dataset.query.replace(/<ent_type="|">/g, '');
+    let isEnglishEntType = this.elements.englishEntTypeSelection.querySelector(`option[value=${entType}]`) !== null;
+    let selection = isEnglishEntType ? this.elements.englishEntTypeSelection : this.elements.germanEntTypeSelection;
+    this.app.resetMaterializeSelection([selection], entType);
+  }
+}

app/static/js/corpus-analysis/query-builder/token-attribute-builder-functions.js

@@ -0,0 +1,329 @@
+nopaque.corpus_analysis.query_builder.TokenAttributeBuilderFunctions = class TokenAttributeBuilderFunctions {
+  constructor(app) {
+    this.app = app;
+    this.elements = app.elements;
+
+    this.elements.positionalAttrSelection.addEventListener('change', () => {
+      this.preparePositionalAttrModal();
+    });
+
+    // Options for positional attribute selection
+    document.querySelectorAll('.positional-attr-options-action-button[data-options-action]').forEach(button => {
+      button.addEventListener('click', () => {this.actionButtonInOptionSectionHandler(button.dataset.optionsAction);});
+    });
+
+    this.elements.tokenSubmitButton.addEventListener('click', () => {this.addTokenToQuery();});
+
+    this.elements.positionalAttrModal = M.Modal.init(
+      document.querySelector('#corpus-analysis-concordance-positional-attr-modal'), 
+      {
+        onOpenStart: () => {
+          this.preparePositionalAttrModal();
+        },
+        onCloseStart: () => {
+          this.resetPositionalAttrModal();
+        }
+      }
+    );
+  }
+
+  resetPositionalAttrModal() {
+    let originalSelectionList = 
+      `
+        <option value="word" selected>word</option>
+        <option value="lemma" >lemma</option>
+        <option value="english-pos">english pos</option>
+        <option value="german-pos">german pos</option>
+        <option value="simple_pos">simple_pos</option>
+        <option value="empty-token">empty token</option>
+      `;
+    this.elements.positionalAttrSelection.innerHTML = originalSelectionList;
+    this.elements.tokenQuery.innerHTML = '';
+    this.elements.tokenBuilderContent.innerHTML = '';
+    this.app.toggleClass(['input-field-options'], 'hide', 'remove');
+    this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
+    this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], "word");
+    this.elements.ignoreCaseCheckbox.checked = false;
+    this.elements.editingModusOn = false;
+    this.elements.editedQueryChipElementIndex = undefined;
+  }
+
+  actionButtonInOptionSectionHandler(elem) {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    switch (elem) {
+      case 'option-group':
+        this.cursorPositionInputfieldHandler(input, '(option1|option2)');
+        let firstIndex = input.value.indexOf('option1');
+        let lastIndex = firstIndex + 'option1'.length;
+        input.setSelectionRange(firstIndex, lastIndex);
+        break;
+      case 'wildcard-char':
+        this.cursorPositionInputfieldHandler(input, '.');
+        input.focus();
+        break;
+      case 'and':
+        this.conditionHandler('and');
+        break;
+      case 'or':
+        this.conditionHandler('or');
+        break;
+      default:
+        break;
+    }
+    this.optionToggleHandler();
+  }
+
+  cursorPositionInputfieldHandler(input, addedInput) {
+    let cursorPosition = input.selectionStart;
+    let textBeforeCursor = input.value.substring(0, cursorPosition);
+    let textAfterCursor = input.value.substring(cursorPosition);
+    let newInputValue = textBeforeCursor + addedInput + textAfterCursor;
+    input.value = newInputValue;
+    let newCursorPosition = cursorPosition + addedInput.length;
+    input.setSelectionRange(newCursorPosition, newCursorPosition);
+  }
+
+  characterIncidenceModifierHandler(elem) {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    this.cursorPositionInputfieldHandler(input, elem.dataset.token);
+  }
+
+  characterNMSubmitHandler(modalId) {
+    let modal = document.querySelector(`#${modalId}`);
+    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
+    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
+    input_m = input_m !== undefined ? ',' + input_m.value : '';
+    let addedInput = `${input_n}${input_m}`;
+
+    let instance = M.Modal.getInstance(modal);
+    instance.close();
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    this.cursorPositionInputfieldHandler(input, `{${addedInput}}`);
+  }
+
+  conditionHandler(conditionText) {
+    let tokenQueryTemplateClone = this.elements.tokenQueryTemplate.content.cloneNode(true);
+    tokenQueryTemplateClone.querySelector('.token-query-template-content').appendChild(this.elements.tokenBuilderContent.firstElementChild);
+    let notSelectedButton = tokenQueryTemplateClone.querySelector(`[data-condition-pretty-text]:not([data-condition-pretty-text="${conditionText}"])`);
+    let deleteButton = tokenQueryTemplateClone.querySelector(`[data-token-query-content-action="delete"]`);
+    deleteButton.addEventListener('click', (event) => {
+      this.deleteTokenQueryRow(event.target);
+    });
+    notSelectedButton.parentNode.removeChild(notSelectedButton);
+    this.elements.tokenQuery.appendChild(tokenQueryTemplateClone);
+
+    let lastTokenQueryRow = this.elements.tokenQuery.lastElementChild;
+    if(lastTokenQueryRow.querySelector('[data-kind-of-token="word"]') || lastTokenQueryRow.querySelector('[data-kind-of-token="lemma"]')) {
+      this.appendIgnoreCaseCheckbox(lastTokenQueryRow.querySelector('.token-query-template-content'), this.elements.ignoreCaseCheckbox.checked);
+    }
+    this.elements.ignoreCaseCheckbox.checked = false;
+    this.setTokenSelection();
+  }
+
+  deleteTokenQueryRow(deleteButton) {
+    let deletedRow = deleteButton.closest('.row');
+    let condition = deletedRow.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
+    if (condition === 'and') {
+      let kindOfToken = deletedRow.querySelector('[data-kind-of-token]').dataset.kindOfToken;
+      switch (kindOfToken) {
+        case 'english-pos' || 'german-pos':
+          this.createOptionElementForPosAttrSelection('english-pos');
+          this.createOptionElementForPosAttrSelection('german-pos');
+          break;
+        default:
+          this.createOptionElementForPosAttrSelection(kindOfToken);
+          break;
+      }
+      M.FormSelect.init(this.elements.positionalAttrSelection);
+    }
+    deletedRow.remove();
+  }
+
+  createOptionElementForPosAttrSelection(kindOfToken) {
+    let option = document.createElement('option');
+    option.value = kindOfToken;
+    option.text = kindOfToken;
+    this.elements.positionalAttrSelection.appendChild(option);
+  }
+
+  appendIgnoreCaseCheckbox(parentElement, checked=false) {
+    let ignoreCaseCheckboxClone = document.querySelector('#ignore-case-checkbox-template').content.cloneNode(true);
+    parentElement.appendChild(ignoreCaseCheckboxClone);
+    M.Tooltip.init(parentElement.querySelectorAll('.tooltipped'));
+    if (checked) {
+      parentElement.querySelector('input[type="checkbox"]').checked = true;
+    }
+  }
+
+  setTokenSelection(selection="word", optionDeleteList=['empty-token']) {
+    optionDeleteList.forEach(option => {
+      if (this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`) !== null) {
+        this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`).remove();
+      }
+    });
+
+    this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], selection);
+    this.preparePositionalAttrModal();
+  }
+
+  preparePositionalAttrModal() {
+    let selection = this.elements.positionalAttrSelection.value;
+    if (selection !== 'empty-token') {
+      let selectionTemplate = document.querySelector(`.token-builder-section[data-token-builder-section="${selection}"]`);
+      let selectionTemplateClone = selectionTemplate.content.cloneNode(true);
+    
+      this.elements.tokenBuilderContent.innerHTML = '';
+      this.elements.tokenBuilderContent.appendChild(selectionTemplateClone);
+      if (this.elements.tokenBuilderContent.querySelector('select') !== null) {
+        let selectElement = this.elements.tokenBuilderContent.querySelector('select');
+        M.FormSelect.init(selectElement);
+        selectElement.addEventListener('change', () => {this.optionToggleHandler();});
+      } else {
+        this.elements.tokenBuilderContent.querySelector('input').addEventListener('input', () => {this.optionToggleHandler();});
+      }
+    }
+    this.optionToggleHandler();
+
+    if (selection === 'word' || selection === 'lemma') {
+      this.app.toggleClass(['input-field-options'], 'hide', 'remove');
+    } else if (selection === 'empty-token'){
+      this.addTokenToQuery();
+    } else {
+      this.app.toggleClass(['input-field-options'], 'hide', 'add');
+    }
+  }
+
+  tokenInputCheck(elem) {
+    return elem.querySelector('select') !== null ? elem.querySelector('select') : elem.querySelector('input');
+  }
+
+  optionToggleHandler() {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    if (input.value === '' && this.elements.editingModusOn === false) {
+      this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
+    } else if (this.elements.positionalAttrSelection.querySelectorAll('option').length === 1) {
+      this.app.toggleClass(['and'], 'disabled', 'add');
+      this.app.toggleClass(['or'], 'disabled', 'remove');
+    } else {
+      this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'remove');
+    }
+  }
+
+  addTokenToQuery() {
+    let tokenQueryPrettyText = '';
+    let tokenQueryCQLText = '';
+    let input;
+    let kindOfToken = this.kindOfTokenCheck(this.elements.positionalAttrSelection.value);
+    
+    // Takes all rows of the token query (if there is a query concatenation).
+    // Adds their contents to tokenQueryPrettyText and tokenQueryCQLText, which will later be expanded with the current input field.
+    let tokenQueryRows = this.elements.tokenQuery.querySelectorAll('.row');
+    tokenQueryRows.forEach(row => {
+      let ignoreCaseCheckbox = row.querySelector('input[type="checkbox"]');
+      let c = ignoreCaseCheckbox !== null && ignoreCaseCheckbox.checked ? ' %c' : '';
+      let tokenQueryRowInput = this.tokenInputCheck(row.querySelector('.token-query-template-content'));
+      let tokenQueryKindOfToken = this.kindOfTokenCheck(tokenQueryRowInput.closest('.input-field').dataset.kindOfToken);
+      let tokenConditionPrettyText = row.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
+      let tokenConditionCQLText = row.querySelector('[data-condition-cql-text]').dataset.conditionCqlText;
+      tokenQueryPrettyText += `${tokenQueryKindOfToken}=${tokenQueryRowInput.value}${c} ${tokenConditionPrettyText} `;
+      tokenQueryCQLText += `${tokenQueryKindOfToken}="${tokenQueryRowInput.value}"${c} ${tokenConditionCQLText}`;
+    });
+    if (kindOfToken === 'empty-token') {
+      tokenQueryPrettyText += 'empty token';
+    } else {
+      let c = this.elements.ignoreCaseCheckbox.checked ? ' %c' : '';
+      input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+      tokenQueryPrettyText += `${kindOfToken}=${input.value}${c}`;
+      tokenQueryCQLText += `${kindOfToken}="${input.value}"${c}`;
+    }
+    // isTokenQueryInvalid looks if a valid value is passed. If the input fields/dropdowns are empty (isTokenQueryInvalid === true), no token is added.
+    if (this.elements.positionalAttrSelection.value !== 'empty-token' && input.value === '') {
+      this.disableTokenSubmit();
+    } else {
+      tokenQueryCQLText = `[${tokenQueryCQLText}]`;
+      this.app.submitQueryChipElement('token', tokenQueryPrettyText, tokenQueryCQLText, null, false, kindOfToken === 'empty-token' ? false : true);
+      this.elements.positionalAttrModal.close();
+    }
+  }
+  
+  kindOfTokenCheck(kindOfToken) {
+    return kindOfToken === 'english-pos' || kindOfToken === 'german-pos' ? 'pos' : kindOfToken;
+  }
+
+  disableTokenSubmit() {
+    this.elements.tokenSubmitButton.classList.add('red');
+    this.elements.noValueMessage.classList.remove('hide');
+    setTimeout(() => {
+      this.elements.tokenSubmitButton.classList.remove('red');
+    }, 500);
+    setTimeout(() => {
+      this.elements.noValueMessage.classList.add('hide');
+    }, 3000);
+  }
+
+  editTokenChipElement(queryElementsContent) {
+    this.elements.positionalAttrModal.open();
+    queryElementsContent.forEach((queryElement) => {
+      this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
+      this.preparePositionalAttrModal();
+      switch (queryElement.tokenAttr) {
+        case 'word':
+        case 'lemma':
+          this.elements.tokenBuilderContent.querySelector('input').value = queryElement.tokenValue;
+          break;
+        case 'english-pos':
+          // English-pos is selected by default. Then it is checked whether the passed token value occurs in the english-pos selection. If not, the selection is reseted and changed to german-pos.
+          let selection = this.elements.tokenBuilderContent.querySelector('select');
+          queryElement.tokenAttr = selection.querySelector(`option[value=${queryElement.tokenValue}]`) ? 'english-pos' : 'german-pos';
+          this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
+          this.preparePositionalAttrModal();
+          this.app.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
+          break;
+        case 'simple_pos':
+          this.app.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
+        default:
+          break;
+      }
+      if (queryElement.ignoreCase) {
+        this.elements.ignoreCaseCheckbox.checked = true;
+      }
+      if (queryElement.condition !== undefined) {
+        this.conditionHandler(queryElement.condition, true);
+      }
+
+    });
+  }
+
+  prepareTokenQueryElementsContent(queryChipElement) {
+    //this regex searches for word or lemma or pos or simple_pos="any string (also quotation marks escaped by backslash) within double quotes" followed by one or no ignore case markers, followed by one or no condition characters.
+    let regex = new RegExp('(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?(\\&|\\|)?', 'gm');
+    let m;
+    let queryElementsContent = [];
+    while ((m = regex.exec(queryChipElement.dataset.query)) !== null) {
+      // this is necessary to avoid infinite loops with zero-width matches
+      if (m.index === regex.lastIndex) {
+          regex.lastIndex++;
+      }
+      let tokenAttr = m[1];
+      // Passes english-pos by default so that the template is added. In editTokenChipElement it is then checked whether it is english-pos or german-pos.
+      if (tokenAttr === 'pos') {
+        tokenAttr = 'english-pos';
+      }
+      let tokenValue = m[2].replace(/(?<!\\)"/g, '');
+      let ignoreCase = false;
+      let condition = undefined;
+      m.forEach((match) => {
+        if (match === "%c") {
+          ignoreCase = true;
+        } else if (match === "&") {
+          condition = "and";
+        } else if (match === "|") {
+          condition = "or";
+        }
+      });
+      queryElementsContent.push({tokenAttr: tokenAttr, tokenValue: tokenValue, ignoreCase: ignoreCase, condition: condition});
+    }
+    return queryElementsContent;
+  }
+
+}

app/static/js/corpus-analysis/reader-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisReader {
+nopaque.corpus_analysis.ReaderExtension = class ReaderExtension {
   name = 'Reader';
 
   constructor(app) {
@@ -112,7 +112,7 @@ class CorpusAnalysisReader {
     if (this.data.corpus.p.pages === 0) {return;}
     let pageElement;
     // First page button. Disables first page button if on first page
-    pageElement = Utils.HTMLToElement(
+    pageElement = nopaque.Utils.HTMLToElement(
       `
         <li class="${this.data.corpus.p.page === 1 ? 'disabled' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.page === 1 ? '' : 'data-target="1"'}>
@@ -123,7 +123,7 @@ class CorpusAnalysisReader {
     );
     this.elements.corpusPagination.appendChild(pageElement);
     // Previous page button. Disables previous page button if on first page
-    pageElement = Utils.HTMLToElement(
+    pageElement = nopaque.Utils.HTMLToElement(
       `
         <li class="${this.data.corpus.p.has_prev ? 'waves-effect' : 'disabled'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.has_prev ? 'data-target="' + this.data.corpus.p.prev_num + '"' : ''}>
@@ -135,7 +135,7 @@ class CorpusAnalysisReader {
     this.elements.corpusPagination.appendChild(pageElement);
     // First page as number. Hides first page button if on first page
     if (this.data.corpus.p.page > 6) {
-      pageElement = Utils.HTMLToElement(
+      pageElement = nopaque.Utils.HTMLToElement(
         `
           <li class="waves-effect">
             <a class="corpus-analysis-action pagination-trigger" data-target="1">1</a>
@@ -143,14 +143,14 @@ class CorpusAnalysisReader {
         `
       );
       this.elements.corpusPagination.appendChild(pageElement);
-      pageElement = Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
+      pageElement = nopaque.Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
       this.elements.corpusPagination.appendChild(pageElement);
     }
 
     // render page buttons (5 before and 5 after current page)
     for (let i = this.data.corpus.p.page - this.settings.pagination.innerWindow; i <= this.data.corpus.p.page; i++) {
       if (i <= 0) {continue;}
-      pageElement = Utils.HTMLToElement(
+      pageElement = nopaque.Utils.HTMLToElement(
         `
           <li class="${i === this.data.corpus.p.page ? 'active' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${i === this.data.corpus.p.page ? '' : 'data-target="' + i + '"'}>${i}</a>
@@ -161,7 +161,7 @@ class CorpusAnalysisReader {
     };
     for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page + this.settings.pagination.innerWindow; i++) {
       if (i > this.data.corpus.p.pages) {break;}
-      pageElement = Utils.HTMLToElement(
+      pageElement = nopaque.Utils.HTMLToElement(
         `
           <li class="${i === this.data.corpus.p.page ? 'active' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${i === this.data.corpus.p.page ? '' : 'data-target="' + i + '"'}>${i}</a>
@@ -172,9 +172,9 @@ class CorpusAnalysisReader {
     };
     // Last page as number. Hides last page button if on last page
     if (this.data.corpus.p.page < this.data.corpus.p.pages - 6) {
-      pageElement = Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
+      pageElement = nopaque.Utils.HTMLToElement("<li style='margin-top: 5px;'>&hellip;</li>");
       this.elements.corpusPagination.appendChild(pageElement);
-      pageElement = Utils.HTMLToElement(
+      pageElement = nopaque.Utils.HTMLToElement(
         `
           <li class="waves-effect">
             <a class="corpus-analysis-action pagination-trigger" data-target="${this.data.corpus.p.pages}">${this.data.corpus.p.pages}</a>
@@ -184,7 +184,7 @@ class CorpusAnalysisReader {
       this.elements.corpusPagination.appendChild(pageElement);
     }
     // Next page button. Disables next page button if on last page
-    pageElement = Utils.HTMLToElement(
+    pageElement = nopaque.Utils.HTMLToElement(
       `
         <li class="${this.data.corpus.p.has_next ? 'waves-effect' : 'disabled'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.has_next ? 'data-target="' + this.data.corpus.p.next_num + '"' : ''}>
@@ -195,7 +195,7 @@ class CorpusAnalysisReader {
     );
     this.elements.corpusPagination.appendChild(pageElement);
     // Last page button. Disables last page button if on last page
-    pageElement = Utils.HTMLToElement(
+    pageElement = nopaque.Utils.HTMLToElement(
       `
         <li class="${this.data.corpus.p.page === this.data.corpus.p.pages ? 'disabled' : 'waves-effect'}">
           <a class="corpus-analysis-action pagination-trigger" ${this.data.corpus.p.page === this.data.corpus.p.pages ? '' : 'data-target="' + this.data.corpus.p.pages + '"'}>

app/static/js/corpus-analysis/static-visualization-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisStaticVisualization {
+nopaque.corpus_analysis.StaticVisualizationExtension = class StaticVisualizationExtension {
   name = 'Static Visualization (beta)';
 
   constructor(app) {
@@ -75,7 +75,7 @@ class CorpusAnalysisStaticVisualization {
 
   getStopwords() {
     this.data.promises.getStopwords = new Promise((resolve, reject) => {
-      Requests.corpora.entity.getStopwords()
+      nopaque.requests.corpora.entity.getStopwords()
         .then((response) => {
           response.json()
             .then((json) => {
@@ -104,7 +104,7 @@ class CorpusAnalysisStaticVisualization {
   renderTextInfoList() {
     let corpusData = this.data.corpus.o.staticData;
     let corpusTextInfoListElement = document.querySelector('.corpus-text-info-list');
-    let corpusTextInfoList = new CorpusTextInfoList(corpusTextInfoListElement);
+    let corpusTextInfoList = new nopaque.resource_lists.CorpusTextInfoList(corpusTextInfoListElement);
     let texts = corpusData.s_attrs.text.lexicon;
     let textData = [];
     for (let i = 0; i < Object.entries(texts).length; i++) {
@@ -213,7 +213,7 @@ class CorpusAnalysisStaticVisualization {
 
   async renderTokenList() {
     let corpusTokenListElement = document.querySelector('.corpus-token-list');
-    let corpusTokenList = new CorpusTokenList(corpusTokenListElement);
+    let corpusTokenList = new nopaque.resource_lists.CorpusTokenList(corpusTokenListElement);
     let filteredData = this.filterData();
     let stopwords = this.data.stopwords;
     if (this.data.stopwords === undefined) {
@@ -358,7 +358,7 @@ class CorpusAnalysisStaticVisualization {
     if (stopwordLanguageSelection.children.length === 0) {
       Object.keys(stopwords).forEach(language => {
         if (language !== 'user_stopwords') {
-          let optionElement = Utils.HTMLToElement(`<option value="${language}" ${language === 'english' ? 'selected' : ''}>${language}</option>`);
+          let optionElement = nopaque.Utils.HTMLToElement(`<option value="${language}" ${language === 'english' ? 'selected' : ''}>${language}</option>`);
           stopwordLanguageSelection.appendChild(optionElement);
         }
       });
@@ -367,7 +367,7 @@ class CorpusAnalysisStaticVisualization {
     // Render user stopwords over input field.
     if (this.data.stopwords['user_stopwords'].length > 0) {
       for (let word of this.data.stopwords['user_stopwords']) {
-        let chipElement = Utils.HTMLToElement(`<div class="chip">${word}<i class="close material-icons">close</i></div>`);
+        let chipElement = nopaque.Utils.HTMLToElement(`<div class="chip">${word}<i class="close material-icons">close</i></div>`);
         chipElement.addEventListener('click', (event) => {
           let removedListItem = event.target.closest('.chip').firstChild.textContent;
           this.data.stopwords['user_stopwords'] = structuredClone(this.data.stopwords['user_stopwords'].filter(item => item !== removedListItem));
@@ -433,7 +433,7 @@ class CorpusAnalysisStaticVisualization {
     let stopwordLanguageChipList = document.querySelector('#stopword-language-chip-list');
     stopwordLanguageChipList.innerHTML = '';
     for (let word of stopwords) {
-      let chipElement = Utils.HTMLToElement(`<div class="chip">${word}<i class="close material-icons">close</i></div>`);
+      let chipElement = nopaque.Utils.HTMLToElement(`<div class="chip">${word}<i class="close material-icons">close</i></div>`);
       chipElement.addEventListener('click', (event) => {
         let removedListItem = event.target.closest('.chip').firstChild.textContent;
         this.data.stopwords[language] = structuredClone(this.data.stopwords[language].filter(item => item !== removedListItem));

app/static/js/cqi/api/index.js

@@ -1 +0,0 @@
-cqi.api = {};

app/static/js/cqi/errors.js

@@ -1,185 +0,0 @@
-cqi.errors = {};
-
-
-/**
- * A base class from which all other errors inherit.
- * If you want to catch all errors that the CQi package might throw,
- * catch this base error.
- */
-cqi.errors.CQiError = class CQiError extends Error {
-  constructor(message) {
-    super(message);
-    this.code = undefined;
-    this.description = undefined;
-  }
-};
-
-
-cqi.errors.Error = class Error extends cqi.errors.CQiError {
-  constructor(message) {
-    super(message);
-    this.code = 2;
-  }
-};
-
-
-cqi.errors.ErrorGeneralError = class ErrorGeneralError extends cqi.errors.Error {
-  constructor(message) {
-    super(message);
-    this.code = 513;
-  }
-};
-
-
-cqi.errors.ErrorConnectRefused = class ErrorConnectRefused extends cqi.errors.Error {
-  constructor(message) {
-    super(message);
-    this.code = 514;
-  }
-};
-
-
-cqi.errors.ErrorUserAbort = class ErrorUserAbort extends cqi.errors.Error {
-  constructor(message) {
-    super(message);
-    this.code = 515;
-  }
-};
-
-
-cqi.errors.ErrorSyntaxError = class ErrorSyntaxError extends cqi.errors.Error {
-  constructor(message) {
-    super(message);
-    this.code = 516;
-  }
-};
-
-
-cqi.errors.CLError = class Error extends cqi.errors.CQiError {
-  constructor(message) {
-    super(message);
-    this.code = 4;
-  }
-};
-
-
-cqi.errors.CLErrorNoSuchAttribute = class CLErrorNoSuchAttribute extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1025;
-    this.description = "CQi server couldn't open attribute";
-  }
-};
-
-
-cqi.errors.CLErrorWrongAttributeType = class CLErrorWrongAttributeType extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1026;
-  }
-};
-
-
-cqi.errors.CLErrorOutOfRange = class CLErrorOutOfRange extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1027;
-  }
-};
-
-
-cqi.errors.CLErrorRegex = class CLErrorRegex extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1028;
-  }
-};
-
-
-cqi.errors.CLErrorCorpusAccess = class CLErrorCorpusAccess extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1029;
-  }
-};
-
-
-cqi.errors.CLErrorOutOfMemory = class CLErrorOutOfMemory extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1030;
-    this.description = 'CQi server has run out of memory; try discarding some other corpora and/or subcorpora';
-  }
-};
-
-
-cqi.errors.CLErrorInternal = class CLErrorInternal extends cqi.errors.CLError {
-  constructor(message) {
-    super(message);
-    this.code = 1031;
-    this.description = "The classical 'please contact technical support' error";
-  }
-};
-
-
-cqi.errors.CQPError = class Error extends cqi.errors.CQiError {
-  constructor(message) {
-    super(message);
-    this.code = 5;
-  }
-};
-
-
-cqi.errors.CQPErrorGeneral = class CQPErrorGeneral extends cqi.errors.CQPError {
-  constructor(message) {
-    super(message);
-    this.code = 1281;
-  }
-};
-
-
-cqi.errors.CQPErrorNoSuchCorpus = class CQPErrorNoSuchCorpus extends cqi.errors.CQPError {
-  constructor(message) {
-    super(message);
-    this.code = 1282;
-  }
-};
-
-
-cqi.errors.CQPErrorInvalidField = class CQPErrorInvalidField extends cqi.errors.CQPError {
-  constructor(message) {
-    super(message);
-    this.code = 1283;
-  }
-};
-
-
-cqi.errors.CQPErrorOutOfRange = class CQPErrorOutOfRange extends cqi.errors.CQPError {
-  constructor(message) {
-    super(message);
-    this.code = 1284;
-    this.description = 'A number is out of range';
-  }
-};
-
-
-cqi.errors.lookup = {
-  2: cqi.errors.Error,
-  513: cqi.errors.ErrorGeneralError,
-  514: cqi.errors.ErrorConnectRefused,
-  515: cqi.errors.ErrorUserAbort,
-  516: cqi.errors.ErrorSyntaxError,
-  4: cqi.errors.CLError,
-  1025: cqi.errors.CLErrorNoSuchAttribute,
-  1026: cqi.errors.CLErrorWrongAttributeType,
-  1027: cqi.errors.CLErrorOutOfRange,
-  1028: cqi.errors.CLErrorRegex,
-  1029: cqi.errors.CLErrorCorpusAccess,
-  1030: cqi.errors.CLErrorOutOfMemory,
-  1031: cqi.errors.CLErrorInternal,
-  5: cqi.errors.CQPError,
-  1281: cqi.errors.CQPErrorGeneral,
-  1282: cqi.errors.CQPErrorNoSuchCorpus,
-  1283: cqi.errors.CQPErrorInvalidField,
-  1284: cqi.errors.CQPErrorOutOfRange
-};

app/static/js/cqi/index.js

@@ -1,6 +0,0 @@
-var cqi = {};
-
-cqi.CONST_FIELD_KEYWORD = 9;
-cqi.CONST_FIELD_MATCH = 16;
-cqi.CONST_FIELD_MATCHEND = 17;
-cqi.CONST_FIELD_TARGET = 0;

app/static/js/cqi/models/index.js

@@ -1 +0,0 @@
-cqi.models = {};

app/static/js/cqi/status.js

@@ -1,51 +0,0 @@
-cqi.status = {};
-
-
-/**
- * A base class from which all other status inherit.
- */
-cqi.status.CQiStatus = class CQiStatus {
-  constructor() {
-    this.code = undefined;
-  }
-};
-
-
-cqi.status.StatusOk = class StatusOk extends cqi.status.CQiStatus {
-  constructor() {
-    super();
-    this.code = 257;
-  }
-};
-
-
-cqi.status.StatusConnectOk = class StatusConnectOk extends cqi.status.CQiStatus {
-  constructor() {
-    super();
-    this.code = 258;
-  }
-};
-
-
-cqi.status.StatusByeOk = class StatusByeOk extends cqi.status.CQiStatus {
-  constructor() {
-    super();
-    this.code = 259;
-  }
-};
-
-
-cqi.status.StatusPingOk = class StatusPingOk extends cqi.status.CQiStatus {
-  constructor() {
-    super();
-    this.code = 260;
-  }
-};
-
-
-cqi.status.lookup = {
-  257: cqi.status.StatusOk,
-  258: cqi.status.StatusConnectOk,
-  259: cqi.status.StatusByeOk,
-  260: cqi.status.StatusPingOk
-};

app/static/js/forms/base-form.js

@@ -0,0 +1,138 @@
+nopaque.forms.BaseForm = class BaseForm {
+  static htmlClass;
+
+  constructor(formElement) {
+    this.formElement = formElement;
+    this.eventListeners = {
+      'requestLoad': []
+    };
+    this.afterRequestListeners = [];
+
+    for (let selectElement of this.formElement.querySelectorAll('select')) {
+      selectElement.removeAttribute('required');
+    }
+
+    this.formElement.addEventListener('submit', (event) => {
+      event.preventDefault();
+      this.submit(event);
+    });
+  }
+
+  addEventListener(eventType, listener) {
+    if (eventType in this.eventListeners) {
+      this.eventListeners[eventType].push(listener);
+    } else {
+      throw `Unknown event type ${eventType}`;
+    }
+  }
+
+  submit(event) {
+    let request = new XMLHttpRequest();
+    let modalElement = nopaque.Utils.HTMLToElement(
+      `
+        <div class="modal">
+          <div class="modal-content">
+            <h4><i class="material-icons left">file_upload</i>Submitting...</h4>
+            <div class="progress">
+              <div class="determinate" style="width: 0%"></div>
+            </div>
+          </div>
+          <div class="modal-footer">
+            <a class="action-button btn red waves-effect waves-light modal-close" data-action="cancel">Cancel</a>
+          </div>
+        </div>
+      `
+    );
+    document.querySelector('#modals').appendChild(modalElement);
+    let modal = M.Modal.init(
+      modalElement,
+      {
+        dismissible: false,
+        onCloseEnd: () => {
+          modal.destroy();
+          modalElement.remove();
+        }
+      }
+    );
+    modal.open();
+
+    // Remove all previous helper text elements that indicate errors
+    let errorHelperTextElements = this.formElement
+      .querySelectorAll('.helper-text[data-helper-text-type="error"]');
+    for (let errorHelperTextElement of errorHelperTextElements) {
+      errorHelperTextElement.remove();
+    }
+
+    // Check if select elements are filled out properly
+    for (let selectElement of this.formElement.querySelectorAll('select')) {
+      if (selectElement.value === '') {
+        let inputFieldElement = selectElement.closest('.input-field');
+        let errorHelperTextElement = nopaque.Utils.HTMLToElement(
+          '<span class="helper-text error-color-text" data-helper-text-type="error">Please select an option.</span>'
+        );
+        inputFieldElement.appendChild(errorHelperTextElement);
+        inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
+        modal.close();
+        return;
+      }
+    }
+
+    // Setup abort handling
+    let cancelElement = modalElement.querySelector('.action-button[data-action="cancel"]');
+    cancelElement.addEventListener('click', (event) => {request.abort();});
+
+    // Setup load handling (after the request completed)
+    request.addEventListener('load', (event) => {
+      for (let listener of this.eventListeners['requestLoad']) {
+        listener(event);
+      }
+      if (request.status === 400) {
+        let responseJson = JSON.parse(request.responseText);
+        for (let [inputName, inputErrors] of Object.entries(responseJson.errors)) {
+          let inputFieldElement = this.formElement
+            .querySelector(`input[name$="${inputName}"], select[name$="${inputName}"]`)
+            .closest('.input-field');
+          for (let inputError of inputErrors) {
+            let errorHelperTextElement = nopaque.Utils.HTMLToElement(
+              `<span class="helper-text error-color-text" data-helper-type="error">${inputError}</span>`
+            );
+            inputFieldElement.appendChild(errorHelperTextElement);
+          }
+        }
+      }
+      if (request.status === 500) {
+        app.flash('Internal Server Error', 'error');
+      }
+      modal.close();
+    });
+
+    // Setup progress handling
+    let progressBarElement = modalElement.querySelector('.progress > .determinate');
+    request.upload.addEventListener('progress', (event) => {
+      let progress = Math.floor(100 * event.loaded / event.total);
+      progressBarElement.style.width = `${progress}%`;
+    });
+
+    request.open(this.formElement.method, this.formElement.action);
+    request.setRequestHeader('Accept', 'application/json');
+    let formData = new FormData(this.formElement);
+    switch (this.formElement.enctype) {
+      case 'application/x-www-form-urlencoded': {
+        let urlSearchParams = new URLSearchParams(formData);
+        request.send(urlSearchParams);
+        break;
+      }
+      case 'multipart/form-data': {
+        request.send(formData);
+        break;
+      }
+      case 'text/plain': {
+        throw 'enctype "text/plain" is not supported';
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+};

app/static/js/forms/create-contribution-form.js

@@ -1,10 +1,5 @@
-Forms.CreateContributionForm = class CreateContributionForm extends Forms.BaseForm {
-  static autoInit() {
-    let createContributionFormElements = document.querySelectorAll('.create-contribution-form');
-    for (let createContributionFormElement of createContributionFormElements) {
-      new Forms.CreateContributionForm(createContributionFormElement);
-    }
-  }
+nopaque.forms.CreateContributionForm = class CreateContributionForm extends nopaque.forms.BaseForm {
+  static htmlClass = 'create-contribution-form';
 
   constructor(formElement) {
     super(formElement);

app/static/js/forms/create-corpus-file-form.js

@@ -1,10 +1,5 @@
-Forms.CreateCorpusFileForm = class CreateCorpusFileForm extends Forms.BaseForm {
-  static autoInit() {
-    let createCorpusFileFormElements = document.querySelectorAll('.create-corpus-file-form');
-    for (let createCorpusFileFormElement of createCorpusFileFormElements) {
-      new Forms.CreateCorpusFileForm(createCorpusFileFormElement);
-    }
-  }
+nopaque.forms.CreateCorpusFileForm = class CreateCorpusFileForm extends nopaque.forms.BaseForm {
+  static htmlClass = 'create-corpus-file-form';
 
   constructor(formElement) {
     super(formElement);

app/static/js/forms/create-job-form.js

@@ -1,10 +1,5 @@
-Forms.CreateJobForm = class CreateJobForm extends Forms.BaseForm {
-  static autoInit() {
-    let createJobFormElements = document.querySelectorAll('.create-job-form');
-    for (let createJobFormElement of createJobFormElements) {
-      new Forms.CreateJobForm(createJobFormElement);
-    }
-  }
+nopaque.forms.CreateJobForm = class CreateJobForm extends nopaque.forms.BaseForm {
+  static htmlClass = 'create-job-form';
 
   constructor(formElement) {
     super(formElement);

app/static/js/forms/index.js

@@ -1,150 +1,18 @@
-var Forms = {};
+nopaque.forms = {};
 
-Forms.autoInit = () => {
-  for (let propertyName in Forms) {
-    let property = Forms[propertyName];
-    // Call the autoInit method of all properties that are subclasses of Forms.BaseForm
-    if (property.prototype instanceof Forms.BaseForm) {
-      property.autoInit();
-    }
-  }
-};
-
-Forms.BaseForm = class BaseForm {
-  static autoInit() {throw 'Not implemented';}
-
-  constructor(formElement) {
-    this.formElement = formElement;
-    this.eventListeners = {
-      'requestLoad': []
-    };
-    this.afterRequestListeners = [];
-
-    for (let selectElement of this.formElement.querySelectorAll('select')) {
-      selectElement.removeAttribute('required');
-    }
-
-    this.formElement.addEventListener('submit', (event) => {
-      event.preventDefault();
-      this.submit(event);
-    });
-  }
-
-  addEventListener(eventType, listener) {
-    if (eventType in this.eventListeners) {
-      this.eventListeners[eventType].push(listener);
-    } else {
-      throw `Unknown event type ${eventType}`;
-    }
-  }
-
-  submit(event) {
-    let request = new XMLHttpRequest();
-    let modalElement = Utils.HTMLToElement(
-      `
-        <div class="modal">
-          <div class="modal-content">
-            <h4><i class="material-icons left">file_upload</i>Submitting...</h4>
-            <div class="progress">
-              <div class="determinate" style="width: 0%"></div>
-            </div>
-          </div>
-          <div class="modal-footer">
-            <a class="action-button btn red waves-effect waves-light modal-close" data-action="cancel">Cancel</a>
-          </div>
-        </div>
-      `
-    );
-    document.querySelector('#modals').appendChild(modalElement);
-    let modal = M.Modal.init(
-      modalElement,
-      {
-        dismissible: false,
-        onCloseEnd: () => {
-          modal.destroy();
-          modalElement.remove();
-        }
-      }
-    );
-    modal.open();
-
-    // Remove all previous helper text elements that indicate errors
-    let errorHelperTextElements = this.formElement
-      .querySelectorAll('.helper-text[data-helper-text-type="error"]');
-    for (let errorHelperTextElement of errorHelperTextElements) {
-      errorHelperTextElement.remove();
-    }
-
-    // Check if select elements are filled out properly
-    for (let selectElement of this.formElement.querySelectorAll('select')) {
-      if (selectElement.value === '') {
-        let inputFieldElement = selectElement.closest('.input-field');
-        let errorHelperTextElement = Utils.HTMLToElement(
-          '<span class="helper-text error-color-text" data-helper-text-type="error">Please select an option.</span>'
-        );
-        inputFieldElement.appendChild(errorHelperTextElement);
-        inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
-        modal.close();
-        return;
-      }
-    }
-
-    // Setup abort handling
-    let cancelElement = modalElement.querySelector('.action-button[data-action="cancel"]');
-    cancelElement.addEventListener('click', (event) => {request.abort();});
-
-    // Setup load handling (after the request completed)
-    request.addEventListener('load', (event) => {
-      for (let listener of this.eventListeners['requestLoad']) {
-        listener(event);
-      }
-      if (request.status === 400) {
-        let responseJson = JSON.parse(request.responseText);
-        for (let [inputName, inputErrors] of Object.entries(responseJson.errors)) {
-          let inputFieldElement = this.formElement
-            .querySelector(`input[name$="${inputName}"], select[name$="${inputName}"]`)
-            .closest('.input-field');
-          for (let inputError of inputErrors) {
-            let errorHelperTextElement = Utils.HTMLToElement(
-              `<span class="helper-text error-color-text" data-helper-type="error">${inputError}</span>`
-            );
-            inputFieldElement.appendChild(errorHelperTextElement);
-          }
-        }
-      }
-      if (request.status === 500) {
-        app.flash('Internal Server Error', 'error');
-      }
-      modal.close();
-    });
-
-    // Setup progress handling
-    let progressBarElement = modalElement.querySelector('.progress > .determinate');
-    request.upload.addEventListener('progress', (event) => {
-      let progress = Math.floor(100 * event.loaded / event.total);
-      progressBarElement.style.width = `${progress}%`;
-    });
-
-    request.open(this.formElement.method, this.formElement.action);
-    request.setRequestHeader('Accept', 'application/json');
-    let formData = new FormData(this.formElement);
-    switch (this.formElement.enctype) {
-      case 'application/x-www-form-urlencoded': {
-        let urlSearchParams = new URLSearchParams(formData);
-        request.send(urlSearchParams);
-        break;
-      }
-      case 'multipart/form-data': {
-        request.send(formData);
-        break;
-      }
-      case 'text/plain': {
-        throw 'enctype "text/plain" is not supported';
-        break;
-      }
-      default: {
-        break;
-      }
+nopaque.forms.AutoInit = () => {
+  for (let propertyName in nopaque.forms) {
+    let property = nopaque.forms[propertyName];
+    // Initialize properties that are subclasses of nopaque.forms.BaseForm.
+    // This does not include nopaque.forms.BaseForm itself.
+    if (property.prototype instanceof nopaque.forms.BaseForm) {
+      // Check if the static htmlClass property is defined.
+      if (property.htmlClass === undefined) {return;}
+      // Gather all HTML elements that have the `this.htmlClass` class
+      // and do not have the no-autoinit class.
+      let formElements = document.querySelectorAll(`.${property.htmlClass}:not(.no-autoinit)`);
+      // Create an instance of this class for each form element.
+      for (let formElement of formElements) {new property(formElement);}
     }
   }
 };

app/static/js/index.js

@@ -0,0 +1,5 @@
+/*
+ * This object functions as a global namespace for nopaque.
+ * All components of nopaque should be attached to this object.
+ */
+var nopaque = {};

app/static/js/requests/admin.js

@@ -1,19 +1,19 @@
 /*****************************************************************************
  * Requests for /admin routes                                                *
 *****************************************************************************/
-Requests.admin = {};
+nopaque.requests.admin = {};
 
-Requests.admin.users = {};
+nopaque.requests.admin.users = {};
 
-Requests.admin.users.entity = {};
+nopaque.requests.admin.users.entity = {};
 
-Requests.admin.users.entity.confirmed = {};
+nopaque.requests.admin.users.entity.confirmed = {};
 
-Requests.admin.users.entity.confirmed.update = (userId, value) => {
+nopaque.requests.admin.users.entity.confirmed.update = (userId, value) => {
   let input = `/admin/users/${userId}/confirmed`;
   let init = {
     method: 'PUT',
     body: JSON.stringify(value)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };

app/static/js/requests/contributions.js

@@ -1,58 +1,58 @@
 /*****************************************************************************
 * Requests for /contributions routes                                         *
 *****************************************************************************/
-Requests.contributions = {};
+nopaque.requests.contributions = {};
 
 
 /*****************************************************************************
 * Requests for /contributions/spacy-nlp-pipeline-models routes               *
 *****************************************************************************/
-Requests.contributions.spacy_nlp_pipeline_models = {};
+nopaque.requests.contributions.spacy_nlp_pipeline_models = {};
 
-Requests.contributions.spacy_nlp_pipeline_models.entity = {};
+nopaque.requests.contributions.spacy_nlp_pipeline_models.entity = {};
 
-Requests.contributions.spacy_nlp_pipeline_models.entity.delete = (spacyNlpPipelineModelId) => {
+nopaque.requests.contributions.spacy_nlp_pipeline_models.entity.delete = (spacyNlpPipelineModelId) => {
   let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}`;
   let init = {
     method: 'DELETE'
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic = {};
+nopaque.requests.contributions.spacy_nlp_pipeline_models.entity.isPublic = {};
 
-Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic.update = (spacyNlpPipelineModelId, value) => {
+nopaque.requests.contributions.spacy_nlp_pipeline_models.entity.isPublic.update = (spacyNlpPipelineModelId, value) => {
   let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}/is_public`;
   let init = {
     method: 'PUT',
     body: JSON.stringify(value)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
 
 /*****************************************************************************
 * Requests for /contributions/tesseract-ocr-pipeline-models routes           *
 *****************************************************************************/
-Requests.contributions.tesseract_ocr_pipeline_models = {};
+nopaque.requests.contributions.tesseract_ocr_pipeline_models = {};
 
-Requests.contributions.tesseract_ocr_pipeline_models.entity = {};
+nopaque.requests.contributions.tesseract_ocr_pipeline_models.entity = {};
 
-Requests.contributions.tesseract_ocr_pipeline_models.entity.delete = (tesseractOcrPipelineModelId) => {
+nopaque.requests.contributions.tesseract_ocr_pipeline_models.entity.delete = (tesseractOcrPipelineModelId) => {
   let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}`;
   let init = {
     method: 'DELETE'
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic = {};
+nopaque.requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic = {};
 
-Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic.update = (tesseractOcrPipelineModelId, value) => {
+nopaque.requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic.update = (tesseractOcrPipelineModelId, value) => {
   let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}/is_public`;
   let init = {
     method: 'PUT',
     body: JSON.stringify(value)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };

app/static/js/requests/corpora.js

@@ -1,102 +1,102 @@
 /*****************************************************************************
 * Requests for /corpora routes                                               *
 *****************************************************************************/
-Requests.corpora = {};
+nopaque.requests.corpora = {};
 
-Requests.corpora.entity = {};
+nopaque.requests.corpora.entity = {};
 
-Requests.corpora.entity.delete = (corpusId) => {
+nopaque.requests.corpora.entity.delete = (corpusId) => {
   let input = `/corpora/${corpusId}`;
   let init = {
     method: 'DELETE'
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.build = (corpusId) => {
+nopaque.requests.corpora.entity.build = (corpusId) => {
   let input = `/corpora/${corpusId}/build`;
   let init = {
     method: 'POST',
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.generateShareLink = (corpusId, role, expiration) => {
+nopaque.requests.corpora.entity.generateShareLink = (corpusId, role, expiration) => {
   let input = `/corpora/${corpusId}/generate-share-link`;
   let init = {
     method: 'POST',
     body: JSON.stringify({role: role, expiration: expiration})
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.getStopwords = () => {
+nopaque.requests.corpora.entity.getStopwords = () => {
   let input = `/corpora/stopwords`;
   let init = {
     method: 'GET'
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.isPublic = {};
+nopaque.requests.corpora.entity.isPublic = {};
 
-Requests.corpora.entity.isPublic.update = (corpusId, isPublic) => {
+nopaque.requests.corpora.entity.isPublic.update = (corpusId, isPublic) => {
   let input = `/corpora/${corpusId}/is_public`;
   let init = {
     method: 'PUT',
     body: JSON.stringify(isPublic)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
 
 /*****************************************************************************
 * Requests for /corpora/<entity>/files routes                                *
 *****************************************************************************/
-Requests.corpora.entity.files = {};
+nopaque.requests.corpora.entity.files = {};
 
-Requests.corpora.entity.files.ent = {};
+nopaque.requests.corpora.entity.files.ent = {};
 
-Requests.corpora.entity.files.ent.delete = (corpusId, corpusFileId) => {
+nopaque.requests.corpora.entity.files.ent.delete = (corpusId, corpusFileId) => {
   let input = `/corpora/${corpusId}/files/${corpusFileId}`;
   let init = {
     method: 'DELETE',
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
 
 /*****************************************************************************
 * Requests for /corpora/<entity>/followers routes                            *
 *****************************************************************************/
-Requests.corpora.entity.followers = {};
+nopaque.requests.corpora.entity.followers = {};
 
-Requests.corpora.entity.followers.add = (corpusId, usernames) => {
+nopaque.requests.corpora.entity.followers.add = (corpusId, usernames) => {
   let input = `/corpora/${corpusId}/followers`;
   let init = {
     method: 'POST',
     body: JSON.stringify(usernames)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.followers.entity = {};
+nopaque.requests.corpora.entity.followers.entity = {};
 
-Requests.corpora.entity.followers.entity.delete = (corpusId, followerId) => {
+nopaque.requests.corpora.entity.followers.entity.delete = (corpusId, followerId) => {
   let input = `/corpora/${corpusId}/followers/${followerId}`;
   let init = {
     method: 'DELETE',
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };
 
-Requests.corpora.entity.followers.entity.role = {};
+nopaque.requests.corpora.entity.followers.entity.role = {};
 
-Requests.corpora.entity.followers.entity.role.update = (corpusId, followerId, value) => {
+nopaque.requests.corpora.entity.followers.entity.role.update = (corpusId, followerId, value) => {
   let input = `/corpora/${corpusId}/followers/${followerId}/role`;
   let init = {
     method: 'PUT',
     body: JSON.stringify(value)
   };
-  return Requests.JSONfetch(input, init);
+  return nopaque.requests.JSONfetch(input, init);
 };

app/static/js/requests/index.js

@@ -1,6 +1,6 @@
-var Requests = {};
+nopaque.requests = {};
 
-Requests.JSONfetch = (input, init={}) => {
+nopaque.requests.JSONfetch = (input, init={}) => {
   return new Promise((resolve, reject) => {
     let fixedInit = {};
     fixedInit.headers = {};
@@ -8,7 +8,7 @@ Requests.JSONfetch = (input, init={}) => {
     if (init.hasOwnProperty('body')) {
       fixedInit.headers['Content-Type'] = 'application/json';
     }
-    fetch(input, Utils.mergeObjectsDeep(init, fixedInit))
+    fetch(input, nopaque.Utils.mergeObjectsDeep(init, fixedInit))
       .then(
         (response) => {
           if (response.ok) {