Merge branch 'access-pipeline' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into access-pipeline

.dockerignore

@@ -8,6 +8,6 @@
 !.flaskenv
 !boot.sh
 !config.py
-!docker-entrypoint.sh
+!docker-nopaque-entrypoint.sh
 !nopaque.py
 !requirements.txt

.gitlab-ci.yml

@@ -1,3 +1,37 @@
+include:
+  - template: Security/Container-Scanning.gitlab-ci.yml
+
+##############################################################################
+# Pipeline stages in order of execution                                      #
+##############################################################################
+stages:
+  - build
+  - publish
+  - sca
+
+##############################################################################
+# Pipeline behavior                                                          #
+##############################################################################
+workflow:
+  rules:
+    # Run the pipeline on commits to the default branch
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      variables:
+        # Set the Docker image tag to `latest`
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest
+      when: always
+    # Run the pipeline on tag creation
+    - if: $CI_COMMIT_TAG
+      variables:
+        # Set the Docker image tag to the Git tag name
+        DOCKER_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+      when: always
+    # Don't run the pipeline on all other occasions
+    - when: never
+
+##############################################################################
+# Default values for pipeline jobs                                           #
+##############################################################################
 default:
   image: docker:24.0.6
   services:
@@ -5,38 +39,46 @@ default:
   tags:
     - docker
 
+##############################################################################
+# CI/CD variables for all jobs in the pipeline                               #
+##############################################################################
 variables:
   DOCKER_TLS_CERTDIR: /certs
+  DOCKER_BUILD_PATH: .
+  DOCKERFILE: Dockerfile
 
-build_image:
+##############################################################################
+# Pipeline jobs                                                              #
+##############################################################################
+build:
   stage: build
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: on_success
-      variables:
-        IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
-    - if: $CI_COMMIT_TAG
-      when: "on_success"
-      variables:
-        IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-    - when: never
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - docker build -t $IMAGE_TAG .
-    - docker push $IMAGE_TAG
+    - docker build --tag $DOCKER_IMAGE --file $DOCKERFILE $DOCKER_BUILD_PATH
+    - docker save $DOCKER_IMAGE > docker_image.tar
+  artifacts:
+    paths:
+      - docker_image.tar
 
-include:
-  - template: Security/Container-Scanning.gitlab-ci.yml
+publish:
+  stage: publish
+  before_script:
+    - docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - docker load --input docker_image.tar
+    - docker push $DOCKER_IMAGE
+  after_script:
+    - docker logout $CI_REGISTRY
 
 container_scanning:
+  stage: sca
   rules:
+    # Run the job on commits to the default branch
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: on_success
-      variables:
-        CS_IMAGE: $CI_REGISTRY_IMAGE:latest
+      when: always
+    # Run the job on tag creation
     - if: $CI_COMMIT_TAG
-      when: on_success
-      variables:
-        CS_IMAGE: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}
+      when: always
+    # Don't run the job on all other occasions
     - when: never
+  variables:
+    CS_IMAGE: $DOCKER_IMAGE

.vscode/extensions.json

@@ -1,7 +1,8 @@
 {
     "recommendations": [
-        "samuelcolvin.jinjahtml",
+        "irongeek.vscode-env",
         "ms-azuretools.vscode-docker",
-        "ms-python.python"
+        "ms-python.python",
+        "samuelcolvin.jinjahtml"
     ]
 }

.vscode/settings.json

@@ -1,13 +1,9 @@
 {
     "editor.rulers": [79],
     "files.insertFinalNewline": true,
-    "python.terminal.activateEnvironment": false,
     "[css]": {
         "editor.tabSize": 2
     },
-    "[scss]": {
-        "editor.tabSize": 2
-    },
     "[html]": {
         "editor.tabSize": 2
     },
@@ -17,7 +13,7 @@
     "[jinja-html]": {
         "editor.tabSize": 2
     },
-    "[jinja-js]": {
+    "[scss]": {
         "editor.tabSize": 2
     }
 }

Dockerfile

@@ -4,11 +4,13 @@ FROM python:3.10.13-slim-bookworm
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
 
 
+# Set environment variables
 ENV LANG="C.UTF-8"
 ENV PYTHONDONTWRITEBYTECODE="1"
 ENV PYTHONUNBUFFERED="1"
 
 
+# Install system dependencies
 RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
@@ -17,37 +19,42 @@ RUN apt-get update \
  && rm --recursive /var/lib/apt/lists/*
 
 
-COPY docker-entrypoint.sh /usr/local/bin/
-
-
+# Create a non-root user
 RUN useradd --create-home --no-log-init nopaque \
  && groupadd docker \
  && usermod --append --groups docker nopaque
 
-
 USER nopaque
 WORKDIR /home/nopaque
 
 
+# Create a Python virtual environment
 ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv"
 RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}"
 ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}"
 
 
+# Install Python dependencies
+COPY --chown=nopaque:nopaque requirements.txt requirements.txt
+RUN python3 -m pip install --requirement requirements.txt \
+ && rm requirements.txt
+
+
+# Install the application
+COPY docker-nopaque-entrypoint.sh /usr/local/bin/
+
 COPY --chown=nopaque:nopaque app app
 COPY --chown=nopaque:nopaque migrations migrations
 COPY --chown=nopaque:nopaque tests tests
 COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py requirements.txt ./
 
-
-RUN python3 -m pip install --requirement requirements.txt \
- && mkdir logs
-
-
-USER root
+RUN mkdir logs
 
 
 EXPOSE 5000
 
 
-ENTRYPOINT ["docker-entrypoint.sh"]
+USER root
+
+
+ENTRYPOINT ["docker-nopaque-entrypoint.sh"]

README.md

@@ -1,5 +1,8 @@
 # nopaque
 
+![release badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/-/badges/release.svg)
+![pipeline badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/badges/master/pipeline.svg?ignore_skipped=true)
+
 nopaque bundles various tools and services that provide humanities scholars with DH methods and thus can support their various individual research processes. Using nopaque, researchers can subject digitized sources to Optical Character Recognition (OCR). The resulting text files can then be used as a data basis for Natural Language Processing (NLP). The texts are automatically subjected to various linguistic annotations. The data processed via NLP can then be summarized in the web application as corpora and analyzed by means of an information retrieval system through complex search queries. The range of functions of the web application will be successively extended according to the needs of the researchers.
 
 ## Prerequisites and requirements

app/__init__.py

@@ -57,6 +57,9 @@ def create_app(config: Config = Config) -> Flask:
     scheduler.init_app(app)
     socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
 
+    from .models.event_listeners import register_event_listeners
+    register_event_listeners()
+
     from .admin import bp as admin_blueprint
     default_breadcrumb_root(admin_blueprint, '.admin')
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
@@ -99,7 +102,7 @@ def create_app(config: Config = Config) -> Flask:
 
     from .users import bp as users_blueprint
     default_breadcrumb_root(users_blueprint, '.users')
-    app.register_blueprint(users_blueprint, url_prefix='/users')
+    app.register_blueprint(users_blueprint, cli_group='user', url_prefix='/users')
 
     from .workshops import bp as workshops_blueprint
     app.register_blueprint(workshops_blueprint, url_prefix='/workshops')

app/contributions/spacy_nlp_pipeline_models/forms.py

@@ -16,8 +16,8 @@ class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
     )
 
     def validate_spacy_model_file(self, field):
-        if not field.data.filename.lower().endswith('.tar.gz'):
-            raise ValidationError('.tar.gz files only!')
+        if not field.data.filename.lower().endswith(('.tar.gz', ('.whl'))):
+            raise ValidationError('.tar.gz or .whl files only!')
 
     def __init__(self, *args, **kwargs):
         if 'prefix' not in kwargs:

app/converters/sandpaper.py

@@ -2,80 +2,69 @@ from flask import current_app
 from app import db
 from app.models import User, Corpus, CorpusFile
 from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
 import json
-import os
 import shutil
 
 
 class SandpaperConverter:
-    def __init__(self, json_db_file, data_dir):
+    def __init__(self, json_db_file: Path, data_dir: Path):
         self.json_db_file = json_db_file
         self.data_dir = data_dir
 
     def run(self):
-        with open(self.json_db_file, 'r') as f:
-            json_db = json.loads(f.read())
+        with self.json_db_file.open('r') as f:
+            json_db: List[Dict] = json.load(f)
 
         for json_user in json_db:
             if not json_user['confirmed']:
                 current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
                 continue
-            user_dir = os.path.join(self.data_dir, str(json_user['id']))
+            user_dir = self.data_dir / f'{json_user["id"]}'
             self.convert_user(json_user, user_dir)
             db.session.commit()
 
 
-    def convert_user(self, json_user, user_dir):
+    def convert_user(self, json_user: Dict, user_dir: Path):
         current_app.logger.info(f'Create User {json_user["username"]}...')
-        user = User(
-            confirmed=json_user['confirmed'],
-            email=json_user['email'],
-            last_seen=datetime.fromtimestamp(json_user['last_seen']),
-            member_since=datetime.fromtimestamp(json_user['member_since']),
-            password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
-            username=json_user['username']
-        )
-        db.session.add(user)
-        db.session.flush(objects=[user])
-        db.session.refresh(user)
         try:
-            user.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+            user = User.create(
+                confirmed=json_user['confirmed'],
+                email=json_user['email'],
+                last_seen=datetime.fromtimestamp(json_user['last_seen']),
+                member_since=datetime.fromtimestamp(json_user['member_since']),
+                password_hash=json_user['password_hash'],  # TODO: Needs to be added manually
+                username=json_user['username']
+            )
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus in json_user['corpora'].values():
             if not json_corpus['files'].values():
                 current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
                 continue
-            corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
+            corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
             self.convert_corpus(json_corpus, user, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus(self, json_corpus, user, corpus_dir):
+    def convert_corpus(self, json_corpus: Dict, user: User, corpus_dir: Path):
         current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
-        corpus = Corpus(
-            user=user,
-            creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
-            description=json_corpus['description'],
-            title=json_corpus['title']
-        )
-        db.session.add(corpus)
-        db.session.flush(objects=[corpus])
-        db.session.refresh(corpus)
         try:
-            corpus.makedirs()
-        except OSError as e:
-            current_app.logger.error(e)
-            db.session.rollback()
+            corpus = Corpus.create(
+                user=user,
+                creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
+                description=json_corpus['description'],
+                title=json_corpus['title']
+            )
+        except OSError:
             raise Exception('Internal Server Error')
         for json_corpus_file in json_corpus['files'].values():
             self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
         current_app.logger.info('Done')
 
 
-    def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
+    def convert_corpus_file(self, json_corpus_file: Dict, corpus: Corpus, corpus_dir: Path):
         current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
         corpus_file = CorpusFile(
             corpus=corpus,
@@ -99,13 +88,13 @@ class SandpaperConverter:
         db.session.refresh(corpus_file)
         try:
             shutil.copy2(
-                os.path.join(corpus_dir, json_corpus_file['filename']),
+                corpus_dir / json_corpus_file['filename'],
                 corpus_file.path
             )
         except:
             current_app.logger.warning(
                 'Can not convert corpus file: '
-                f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
+                f'{corpus_dir / json_corpus_file["filename"]}'
                 ' -> '
                 f'{corpus_file.path}'
             )

app/corpora/cli.py

@@ -1,7 +1,7 @@
-from app.models import Corpus, CorpusStatus
-import os
+from flask import current_app
 import shutil
 from app import db
+from app.models import Corpus, CorpusStatus
 from . import bp
 
 
@@ -18,10 +18,17 @@ def reset():
     ]
     for corpus in [x for x in Corpus.query.all() if x.status in status]:
         print(f'Resetting corpus {corpus}')
-        shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
-        os.mkdir(os.path.join(corpus.path, 'cwb'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'data'))
-        os.mkdir(os.path.join(corpus.path, 'cwb', 'registry'))
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
         corpus.status = CorpusStatus.UNPREPARED
         corpus.num_analysis_sessions = 0
     db.session.commit()

app/corpora/cqi_over_sio/extensions.py

@@ -12,7 +12,6 @@ from typing import Dict, List
 import gzip
 import json
 import math
-import os
 from app import db
 from app.models import Corpus
 from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
@@ -42,9 +41,9 @@ def ext_corpus_static_data(corpus: str) -> Dict:
     db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
     db_corpus: Corpus = Corpus.query.get(db_corpus_id)
 
-    static_data_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
-    if os.path.exists(static_data_file_path):
-        with open(static_data_file_path, 'rb') as f:
+    static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
+    if static_data_file_path.exists():
+        with static_data_file_path.open('rb') as f:
             return f.read()
 
     cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']

app/corpora/files/routes.py

@@ -7,7 +7,6 @@ from flask import (
     url_for
 )
 from flask_breadcrumbs import register_breadcrumb
-import os
 from app import db
 from app.models import Corpus, CorpusFile, CorpusStatus
 from ..decorators import corpus_follower_permission_required
@@ -92,8 +91,8 @@ def corpus_file(corpus_id, corpus_file_id):
 def download_corpus_file(corpus_id, corpus_file_id):
     corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
     return send_from_directory(
-        os.path.dirname(corpus_file.path),
-        os.path.basename(corpus_file.path),
+        corpus_file.path.parent,
+        corpus_file.path.name,
         as_attachment=True,
         attachment_filename=corpus_file.filename,
         mimetype=corpus_file.mimetype

app/corpora/routes.py

@@ -97,14 +97,14 @@ def analysis(corpus_id):
     )
 
 
-# @bp.route('/<hashid:corpus_id>/follow/<token>')
-# def follow_corpus(corpus_id, token):
-#     corpus = Corpus.query.get_or_404(corpus_id)
-#     if current_user.follow_corpus_by_token(token):
-#         db.session.commit()
-#         flash(f'You are following "{corpus.title}" now', category='corpus')
-#         return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
-#     abort(403)
+@bp.route('/<hashid:corpus_id>/follow/<token>')
+def follow_corpus(corpus_id, token):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if current_user.follow_corpus_by_token(token):
+        db.session.commit()
+        flash(f'You are following "{corpus.title}" now', category='corpus')
+        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    abort(403)
 
 
 @bp.route('/import', methods=['GET', 'POST'])

app/daemon/corpus_utils.py

@@ -45,7 +45,7 @@ def _create_build_corpus_service(corpus):
     ''' ## Constraints ## '''
     constraints = ['node.role==worker']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1853'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
     ''' ## Labels ## '''
     labels = {
         'origin': current_app.config['SERVER_NAME'],
@@ -139,7 +139,7 @@ def _create_cqpserver_container(corpus):
     ''' ## Entrypoint ## '''
     entrypoint = ['bash', '-c']
     ''' ## Image ## '''
-    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1853'
+    image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
     ''' ## Name ## '''
     name = f'cqpserver_{corpus.id}'
     ''' ## Network ## '''

app/ext/flask_sqlalchemy/__init__.py

@@ -0,0 +1,2 @@
+from .container_column import ContainerColumn
+from .int_enum_column import IntEnumColumn

app/ext/flask_sqlalchemy/container_column.py

@@ -0,0 +1,21 @@
+import json
+from app import db
+
+
+class ContainerColumn(db.TypeDecorator):
+    impl = db.String
+
+    def __init__(self, container_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.container_type = container_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.container_type):
+            return json.dumps(value)
+        elif isinstance(value, str) and isinstance(json.loads(value), self.container_type):
+            return value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return json.loads(value)

app/ext/flask_sqlalchemy/int_enum_column.py

@@ -0,0 +1,22 @@
+from app import db
+
+
+class IntEnumColumn(db.TypeDecorator):
+    impl = db.Integer
+
+    def __init__(self, enum_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_type = enum_type
+
+    def process_bind_param(self, value, dialect):
+        if isinstance(value, self.enum_type) and isinstance(value.value, int):
+            return value.value
+        elif isinstance(value, int):
+            return self.enum_type(value).value
+        elif isinstance(value, str):
+            return self.enum_type[value].value
+        else:
+            return TypeError()
+
+    def process_result_value(self, value, dialect):
+        return self.enum_type(value)

app/jobs/json_routes.py

@@ -1,7 +1,6 @@
 from flask import abort, current_app
 from flask_login import current_user
 from threading import Thread
-import os
 from app import db
 from app.decorators import admin_required, content_negotiation
 from app.models import Job, JobStatus
@@ -39,7 +38,7 @@ def job_log(job_id):
     if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
         response = {'errors': {'message': 'Job status is not completed or failed'}}
         return response, 409
-    with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+    with open(job.path / 'pipeline_data' / 'logs' / 'pyflow_log.txt') as log_file:
         log = log_file.read()
     response_data = {
         'jobLog': log

app/jobs/routes.py

@@ -7,7 +7,6 @@ from flask import (
 )
 from flask_breadcrumbs import register_breadcrumb
 from flask_login import current_user
-import os
 from app.models import Job, JobInput, JobResult
 from . import bp
 from .utils import job_dynamic_list_constructor as job_dlc
@@ -38,8 +37,8 @@ def download_job_input(job_id, job_input_id):
     if not (job_input.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_input.path),
-        os.path.basename(job_input.path),
+        job_input.path.parent,
+        job_input.path.name,
         as_attachment=True,
         attachment_filename=job_input.filename,
         mimetype=job_input.mimetype
@@ -52,8 +51,8 @@ def download_job_result(job_id, job_result_id):
     if not (job_result.job.user == current_user or current_user.is_administrator()):
         abort(403)
     return send_from_directory(
-        os.path.dirname(job_result.path),
-        os.path.basename(job_result.path),
+        job_result.path.parent,
+        job_result.path.name,
         as_attachment=True,
         attachment_filename=job_result.filename,
         mimetype=job_result.mimetype

app/main/cli.py

@@ -1,6 +1,7 @@
 from flask import current_app
 from flask_migrate import upgrade
-import os
+from pathlib import Path
+from typing import List
 from app.models import (
     CorpusFollowerRole,
     Role,
@@ -17,16 +18,15 @@ def deploy():
     # Make default directories
     print('Make default directories')
     base_dir = current_app.config['NOPAQUE_DATA_DIR']
-    default_dirs = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
+    default_dirs: List[Path] = [
+        base_dir / 'tmp',
+        base_dir / 'users'
     ]
-    for dir in default_dirs:
-        if os.path.exists(dir):
-            if not os.path.isdir(dir):
-                raise NotADirectoryError(f'{dir} is not a directory')
-        else:
-            os.mkdir(dir)
+    for default_dir in default_dirs:
+        if not default_dir.exists():
+            default_dir.mkdir()
+        if not default_dir.is_dir():
+            raise NotADirectoryError(f'{default_dir} is not a directory')
 
     # migrate database to latest revision
     print('Migrate database to latest revision')

app/main/routes.py

@@ -45,12 +45,6 @@ def dashboard():
     )
 
 
-# @bp.route('/user_manual')
-# @register_breadcrumb(bp, '.user_manual', '<i class="material-icons left">help</i>User manual')
-# def user_manual():
-#     return render_template('main/user_manual.html.j2', title='User manual')
-
-
 @bp.route('/news')
 @register_breadcrumb(bp, '.news', '<i class="material-icons left">email</i>News')
 def news():

app/models/__init__.py

@@ -0,0 +1,19 @@
+from .avatar import *
+from .corpus_file import *
+from .corpus_follower_association import *
+from .corpus_follower_role import *
+from .corpus import *
+from .job_input import *
+from .job_result import *
+from .job import *
+from .role import *
+from .spacy_nlp_pipeline_model import *
+from .tesseract_ocr_pipeline_model import *
+from .token import *
+from .user import *
+from app import login
+
+
+@login.user_loader
+def load_user(user_id):
+    return User.query.get(int(user_id))

app/models/avatar.py

@@ -0,0 +1,40 @@
+from flask import current_app
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class Avatar(HashidMixin, FileMixin, db.Model):
+    __tablename__ = 'avatars'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Relationships
+    user = db.relationship('User', back_populates='avatar')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'avatar'
+        # return os.path.join(self.user.path, 'avatar')
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus.py

@@ -0,0 +1,200 @@
+from datetime import datetime
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from sqlalchemy.ext.associationproxy import association_proxy
+from typing import Union
+from pathlib import Path
+import shutil
+import xml.etree.ElementTree as ET
+from app import db
+from app.converters.vrt import normalize_vrt_file
+from app.ext.flask_sqlalchemy import IntEnumColumn
+from .corpus_follower_association import CorpusFollowerAssociation
+
+
+class CorpusStatus(IntEnum):
+    UNPREPARED = 1
+    SUBMITTED = 2
+    QUEUED = 3
+    BUILDING = 4
+    BUILT = 5
+    FAILED = 6
+    STARTING_ANALYSIS_SESSION = 7
+    RUNNING_ANALYSIS_SESSION = 8
+    CANCELING_ANALYSIS_SESSION = 9
+
+    @staticmethod
+    def get(corpus_status: Union['CorpusStatus', int, str]) -> 'CorpusStatus':
+        if isinstance(corpus_status, CorpusStatus):
+            return corpus_status
+        if isinstance(corpus_status, int):
+            return CorpusStatus(corpus_status)
+        if isinstance(corpus_status, str):
+            return CorpusStatus[corpus_status]
+        raise TypeError('corpus_status must be CorpusStatus, int, or str')
+
+
+class Corpus(HashidMixin, db.Model):
+    '''
+    Class to define a corpus.
+    '''
+    __tablename__ = 'corpora'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
+    description = db.Column(db.String(255))
+    status = db.Column(
+        IntEnumColumn(CorpusStatus),
+        default=CorpusStatus.UNPREPARED
+    )
+    title = db.Column(db.String(32))
+    num_analysis_sessions = db.Column(db.Integer, default=0)
+    num_tokens = db.Column(db.Integer, default=0)
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    files = db.relationship(
+        'CorpusFile',
+        back_populates='corpus',
+        lazy='dynamic',
+        cascade='all, delete-orphan'
+    )
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='corpus',
+        cascade='all, delete-orphan'
+    )
+    followers = association_proxy(
+        'corpus_follower_associations',
+        'follower',
+        creator=lambda u: CorpusFollowerAssociation(follower=u)
+    )
+    user = db.relationship('User', back_populates='corpora')
+    # "static" attributes
+    max_num_tokens = 2_147_483_647
+
+    def __repr__(self):
+        return f'<Corpus {self.title}>'
+
+    @property
+    def analysis_url(self):
+        return url_for('corpora.analysis', corpus_id=self.id)
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/corpora/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'corpora' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for('corpora.corpus', corpus_id=self.id)
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def create(**kwargs):
+        corpus = Corpus(**kwargs)
+        db.session.add(corpus)
+        db.session.flush(objects=[corpus])
+        db.session.refresh(corpus)
+        corpus_files_dir = corpus.path / 'files'
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            corpus.path.mkdir()
+            corpus_files_dir.mkdir()
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return corpus
+
+    def build(self):
+        corpus_cwb_dir = self.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus_cwb_dir, ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            self.status = CorpusStatus.FAILED
+            raise
+        corpus_element = ET.fromstring('<corpus>\n</corpus>')
+        for corpus_file in self.files:
+            normalized_vrt_path = corpus_cwb_dir / f'{corpus_file.id}.norm.vrt'
+            try:
+                normalize_vrt_file(corpus_file.path, normalized_vrt_path)
+            except:
+                self.status = CorpusStatus.FAILED
+                return
+            element_tree = ET.parse(normalized_vrt_path)
+            text_element = element_tree.getroot()
+            text_element.set('author', corpus_file.author)
+            text_element.set('title', corpus_file.title)
+            text_element.set(
+                'publishing_year',
+                f'{corpus_file.publishing_year}'
+            )
+            text_element.set('address', corpus_file.address or 'NULL')
+            text_element.set('booktitle', corpus_file.booktitle or 'NULL')
+            text_element.set('chapter', corpus_file.chapter or 'NULL')
+            text_element.set('editor', corpus_file.editor or 'NULL')
+            text_element.set('institution', corpus_file.institution or 'NULL')
+            text_element.set('journal', corpus_file.journal or 'NULL')
+            text_element.set('pages', f'{corpus_file.pages}' or 'NULL')
+            text_element.set('publisher', corpus_file.publisher or 'NULL')
+            text_element.set('school', corpus_file.school or 'NULL')
+            text_element.tail = '\n'
+            # corpus_element.insert(1, text_element)
+            corpus_element.append(text_element)
+        ET.ElementTree(corpus_element).write(
+            corpus_cwb_dir / 'corpus.vrt',
+            encoding='utf-8'
+        )
+        self.status = CorpusStatus.SUBMITTED
+
+    def delete(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'description': self.description,
+            'max_num_tokens': self.max_num_tokens,
+            'num_analysis_sessions': self.num_analysis_sessions,
+            'num_tokens': self.num_tokens,
+            'status': self.status.name,
+            'title': self.title,
+            'is_public': self.is_public
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
+            json_serializeable['files'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.files
+            }
+        return json_serializeable

app/models/corpus_file.py

@@ -0,0 +1,102 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .corpus import CorpusStatus
+from .file_mixin import FileMixin
+
+
+class CorpusFile(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'corpus_files'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    # Fields
+    author = db.Column(db.String(255))
+    description = db.Column(db.String(255))
+    publishing_year = db.Column(db.Integer)
+    title = db.Column(db.String(255))
+    address = db.Column(db.String(255))
+    booktitle = db.Column(db.String(255))
+    chapter = db.Column(db.String(255))
+    editor = db.Column(db.String(255))
+    institution = db.Column(db.String(255))
+    journal = db.Column(db.String(255))
+    pages = db.Column(db.String(255))
+    publisher = db.Column(db.String(255))
+    school = db.Column(db.String(255))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='files'
+    )
+
+    @property
+    def download_url(self):
+        return url_for(
+            'corpora.download_corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.corpus.jsonpatch_path}/files/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.corpus.path / 'files' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'corpora.corpus_file',
+            corpus_id=self.corpus_id,
+            corpus_file_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.corpus.user.hashid
+
+    @property
+    def user_id(self):
+        return self.corpus.user_id
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+        self.corpus.status = CorpusStatus.UNPREPARED
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'address': self.address,
+            'author': self.author,
+            'description': self.description,
+            'booktitle': self.booktitle,
+            'chapter': self.chapter,
+            'editor': self.editor,
+            'institution': self.institution,
+            'journal': self.journal,
+            'pages': self.pages,
+            'publisher': self.publisher,
+            'publishing_year': self.publishing_year,
+            'school': self.school,
+            'title': self.title,
+            **self.file_mixin_to_json_serializeable(
+                backrefs=backrefs,
+                relationships=relationships
+            )
+        }
+        if backrefs:
+            json_serializeable['corpus'] = \
+                self.corpus.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_association.py

@@ -0,0 +1,47 @@
+from flask_hashids import HashidMixin
+from app import db
+from .corpus_follower_role import CorpusFollowerRole
+
+
+class CorpusFollowerAssociation(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_associations'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
+    follower_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    role_id = db.Column(db.Integer, db.ForeignKey('corpus_follower_roles.id'))
+    # Relationships
+    corpus = db.relationship(
+        'Corpus',
+        back_populates='corpus_follower_associations'
+    )
+    follower = db.relationship(
+        'User',
+        back_populates='corpus_follower_associations'
+    )
+    role = db.relationship(
+        'CorpusFollowerRole',
+        back_populates='corpus_follower_associations'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = CorpusFollowerRole.query.filter_by(default=True).first()
+        super().__init__(**kwargs)
+
+    def __repr__(self):
+        return f'<CorpusFollowerAssociation {self.follower.__repr__()} ~ {self.role.__repr__()} ~ {self.corpus.__repr__()}>'
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'corpus': self.corpus.to_json_serializeable(backrefs=True),
+            'follower': self.follower.to_json_serializeable(),
+            'role': self.role.to_json_serializeable()
+        }
+        if backrefs:
+            pass
+        if relationships:
+            pass
+        return json_serializeable

app/models/corpus_follower_role.py

@@ -0,0 +1,107 @@
+from flask_hashids import HashidMixin
+from enum import IntEnum
+from typing import Union
+from app import db
+
+
+class CorpusFollowerPermission(IntEnum):
+    VIEW = 1
+    MANAGE_FILES = 2
+    MANAGE_FOLLOWERS = 4
+    MANAGE_CORPUS = 8
+
+    @staticmethod
+    def get(corpus_follower_permission: Union['CorpusFollowerPermission', int, str]) -> 'CorpusFollowerPermission':
+        if isinstance(corpus_follower_permission, CorpusFollowerPermission):
+            return corpus_follower_permission
+        if isinstance(corpus_follower_permission, int):
+            return CorpusFollowerPermission(corpus_follower_permission)
+        if isinstance(corpus_follower_permission, str):
+            return CorpusFollowerPermission[corpus_follower_permission]
+        raise TypeError('corpus_follower_permission must be CorpusFollowerPermission, int, or str')
+
+
+class CorpusFollowerRole(HashidMixin, db.Model):
+    __tablename__ = 'corpus_follower_roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='role'
+    )
+
+    def __repr__(self):
+        return f'<CorpusFollowerRole {self.name}>'
+
+    def has_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        return self.permissions & perm.value == perm.value
+    
+    def add_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if not self.has_permission(perm):
+            self.permissions += perm.value
+    
+    def remove_permission(self, permission: Union[CorpusFollowerPermission, int, str]):
+        perm = CorpusFollowerPermission.get(permission)
+        if self.has_permission(perm):
+            self.permissions -= perm.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name
+                for x in CorpusFollowerPermission
+                if self.has_permission(x)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['corpus_follower_association'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpus_follower_association
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'Anonymous': [],
+            'Viewer': [
+                CorpusFollowerPermission.VIEW
+            ],
+            'Contributor': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES
+            ],
+            'Administrator': [
+                CorpusFollowerPermission.VIEW,
+                CorpusFollowerPermission.MANAGE_FILES,
+                CorpusFollowerPermission.MANAGE_FOLLOWERS,
+                CorpusFollowerPermission.MANAGE_CORPUS
+
+            ]
+        }
+        default_role_name = 'Viewer'
+        for role_name, permissions in roles.items():
+            role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+            if role is None:
+                role = CorpusFollowerRole(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()

app/models/default_records/spacy_nlp_pipeline_model.yml

@@ -8,7 +8,7 @@
   pipeline_name: 'ca_core_news_md'
   version: '3.2.0'
   compatible_service_versions:
-    - '0.1.0' 
+    - '0.1.0'
 - title: 'German'
   description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
   url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.2.0/de_core_news_md-3.2.0.tar.gz'
@@ -19,7 +19,7 @@
   pipeline_name: 'de_core_news_md'
   version: '3.2.0'
   compatible_service_versions:
-    - '0.1.0' 
+    - '0.1.0'
 - title: 'Greek'
   description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
   url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.2.0/el_core_news_md-3.2.0.tar.gz'
@@ -180,7 +180,7 @@
   version: '3.4.0'
   compatible_service_versions:
     - '0.1.1'
-    - '0.1.2' 
+    - '0.1.2'
 - title: 'Italian'
   description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner'
   url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz'

app/models/default_records/tesseract_ocr_pipeline_model.yml

@@ -9,6 +9,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Amharic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata'
@@ -20,6 +21,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Arabic'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata'
@@ -31,6 +33,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Assamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata'
@@ -42,6 +45,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Azerbaijani'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata'
@@ -53,6 +57,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Azerbaijani - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata'
@@ -64,6 +69,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Belarusian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata'
@@ -75,6 +81,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bengali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata'
@@ -86,6 +93,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tibetan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata'
@@ -97,6 +105,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bosnian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata'
@@ -108,6 +117,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Bulgarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata'
@@ -119,6 +129,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Catalan; Valencian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata'
@@ -130,6 +141,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Cebuano'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata'
@@ -141,6 +153,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Czech'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata'
@@ -152,6 +165,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Chinese - Simplified'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata'
@@ -163,6 +177,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Chinese - Traditional'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata'
@@ -174,6 +189,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Cherokee'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata'
@@ -185,6 +201,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Welsh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata'
@@ -196,6 +213,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Danish'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata'
@@ -207,6 +225,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'German'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata'
@@ -218,6 +237,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Dzongkha'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata'
@@ -229,6 +249,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Greek, Modern (1453-)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata'
@@ -240,6 +261,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'English'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata'
@@ -251,6 +273,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'English, Middle (1100-1500)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata'
@@ -262,6 +285,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Esperanto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata'
@@ -273,6 +297,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Estonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata'
@@ -284,6 +309,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Basque'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata'
@@ -295,6 +321,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Persian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata'
@@ -306,6 +333,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Finnish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata'
@@ -317,6 +345,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'French'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata'
@@ -328,6 +357,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'German Fraktur'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata'
@@ -339,6 +369,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'French, Middle (ca. 1400-1600)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata'
@@ -350,6 +381,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Irish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata'
@@ -361,6 +393,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Galician'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata'
@@ -372,6 +405,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Greek, Ancient (-1453)'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata'
@@ -383,6 +417,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Gujarati'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata'
@@ -394,6 +429,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Haitian; Haitian Creole'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata'
@@ -405,6 +441,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hebrew'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata'
@@ -416,6 +453,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hindi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata'
@@ -427,6 +465,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Croatian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata'
@@ -438,6 +477,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Hungarian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata'
@@ -449,6 +489,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Inuktitut'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata'
@@ -460,6 +501,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Indonesian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata'
@@ -471,6 +513,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Icelandic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata'
@@ -482,6 +525,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Italian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata'
@@ -493,6 +537,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'Italian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata'
@@ -504,6 +549,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Javanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata'
@@ -515,6 +561,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Japanese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata'
@@ -526,6 +573,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kannada'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata'
@@ -537,6 +585,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Georgian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata'
@@ -548,6 +597,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Georgian - Old'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata'
@@ -559,6 +609,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kazakh'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata'
@@ -570,6 +621,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Central Khmer'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata'
@@ -581,6 +633,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kirghiz; Kyrgyz'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata'
@@ -592,6 +645,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Korean'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata'
@@ -603,6 +657,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Kurdish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata'
@@ -614,6 +669,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Lao'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata'
@@ -625,6 +681,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata'
@@ -636,6 +693,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Latvian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata'
@@ -647,6 +705,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Lithuanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata'
@@ -658,6 +717,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Malayalam'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata'
@@ -669,6 +729,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Marathi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata'
@@ -680,6 +741,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Macedonian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata'
@@ -691,6 +753,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Maltese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata'
@@ -702,6 +765,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Malay'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata'
@@ -713,6 +777,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Burmese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata'
@@ -724,6 +789,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Nepali'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata'
@@ -735,6 +801,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Dutch; Flemish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata'
@@ -746,6 +813,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Norwegian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata'
@@ -757,6 +825,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Oriya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata'
@@ -768,6 +837,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Panjabi; Punjabi'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata'
@@ -779,6 +849,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Polish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata'
@@ -790,6 +861,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Portuguese'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata'
@@ -801,6 +873,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Pushto; Pashto'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata'
@@ -812,6 +885,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Romanian; Moldavian; Moldovan'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata'
@@ -823,6 +897,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Russian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata'
@@ -834,6 +909,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Sanskrit'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata'
@@ -845,6 +921,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Sinhala; Sinhalese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata'
@@ -856,6 +933,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Slovak'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata'
@@ -867,6 +945,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Slovenian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata'
@@ -878,6 +957,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 - title: 'Spanish; Castilian'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata'
@@ -889,6 +969,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 - title: 'Spanish; Castilian - Old'
   description: ''
   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata'
@@ -900,6 +981,7 @@
   compatible_service_versions:
     - '0.1.0'
     - '0.1.1'
+    - '0.1.2'
 # - title: 'Albanian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata'
@@ -911,6 +993,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Serbian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata'
@@ -922,6 +1005,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Serbian - Latin'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata'
@@ -933,6 +1017,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Swahili'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata'
@@ -944,6 +1029,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Swedish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata'
@@ -955,6 +1041,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Syriac'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata'
@@ -966,6 +1053,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tamil'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata'
@@ -977,6 +1065,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Telugu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata'
@@ -988,6 +1077,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tajik'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata'
@@ -999,6 +1089,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tagalog'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata'
@@ -1010,6 +1101,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Thai'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata'
@@ -1021,6 +1113,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Tigrinya'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata'
@@ -1032,6 +1125,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Turkish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata'
@@ -1043,6 +1137,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uighur; Uyghur'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata'
@@ -1054,6 +1149,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Ukrainian'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata'
@@ -1065,6 +1161,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Urdu'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata'
@@ -1076,6 +1173,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uzbek'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata'
@@ -1087,6 +1185,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Uzbek - Cyrillic'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata'
@@ -1098,6 +1197,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Vietnamese'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata'
@@ -1109,6 +1209,7 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'
 # - title: 'Yiddish'
 #   description: ''
 #   url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata'
@@ -1120,3 +1221,4 @@
 #   compatible_service_versions:
 #     - '0.1.0'
 #     - '0.1.1'
+#     - '0.1.2'

app/models/event_listeners.py

@@ -0,0 +1,133 @@
+from datetime import datetime
+from enum import Enum
+from app import db, mail, socketio
+from app.email import create_message
+from .corpus_file import CorpusFile
+from .corpus_follower_association import CorpusFollowerAssociation
+from .corpus import Corpus
+from .job_input import JobInput
+from .job_result import JobResult
+from .job import Job, JobStatus
+from .spacy_nlp_pipeline_model import SpaCyNLPPipelineModel
+from .tesseract_ocr_pipeline_model import TesseractOCRPipelineModel
+from .user import UserSettingJobStatusMailNotificationLevel
+
+
+def register_event_listeners():
+    resources = [
+        Corpus,
+        CorpusFile,
+        Job,
+        JobInput,
+        JobResult,
+        SpaCyNLPPipelineModel,
+        TesseractOCRPipelineModel
+    ]
+
+    for resource in resources:
+        db.event.listen(resource, 'after_delete', resource_after_delete)
+        db.event.listen(resource, 'after_insert', resource_after_insert)
+        db.event.listen(resource, 'after_update', resource_after_update)
+
+    db.event.listen(CorpusFollowerAssociation, 'after_delete', cfa_after_delete)
+    db.event.listen(CorpusFollowerAssociation, 'after_insert', cfa_after_insert)
+
+    db.event.listen(Job, 'after_update', job_after_update)
+
+
+def resource_after_delete(mapper, connection, resource):
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': resource.jsonpatch_path
+        }
+    ]
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def cfa_after_delete(mapper, connection, cfa):
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'remove',
+            'path': jsonpatch_path
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def resource_after_insert(mapper, connection, resource):
+    jsonpatch_value = resource.to_json_serializeable()
+    for attr in mapper.relationships:
+        jsonpatch_value[attr.key] = {}
+    jsonpatch = [
+        {
+            'op': 'add',
+            'path': resource.jsonpatch_path,
+            'value': jsonpatch_value
+        }
+    ]
+    room = f'/users/{resource.user_hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def cfa_after_insert(mapper, connection, cfa):
+    jsonpatch_value = cfa.to_json_serializeable()
+    jsonpatch_path = f'/users/{cfa.corpus.user.hashid}/corpora/{cfa.corpus.hashid}/corpus_follower_associations/{cfa.hashid}'
+    jsonpatch = [
+        {
+            'op': 'add',
+            'path': jsonpatch_path,
+            'value': jsonpatch_value
+        }
+    ]
+    room = f'/users/{cfa.corpus.user.hashid}'
+    socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def resource_after_update(mapper, connection, resource):
+    jsonpatch = []
+    for attr in db.inspect(resource).attrs:
+        if attr.key in mapper.relationships:
+            continue
+        if not attr.load_history().has_changes():
+            continue
+        jsonpatch_path = f'{resource.jsonpatch_path}/{attr.key}'
+        if isinstance(attr.value, datetime):
+            jsonpatch_value = f'{attr.value.isoformat()}Z'
+        elif isinstance(attr.value, Enum):
+            jsonpatch_value = attr.value.name
+        else:
+            jsonpatch_value = attr.value
+        jsonpatch.append(
+            {
+                'op': 'replace',
+                'path': jsonpatch_path,
+                'value': jsonpatch_value
+            }
+        )
+    if jsonpatch:
+        room = f'/users/{resource.user_hashid}'
+        socketio.emit('PATCH', jsonpatch, room=room)
+
+
+def job_after_update(mapper, connection, job):
+    for attr in db.inspect(job).attrs:
+        if attr.key != 'status':
+            continue
+        if not attr.load_history().has_changes():
+            return
+        if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:
+            return
+        if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:
+            if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+                return
+        msg = create_message(
+            job.user.email,
+            f'Status update for your Job "{job.title}"',
+            'tasks/email/notification',
+            job=job
+        )
+        mail.send(msg)

app/models/file_mixin.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+from flask import current_app
+from werkzeug.utils import secure_filename
+from app import db
+
+
+class FileMixin:
+    '''
+    Mixin for db.Model classes. All file related models should use this.
+    '''
+    creation_date = db.Column(db.DateTime, default=datetime.utcnow)
+    filename = db.Column(db.String(255))
+    mimetype = db.Column(db.String(255))
+
+    def file_mixin_to_json_serializeable(self, backrefs=False, relationships=False):
+        return {
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'filename': self.filename,
+            'mimetype': self.mimetype
+        }
+    
+    @classmethod
+    def create(cls, file_storage, **kwargs):
+        filename = kwargs.pop('filename', file_storage.filename)
+        mimetype = kwargs.pop('mimetype', file_storage.mimetype)
+        obj = cls(
+            filename=secure_filename(filename),
+            mimetype=mimetype,
+            **kwargs
+        )
+        db.session.add(obj)
+        db.session.flush(objects=[obj])
+        db.session.refresh(obj)
+        try:
+            file_storage.save(obj.path)
+        except (AttributeError, OSError) as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise e
+        return obj

app/models/job.py

@@ -0,0 +1,172 @@
+from datetime import datetime
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from time import sleep
+from typing import Union
+from pathlib import Path
+import shutil
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn, IntEnumColumn
+
+
+class JobStatus(IntEnum):
+    INITIALIZING = 1
+    SUBMITTED = 2
+    QUEUED = 3
+    RUNNING = 4
+    CANCELING = 5
+    CANCELED = 6
+    COMPLETED = 7
+    FAILED = 8
+
+    @staticmethod
+    def get(job_status: Union['JobStatus', int, str]) -> 'JobStatus':
+        if isinstance(job_status, JobStatus):
+            return job_status
+        if isinstance(job_status, int):
+            return JobStatus(job_status)
+        if isinstance(job_status, str):
+            return JobStatus[job_status]
+        raise TypeError('job_status must be JobStatus, int, or str')
+
+
+class Job(HashidMixin, db.Model):
+    '''
+    Class to define Jobs.
+    '''
+    __tablename__ = 'jobs'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    creation_date = \
+        db.Column(db.DateTime(), default=datetime.utcnow)
+    description = db.Column(db.String(255))
+    end_date = db.Column(db.DateTime())
+    service = db.Column(db.String(64))
+    service_args = db.Column(ContainerColumn(dict, 255))
+    service_version = db.Column(db.String(16))
+    status = db.Column(
+        IntEnumColumn(JobStatus),
+        default=JobStatus.INITIALIZING
+    )
+    title = db.Column(db.String(32))
+    # Relationships
+    inputs = db.relationship(
+        'JobInput',
+        back_populates='job',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    results = db.relationship(
+        'JobResult',
+        back_populates='job',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    user = db.relationship(
+        'User',
+        back_populates='jobs'
+    )
+
+    def __repr__(self):
+        return f'<Job {self.title}>'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/jobs/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'jobs' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for('jobs.job', job_id=self.id)
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def create(**kwargs):
+        job = Job(**kwargs)
+        db.session.add(job)
+        db.session.flush(objects=[job])
+        db.session.refresh(job)
+        job_inputs_dir = job.path / 'inputs'
+        job_pipeline_data_dir = job.path / 'pipeline_data'
+        job_results_dir = job.path / 'results'
+        try:
+            job.path.mkdir()
+            job_inputs_dir.mkdir()
+            job_pipeline_data_dir.mkdir()
+            job_results_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return job
+
+    def delete(self):
+        ''' Delete the job and its inputs and results from the database. '''
+        if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:  # noqa
+            self.status = JobStatus.CANCELING
+            db.session.commit()
+            while self.status != JobStatus.CANCELED:
+                # In case the daemon handled a job in any way
+                if self.status != JobStatus.CANCELING:
+                    self.status = JobStatus.CANCELING
+                    db.session.commit()
+                sleep(1)
+                db.session.refresh(self)
+        try:
+            shutil.rmtree(self.path)
+        except OSError as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise e
+        db.session.delete(self)
+
+    def restart(self):
+        ''' Restart a job - only if the status is failed '''
+        if self.status != JobStatus.FAILED:
+            raise Exception('Job status is not "failed"')
+        shutil.rmtree(self.path / 'results', ignore_errors=True)
+        shutil.rmtree(self.path / 'pyflow.data', ignore_errors=True)
+        for result in self.results:
+            db.session.delete(result)
+        self.end_date = None
+        self.status = JobStatus.SUBMITTED
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'creation_date': f'{self.creation_date.isoformat()}Z',
+            'description': self.description,
+            'end_date': (
+                None if self.end_date is None
+                else f'{self.end_date.isoformat()}Z'
+            ),
+            'service': self.service,
+            'service_args': self.service_args,
+            'service_version': self.service_version,
+            'status': self.status.name,
+            'title': self.title
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['inputs'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.inputs
+            }
+            json_serializeable['results'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.results
+            }
+        return json_serializeable

app/models/job_input.py

@@ -0,0 +1,65 @@
+from flask import url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class JobInput(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'job_inputs'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
+    # Relationships
+    job = db.relationship(
+        'Job',
+        back_populates='inputs'
+    )
+
+    def __repr__(self):
+        return f'<JobInput {self.filename}>'
+
+    @property
+    def content_url(self):
+        return url_for(
+            'jobs.download_job_input',
+            job_id=self.job.id,
+            job_input_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.job.jsonpatch_path}/inputs/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.job.path / 'inputs' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'jobs.job',
+            job_id=self.job_id,
+            _anchor=f'job-{self.job.hashid}-input-{self.hashid}'
+        )
+
+    @property
+    def user_hashid(self):
+        return self.job.user.hashid
+
+    @property
+    def user_id(self):
+        return self.job.user.id
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['job'] = \
+                self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/job_result.py

@@ -0,0 +1,71 @@
+from flask import url_for
+from flask_hashids import HashidMixin
+from pathlib import Path
+from app import db
+from .file_mixin import FileMixin
+
+
+class JobResult(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'job_results'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
+    # Fields
+    description = db.Column(db.String(255))
+    # Relationships
+    job = db.relationship(
+        'Job',
+        back_populates='results'
+    )
+
+    def __repr__(self):
+        return f'<JobResult {self.filename}>'
+
+    @property
+    def download_url(self):
+        return url_for(
+            'jobs.download_job_result',
+            job_id=self.job_id,
+            job_result_id=self.id
+        )
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.job.jsonpatch_path}/results/{self.hashid}'
+
+    @property
+    def path(self) -> Path:
+        return self.job.path / 'results' / f'{self.id}'
+
+    @property
+    def url(self):
+        return url_for(
+            'jobs.job',
+            job_id=self.job_id,
+            _anchor=f'job-{self.job.hashid}-result-{self.hashid}'
+        )
+
+    @property
+    def user_hashid(self):
+        return self.job.user.hashid
+
+    @property
+    def user_id(self):
+        return self.job.user.id
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'description': self.description,
+            **self.file_mixin_to_json_serializeable(
+                backrefs=backrefs,
+                relationships=relationships
+            )
+        }
+        if backrefs:
+            json_serializeable['job'] = \
+                self.job.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/role.py

@@ -0,0 +1,100 @@
+from enum import IntEnum
+from flask_hashids import HashidMixin
+from typing import Union
+from app import db
+
+
+class Permission(IntEnum):
+    '''
+    Defines User permissions as integers by the power of 2. User permission
+    can be evaluated using the bitwise operator &.
+    '''
+    ADMINISTRATE = 1
+    CONTRIBUTE = 2
+    USE_API = 4
+
+    @staticmethod
+    def get(permission: Union['Permission', int, str]) -> 'Permission':
+        if isinstance(permission, Permission):
+            return permission
+        if isinstance(permission, int):
+            return Permission(permission)
+        if isinstance(permission, str):
+            return Permission[permission]
+        raise TypeError('permission must be Permission, int, or str')
+
+
+class Role(HashidMixin, db.Model):
+    __tablename__ = 'roles'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Fields
+    name = db.Column(db.String(64), unique=True)
+    default = db.Column(db.Boolean, default=False, index=True)
+    permissions = db.Column(db.Integer, default=0)
+    # Relationships
+    users = db.relationship('User', back_populates='role', lazy='dynamic')
+
+    def __repr__(self):
+        return f'<Role {self.name}>'
+
+    def has_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        return self.permissions & p.value == p.value
+    
+    def add_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if not self.has_permission(p):
+            self.permissions += p.value
+    
+    def remove_permission(self, permission: Union[Permission, int, str]):
+        p = Permission.get(permission)
+        if self.has_permission(p):
+            self.permissions -= p.value
+
+    def reset_permissions(self):
+        self.permissions = 0
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'default': self.default,
+            'name': self.name,
+            'permissions': [
+                x.name for x in Permission
+                if self.has_permission(x.value)
+            ]
+        }
+        if backrefs:
+            pass
+        if relationships:
+            json_serializeable['users'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.users
+            }
+        return json_serializeable
+
+    @staticmethod
+    def insert_defaults():
+        roles = {
+            'User': [],
+            'API user': [Permission.USE_API],
+            'Contributor': [Permission.CONTRIBUTE],
+            'Administrator': [
+                Permission.ADMINISTRATE,
+                Permission.CONTRIBUTE,
+                Permission.USE_API
+            ],
+            'System user': []
+        }
+        default_role_name = 'User'
+        for role_name, permissions in roles.items():
+            role = Role.query.filter_by(name=role_name).first()
+            if role is None:
+                role = Role(name=role_name)
+            role.reset_permissions()
+            for permission in permissions:
+                role.add_permission(permission)
+            role.default = role.name == default_role_name
+            db.session.add(role)
+        db.session.commit()

app/models/spacy_nlp_pipeline_model.py

@@ -0,0 +1,136 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from tqdm import tqdm
+from pathlib import Path
+import requests
+import yaml
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn
+from .file_mixin import FileMixin
+from .user import User
+
+
+class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'spacy_nlp_pipeline_models'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    title = db.Column(db.String(64))
+    description = db.Column(db.String(255))
+    version = db.Column(db.String(16))
+    compatible_service_versions = db.Column(ContainerColumn(list, 255))
+    publisher = db.Column(db.String(128))
+    publisher_url = db.Column(db.String(512))
+    publishing_url = db.Column(db.String(512))
+    publishing_year = db.Column(db.Integer)
+    pipeline_name = db.Column(db.String(64))
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    user = db.relationship('User', back_populates='spacy_nlp_pipeline_models')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'spacy_nlp_pipeline_models' / f'{self.id}'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}'
+
+    @property
+    def url(self):
+        return url_for(
+            'contributions.spacy_nlp_pipeline_model',
+            spacy_nlp_pipeline_model_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def insert_defaults(force_download=False):
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        default_records_file = Path(__file__).parent / 'default_records' / 'spacy_nlp_pipeline_model.yml'
+        with default_records_file.open('r') as f:
+            default_records = yaml.safe_load(f)
+        for m in default_records:
+            model = SpaCyNLPPipelineModel.query.filter_by(title=m['title'], version=m['version']).first()  # noqa
+            if model is not None:
+                model.compatible_service_versions = m['compatible_service_versions']
+                model.description = m['description']
+                model.filename = m['url'].split('/')[-1]
+                model.publisher = m['publisher']
+                model.publisher_url = m['publisher_url']
+                model.publishing_url = m['publishing_url']
+                model.publishing_year = m['publishing_year']
+                model.is_public = True
+                model.title = m['title']
+                model.version = m['version']
+                model.pipeline_name = m['pipeline_name']
+            else:
+                model = SpaCyNLPPipelineModel(
+                    compatible_service_versions=m['compatible_service_versions'],
+                    description=m['description'],
+                    filename=m['url'].split('/')[-1],
+                    publisher=m['publisher'],
+                    publisher_url=m['publisher_url'],
+                    publishing_url=m['publishing_url'],
+                    publishing_year=m['publishing_year'],
+                    is_public=True,
+                    title=m['title'],
+                    user=nopaque_user,
+                    version=m['version'],
+                    pipeline_name=m['pipeline_name']
+                )
+                db.session.add(model)
+                db.session.flush(objects=[model])
+                db.session.refresh(model)
+            if not model.path.exists() or force_download:
+                r = requests.get(m['url'], stream=True)
+                pbar = tqdm(
+                    desc=f'{model.title} ({model.filename})',
+                    unit="B",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                    total=int(r.headers['Content-Length'])
+                )
+                pbar.clear()
+                with open(model.path, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=1024):
+                        if chunk:  # filter out keep-alive new chunks
+                            pbar.update(len(chunk))
+                            f.write(chunk)
+                    pbar.close()
+        db.session.commit()
+    
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'compatible_service_versions': self.compatible_service_versions,
+            'description': self.description,
+            'publisher': self.publisher,
+            'publisher_url': self.publisher_url,
+            'publishing_url': self.publishing_url,
+            'publishing_year': self.publishing_year,
+            'pipeline_name': self.pipeline_name,
+            'is_public': self.is_public,
+            'title': self.title,
+            'version': self.version,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/tesseract_ocr_pipeline_model.py

@@ -0,0 +1,132 @@
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from tqdm import tqdm
+from pathlib import Path
+import requests
+import yaml
+from app import db
+from app.ext.flask_sqlalchemy import ContainerColumn
+from .file_mixin import FileMixin
+from .user import User
+
+
+class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
+    __tablename__ = 'tesseract_ocr_pipeline_models'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    title = db.Column(db.String(64))
+    description = db.Column(db.String(255))
+    version = db.Column(db.String(16))
+    compatible_service_versions = db.Column(ContainerColumn(list, 255))
+    publisher = db.Column(db.String(128))
+    publisher_url = db.Column(db.String(512))
+    publishing_url = db.Column(db.String(512))
+    publishing_year = db.Column(db.Integer)
+    is_public = db.Column(db.Boolean, default=False)
+    # Relationships
+    user = db.relationship('User', back_populates='tesseract_ocr_pipeline_models')
+
+    @property
+    def path(self) -> Path:
+        return self.user.path / 'tesseract_ocr_pipeline_models' / f'{self.id}'
+
+    @property
+    def jsonpatch_path(self):
+        return f'{self.user.jsonpatch_path}/tesseract_ocr_pipeline_models/{self.hashid}'
+
+    @property
+    def url(self):
+        return url_for(
+            'contributions.tesseract_ocr_pipeline_model',
+            tesseract_ocr_pipeline_model_id=self.id
+        )
+
+    @property
+    def user_hashid(self):
+        return self.user.hashid
+
+    @staticmethod
+    def insert_defaults(force_download=False):
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        default_records_file = Path(__file__).parent / 'default_records' / 'tesseract_ocr_pipeline_model.yml'
+        with default_records_file.open('r') as f:
+            default_records = yaml.safe_load(f)
+        for m in default_records:
+            model = TesseractOCRPipelineModel.query.filter_by(title=m['title'], version=m['version']).first()  # noqa
+            if model is not None:
+                model.compatible_service_versions = m['compatible_service_versions']
+                model.description = m['description']
+                model.filename = f'{model.id}.traineddata'
+                model.publisher = m['publisher']
+                model.publisher_url = m['publisher_url']
+                model.publishing_url = m['publishing_url']
+                model.publishing_year = m['publishing_year']
+                model.is_public = True
+                model.title = m['title']
+                model.version = m['version']
+            else:
+                model = TesseractOCRPipelineModel(
+                    compatible_service_versions=m['compatible_service_versions'],
+                    description=m['description'],
+                    publisher=m['publisher'],
+                    publisher_url=m['publisher_url'],
+                    publishing_url=m['publishing_url'],
+                    publishing_year=m['publishing_year'],
+                    is_public=True,
+                    title=m['title'],
+                    user=nopaque_user,
+                    version=m['version']
+                )
+                db.session.add(model)
+                db.session.flush(objects=[model])
+                db.session.refresh(model)
+                model.filename = f'{model.id}.traineddata'
+            if not model.path.exists() or force_download:
+                r = requests.get(m['url'], stream=True)
+                pbar = tqdm(
+                    desc=f'{model.title} ({model.filename})',
+                    unit="B",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                    total=int(r.headers['Content-Length'])
+                )
+                pbar.clear()
+                with open(model.path, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=1024):
+                        if chunk:  # filter out keep-alive new chunks
+                            pbar.update(len(chunk))
+                            f.write(chunk)
+                    pbar.close()
+        db.session.commit()
+
+    def delete(self):
+        try:
+            self.path.unlink(missing_ok=True)
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        db.session.delete(self)
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'compatible_service_versions': self.compatible_service_versions,
+            'description': self.description,
+            'publisher': self.publisher,
+            'publisher_url': self.publisher_url,
+            'publishing_url': self.publishing_url,
+            'publishing_year': self.publishing_year,
+            'is_public': self.is_public,
+            'title': self.title,
+            'version': self.version,
+            **self.file_mixin_to_json_serializeable()
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable

app/models/token.py

@@ -0,0 +1,48 @@
+from datetime import datetime, timedelta
+from app import db
+
+
+class Token(db.Model):
+    __tablename__ = 'tokens'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+    # Fields
+    access_token = db.Column(db.String(64), index=True)
+    access_expiration = db.Column(db.DateTime)
+    refresh_token = db.Column(db.String(64), index=True)
+    refresh_expiration = db.Column(db.DateTime)
+    # Relationships
+    user = db.relationship('User', back_populates='tokens')
+
+    def expire(self):
+        self.access_expiration = datetime.utcnow()
+        self.refresh_expiration = datetime.utcnow()
+
+    def to_json_serializeable(self, backrefs=False, relationships=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'access_token': self.access_token,
+            'access_expiration': (
+                None if self.access_expiration is None
+                else f'{self.access_expiration.isoformat()}Z'
+            ),
+            'refresh_token': self.refresh_token,
+            'refresh_expiration': (
+                None if self.refresh_expiration is None
+                else f'{self.refresh_expiration.isoformat()}Z'
+            )
+        }
+        if backrefs:
+            json_serializeable['user'] = \
+                self.user.to_json_serializeable(backrefs=True)
+        if relationships:
+            pass
+        return json_serializeable
+
+    @staticmethod
+    def clean():
+        """Remove any tokens that have been expired for more than a day."""
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        Token.query.filter(Token.refresh_expiration < yesterday).delete()

app/models/user.py

@@ -0,0 +1,452 @@
+from datetime import datetime, timedelta
+from enum import IntEnum
+from flask import current_app, url_for
+from flask_hashids import HashidMixin
+from flask_login import UserMixin
+from sqlalchemy.ext.associationproxy import association_proxy
+from pathlib import Path
+from typing import Union
+from werkzeug.security import generate_password_hash, check_password_hash
+import jwt
+import re
+import secrets
+import shutil
+from app import db, hashids
+from app.ext.flask_sqlalchemy import IntEnumColumn
+from .corpus import Corpus
+from .corpus_follower_association import CorpusFollowerAssociation
+from .corpus_follower_role import CorpusFollowerRole
+from .role import Permission, Role
+from .token import Token
+
+
+class ProfilePrivacySettings(IntEnum):
+    SHOW_EMAIL = 1
+    SHOW_LAST_SEEN = 2
+    SHOW_MEMBER_SINCE = 4
+
+    @staticmethod
+    def get(profile_privacy_setting: Union['ProfilePrivacySettings', int, str]) -> 'ProfilePrivacySettings':
+        if isinstance(profile_privacy_setting, ProfilePrivacySettings):
+            return profile_privacy_setting
+        if isinstance(profile_privacy_setting, int):
+            return ProfilePrivacySettings(profile_privacy_setting)
+        if isinstance(profile_privacy_setting, str):
+            return ProfilePrivacySettings[profile_privacy_setting]
+        raise TypeError('profile_privacy_setting must be ProfilePrivacySettings, int, or str')
+
+
+class UserSettingJobStatusMailNotificationLevel(IntEnum):
+    NONE = 1
+    END = 2
+    ALL = 3
+
+
+class User(HashidMixin, UserMixin, db.Model):
+    __tablename__ = 'users'
+    # Primary key
+    id = db.Column(db.Integer, primary_key=True)
+    # Foreign keys
+    role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
+    # Fields
+    email = db.Column(db.String(254), index=True, unique=True)
+    username = db.Column(db.String(64), index=True, unique=True)
+    username_pattern = re.compile(r'^[A-Za-zÄÖÜäöüß0-9_.]*$')
+    password_hash = db.Column(db.String(128))
+    confirmed = db.Column(db.Boolean, default=False)
+    terms_of_use_accepted = db.Column(db.Boolean, default=False)
+    member_since = db.Column(db.DateTime(), default=datetime.utcnow)
+    setting_job_status_mail_notification_level = db.Column(
+        IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
+        default=UserSettingJobStatusMailNotificationLevel.END
+    )
+    last_seen = db.Column(db.DateTime())
+    full_name = db.Column(db.String(64))
+    about_me = db.Column(db.String(256))
+    location = db.Column(db.String(64))
+    website = db.Column(db.String(128))
+    organization = db.Column(db.String(128))
+    is_public = db.Column(db.Boolean, default=False)
+    profile_privacy_settings = db.Column(db.Integer(), default=0)
+    # Relationships
+    avatar = db.relationship(
+        'Avatar',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        uselist=False
+    )
+    corpora = db.relationship(
+        'Corpus',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    corpus_follower_associations = db.relationship(
+        'CorpusFollowerAssociation',
+        back_populates='follower',
+        cascade='all, delete-orphan'
+    )
+    followed_corpora = association_proxy(
+        'corpus_follower_associations',
+        'corpus',
+        creator=lambda c: CorpusFollowerAssociation(corpus=c)
+    )
+    jobs = db.relationship(
+        'Job',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    role = db.relationship(
+        'Role',
+        back_populates='users'
+    )
+    spacy_nlp_pipeline_models = db.relationship(
+        'SpaCyNLPPipelineModel',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    tesseract_ocr_pipeline_models = db.relationship(
+        'TesseractOCRPipelineModel',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+    tokens = db.relationship(
+        'Token',
+        back_populates='user',
+        cascade='all, delete-orphan',
+        lazy='dynamic'
+    )
+
+    def __init__(self, **kwargs):
+        if 'role' not in kwargs:
+            kwargs['role'] = (
+                Role.query.filter_by(name='Administrator').first()
+                if kwargs['email'] == current_app.config['NOPAQUE_ADMIN']
+                else Role.query.filter_by(default=True).first()
+            )
+        super().__init__(**kwargs)
+
+    def __repr__(self):
+        return f'<User {self.username}>'
+
+    @property
+    def jsonpatch_path(self):
+        return f'/users/{self.hashid}'
+
+    @property
+    def password(self):
+        raise AttributeError('password is not a readable attribute')
+
+    @password.setter
+    def password(self, password):
+        self.password_hash = generate_password_hash(password)
+
+    @property
+    def path(self) -> Path:
+        return current_app.config.get('NOPAQUE_DATA_DIR') / 'users' / f'{self.id}'
+
+    @staticmethod
+    def create(**kwargs):
+        user = User(**kwargs)
+        db.session.add(user)
+        db.session.flush(objects=[user])
+        db.session.refresh(user)
+        user_spacy_nlp_pipeline_models_dir = user.path / 'spacy_nlp_pipeline_models'
+        user_tesseract_ocr_pipeline_models_dir = user.path / 'tesseract_ocr_pipeline_models'
+        user_corpora_dir = user.path / 'corpora'
+        user_jobs_dir = user.path / 'jobs'
+        try:
+            user.path.mkdir()
+            user_spacy_nlp_pipeline_models_dir.mkdir()
+            user_tesseract_ocr_pipeline_models_dir.mkdir()
+            user_corpora_dir.mkdir()
+            user_jobs_dir.mkdir()
+        except OSError as e:
+            # TODO: Potential leftover cleanup
+            current_app.logger.error(e)
+            db.session.rollback()
+            raise
+        return user
+
+    @staticmethod
+    def insert_defaults():
+        nopaque_user = User.query.filter_by(username='nopaque').first()
+        system_user_role = Role.query.filter_by(name='System user').first()
+        if nopaque_user is None:
+            nopaque_user = User.create(
+                username='nopaque',
+                role=system_user_role
+            )
+            db.session.add(nopaque_user)
+        elif nopaque_user.role != system_user_role:
+            nopaque_user.role = system_user_role
+        db.session.commit()
+
+    @staticmethod
+    def reset_password(token, new_password):
+        try:
+            payload = jwt.decode(
+                token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'User.reset_password':
+            return False
+        user_hashid = payload.get('sub')
+        user_id = hashids.decode(user_hashid)
+        user = User.query.get(user_id)
+        if user is None:
+            return False
+        user.password = new_password
+        db.session.add(user)
+        return True
+
+    @staticmethod
+    def verify_access_token(access_token, refresh_token=None):
+        token = Token.query.filter(Token.access_token == access_token).first()
+        if token is not None:
+            if token.access_expiration > datetime.utcnow():
+                token.user.ping()
+                db.session.commit()
+                if token.user.role.name != 'System user':
+                    return token.user
+
+    @staticmethod
+    def verify_refresh_token(refresh_token, access_token):
+        token = Token.query.filter((Token.refresh_token == refresh_token) & (Token.access_token == access_token)).first()
+        if token is not None:
+            if token.refresh_expiration > datetime.utcnow():
+                return token
+            # someone tried to refresh with an expired token
+            # revoke all tokens from this user as a precaution
+            token.user.revoke_auth_tokens()
+            db.session.commit()
+
+    def can(self, permission):
+        return self.role is not None and self.role.has_permission(permission)
+
+    def confirm(self, confirmation_token):
+        try:
+            payload = jwt.decode(
+                confirmation_token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'user.confirm':
+            return False
+        if payload.get('sub') != self.hashid:
+            return False
+        self.confirmed = True
+        db.session.add(self)
+        return True
+
+    def delete(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        db.session.delete(self)
+
+    def generate_auth_token(self):
+        return Token(
+            access_token=secrets.token_urlsafe(),
+            access_expiration=datetime.utcnow() + timedelta(minutes=15),
+            refresh_token=secrets.token_urlsafe(),
+            refresh_expiration=datetime.utcnow() + timedelta(days=7),
+            user=self
+        )
+
+    def generate_confirm_token(self, expiration=3600):
+        now = datetime.utcnow()
+        payload = {
+            'exp': now + timedelta(seconds=expiration),
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'user.confirm',
+            'sub': self.hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+
+    def generate_reset_password_token(self, expiration=3600):
+        now = datetime.utcnow()
+        payload = {
+            'exp': now + timedelta(seconds=expiration),
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'User.reset_password',
+            'sub': self.hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+
+    def is_administrator(self):
+        return self.can(Permission.ADMINISTRATE)
+
+    def ping(self):
+        self.last_seen = datetime.utcnow()
+
+    def revoke_auth_tokens(self):
+        for token in self.tokens:
+            db.session.delete(token)
+
+    def verify_password(self, password):
+        if self.role.name == 'System user':
+            return False
+        return check_password_hash(self.password_hash, password)
+
+    #region Profile Privacy settings
+    def has_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        return self.profile_privacy_settings & s.value == s.value
+    
+    def add_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        if not self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings += s.value
+
+    def remove_profile_privacy_setting(self, setting):
+        s = ProfilePrivacySettings.get(setting)
+        if self.has_profile_privacy_setting(s):
+            self.profile_privacy_settings -= s.value
+
+    def reset_profile_privacy_settings(self):
+        self.profile_privacy_settings = 0
+    #endregion Profile Privacy settings
+
+    def follow_corpus(self, corpus, role=None):
+        if role is None:
+            cfr = CorpusFollowerRole.query.filter_by(default=True).first()
+        else:
+            cfr = role
+        if self.is_following_corpus(corpus):
+            cfa = CorpusFollowerAssociation.query.filter_by(corpus=corpus, follower=self).first()
+            if cfa.role != cfr:
+                cfa.role = cfr
+        else:
+            cfa = CorpusFollowerAssociation(corpus=corpus, role=cfr, follower=self)
+            db.session.add(cfa)
+
+    def unfollow_corpus(self, corpus):
+        if not self.is_following_corpus(corpus):
+            return
+        self.followed_corpora.remove(corpus)
+
+    def is_following_corpus(self, corpus):
+        return corpus in self.followed_corpora
+    
+    def generate_follow_corpus_token(self, corpus_hashid, role_name, expiration=7):
+        now = datetime.utcnow()
+        payload = {
+            'exp': expiration,
+            'iat': now,
+            'iss': current_app.config['SERVER_NAME'],
+            'purpose': 'User.follow_corpus',
+            'role_name': role_name,
+            'sub': corpus_hashid
+        }
+        return jwt.encode(
+            payload,
+            current_app.config['SECRET_KEY'],
+            algorithm='HS256'
+        )
+    
+    def follow_corpus_by_token(self, token):
+        try:
+            payload = jwt.decode(
+                token,
+                current_app.config['SECRET_KEY'],
+                algorithms=['HS256'],
+                issuer=current_app.config['SERVER_NAME'],
+                options={'require': ['exp', 'iat', 'iss', 'purpose', 'role_name', 'sub']}
+            )
+        except jwt.PyJWTError:
+            return False
+        if payload.get('purpose') != 'User.follow_corpus':
+            return False
+        corpus_hashid = payload.get('sub')
+        corpus_id = hashids.decode(corpus_hashid)
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if corpus is None:
+            return False
+        role_name = payload.get('role_name')
+        role = CorpusFollowerRole.query.filter_by(name=role_name).first()
+        if role is None:
+            return False
+        self.follow_corpus(corpus, role)
+        # db.session.add(self)
+        return True
+
+    def to_json_serializeable(self, backrefs=False, relationships=False, filter_by_privacy_settings=False):
+        json_serializeable = {
+            'id': self.hashid,
+            'confirmed': self.confirmed,
+            'avatar': url_for('users.user_avatar', user_id=self.id),
+            'email': self.email,
+            'last_seen': (
+                None if self.last_seen is None
+                else f'{self.last_seen.isoformat()}Z'
+            ),
+            'member_since': f'{self.member_since.isoformat()}Z',
+            'username': self.username,
+            'full_name': self.full_name,
+            'about_me': self.about_me,
+            'website': self.website,
+            'location': self.location,
+            'organization': self.organization,
+            'job_status_mail_notification_level': \
+                    self.setting_job_status_mail_notification_level.name,
+            'profile_privacy_settings': {
+                'is_public': self.is_public,
+                'show_email': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL),
+                'show_last_seen': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN),
+                'show_member_since': self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE)
+            }
+        }
+        if backrefs:
+            json_serializeable['role'] = \
+                self.role.to_json_serializeable(backrefs=True)
+        if relationships:
+            json_serializeable['corpus_follower_associations'] = {
+                x.hashid: x.to_json_serializeable()
+                for x in self.corpus_follower_associations
+            }
+            json_serializeable['corpora'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.corpora
+            }
+            json_serializeable['jobs'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.jobs
+            }
+            json_serializeable['tesseract_ocr_pipeline_models'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.tesseract_ocr_pipeline_models
+            }
+            json_serializeable['spacy_nlp_pipeline_models'] = {
+                x.hashid: x.to_json_serializeable(relationships=True)
+                for x in self.spacy_nlp_pipeline_models
+            }
+
+        if filter_by_privacy_settings:
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL):
+                json_serializeable.pop('email')
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_LAST_SEEN):
+                json_serializeable.pop('last_seen')
+            if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_MEMBER_SINCE):
+                json_serializeable.pop('member_since')
+        return json_serializeable

app/services/__init__.py

@@ -1,12 +1,11 @@
 from flask import Blueprint
 from flask_login import login_required
-import os
+from pathlib import Path
 import yaml
 
 
-services_file = \
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'services.yml')
-with open(services_file, 'r') as f:
+services_file = Path(__file__).parent / 'services.yml'
+with services_file.open('r') as f:
     SERVICES = yaml.safe_load(f)
 
 bp = Blueprint('services', __name__)

app/services/routes.py

@@ -6,6 +6,7 @@ from app import db, hashids
 from app.models import (
     Job,
     JobInput,
+    JobResult,
     JobStatus,
     TesseractOCRPipelineModel,
     SpaCyNLPPipelineModel
@@ -74,6 +75,8 @@ def tesseract_ocr_pipeline():
     version = request.args.get('version', service_manifest['latest_version'])
     if version not in service_manifest['versions']:
         abort(404)
+    job_results = JobResult.query.all()
+    choosable_job_ids = [job_result.job.hashid for job_result in job_results if job_result.job.service == "file-setup-pipeline" and job_result.filename.endswith('.pdf')]
     form = CreateTesseractOCRPipelineJobForm(prefix='create-job-form', version=version)
     if form.is_submitted():
         if not form.validate():
@@ -111,6 +114,7 @@ def tesseract_ocr_pipeline():
     return render_template(
         'services/tesseract_ocr_pipeline.html.j2',
         title=service_manifest['name'],
+        choosable_job_ids=choosable_job_ids,
         form=form,
         tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models,
         user_tesseract_ocr_pipeline_models_count=user_tesseract_ocr_pipeline_models_count

app/services/services.yml

@@ -10,7 +10,7 @@ file-setup-pipeline:
 tesseract-ocr-pipeline:
   name: 'Tesseract OCR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.1'
+  latest_version: '0.1.2'
   versions:
     0.1.0:
       methods:
@@ -23,6 +23,12 @@ tesseract-ocr-pipeline:
         - 'ocropus_nlbin_threshold'
       publishing_year: 2022
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
+    0.1.2:
+      methods:
+        - 'binarization'
+        - 'ocropus_nlbin_threshold'
+      publishing_year: 2023
+      url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.2'
 transkribus-htr-pipeline:
   name: 'Transkribus HTR Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
@@ -41,7 +47,7 @@ transkribus-htr-pipeline:
 spacy-nlp-pipeline:
   name: 'SpaCy NLP Pipeline'
   publisher: 'Bielefeld University - CRC 1288 - INF'
-  latest_version: '0.1.2'
+  latest_version: '0.1.1'
   versions:
     0.1.0:
       methods:
@@ -56,5 +62,5 @@ spacy-nlp-pipeline:
     0.1.2:
       methods:
         - 'encoding_detection'
-      publishing_year: 2022
+      publishing_year: 2024
       url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2'

app/static/js/CorpusAnalysis/query-builder/general-query-builder-functions.js

@@ -1,495 +0,0 @@
-class GeneralQueryBuilderFunctions {
-  constructor(elements) {
-    this.elements = elements;
-  }
-
-  toggleClass(elements, className, action){
-    elements.forEach(element => {
-      document.querySelector(`[data-toggle-area="${element}"]`).classList[action](className);
-    });
-  }
-
-  resetQueryInputField() {
-    this.elements.queryInputField.innerHTML = '';
-    this.addPlaceholder();
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  updateChipList() {
-    this.elements.queryChipElements = this.elements.queryInputField.querySelectorAll('.query-component');
-  }
-
-  removePlaceholder() {
-    let placeholder = this.elements.queryInputField.querySelector('#corpus-analysis-concordance-query-builder-input-field-placeholder');  
-    if (placeholder && this.elements.queryInputField !== undefined) {
-      this.elements.queryInputField.innerHTML = '';
-    }
-  }
-
-  addPlaceholder() {
-    let placeholder = nopaque.Utils.HTMLToElement('<span id="corpus-analysis-concordance-query-builder-input-field-placeholder">Click on a button to add a query component</span>');
-    this.elements.queryInputField.appendChild(placeholder);
-  }
-  
-  resetMaterializeSelection(selectionElements, value = "default") {
-    selectionElements.forEach(selectionElement => {
-      if (selectionElement.querySelector(`option[value=${value}]`) !== null) {
-        selectionElement.querySelector(`option[value=${value}]`).selected = true;
-      }
-      let instance = M.FormSelect.getInstance(selectionElement);
-      instance.destroy();
-      M.FormSelect.init(selectionElement);
-    })
-  }
-
-  submitQueryChipElement(dataType = undefined, prettyQueryText = undefined, queryText = undefined, index = null, isClosingTag = false, isEditable = false) {
-    if (this.elements.editingModusOn) {
-      let editedQueryChipElement = this.elements.queryChipElements[this.elements.editedQueryChipElementIndex];
-      editedQueryChipElement.dataset.type = dataType;
-      editedQueryChipElement.dataset.query = queryText;
-      editedQueryChipElement.firstChild.textContent = prettyQueryText;
-      this.updateChipList();
-      this.queryPreviewBuilder();
-    } else {
-      this.queryChipFactory(dataType, prettyQueryText, queryText, index, isClosingTag, isEditable);
-    }
-  }
-
-  queryChipFactory(dataType, prettyQueryText, queryText, index = null, isClosingTag = false, isEditable = false) {
-    // Creates a new query chip element, adds Eventlisteners for selection, deletion and drag and drop and appends it to the query input field.
-    queryText = nopaque.Utils.escape(queryText);
-    prettyQueryText = nopaque.Utils.escape(prettyQueryText);
-    let queryChipElement = nopaque.Utils.HTMLToElement(
-      `
-        <span class="chip query-component" data-type="${dataType}" data-query="${queryText}" draggable="true" data-closing-tag="${isClosingTag}">
-          ${prettyQueryText}${isEditable ? '<i class="material-icons chip-action-button" data-chip-action="edit" style="padding-left:5px; font-size:18px; cursor:pointer;">edit</i>': ''}
-          ${isClosingTag ? '<i class="material-icons chip-action-button" data-chip-action="lock" style="padding-top:5px; font-size:20px; cursor:pointer;">lock_open</i>' : '<i class="material-icons close chip-action-button" data-chip-action="delete">close</i>'}
-        </span>
-      `
-    );
-    this.actionListeners(queryChipElement);
-    queryChipElement.addEventListener('dragstart', this.handleDragStart.bind(this, queryChipElement));
-    queryChipElement.addEventListener('dragend', this.handleDragEnd);
-
-    // Ensures that metadata is always at the end of the query and if an index is given, inserts the query chip at the given index and if there is a closing tag, inserts the query chip before the closing tag.
-    this.removePlaceholder();
-    let lastChild = this.elements.queryInputField.lastChild;
-    let isLastChildTextAnnotation = lastChild && lastChild.dataset.type === 'text-annotation';
-    if (!index) {
-      let closingTagElement = this.elements.queryInputField.querySelector('[data-closing-tag="true"]');
-      if (closingTagElement) {
-        index = Array.from(this.elements.queryInputField.children).indexOf(closingTagElement);
-      }
-    }
-    if (dataType !== 'text-annotation' && index) {
-      this.elements.queryInputField.insertBefore(queryChipElement, this.elements.queryChipElements[index]);
-    } else if (dataType !== 'text-annotation' && isLastChildTextAnnotation) {
-      this.elements.queryInputField.insertBefore(queryChipElement, lastChild);
-    } else {
-      this.elements.queryInputField.appendChild(queryChipElement);
-    }
-
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  actionListeners(queryChipElement) {
-    let notQuantifiableDataTypes = ['start-sentence', 'end-sentence', 'start-entity', 'start-empty-entity', 'end-entity', 'token-incidence-modifier'];
-    queryChipElement.addEventListener('click', (event) => {
-      if (event.target.classList.contains('chip')) {
-        if (!notQuantifiableDataTypes.includes(queryChipElement.dataset.type)) {
-          this.selectChipElement(queryChipElement);
-        }
-      }
-    });
-    let chipActionButtons = queryChipElement.querySelectorAll('.chip-action-button');
-    // chipActionButtons.forEach(button => {
-    for (let button of chipActionButtons) {
-      button.addEventListener('click', (event) => {
-      if (event.target.dataset.chipAction === 'delete') {
-        this.deleteChipElement(queryChipElement);
-      } else if (event.target.dataset.chipAction === 'edit') {
-        this.editChipElement(queryChipElement);
-      } else if (event.target.dataset.chipAction === 'lock') {
-        this.lockClosingChipElement(queryChipElement);
-      }
-      });
-    // });
-    }
-  }
-
-  editChipElement(queryChipElement) {
-    this.elements.editingModusOn = true;
-    this.elements.editedQueryChipElementIndex = Array.from(this.elements.queryInputField.children).indexOf(queryChipElement);
-    switch (queryChipElement.dataset.type) {
-      case 'start-entity':
-        this.editStartEntityChipElement(queryChipElement);
-        break;
-      case 'text-annotation':
-        this.editTextAnnotationChipElement(queryChipElement);
-        break;
-      case 'token':
-        let queryElementsContent = this.prepareQueryElementsContent(queryChipElement);
-        this.editTokenChipElement(queryElementsContent);
-        break;
-      default:
-        break;
-    }
-  }
-
-  editStartEntityChipElement(queryChipElement) {
-    this.elements.structuralAttrModal.open();
-    this.toggleClass(['entity-builder'], 'hide', 'remove');
-    this.toggleEditingAreaStructuralAttrModal('add');
-    let entType = queryChipElement.dataset.query.replace(/<ent_type="|">/g, '');
-    let isEnglishEntType = this.elements.englishEntTypeSelection.querySelector(`option[value=${entType}]`) !== null;
-    let selection = isEnglishEntType ? this.elements.englishEntTypeSelection : this.elements.germanEntTypeSelection;
-    this.resetMaterializeSelection([selection], entType);
-  }
-
-  editTextAnnotationChipElement(queryChipElement) {
-    this.elements.structuralAttrModal.open();
-    this.toggleClass(['text-annotation-builder'], 'hide', 'remove');
-    this.structuralAttributeBuilderFunctions.toggleEditingAreaStructuralAttrModal('add');
-    let [textAnnotationSelection, textAnnotationContent] = queryChipElement.dataset.query
-      .replace(/:: ?match\.text_|"|"/g, '')
-      .split('=');
-    this.resetMaterializeSelection([this.elements.textAnnotationSelection], textAnnotationSelection);
-    this.elements.textAnnotationInput.value = textAnnotationContent;
-  }
-
-  prepareQueryElementsContent(queryChipElement) {
-    //this regex searches for word or lemma or pos or simple_pos="any string within single or double quotes" followed by one or no ignore case markers, followed by one or no condition characters.
-    let regex = new RegExp('(word|lemma|pos|simple_pos)=(("[^"]+")|(\\\\u0027[^\\\\u0027]+\\\\u0027)) ?(%c)? ?(\\&|\\|)?', 'gm');
-    let m;
-    let queryElementsContent = [];
-    while ((m = regex.exec(queryChipElement.dataset.query)) !== null) {
-      // this is necessary to avoid infinite loops with zero-width matches
-      if (m.index === regex.lastIndex) {
-          regex.lastIndex++;
-      }
-      let tokenAttr = m[1];
-      // Passes english-pos by default so that the template is added. In editTokenChipElement it is then checked whether it is english-pos or german-pos.
-      if (tokenAttr === 'pos') {
-        tokenAttr = 'english-pos';
-      }
-      let tokenValue = m[2].replace(/"|'/g, '');
-      let ignoreCase = false;
-      let condition = undefined;
-      m.forEach((match) => {
-        if (match === "%c") {
-          ignoreCase = true;
-        } else if (match === "&") {
-          condition = "and";
-        } else if (match === "|") {
-          condition = "or";
-        }
-      });
-      queryElementsContent.push({tokenAttr: tokenAttr, tokenValue: tokenValue, ignoreCase: ignoreCase, condition: condition});
-    }
-    return queryElementsContent;
-  }
-
-  editTokenChipElement(queryElementsContent) {
-    this.elements.positionalAttrModal.open();
-    queryElementsContent.forEach((queryElement) => {
-      this.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
-      this.preparePositionalAttrModal();
-      switch (queryElement.tokenAttr) {
-        case 'word':
-        case 'lemma':
-          this.elements.tokenBuilderContent.querySelector('input').value = queryElement.tokenValue;
-          break;
-        case 'english-pos':
-          // English-pos is selected by default. Then it is checked whether the passed token value occurs in the english-pos selection. If not, the selection is reseted and changed to german-pos.
-          let selection = this.elements.tokenBuilderContent.querySelector('select');
-          queryElement.tokenAttr = selection.querySelector(`option[value=${queryElement.tokenValue}]`) ? 'english-pos' : 'german-pos';
-          this.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
-          this.preparePositionalAttrModal();
-          this.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
-          break;
-        case 'simple_pos':
-          this.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
-        default:
-          break;
-      }
-      if (queryElement.ignoreCase) {
-        this.elements.ignoreCaseCheckbox.checked = true;
-      }
-      if (queryElement.condition !== undefined) {
-        this.conditionHandler(queryElement.condition, true);
-      }
-
-    });
-  }
-
-  lockClosingChipElement(queryChipElement) {
-    queryChipElement.dataset.closingTag = 'false';
-    let lockIcon = queryChipElement.querySelector('[data-chip-action="lock"]');
-    lockIcon.textContent = 'lock';
-    //TODO: Write unlock-Function?
-    lockIcon.dataset.chipAction = 'unlock';
-  }
-  
-  deleteChipElement(attr) {
-    let elementIndex = Array.from(this.elements.queryInputField.children).indexOf(attr);
-    switch (attr.dataset.type) {
-      case 'start-sentence':
-        this.deletingClosingTagHandler(elementIndex, 'end-sentence');
-        break;
-      case 'start-entity':
-        this.deletingClosingTagHandler(elementIndex, 'end-entity');
-        break;
-      case 'token':
-        let nextElement = Array.from(this.elements.queryInputField.children)[elementIndex+1];
-        if (nextElement !== undefined && nextElement.dataset.type === 'token-incidence-modifier') {
-          this.deleteChipElement(nextElement);
-        }
-      default:
-        break;
-    }
-    this.elements.queryInputField.removeChild(attr);
-    if (this.elements.queryInputField.children.length === 0) {
-      this.addPlaceholder();
-    }
-    this.updateChipList();
-    this.queryPreviewBuilder();
-  }
-
-  deletingClosingTagHandler(elementIndex, closingTagType) {
-    let closingTags = this.elements.queryInputField.querySelectorAll(`[data-type="${closingTagType}"]`);
-    for (let i = 0; i < closingTags.length; i++) {
-      let closingTag = closingTags[i];
-    
-      if (Array.from(this.elements.queryInputField.children).indexOf(closingTag) > elementIndex) {
-        this.deleteChipElement(closingTag);
-        break;
-      }
-    }
-  }
-
-  handleDragStart(queryChipElement, event) {
-    // is called when a query chip is dragged. It creates a dropzone (in form of a chip) for the dragged chip and adds it to the query input field.
-    let queryChips = this.elements.queryInputField.querySelectorAll('.query-component');
-    if (queryChipElement.dataset.type === 'token-incidence-modifier') {
-      queryChips = this.elements.queryInputField.querySelectorAll('.query-component[data-type="token"]');
-    }
-    setTimeout(() => {
-      let targetChipElement = nopaque.Utils.HTMLToElement('<span class="chip drop-target">Drop here</span>');
-      for (let element of queryChips) {
-        if (element === this.elements.queryInputField.querySelectorAll('.query-component')[0]) {
-          let secondTargetChipClone = targetChipElement.cloneNode(true);
-          element.insertAdjacentElement('beforebegin', secondTargetChipClone);
-          this.addDragDropListeners(secondTargetChipClone, queryChipElement);
-        }
-        if (element === queryChipElement || element.nextSibling === queryChipElement) {continue;}
-
-        let targetChipClone = targetChipElement.cloneNode(true);
-        element.insertAdjacentElement('afterend', targetChipClone);
-
-        this.addDragDropListeners(targetChipClone, queryChipElement);
-      }
-    }, 0);
-  }
-
-  handleDragEnd(event) {
-    document.querySelectorAll('.drop-target').forEach(target => target.remove());
-  }
-
-  addDragDropListeners(targetChipClone, queryChipElement) {
-    targetChipClone.addEventListener('dragover', (event) => {
-      event.preventDefault();
-    });
-    targetChipClone.addEventListener('dragenter', (event) => {
-      event.preventDefault();
-      event.target.style.borderStyle = 'solid dotted';
-    });
-    targetChipClone.addEventListener('dragleave', (event) => {
-      event.preventDefault();
-      event.target.style.borderStyle = 'hidden';
-    });
-    targetChipClone.addEventListener('drop', (event) => {
-      let dropzone = event.target;
-      dropzone.parentElement.replaceChild(queryChipElement, dropzone);
-      this.updateChipList();
-      this.queryPreviewBuilder();
-    });
-  }
-
-  queryPreviewBuilder() {
-    // Builds the query preview in the form of pure CQL and displays it in the query preview field.
-    let queryPreview = document.querySelector('#corpus-analysis-concordance-query-preview');
-    let queryInputFieldContent = [];
-    this.elements.queryChipElements.forEach(element => {
-      let queryElement = element.dataset.query;
-      if (queryElement !== undefined) {
-        queryElement = nopaque.Utils.escape(queryElement);
-      }
-      queryInputFieldContent.push(queryElement);
-    });
-  
-    let queryString = queryInputFieldContent.join(' ');
-    let replacements = {
-      ' +': '+',
-      ' *': '*',
-      ' ?': '?',
-      ' {': '{'
-    };
-
-    for (let key in replacements) {
-      queryString = queryString.replace(key, replacements[key]);
-    }
-    queryString += ';';
-  
-    queryPreview.innerHTML = queryString;
-    queryPreview.parentNode.classList.toggle('hide', queryString === ';');
-  }
-
-  selectChipElement(attr) {
-    document.querySelectorAll('.chip.teal').forEach(element => {
-        if (element !== attr) {
-          element.classList.remove('teal', 'lighten-2');
-          this.toggleClass(['token-incidence-modifiers'], 'disabled', 'add');
-        }
-    });
-
-    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'toggle');
-    attr.classList.toggle('teal');
-    attr.classList.toggle('lighten-5');
-  }
-
-  tokenIncidenceModifierHandler(incidenceModifier, incidenceModifierPretty) {
-    // Adds a token incidence modifier to the query input field.
-    let selectedChip = this.elements.queryInputField.querySelector('.chip.teal');
-    let selectedChipIndex = Array.from(this.elements.queryInputField.children).indexOf(selectedChip);
-    this.submitQueryChipElement('token-incidence-modifier', incidenceModifierPretty, incidenceModifier, selectedChipIndex+1);
-    this.selectChipElement(selectedChip);
-  }
-
-  tokenNMSubmitHandler(modalId) {
-    // Adds a token incidence modifier (exactly n or between n and m) to the query input field.
-    let modal = document.querySelector(`#${modalId}`);
-    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
-    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
-    input_m = input_m !== undefined ? input_m.value : '';
-    let input = `{${input_n}${input_m !== '' ? ',' : ''}${input_m}}`;
-    let pretty_input = `between ${input_n} and ${input_m} (${input})`;
-    if (input_m === '') {
-     pretty_input = `exactly ${input_n} (${input})`;
-    }
-
-    let instance = M.Modal.getInstance(modal);
-    instance.close();
-
-    this.tokenIncidenceModifierHandler(input, pretty_input);
-  }
-
-  //#region Functions from other classes
-
-  //TODO: Move these functions back to their og classes and make it work.
-
-  toggleEditingAreaStructuralAttrModal(action) {
-    // If the user edits a query chip element, the corresponding editing area is displayed and the other areas are hidden or disabled.
-    this.toggleClass(['sentence-button', 'entity-button', 'text-annotation-button', 'any-type-entity-button'], 'disabled', action);
-  }
-
-  preparePositionalAttrModal() {
-    let selection = this.elements.positionalAttrSelection.value;
-    if (selection !== 'empty-token') {
-      let selectionTemplate = document.querySelector(`.token-builder-section[data-token-builder-section="${selection}"]`);
-      let selectionTemplateClone = selectionTemplate.content.cloneNode(true);
-    
-      this.elements.tokenBuilderContent.innerHTML = '';
-      this.elements.tokenBuilderContent.appendChild(selectionTemplateClone);
-      if (this.elements.tokenBuilderContent.querySelector('select') !== null) {
-        let selectElement = this.elements.tokenBuilderContent.querySelector('select');
-        M.FormSelect.init(selectElement);
-        selectElement.addEventListener('change', () => {this.optionToggleHandler();});
-      } else {
-        this.elements.tokenBuilderContent.querySelector('input').addEventListener('input', () => {this.optionToggleHandler();});
-      }
-    }
-    this.optionToggleHandler();
-
-    if (selection === 'word' || selection === 'lemma') {
-      this.toggleClass(['input-field-options'], 'hide', 'remove');
-    } else if (selection === 'empty-token'){
-      this.addTokenToQuery();
-    } else {
-      this.toggleClass(['input-field-options'], 'hide', 'add');
-    }
-  }
-
-  tokenInputCheck(elem) {
-    return elem.querySelector('select') !== null ? elem.querySelector('select') : elem.querySelector('input');
-  }
-
-  optionToggleHandler() {
-    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
-    if (input.value === '' && this.elements.editingModusOn === false) {
-      this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-    } else if (this.elements.positionalAttrSelection.querySelectorAll('option').length === 1) {
-      this.toggleClass(['and'], 'disabled', 'add');
-      this.toggleClass(['or'], 'disabled', 'remove');
-    } else {
-      this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'remove');
-    }
-  }
-
-  addTokenToQuery() {
-    let tokenQueryPrettyText = '';
-    let tokenQueryCQLText = '';
-    let input;
-    let kindOfToken = this.kindOfTokenCheck(this.elements.positionalAttrSelection.value);
-    
-    // Takes all rows of the token query (if there is a query concatenation).
-    // Adds their contents to tokenQueryPrettyText and tokenQueryCQLText, which will later be expanded with the current input field.
-    let tokenQueryRows = this.elements.tokenQuery.querySelectorAll('.row');
-    tokenQueryRows.forEach(row => {
-      let ignoreCaseCheckbox = row.querySelector('input[type="checkbox"]');
-      let c = ignoreCaseCheckbox !== null && ignoreCaseCheckbox.checked ? ' %c' : '';
-      let tokenQueryRowInput = this.tokenInputCheck(row.querySelector('.token-query-template-content'));
-      let tokenQueryKindOfToken = this.kindOfTokenCheck(tokenQueryRowInput.closest('.input-field').dataset.kindOfToken);
-      let tokenConditionPrettyText = row.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
-      let tokenConditionCQLText = row.querySelector('[data-condition-cql-text]').dataset.conditionCqlText;
-      tokenQueryPrettyText += `${tokenQueryKindOfToken}=${tokenQueryRowInput.value}${c} ${tokenConditionPrettyText} `;
-      tokenQueryCQLText += `${tokenQueryKindOfToken}="${tokenQueryRowInput.value}"${c} ${tokenConditionCQLText}`;
-    });
-    if (kindOfToken === 'empty-token') {
-      tokenQueryPrettyText += 'empty token';
-    } else {
-      let c = this.elements.ignoreCaseCheckbox.checked ? ' %c' : '';
-      input = this.tokenInputCheck(this.elements.tokenBuilderContent);
-      tokenQueryPrettyText += `${kindOfToken}=${input.value}${c}`;
-      tokenQueryCQLText += `${kindOfToken}="${input.value}"${c}`;
-    }
-    // isTokenQueryInvalid looks if a valid value is passed. If the input fields/dropdowns are empty (isTokenQueryInvalid === true), no token is added.
-    if (this.elements.positionalAttrSelection.value !== 'empty-token' && input.value === '') {
-      this.disableTokenSubmit();
-    } else {
-      tokenQueryCQLText = `[${tokenQueryCQLText}]`;
-      this.submitQueryChipElement('token', tokenQueryPrettyText, tokenQueryCQLText, null, false, kindOfToken === 'empty-token' ? false : true);
-      this.elements.positionalAttrModal.close();
-    }
-  }
-  
-  kindOfTokenCheck(kindOfToken) {
-    return kindOfToken === 'english-pos' || kindOfToken === 'german-pos' ? 'pos' : kindOfToken;
-  }
-
-  disableTokenSubmit() {
-    this.elements.tokenSubmitButton.classList.add('red');
-    this.elements.noValueMessage.classList.remove('hide');
-    setTimeout(() => {
-      this.elements.tokenSubmitButton.classList.remove('red');
-    }, 500);
-    setTimeout(() => {
-      this.elements.noValueMessage.classList.add('hide');
-    }, 3000);
-  }
-
-  //#endregion Functions from other classes
-
-}
-

app/static/js/CorpusAnalysis/query-builder/index.js

@@ -1,192 +0,0 @@
-class ConcordanceQueryBuilder {
-
-  constructor() {
-    this.elements = new ElementReferencesQueryBuilder();
-    this.generalFunctions = new GeneralQueryBuilderFunctions(this.elements);
-    this.tokenAttributeBuilderFunctions = new TokenAttributeBuilderFunctions(this.elements);
-    this.structuralAttributeBuilderFunctions = new StructuralAttributeBuilderFunctions(this.elements);
-
-    this.incidenceModifierEventListeners();
-    this.nAndMInputSubmitEventListeners();
-
-    let queryBuilderDisplay = document.querySelector("#corpus-analysis-concordance-query-builder-display");
-    let expertModeDisplay = document.querySelector("#corpus-analysis-concordance-expert-mode-display");
-    let expertModeSwitch = document.querySelector("#corpus-analysis-concordance-expert-mode-switch");
-    
-    expertModeSwitch.addEventListener("change", () => {
-      const isChecked = expertModeSwitch.checked;
-      if (isChecked) {
-        queryBuilderDisplay.classList.add("hide");
-        expertModeDisplay.classList.remove("hide");
-        this.switchToExpertModeParser();
-      } else {
-        queryBuilderDisplay.classList.remove("hide");
-        expertModeDisplay.classList.add("hide");
-        this.switchToQueryBuilderParser();
-      }
-    });
-    
-  }
-
-  incidenceModifierEventListeners() {
-    // Eventlisteners for the incidence modifiers. There are two different types of incidence modifiers: token and character incidence modifiers.
-    document.querySelectorAll('.incidence-modifier-selection').forEach(button => {
-      let dropdownId = button.parentNode.parentNode.id;
-      if (dropdownId === 'corpus-analysis-concordance-token-incidence-modifiers-dropdown') {
-        button.addEventListener('click', () => this.generalFunctions.tokenIncidenceModifierHandler(button.dataset.token, button.innerHTML));
-      } else if (dropdownId === 'corpus-analysis-concordance-character-incidence-modifiers-dropdown') {
-        button.addEventListener('click', () => this.tokenAttributeBuilderFunctions.characterIncidenceModifierHandler(button));
-      }
-    });
-  }
-
-  nAndMInputSubmitEventListeners() {
-    // Eventlisteners for the submit of n- and m-values of the incidence modifier modal for "exactly n" or "between n and m".
-    document.querySelectorAll('.n-m-submit-button').forEach(button => {
-      let modalId = button.dataset.modalId;
-      if (modalId === 'corpus-analysis-concordance-exactly-n-token-modal' || modalId === 'corpus-analysis-concordance-between-nm-token-modal') {
-        button.addEventListener('click', () => this.generalFunctions.tokenNMSubmitHandler(modalId));
-      } else if (modalId === 'corpus-analysis-concordance-exactly-n-character-modal' || modalId === 'corpus-analysis-concordance-between-nm-character-modal') {
-        button.addEventListener('click', () => this.tokenAttributeBuilderFunctions.characterNMSubmitHandler(modalId));
-      }
-    });
-  }
-
-  switchToExpertModeParser() {
-    let expertModeInputField = document.querySelector('#corpus-analysis-concordance-form-query');
-    expertModeInputField.value = '';
-    let queryBuilderInputFieldValue = Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
-    if (queryBuilderInputFieldValue !== "" && queryBuilderInputFieldValue !== ";") {
-      expertModeInputField.value = queryBuilderInputFieldValue;
-    }
-  }
-
-  switchToQueryBuilderParser() {
-    this.generalFunctions.resetQueryInputField();
-    let expertModeInputFieldValue = document.querySelector('#corpus-analysis-concordance-form-query').value;
-    let chipElements = this.parseTextToChip(expertModeInputFieldValue);
-    let closingTagElements = ['end-sentence', 'end-entity'];
-    let editableElements = ['start-entity', 'text-annotation', 'token'];
-    for (let chipElement of chipElements) {
-      let isClosingTag = closingTagElements.includes(chipElement['type']);
-      let isEditable = editableElements.includes(chipElement['type']);
-      if (chipElement['query'] === '[]'){
-        isEditable = false;
-      }
-      this.generalFunctions.submitQueryChipElement(chipElement['type'], chipElement['pretty'], chipElement['query'], null, isClosingTag, isEditable);
-    }
-  }
-
-  parseTextToChip(query) {
-    const parsingElementDict = {
-      '<s>': {
-        pretty: 'Sentence Start',
-        type: 'start-sentence'
-      },
-      '<\/s>': {
-        pretty: 'Sentence End',
-        type: 'end-sentence'
-      },
-      '<ent>': {
-        pretty: 'Entity Start',
-        type: 'start-empty-entity'
-      },
-      '<ent_type="([A-Z]+)">': {
-        pretty: '',
-        type: 'start-entity'
-      },
-      '<\\\/ent(_type)?>': {
-        pretty: 'Entity End',
-        type: 'end-entity'
-      },
-      ':: ?match\\.text_[A-Za-z]+="[^"]+"': {
-        pretty: '',
-        type: 'text-annotation'
-      },
-      '\\[(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?)*\\]': {
-        pretty: '',
-        type: 'token'
-      },
-      '\\[\\]': {
-        pretty: 'Empty Token',
-        type: 'token'
-      },
-      '(?<!\\[) ?\\+ ?(?![^\\]]\\])': {
-        pretty: ' one or more (+)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\* ?(?![^\\]]\\])': {
-        pretty: 'zero or more (*)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\? ?(?![^\\]]\\])': {
-        pretty: 'zero or one (?)',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])': {
-        pretty: '',
-        type: 'token-incidence-modifier'
-      },
-      '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])': {
-        pretty: '',
-        type: 'token-incidence-modifier'
-      }
-    }
-  
-    let chipElements = [];
-    let regexPattern = Object.keys(parsingElementDict).map(pattern => `(${pattern})`).join('|');
-    const regex = new RegExp(regexPattern, 'gi');
-    let match;
-  
-    while ((match = regex.exec(query)) !== null) {
-      // this is necessary to avoid infinite loops with zero-width matches
-      if (match.index === regex.lastIndex) {
-        regex.lastIndex++;
-      }
-      let stringElement = match[0];
-      for (let [pattern, chipElement] of Object.entries(parsingElementDict)) {
-        const parsingRegex = new RegExp(pattern, 'gi');
-        if (parsingRegex.exec(stringElement)) {
-          // Creating the pretty text for the chip element
-          let prettyText;
-          switch (pattern) {
-            case '<ent_type="([A-Z]+)">':
-              prettyText = `Entity Type=${stringElement.replace(/<ent_type="|">/g, '')}`;
-              break;
-            case ':: ?match\\.text_[A-Za-z]+="[^"]+"':
-              prettyText = stringElement.replace(/:: ?match\.text_|"|"/g, '');
-              break;
-            case '\\[(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=(("[^"]+")|(\\u0027[^\\u0027]+\\u0027)) ?(%c)? ?)*\\]':
-              let doubleQuotes = /(word|lemma|pos|simple_pos)="[^"]+"/gi;
-              let singleQuotes = /(word|lemma|pos|simple_pos)='[^']+'/gi;
-              if (doubleQuotes.exec(stringElement)) {
-                prettyText = stringElement.replace(/^\[|\]$|"/g, '');
-              } else if (singleQuotes.exec(stringElement)) {
-                prettyText = stringElement.replace(/^\[|\]$|'/g, '');
-              }
-              prettyText = prettyText.replace(/\&/g, ' and ').replace(/\|/g, ' or ');
-              break;
-            case '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])':
-              prettyText = `exactly ${stringElement.replace(/{|}/g, '')} (${stringElement})`;
-              break;
-            case '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])':
-              prettyText = `between${stringElement.replace(/{|}/g, ' ').replace(',', ' and ')}(${stringElement})`;
-              break;
-            default:
-              prettyText = chipElement.pretty;
-              break;
-          }
-          chipElements.push({
-            type: chipElement.type,
-            pretty: prettyText,
-            query: stringElement
-          });
-          break;
-        }
-      }
-    }
-  
-    return chipElements;
-  }
-  
-}

app/static/js/CorpusAnalysis/query-builder/structural-attribute-builder-functions.js

@@ -1,98 +0,0 @@
-class StructuralAttributeBuilderFunctions extends GeneralQueryBuilderFunctions {
-  constructor(elements) {
-    super(elements);
-
-    this.structuralAttrModalEventlisteners();
-
-    document.querySelector('#corpus-analysis-concordance-text-annotation-submit').addEventListener('click', () => this.textAnnotationSubmitHandler());
-
-    this.elements.structuralAttrModal = M.Modal.init(
-      document.querySelector('#corpus-analysis-concordance-structural-attr-modal'), 
-      {
-        onCloseStart: () => {
-          this.resetStructuralAttrModal();
-        }
-      }
-    );
-  }
-
-  structuralAttrModalEventlisteners() {
-    document.querySelectorAll('[data-structural-attr-modal-action-button]').forEach(button => {
-      button.addEventListener('click', () => {
-        this.actionButtonInStrucAttrModalHandler(button.dataset.structuralAttrModalActionButton);
-      });
-    });
-    document.querySelector('.ent-type-selection-action[data-ent-type="any"]').addEventListener('click', () => {
-      this.submitQueryChipElement('start-empty-entity', 'Entity Start', '<ent>');
-      this.submitQueryChipElement('end-entity', 'Entity End', '</ent>', null, true);
-      this.elements.structuralAttrModal.close();
-    });
-    document.querySelector('.ent-type-selection-action[data-ent-type="english"]').addEventListener('change', (event) => {
-      this.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true);
-      if (!this.elements.editingModusOn) {
-        this.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
-      }
-      this.elements.structuralAttrModal.close();
-    });
-    document.querySelector('.ent-type-selection-action[data-ent-type="german"]').addEventListener('change', (event) => {
-      this.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true); 
-      if (!this.elements.editingModusOn) {
-        this.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
-      }
-      this.elements.structuralAttrModal.close();
-    });
-  }
-
-  resetStructuralAttrModal() {
-    this.resetMaterializeSelection([this.elements.englishEntTypeSelection, this.elements.germanEntTypeSelection]);
-    this.resetMaterializeSelection([this.elements.textAnnotationSelection], 'address');
-    this.elements.textAnnotationInput.value = '';
-
-    this.toggleClass(['entity-builder', 'text-annotation-builder'], 'hide', 'add');
-    this.toggleEditingAreaStructuralAttrModal('remove');
-    this.elements.editingModusOn = false;
-    this.elements.editedQueryChipElementIndex = undefined;
-  }
-
-  actionButtonInStrucAttrModalHandler(action) {
-    switch (action) {
-      case 'sentence':
-        this.submitQueryChipElement('start-sentence', 'Sentence Start', '<s>');
-        this.submitQueryChipElement('end-sentence', 'Sentence End', '</s>', null, true);
-        this.elements.structuralAttrModal.close();
-        break;
-      case 'entity':
-        this.toggleClass(['entity-builder'], 'hide', 'toggle');
-        this.toggleClass(['text-annotation-builder'], 'hide', 'add');
-        break;
-      case 'meta-data':
-        this.toggleClass(['text-annotation-builder'], 'hide', 'toggle');
-        this.toggleClass(['entity-builder'], 'hide', 'add');
-        break;
-      default:
-        break;
-    }
-  }
-
-  textAnnotationSubmitHandler() {
-    let noValueMetadataMessage = document.querySelector('#corpus-analysis-concordance-no-value-metadata-message');
-    let textAnnotationSubmit = document.querySelector('#corpus-analysis-concordance-text-annotation-submit');
-    let textAnnotationInput = document.querySelector('#corpus-analysis-concordance-text-annotation-input');
-    let textAnnotationOptions = document.querySelector('#corpus-analysis-concordance-text-annotation-options');
-
-    if (textAnnotationInput.value === '') {
-      textAnnotationSubmit.classList.add('red');
-      noValueMetadataMessage.classList.remove('hide');
-      setTimeout(() => {
-        textAnnotationSubmit.classList.remove('red');
-      }, 500);
-      setTimeout(() => {
-        noValueMetadataMessage.classList.add('hide');
-      }, 3000);
-    } else {
-      let queryText = `:: match.text_${textAnnotationOptions.value}="${textAnnotationInput.value}"`;
-      this.submitQueryChipElement('text-annotation', `${textAnnotationOptions.value}=${textAnnotationInput.value}`, queryText, null, false, true);
-      this.elements.structuralAttrModal.close();
-    }
-  }
-}

app/static/js/CorpusAnalysis/query-builder/token-attribute-builder-functions.js

@@ -1,182 +0,0 @@
-class TokenAttributeBuilderFunctions extends GeneralQueryBuilderFunctions {
-  constructor(elements) {
-    super(elements);
-
-    this.elements.positionalAttrSelection.addEventListener('change', () => {
-      this.preparePositionalAttrModal();
-    });
-
-    // Options for positional attribute selection
-    document.querySelectorAll('.positional-attr-options-action-button[data-options-action]').forEach(button => {
-      button.addEventListener('click', () => {this.actionButtonInOptionSectionHandler(button.dataset.optionsAction);});
-    });
-
-    this.elements.tokenSubmitButton.addEventListener('click', () => {this.addTokenToQuery();});
-
-    this.elements.positionalAttrModal = M.Modal.init(
-      document.querySelector('#corpus-analysis-concordance-positional-attr-modal'), 
-      {
-        onOpenStart: () => {
-          this.preparePositionalAttrModal();
-        },
-        onCloseStart: () => {
-          this.resetPositionalAttrModal();
-        }
-      }
-    );
-  }
-
-  resetPositionalAttrModal() {
-    let originalSelectionList = 
-      `
-        <option value="word" selected>word</option>
-        <option value="lemma" >lemma</option>
-        <option value="english-pos">english pos</option>
-        <option value="german-pos">german pos</option>
-        <option value="simple_pos">simple_pos</option>
-        <option value="empty-token">empty token</option>
-      `;
-    this.elements.positionalAttrSelection.innerHTML = originalSelectionList;
-    this.elements.tokenQuery.innerHTML = '';
-    this.elements.tokenBuilderContent.innerHTML = '';
-    this.toggleClass(['input-field-options'], 'hide', 'remove');
-    this.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
-    this.resetMaterializeSelection([this.elements.positionalAttrSelection], "word");
-    this.elements.ignoreCaseCheckbox.checked = false;
-    this.elements.editingModusOn = false;
-    this.elements.editedQueryChipElementIndex = undefined;
-  }
-
-  actionButtonInOptionSectionHandler(elem) {
-    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
-    switch (elem) {
-      case 'option-group':
-        input.value  += '(option1|option2)';
-        let firstIndex = input.value.indexOf('option1');
-        let lastIndex = firstIndex + 'option1'.length;
-        input.focus();
-        input.setSelectionRange(firstIndex, lastIndex);
-        break;
-      case 'wildcard-char':
-        input.value += '.';
-        break;
-      case 'and':
-        this.conditionHandler('and');
-        break;
-      case 'or':
-        this.conditionHandler('or');
-        break;
-      default:
-        break;
-    }
-    this.optionToggleHandler();
-  }
-
-  characterIncidenceModifierHandler(elem) {
-    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
-    input.value += elem.dataset.token;
-  }
-
-  characterNMSubmitHandler(modalId) {
-    let modal = document.querySelector(`#${modalId}`);
-    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
-    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
-    input_m = input_m !== undefined ? ',' + input_m.value : '';
-    let input = `${input_n}${input_m}`;
-
-    let instance = M.Modal.getInstance(modal);
-    instance.close();
-    let tokenInput = this.tokenInputCheck(this.elements.tokenBuilderContent);
-    tokenInput.value += '{' + input + '}';
-  }
-
-  conditionHandler(conditionText) {
-    let tokenQueryTemplateClone = this.elements.tokenQueryTemplate.content.cloneNode(true);
-    tokenQueryTemplateClone.querySelector('.token-query-template-content').appendChild(this.elements.tokenBuilderContent.firstElementChild);
-    let notSelectedButton = tokenQueryTemplateClone.querySelector(`[data-condition-pretty-text]:not([data-condition-pretty-text="${conditionText}"])`);
-    let deleteButton = tokenQueryTemplateClone.querySelector(`[data-token-query-content-action="delete"]`);
-    deleteButton.addEventListener('click', (event) => {
-      this.deleteTokenQueryRow(event.target);
-    });
-    notSelectedButton.parentNode.removeChild(notSelectedButton);
-    this.elements.tokenQuery.appendChild(tokenQueryTemplateClone);
-
-    // Deleting the options which do not make sense in the context of the condition like "word" AND "word". Also sets selection default.
-    let selectionDefault = "word";
-    let optionDeleteList = ['empty-token'];
-    if (conditionText === 'and') {
-      switch (this.elements.positionalAttrSelection.value) {
-        case 'english-pos' || 'german-pos':
-          optionDeleteList.push('english-pos', 'german-pos');
-          break;
-        default:
-          optionDeleteList.push(this.elements.positionalAttrSelection.value);
-          break;
-      }
-    } else {
-      let originalSelectionList = 
-      `
-        <option value="word" selected>word</option>
-        <option value="lemma" >lemma</option>
-        <option value="english-pos">english pos</option>
-        <option value="german-pos">german pos</option>
-        <option value="simple_pos">simple_pos</option>
-      `;
-      this.elements.positionalAttrSelection.innerHTML = originalSelectionList;
-      M.FormSelect.init(this.elements.positionalAttrSelection);
-    }
-    let lastTokenQueryRow = this.elements.tokenQuery.lastElementChild;
-    if(lastTokenQueryRow.querySelector('[data-kind-of-token="word"]') || lastTokenQueryRow.querySelector('[data-kind-of-token="lemma"]')) {
-      this.appendIgnoreCaseCheckbox(lastTokenQueryRow.querySelector('.token-query-template-content'), this.elements.ignoreCaseCheckbox.checked);
-    }
-    this.elements.ignoreCaseCheckbox.checked = false;
-    this.setTokenSelection(selectionDefault, optionDeleteList);
-  }
-
-  deleteTokenQueryRow(deleteButton) {
-    let deletedRow = deleteButton.closest('.row');
-    let condition = deletedRow.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
-    if (condition === 'and') {
-      let kindOfToken = deletedRow.querySelector('[data-kind-of-token]').dataset.kindOfToken;
-      switch (kindOfToken) {
-        case 'english-pos' || 'german-pos':
-          this.createOptionElementForPosAttrSelection('english-pos');
-          this.createOptionElementForPosAttrSelection('german-pos');
-          break;
-        default:
-          this.createOptionElementForPosAttrSelection(kindOfToken);
-          break;
-      }
-      M.FormSelect.init(this.elements.positionalAttrSelection);
-    }
-    deletedRow.remove();
-  }
-
-  createOptionElementForPosAttrSelection(kindOfToken) {
-    let option = document.createElement('option');
-    option.value = kindOfToken;
-    option.text = kindOfToken;
-    this.elements.positionalAttrSelection.appendChild(option);
-  }
-
-  appendIgnoreCaseCheckbox(parentElement, checked = false) {
-    let ignoreCaseCheckboxClone = document.querySelector('#ignore-case-checkbox-template').content.cloneNode(true);
-    parentElement.appendChild(ignoreCaseCheckboxClone);
-    M.Tooltip.init(parentElement.querySelectorAll('.tooltipped'));
-    if (checked) {
-      parentElement.querySelector('input[type="checkbox"]').checked = true;
-    }
-  }
-
-  setTokenSelection(selection, optionDeleteList) {
-    optionDeleteList.forEach(option => {
-      if (this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`) !== null) {
-        this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`).remove();
-      }
-    });
-
-    this.resetMaterializeSelection([this.elements.positionalAttrSelection], selection);
-    this.preparePositionalAttrModal();
-  }
-
-}

app/static/js/app.js

@@ -168,6 +168,14 @@ nopaque.App = class App {
             let manualModalTocElement = document.querySelector('#manual-modal-toc');
             let manualModalToc = M.Tabs.getInstance(manualModalTocElement);
             manualModalToc.select(modalTriggerElement.dataset.manualModalChapter);
+            // TODO: Make this work.
+            // if ('manualModalChapterAnchor' in modalTriggerElement.dataset) {
+            //   let manualModalChapterAnchor = document.querySelector(`#${modalTriggerElement.dataset.manualModalChapterAnchor}`);
+            //   let xCoord = manualModalChapterAnchor.getBoundingClientRect().left;
+            //   let yCoord = manualModalChapterAnchor.getBoundingClientRect().top;
+            //   let modalContentElement = modalElement.querySelector('.modal-content');
+            //   modalContentElement.scroll(xCoord, yCoord);
+            // }
           }
         }
       }

app/static/js/corpus-analysis/app.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisApp {
+nopaque.corpus_analysis.App = class App {
   constructor(corpusId) {
     this.corpusId = corpusId;
 

app/static/js/corpus-analysis/concordance-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisConcordance {
+nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension {
   name = 'Concordance';
 
   constructor(app) {

app/static/js/corpus-analysis/query-builder/element-references.js

@@ -1,18 +1,18 @@
-class ElementReferencesQueryBuilder {
+nopaque.corpus_analysis.query_builder.ElementReferences = class ElementReferences {
   constructor() {
     // General Elements
     this.queryInputField = document.querySelector('#corpus-analysis-concordance-query-builder-input-field');
     this.queryChipElements = [];
+    this.queryElementTarget = document.querySelector('.query-element-target')
     this.editingModusOn = false;
     this.editedQueryChipElementIndex = undefined;
+    this.deleteQueryButton = document.querySelector('#corpus-analysis-concordance-delete-query-button');
 
     // Structural Attribute Builder Elements
     this.structuralAttrModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-structural-attr-modal'));
     this.englishEntTypeSelection = document.querySelector('#corpus-analysis-concordance-english-ent-type-selection');
     this.germanEntTypeSelection = document.querySelector('#corpus-analysis-concordance-german-ent-type-selection');
-    this.textAnnotationSelection = document.querySelector('#corpus-analysis-concordance-text-annotation-options');
-    this.textAnnotationInput = document.querySelector('#corpus-analysis-concordance-text-annotation-input');
-
+    
     // Token Attribute Builder Elements
     this.positionalAttrModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-positional-attr-modal'));
     this.positionalAttrSelection = document.querySelector('#corpus-analysis-concordance-positional-attr-selection');
@@ -25,4 +25,4 @@ class ElementReferencesQueryBuilder {
 
     this.ignoreCaseCheckbox = document.querySelector('#corpus-analysis-concordance-ignore-case-checkbox');
   }
-}
+};

app/static/js/corpus-analysis/query-builder/index.js

@@ -0,0 +1 @@
+nopaque.corpus_analysis.query_builder = {};

app/static/js/corpus-analysis/query-builder/query-builder.js

@@ -0,0 +1,500 @@
+nopaque.corpus_analysis.query_builder.QueryBuilder = class QueryBuilder {
+  constructor() {
+    this.elements = new nopaque.corpus_analysis.query_builder.ElementReferences();
+
+    this.addEventListenersToQueryElementTarget();
+    this.addEventListenersToIncidenceModifier();
+    this.addEventListenersToNAndMInputSubmit();
+
+    this.elements.deleteQueryButton.addEventListener('click', () => {this.resetQueryInputField()});
+    this.expertModeQueryBuilderSwitchHandler();
+
+    this.extensions = {
+      structuralAttributeBuilderFunctions: new nopaque.corpus_analysis.query_builder.StructuralAttributeBuilderFunctions(this),
+      tokenAttributeBuilderFunctions: new nopaque.corpus_analysis.query_builder.TokenAttributeBuilderFunctions(this),
+    };
+
+    this.dropdown = M.Dropdown.init(
+      document.querySelector('.dropdown-trigger[data-toggle-area="token-incidence-modifiers"]'), 
+      {
+        onCloseStart: () => {
+          this.unselectChipElement(this.elements.queryInputField.querySelector('.chip.teal'));
+        }
+      }
+    )
+  }
+
+  addEventListenersToQueryElementTarget() {
+    this.elements.queryElementTarget.addEventListener('click', () => {
+      this.elements.positionalAttrModal.open();
+    });
+    this.elements.queryElementTarget.addEventListener('dragstart', this.handleDragStart.bind(this, this.elements.queryElementTarget));
+    this.elements.queryElementTarget.addEventListener('dragend', this.handleDragEnd);
+  }
+
+  addEventListenersToIncidenceModifier() {
+    // Eventlisteners for the incidence modifiers. There are two different types of incidence modifiers: token and character incidence modifiers.
+    document.querySelectorAll('.incidence-modifier-selection').forEach(button => {
+      let dropdownId = button.parentNode.parentNode.id;
+      if (dropdownId === 'corpus-analysis-concordance-token-incidence-modifiers-dropdown') {
+        button.addEventListener('click', () => this.tokenIncidenceModifierHandler(button.dataset.token, button.innerHTML));
+      } else if (dropdownId === 'corpus-analysis-concordance-character-incidence-modifiers-dropdown') {
+        button.addEventListener('click', () => this.extensions.tokenAttributeBuilderFunctions.characterIncidenceModifierHandler(button));
+      }
+    });
+  }
+
+  addEventListenersToNAndMInputSubmit() {
+    // Eventlisteners for the submit of n- and m-values of the incidence modifier modal for "exactly n" or "between n and m".
+    document.querySelectorAll('.n-m-submit-button').forEach(button => {
+      let modalId = button.dataset.modalId;
+      if (modalId === 'corpus-analysis-concordance-exactly-n-token-modal' || modalId === 'corpus-analysis-concordance-between-nm-token-modal') {
+        button.addEventListener('click', () => this.tokenNMSubmitHandler(modalId));
+      } else if (modalId === 'corpus-analysis-concordance-exactly-n-character-modal' || modalId === 'corpus-analysis-concordance-between-nm-character-modal') {
+        button.addEventListener('click', () => this.extensions.tokenAttributeBuilderFunctions.characterNMSubmitHandler(modalId));
+      }
+    });
+  }
+
+  toggleClass(elements, className, action) {
+    elements.forEach(element => {
+      document.querySelector(`[data-toggle-area="${element}"]`).classList[action](className);
+    });
+  }
+
+  resetQueryInputField() {
+    this.elements.queryInputField.innerHTML = '';
+    this.addQueryElementTarget();
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  addQueryElementTarget() {
+    let queryElementTarget = nopaque.Utils.HTMLToElement(
+      `
+      <a class="query-element-target btn-floating btn-small blue-grey lighten-4 waves-effect waves-light tooltipped" style="margin-bottom:10px; margin-right:5px;" draggable="true" data-position="bottom" data-tooltip="Add an Element to your query">
+        <i class="material-icons">add</i>
+      </a>
+      `
+    );
+    this.elements.queryInputField.appendChild(queryElementTarget);
+    this.elements.queryElementTarget = queryElementTarget;
+    this.addEventListenersToQueryElementTarget();
+  }
+
+  updateChipList() {
+    this.elements.queryChipElements = this.elements.queryInputField.querySelectorAll('.query-component');
+  }
+  
+  resetMaterializeSelection(selectionElements, value = "default") {
+    selectionElements.forEach(selectionElement => {
+      if (selectionElement.querySelector(`option[value=${value}]`) !== null) {
+        selectionElement.querySelector(`option[value=${value}]`).selected = true;
+      }
+      let instance = M.FormSelect.getInstance(selectionElement);
+      instance.destroy();
+      M.FormSelect.init(selectionElement);
+    })
+  }
+
+  submitQueryChipElement(dataType=undefined, prettyQueryText=undefined, queryText=undefined, index=null, isClosingTag=false, isEditable=false) {
+    if (this.elements.editingModusOn) {
+      let editedQueryChipElement = this.elements.queryChipElements[this.elements.editedQueryChipElementIndex];
+      editedQueryChipElement.dataset.type = dataType;
+      editedQueryChipElement.dataset.query = queryText;
+      editedQueryChipElement.firstChild.textContent = prettyQueryText;
+      this.updateChipList();
+      this.queryPreviewBuilder();
+    } else {
+      this.queryChipFactory(dataType, prettyQueryText, queryText, index, isClosingTag, isEditable);
+    }
+  }
+
+  queryChipFactory(dataType, prettyQueryText, queryText, index=null, isClosingTag=false, isEditable=false) {
+    // Creates a new query chip element, adds Eventlisteners for selection, deletion and drag and drop and appends it to the query input field.
+    queryText = nopaque.Utils.escape(queryText);
+    prettyQueryText = nopaque.Utils.escape(prettyQueryText);
+    let queryChipElement = nopaque.Utils.HTMLToElement(
+      `
+        <span class="chip query-component" data-type="${dataType}" data-query="${queryText}" draggable="true"">
+          ${prettyQueryText}${isEditable ? '<i class="material-icons chip-action-button" data-chip-action="edit" style="padding-left:5px; font-size:18px; cursor:pointer;">edit</i>': ''}
+          ${isClosingTag ? '' : '<i class="material-icons close chip-action-button" data-chip-action="delete">close</i>'}
+        </span>
+      `
+    );
+    this.addActionListeners(queryChipElement);
+    queryChipElement.addEventListener('dragstart', this.handleDragStart.bind(this, queryChipElement));
+    queryChipElement.addEventListener('dragend', this.handleDragEnd);
+    // If an index is given, inserts the query chip after the given index (only relevant for Incidence Modifier) and if there is a closing tag, inserts the query chip before the closing tag.
+    if (index !== null) {
+      this.updateChipList();
+      this.elements.queryChipElements[index].after(queryChipElement);
+    } else {
+      this.elements.queryInputField.insertBefore(queryChipElement, this.elements.queryElementTarget);
+    }
+    if (isClosingTag) {
+      this.moveQueryElementTarget(queryChipElement);
+    }
+
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  moveQueryElementTarget(element) {
+    this.elements.queryInputField.insertBefore(this.elements.queryElementTarget, element);
+  }
+
+  addActionListeners(queryChipElement) {
+    let notQuantifiableDataTypes = ['start-sentence', 'end-sentence', 'start-entity', 'start-empty-entity', 'end-entity', 'token-incidence-modifier'];
+    queryChipElement.addEventListener('click', (event) => {
+      if (event.target.classList.contains('chip')) {
+        if (!notQuantifiableDataTypes.includes(queryChipElement.dataset.type)) {
+          this.selectChipElement(queryChipElement);
+        }
+      }
+    });
+    let chipActionButtons = queryChipElement.querySelectorAll('.chip-action-button');
+    chipActionButtons.forEach(button => {
+      button.addEventListener('click', (event) => {
+      if (event.target.dataset.chipAction === 'delete') {
+        this.deleteChipElement(queryChipElement);
+      } else if (event.target.dataset.chipAction === 'edit') {
+        this.editChipElement(queryChipElement);
+      }
+      });
+    });
+  }
+
+  editChipElement(queryChipElement) {
+    this.elements.editingModusOn = true;
+    this.elements.editedQueryChipElementIndex = Array.from(this.elements.queryInputField.children).indexOf(queryChipElement);
+    switch (queryChipElement.dataset.type) {
+      case 'start-entity':
+        this.extensions.structuralAttributeBuilderFunctions.editStartEntityChipElement(queryChipElement);
+        break;
+      case 'token':
+        let queryElementsContent = this.extensions.tokenAttributeBuilderFunctions.prepareTokenQueryElementsContent(queryChipElement);
+        this.extensions.tokenAttributeBuilderFunctions.editTokenChipElement(queryElementsContent);
+        break;
+      default:
+        break;
+    }
+  }
+
+  deleteChipElement(attr) {
+    let elementIndex = Array.from(this.elements.queryInputField.children).indexOf(attr);
+    switch (attr.dataset.type) {
+      case 'start-sentence':
+        this.deleteClosingTagHandler(elementIndex, 'end-sentence');
+        break;
+      case 'start-empty-entity':
+      case 'start-entity':
+        this.deleteClosingTagHandler(elementIndex, 'end-entity');
+        break;
+      case 'token':
+        let nextElement = Array.from(this.elements.queryInputField.children)[elementIndex+1];
+        if (nextElement !== undefined && nextElement.dataset.type === 'token-incidence-modifier') {
+          this.deleteChipElement(nextElement);
+        }
+      default:
+        break;
+    }
+    this.elements.queryInputField.removeChild(attr);
+    this.updateChipList();
+    this.queryPreviewBuilder();
+  }
+
+  deleteClosingTagHandler(elementIndex, closingTagType) {
+    let closingTags = this.elements.queryInputField.querySelectorAll(`[data-type="${closingTagType}"]`);
+    for (let i = 0; i < closingTags.length; i++) {
+      let closingTag = closingTags[i];
+    
+      if (Array.from(this.elements.queryInputField.children).indexOf(closingTag) > elementIndex) {
+        this.deleteChipElement(closingTag);
+        break;
+      }
+    }
+  }
+
+  handleDragStart(queryChipElement) {
+    // is called when a query chip is dragged. It creates a dropzone (in form of a chip) for the dragged chip and adds it to the query input field.
+    let queryChips = this.elements.queryInputField.querySelectorAll('.query-component');
+    if (queryChipElement.dataset.type === 'token-incidence-modifier') {
+      queryChips = this.elements.queryInputField.querySelectorAll('.query-component[data-type="token"]');
+    }
+    setTimeout(() => {
+      let targetChipElement = nopaque.Utils.HTMLToElement('<span class="chip drop-target">Drop here</span>');
+      for (let element of queryChips) {
+        if (element === this.elements.queryInputField.querySelectorAll('.query-component')[0]) {
+          let secondTargetChipClone = targetChipElement.cloneNode(true);
+          element.insertAdjacentElement('beforebegin', secondTargetChipClone);
+          this.addDragDropListeners(secondTargetChipClone, queryChipElement);
+        }
+        if (element === queryChipElement || element.nextSibling === queryChipElement) {continue;}
+
+        let targetChipClone = targetChipElement.cloneNode(true);
+        element.insertAdjacentElement('afterend', targetChipClone);
+        //TODO: Change to two different functions for drag and drop
+        this.addDragDropListeners(targetChipClone, queryChipElement);
+      }
+    }, 0);
+  }
+
+  handleDragEnd(event) {
+    // is called when a query chip is dropped. It removes the dropzones and initializes the tooltips if the dragged element is the query element target.
+    if (event.target.classList.contains('query-element-target')) {
+      M.Tooltip.init(event.target);
+    }
+    document.querySelectorAll('.drop-target').forEach(target => target.remove());
+  }
+
+  addDragDropListeners(targetChipClone, queryChipElement) {
+    targetChipClone.addEventListener('dragover', (event) => {
+      event.preventDefault();
+    });
+    targetChipClone.addEventListener('dragenter', (event) => {
+      event.preventDefault();
+      event.target.style.borderStyle = 'solid dotted';
+    });
+    targetChipClone.addEventListener('dragleave', (event) => {
+      event.preventDefault();
+      event.target.style.borderStyle = 'hidden';
+    });
+    targetChipClone.addEventListener('drop', (event) => {
+      let dropzone = event.target;
+      dropzone.parentElement.replaceChild(queryChipElement, dropzone);
+      this.updateChipList();
+      this.queryPreviewBuilder();
+    });
+  }
+
+  queryPreviewBuilder() {
+    // Builds the query preview in the form of pure CQL and displays it in the query preview field.
+    let queryPreview = document.querySelector('#corpus-analysis-concordance-query-preview');
+    let queryInputFieldContent = [];
+    this.elements.queryChipElements.forEach(element => {
+      let queryElement = element.dataset.query;
+      if (queryElement !== undefined) {
+        queryElement = nopaque.Utils.escape(queryElement);
+      }
+      queryInputFieldContent.push(queryElement);
+    });
+  
+    let queryString = queryInputFieldContent.join(' ');
+    let replacements = {
+      ' +': '+',
+      ' *': '*',
+      ' ?': '?',
+      ' {': '{'
+    };
+
+    for (let key in replacements) {
+      queryString = queryString.replace(key, replacements[key]);
+    }
+    queryString += ';';
+  
+    queryPreview.innerHTML = queryString;
+    queryPreview.parentNode.classList.toggle('hide', queryString === ';');
+  }
+
+  selectChipElement(attr) {
+    if (attr.classList.contains('teal')) {
+      return;
+    }
+
+    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'toggle');
+    attr.classList.toggle('teal');
+    attr.classList.toggle('lighten-5');
+
+    M.Dropdown.getInstance(document.querySelector('.dropdown-trigger[data-toggle-area="token-incidence-modifiers"]')).open();
+
+  }
+
+  unselectChipElement(attr) {
+    let nModalInstance = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-exactly-n-token-modal'));
+    let nmModalInstance = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-between-nm-token-modal'));
+    if (nModalInstance.isOpen || nmModalInstance.isOpen) {
+      return;
+    }
+    attr.classList.remove('teal', 'lighten-5');
+    this.toggleClass(['token-incidence-modifiers'], 'disabled', 'add');
+  }
+
+  tokenIncidenceModifierHandler(incidenceModifier, incidenceModifierPretty, nOrNM = false) {
+    // Adds a token incidence modifier to the query input field.
+    let selectedChip = this.elements.queryInputField.querySelector('.chip.teal');
+    let selectedChipIndex = Array.from(this.elements.queryChipElements).indexOf(selectedChip);
+    if (nOrNM) {
+      this.unselectChipElement(selectedChip);
+    }
+    this.submitQueryChipElement('token-incidence-modifier', incidenceModifierPretty, incidenceModifier, selectedChipIndex);
+  }
+
+  tokenNMSubmitHandler(modalId) {
+    // Adds a token incidence modifier (exactly n or between n and m) to the query input field.
+    let modal = document.querySelector(`#${modalId}`);
+    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
+    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
+    input_m = input_m !== undefined ? input_m.value : '';
+    let input = `{${input_n}${input_m !== '' ? ',' : ''}${input_m}}`;
+    let pretty_input = `between ${input_n} and ${input_m} (${input})`;
+    if (input_m === '') {
+     pretty_input = `exactly ${input_n} (${input})`;
+    }
+
+    let instance = M.Modal.getInstance(modal);
+    instance.close();
+
+    this.tokenIncidenceModifierHandler(input, pretty_input, true);
+  }
+
+  expertModeQueryBuilderSwitchHandler() {
+    let queryBuilderDisplay = document.querySelector("#corpus-analysis-concordance-query-builder-display");
+    let expertModeDisplay = document.querySelector("#corpus-analysis-concordance-expert-mode-display");
+    let expertModeSwitch = document.querySelector("#corpus-analysis-concordance-expert-mode-switch");
+    let submitModal = M.Modal.getInstance(document.querySelector('#corpus-analysis-concordance-switch-to-query-builder-submit-modal'));
+    
+    let confirmSwitchToQueryBuilderButton = document.querySelector('.switch-action[data-switch-action="confirm"]');
+    confirmSwitchToQueryBuilderButton.addEventListener("click", () => {
+      queryBuilderDisplay.classList.remove("hide");
+      expertModeDisplay.classList.add("hide");
+      this.switchToQueryBuilderParser();
+    });
+
+    expertModeSwitch.addEventListener("change", () => {
+      const isChecked = expertModeSwitch.checked;
+      if (isChecked) {
+        queryBuilderDisplay.classList.add("hide");
+        expertModeDisplay.classList.remove("hide");
+        this.switchToExpertModeParser();
+      } else {
+        submitModal.open();
+      }
+    });
+  }
+
+  switchToExpertModeParser() {
+    let expertModeInputField = document.querySelector('#corpus-analysis-concordance-form-query');
+    expertModeInputField.value = '';
+    let queryBuilderInputFieldValue = nopaque.Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim());
+    if (queryBuilderInputFieldValue !== "" && queryBuilderInputFieldValue !== ";") {
+      expertModeInputField.value = queryBuilderInputFieldValue;
+    }
+  }
+
+  switchToQueryBuilderParser() {
+    this.resetQueryInputField();
+    let expertModeInputFieldValue = document.querySelector('#corpus-analysis-concordance-form-query').value;
+    let chipElements = this.parseTextToChip(expertModeInputFieldValue);
+    let editableElements = ['start-entity', 'token'];
+    for (let chipElement of chipElements) {
+      let isEditable = editableElements.includes(chipElement['type']);
+      if (chipElement['query'] === '[]'){
+        isEditable = false;
+      }
+      this.submitQueryChipElement(chipElement['type'], chipElement['pretty'], chipElement['query'], null, false, isEditable);
+    }
+  }
+
+  parseTextToChip(query) {
+    const parsingElementDict = {
+      '<s>': {
+        pretty: 'Sentence Start',
+        type: 'start-sentence'
+      },
+      '<\/s>': {
+        pretty: 'Sentence End',
+        type: 'end-sentence'
+      },
+      '<ent>': {
+        pretty: 'Entity Start',
+        type: 'start-empty-entity'
+      },
+      '<ent_type="([A-Z]+)">': {
+        pretty: '',
+        type: 'start-entity'
+      },
+      '<\\\/ent(_type)?>': {
+        pretty: 'Entity End',
+        type: 'end-entity'
+      },
+      '\\[(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?)*\\]': {
+        pretty: '',
+        type: 'token'
+      },
+      '\\[\\]': {
+        pretty: 'Empty Token',
+        type: 'token'
+      },
+      '(?<!\\[) ?\\+ ?(?![^\\]]\\])': {
+        pretty: ' one or more (+)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\* ?(?![^\\]]\\])': {
+        pretty: 'zero or more (*)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\? ?(?![^\\]]\\])': {
+        pretty: 'zero or one (?)',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])': {
+        pretty: '',
+        type: 'token-incidence-modifier'
+      },
+      '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])': {
+        pretty: '',
+        type: 'token-incidence-modifier'
+      }
+    }
+  
+    let chipElements = [];
+    let regexPattern = Object.keys(parsingElementDict).map(pattern => `(${pattern})`).join('|');
+    const regex = new RegExp(regexPattern, 'gi');
+    let match;
+  
+    while ((match = regex.exec(query)) !== null) {
+      // this is necessary to avoid infinite loops with zero-width matches
+      if (match.index === regex.lastIndex) {
+        regex.lastIndex++;
+      }
+      let stringElement = match[0];
+      for (let [pattern, chipElement] of Object.entries(parsingElementDict)) {
+        const parsingRegex = new RegExp(pattern, 'gi');
+        if (parsingRegex.exec(stringElement)) {
+          // Creating the pretty text for the chip element
+          let prettyText;
+          switch (pattern) {
+            case '<ent_type="([A-Z]+)">':
+              prettyText = `Entity Type=${stringElement.replace(/<ent_type="|">/g, '')}`;
+              break;
+            case ':: ?match\\.text_[A-Za-z]+="[^"]+"':
+              prettyText = stringElement.replace(/:: ?match\.text_|"|"/g, '');
+              break;
+            case '\\[(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?((\\&|\\|) ?(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?)*\\]':
+              prettyText = stringElement.replace(/^\[|\]$|(?<!\\)"/g, '');
+              prettyText = prettyText.replace(/\&/g, ' and ').replace(/\|/g, ' or ');
+              break;
+            case '(?<!\\[) ?\\{[0-9]+} ?(?![^\\]]\\])':
+              prettyText = `exactly ${stringElement.replace(/{|}/g, '')} (${stringElement})`;
+              break;
+            case '(?<!\\[) ?\\{[0-9]+(,[0-9]+)?} ?(?![^\\]]\\])':
+              prettyText = `between${stringElement.replace(/{|}/g, ' ').replace(',', ' and ')}(${stringElement})`;
+              break;
+            default:
+              prettyText = chipElement.pretty;
+              break;
+          }
+          chipElements.push({
+            type: chipElement.type,
+            pretty: prettyText,
+            query: stringElement
+          });
+          break;
+        }
+      }
+    }
+  
+    return chipElements;
+  }
+};

app/static/js/corpus-analysis/query-builder/structural-attribute-builder-functions.js

@@ -0,0 +1,82 @@
+nopaque.corpus_analysis.query_builder.StructuralAttributeBuilderFunctions = class StructuralAttributeBuilderFunctions {
+  constructor(app) {
+    this.app = app;
+    this.elements = app.elements;
+
+    this.structuralAttrModalEventlisteners();
+
+    this.elements.structuralAttrModal = M.Modal.init(
+      document.querySelector('#corpus-analysis-concordance-structural-attr-modal'), 
+      {
+        onCloseStart: () => {
+          this.resetStructuralAttrModal();
+        }
+      }
+    );
+  }
+
+  structuralAttrModalEventlisteners() {
+    document.querySelectorAll('[data-structural-attr-modal-action-button]').forEach(button => {
+      button.addEventListener('click', () => {
+        this.actionButtonInStrucAttrModalHandler(button.dataset.structuralAttrModalActionButton);
+      });
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="any"]').addEventListener('click', () => {
+      this.app.submitQueryChipElement('start-empty-entity', 'Entity Start', '<ent>');
+      this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent>', null, true);
+      this.elements.structuralAttrModal.close();
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="english"]').addEventListener('change', (event) => {
+      this.app.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true);
+      if (!this.elements.editingModusOn) {
+        this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
+      }
+      this.elements.structuralAttrModal.close();
+    });
+    document.querySelector('.ent-type-selection-action[data-ent-type="german"]').addEventListener('change', (event) => {
+      this.app.submitQueryChipElement('start-entity', `Entity Type=${event.target.value}`, `<ent_type="${event.target.value}">`, null, false, true); 
+      if (!this.elements.editingModusOn) {
+        this.app.submitQueryChipElement('end-entity', 'Entity End', '</ent_type>', null, true);
+      }
+      this.elements.structuralAttrModal.close();
+    });
+  }
+
+  resetStructuralAttrModal() {
+    this.app.resetMaterializeSelection([this.elements.englishEntTypeSelection, this.elements.germanEntTypeSelection]);
+    this.app.toggleClass(['entity-builder'], 'hide', 'add');
+    this.toggleEditingAreaStructuralAttrModal('remove');
+    this.elements.editingModusOn = false;
+    this.elements.editedQueryChipElementIndex = undefined;
+  }
+
+  actionButtonInStrucAttrModalHandler(action) {
+    switch (action) {
+      case 'sentence':
+        this.app.submitQueryChipElement('start-sentence', 'Sentence Start', '<s>');
+        this.app.submitQueryChipElement('end-sentence', 'Sentence End', '</s>', null, true);
+        this.elements.structuralAttrModal.close();
+        break;
+      case 'entity':
+        this.app.toggleClass(['entity-builder'], 'hide', 'toggle');
+        break;
+      default:
+        break;
+    }
+  }
+
+  toggleEditingAreaStructuralAttrModal(action) {
+    // If the user edits a query chip element, the corresponding editing area is displayed and the other areas are hidden or disabled.
+    this.app.toggleClass(['sentence-button', 'entity-button', 'any-type-entity-button'], 'disabled', action);
+  }
+
+  editStartEntityChipElement(queryChipElement) {
+    this.elements.structuralAttrModal.open();
+    this.app.toggleClass(['entity-builder'], 'hide', 'remove');
+    this.toggleEditingAreaStructuralAttrModal('add');
+    let entType = queryChipElement.dataset.query.replace(/<ent_type="|">/g, '');
+    let isEnglishEntType = this.elements.englishEntTypeSelection.querySelector(`option[value=${entType}]`) !== null;
+    let selection = isEnglishEntType ? this.elements.englishEntTypeSelection : this.elements.germanEntTypeSelection;
+    this.app.resetMaterializeSelection([selection], entType);
+  }
+}

app/static/js/corpus-analysis/query-builder/token-attribute-builder-functions.js

@@ -0,0 +1,329 @@
+nopaque.corpus_analysis.query_builder.TokenAttributeBuilderFunctions = class TokenAttributeBuilderFunctions {
+  constructor(app) {
+    this.app = app;
+    this.elements = app.elements;
+
+    this.elements.positionalAttrSelection.addEventListener('change', () => {
+      this.preparePositionalAttrModal();
+    });
+
+    // Options for positional attribute selection
+    document.querySelectorAll('.positional-attr-options-action-button[data-options-action]').forEach(button => {
+      button.addEventListener('click', () => {this.actionButtonInOptionSectionHandler(button.dataset.optionsAction);});
+    });
+
+    this.elements.tokenSubmitButton.addEventListener('click', () => {this.addTokenToQuery();});
+
+    this.elements.positionalAttrModal = M.Modal.init(
+      document.querySelector('#corpus-analysis-concordance-positional-attr-modal'), 
+      {
+        onOpenStart: () => {
+          this.preparePositionalAttrModal();
+        },
+        onCloseStart: () => {
+          this.resetPositionalAttrModal();
+        }
+      }
+    );
+  }
+
+  resetPositionalAttrModal() {
+    let originalSelectionList = 
+      `
+        <option value="word" selected>word</option>
+        <option value="lemma" >lemma</option>
+        <option value="english-pos">english pos</option>
+        <option value="german-pos">german pos</option>
+        <option value="simple_pos">simple_pos</option>
+        <option value="empty-token">empty token</option>
+      `;
+    this.elements.positionalAttrSelection.innerHTML = originalSelectionList;
+    this.elements.tokenQuery.innerHTML = '';
+    this.elements.tokenBuilderContent.innerHTML = '';
+    this.app.toggleClass(['input-field-options'], 'hide', 'remove');
+    this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
+    this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], "word");
+    this.elements.ignoreCaseCheckbox.checked = false;
+    this.elements.editingModusOn = false;
+    this.elements.editedQueryChipElementIndex = undefined;
+  }
+
+  actionButtonInOptionSectionHandler(elem) {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    switch (elem) {
+      case 'option-group':
+        this.cursorPositionInputfieldHandler(input, '(option1|option2)');
+        let firstIndex = input.value.indexOf('option1');
+        let lastIndex = firstIndex + 'option1'.length;
+        input.setSelectionRange(firstIndex, lastIndex);
+        break;
+      case 'wildcard-char':
+        this.cursorPositionInputfieldHandler(input, '.');
+        input.focus();
+        break;
+      case 'and':
+        this.conditionHandler('and');
+        break;
+      case 'or':
+        this.conditionHandler('or');
+        break;
+      default:
+        break;
+    }
+    this.optionToggleHandler();
+  }
+
+  cursorPositionInputfieldHandler(input, addedInput) {
+    let cursorPosition = input.selectionStart;
+    let textBeforeCursor = input.value.substring(0, cursorPosition);
+    let textAfterCursor = input.value.substring(cursorPosition);
+    let newInputValue = textBeforeCursor + addedInput + textAfterCursor;
+    input.value = newInputValue;
+    let newCursorPosition = cursorPosition + addedInput.length;
+    input.setSelectionRange(newCursorPosition, newCursorPosition);
+  }
+
+  characterIncidenceModifierHandler(elem) {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    this.cursorPositionInputfieldHandler(input, elem.dataset.token);
+  }
+
+  characterNMSubmitHandler(modalId) {
+    let modal = document.querySelector(`#${modalId}`);
+    let input_n = modal.querySelector('.n-m-input[data-value-type="n"]').value;
+    let input_m = modal.querySelector('.n-m-input[data-value-type="m"]') || undefined;
+    input_m = input_m !== undefined ? ',' + input_m.value : '';
+    let addedInput = `${input_n}${input_m}`;
+
+    let instance = M.Modal.getInstance(modal);
+    instance.close();
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    this.cursorPositionInputfieldHandler(input, `{${addedInput}}`);
+  }
+
+  conditionHandler(conditionText) {
+    let tokenQueryTemplateClone = this.elements.tokenQueryTemplate.content.cloneNode(true);
+    tokenQueryTemplateClone.querySelector('.token-query-template-content').appendChild(this.elements.tokenBuilderContent.firstElementChild);
+    let notSelectedButton = tokenQueryTemplateClone.querySelector(`[data-condition-pretty-text]:not([data-condition-pretty-text="${conditionText}"])`);
+    let deleteButton = tokenQueryTemplateClone.querySelector(`[data-token-query-content-action="delete"]`);
+    deleteButton.addEventListener('click', (event) => {
+      this.deleteTokenQueryRow(event.target);
+    });
+    notSelectedButton.parentNode.removeChild(notSelectedButton);
+    this.elements.tokenQuery.appendChild(tokenQueryTemplateClone);
+
+    let lastTokenQueryRow = this.elements.tokenQuery.lastElementChild;
+    if(lastTokenQueryRow.querySelector('[data-kind-of-token="word"]') || lastTokenQueryRow.querySelector('[data-kind-of-token="lemma"]')) {
+      this.appendIgnoreCaseCheckbox(lastTokenQueryRow.querySelector('.token-query-template-content'), this.elements.ignoreCaseCheckbox.checked);
+    }
+    this.elements.ignoreCaseCheckbox.checked = false;
+    this.setTokenSelection();
+  }
+
+  deleteTokenQueryRow(deleteButton) {
+    let deletedRow = deleteButton.closest('.row');
+    let condition = deletedRow.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
+    if (condition === 'and') {
+      let kindOfToken = deletedRow.querySelector('[data-kind-of-token]').dataset.kindOfToken;
+      switch (kindOfToken) {
+        case 'english-pos' || 'german-pos':
+          this.createOptionElementForPosAttrSelection('english-pos');
+          this.createOptionElementForPosAttrSelection('german-pos');
+          break;
+        default:
+          this.createOptionElementForPosAttrSelection(kindOfToken);
+          break;
+      }
+      M.FormSelect.init(this.elements.positionalAttrSelection);
+    }
+    deletedRow.remove();
+  }
+
+  createOptionElementForPosAttrSelection(kindOfToken) {
+    let option = document.createElement('option');
+    option.value = kindOfToken;
+    option.text = kindOfToken;
+    this.elements.positionalAttrSelection.appendChild(option);
+  }
+
+  appendIgnoreCaseCheckbox(parentElement, checked=false) {
+    let ignoreCaseCheckboxClone = document.querySelector('#ignore-case-checkbox-template').content.cloneNode(true);
+    parentElement.appendChild(ignoreCaseCheckboxClone);
+    M.Tooltip.init(parentElement.querySelectorAll('.tooltipped'));
+    if (checked) {
+      parentElement.querySelector('input[type="checkbox"]').checked = true;
+    }
+  }
+
+  setTokenSelection(selection="word", optionDeleteList=['empty-token']) {
+    optionDeleteList.forEach(option => {
+      if (this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`) !== null) {
+        this.elements.positionalAttrSelection.querySelector(`option[value=${option}]`).remove();
+      }
+    });
+
+    this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], selection);
+    this.preparePositionalAttrModal();
+  }
+
+  preparePositionalAttrModal() {
+    let selection = this.elements.positionalAttrSelection.value;
+    if (selection !== 'empty-token') {
+      let selectionTemplate = document.querySelector(`.token-builder-section[data-token-builder-section="${selection}"]`);
+      let selectionTemplateClone = selectionTemplate.content.cloneNode(true);
+    
+      this.elements.tokenBuilderContent.innerHTML = '';
+      this.elements.tokenBuilderContent.appendChild(selectionTemplateClone);
+      if (this.elements.tokenBuilderContent.querySelector('select') !== null) {
+        let selectElement = this.elements.tokenBuilderContent.querySelector('select');
+        M.FormSelect.init(selectElement);
+        selectElement.addEventListener('change', () => {this.optionToggleHandler();});
+      } else {
+        this.elements.tokenBuilderContent.querySelector('input').addEventListener('input', () => {this.optionToggleHandler();});
+      }
+    }
+    this.optionToggleHandler();
+
+    if (selection === 'word' || selection === 'lemma') {
+      this.app.toggleClass(['input-field-options'], 'hide', 'remove');
+    } else if (selection === 'empty-token'){
+      this.addTokenToQuery();
+    } else {
+      this.app.toggleClass(['input-field-options'], 'hide', 'add');
+    }
+  }
+
+  tokenInputCheck(elem) {
+    return elem.querySelector('select') !== null ? elem.querySelector('select') : elem.querySelector('input');
+  }
+
+  optionToggleHandler() {
+    let input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+    if (input.value === '' && this.elements.editingModusOn === false) {
+      this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'add');
+    } else if (this.elements.positionalAttrSelection.querySelectorAll('option').length === 1) {
+      this.app.toggleClass(['and'], 'disabled', 'add');
+      this.app.toggleClass(['or'], 'disabled', 'remove');
+    } else {
+      this.app.toggleClass(['incidence-modifiers', 'or', 'and'], 'disabled', 'remove');
+    }
+  }
+
+  addTokenToQuery() {
+    let tokenQueryPrettyText = '';
+    let tokenQueryCQLText = '';
+    let input;
+    let kindOfToken = this.kindOfTokenCheck(this.elements.positionalAttrSelection.value);
+    
+    // Takes all rows of the token query (if there is a query concatenation).
+    // Adds their contents to tokenQueryPrettyText and tokenQueryCQLText, which will later be expanded with the current input field.
+    let tokenQueryRows = this.elements.tokenQuery.querySelectorAll('.row');
+    tokenQueryRows.forEach(row => {
+      let ignoreCaseCheckbox = row.querySelector('input[type="checkbox"]');
+      let c = ignoreCaseCheckbox !== null && ignoreCaseCheckbox.checked ? ' %c' : '';
+      let tokenQueryRowInput = this.tokenInputCheck(row.querySelector('.token-query-template-content'));
+      let tokenQueryKindOfToken = this.kindOfTokenCheck(tokenQueryRowInput.closest('.input-field').dataset.kindOfToken);
+      let tokenConditionPrettyText = row.querySelector('[data-condition-pretty-text]').dataset.conditionPrettyText;
+      let tokenConditionCQLText = row.querySelector('[data-condition-cql-text]').dataset.conditionCqlText;
+      tokenQueryPrettyText += `${tokenQueryKindOfToken}=${tokenQueryRowInput.value}${c} ${tokenConditionPrettyText} `;
+      tokenQueryCQLText += `${tokenQueryKindOfToken}="${tokenQueryRowInput.value}"${c} ${tokenConditionCQLText}`;
+    });
+    if (kindOfToken === 'empty-token') {
+      tokenQueryPrettyText += 'empty token';
+    } else {
+      let c = this.elements.ignoreCaseCheckbox.checked ? ' %c' : '';
+      input = this.tokenInputCheck(this.elements.tokenBuilderContent);
+      tokenQueryPrettyText += `${kindOfToken}=${input.value}${c}`;
+      tokenQueryCQLText += `${kindOfToken}="${input.value}"${c}`;
+    }
+    // isTokenQueryInvalid looks if a valid value is passed. If the input fields/dropdowns are empty (isTokenQueryInvalid === true), no token is added.
+    if (this.elements.positionalAttrSelection.value !== 'empty-token' && input.value === '') {
+      this.disableTokenSubmit();
+    } else {
+      tokenQueryCQLText = `[${tokenQueryCQLText}]`;
+      this.app.submitQueryChipElement('token', tokenQueryPrettyText, tokenQueryCQLText, null, false, kindOfToken === 'empty-token' ? false : true);
+      this.elements.positionalAttrModal.close();
+    }
+  }
+  
+  kindOfTokenCheck(kindOfToken) {
+    return kindOfToken === 'english-pos' || kindOfToken === 'german-pos' ? 'pos' : kindOfToken;
+  }
+
+  disableTokenSubmit() {
+    this.elements.tokenSubmitButton.classList.add('red');
+    this.elements.noValueMessage.classList.remove('hide');
+    setTimeout(() => {
+      this.elements.tokenSubmitButton.classList.remove('red');
+    }, 500);
+    setTimeout(() => {
+      this.elements.noValueMessage.classList.add('hide');
+    }, 3000);
+  }
+
+  editTokenChipElement(queryElementsContent) {
+    this.elements.positionalAttrModal.open();
+    queryElementsContent.forEach((queryElement) => {
+      this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
+      this.preparePositionalAttrModal();
+      switch (queryElement.tokenAttr) {
+        case 'word':
+        case 'lemma':
+          this.elements.tokenBuilderContent.querySelector('input').value = queryElement.tokenValue;
+          break;
+        case 'english-pos':
+          // English-pos is selected by default. Then it is checked whether the passed token value occurs in the english-pos selection. If not, the selection is reseted and changed to german-pos.
+          let selection = this.elements.tokenBuilderContent.querySelector('select');
+          queryElement.tokenAttr = selection.querySelector(`option[value=${queryElement.tokenValue}]`) ? 'english-pos' : 'german-pos';
+          this.app.resetMaterializeSelection([this.elements.positionalAttrSelection], queryElement.tokenAttr);
+          this.preparePositionalAttrModal();
+          this.app.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
+          break;
+        case 'simple_pos':
+          this.app.resetMaterializeSelection([this.elements.tokenBuilderContent.querySelector('select')], queryElement.tokenValue);
+        default:
+          break;
+      }
+      if (queryElement.ignoreCase) {
+        this.elements.ignoreCaseCheckbox.checked = true;
+      }
+      if (queryElement.condition !== undefined) {
+        this.conditionHandler(queryElement.condition, true);
+      }
+
+    });
+  }
+
+  prepareTokenQueryElementsContent(queryChipElement) {
+    //this regex searches for word or lemma or pos or simple_pos="any string (also quotation marks escaped by backslash) within double quotes" followed by one or no ignore case markers, followed by one or no condition characters.
+    let regex = new RegExp('(word|lemma|pos|simple_pos)=("(?:[^"\\\\]|\\\\")*") ?(%c)? ?(\\&|\\|)?', 'gm');
+    let m;
+    let queryElementsContent = [];
+    while ((m = regex.exec(queryChipElement.dataset.query)) !== null) {
+      // this is necessary to avoid infinite loops with zero-width matches
+      if (m.index === regex.lastIndex) {
+          regex.lastIndex++;
+      }
+      let tokenAttr = m[1];
+      // Passes english-pos by default so that the template is added. In editTokenChipElement it is then checked whether it is english-pos or german-pos.
+      if (tokenAttr === 'pos') {
+        tokenAttr = 'english-pos';
+      }
+      let tokenValue = m[2].replace(/(?<!\\)"/g, '');
+      let ignoreCase = false;
+      let condition = undefined;
+      m.forEach((match) => {
+        if (match === "%c") {
+          ignoreCase = true;
+        } else if (match === "&") {
+          condition = "and";
+        } else if (match === "|") {
+          condition = "or";
+        }
+      });
+      queryElementsContent.push({tokenAttr: tokenAttr, tokenValue: tokenValue, ignoreCase: ignoreCase, condition: condition});
+    }
+    return queryElementsContent;
+  }
+
+}

app/static/js/corpus-analysis/reader-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisReader {
+nopaque.corpus_analysis.ReaderExtension = class ReaderExtension {
   name = 'Reader';
 
   constructor(app) {

app/static/js/corpus-analysis/static-visualization-extension.js

@@ -1,4 +1,4 @@
-class CorpusAnalysisStaticVisualization {
+nopaque.corpus_analysis.StaticVisualizationExtension = class StaticVisualizationExtension {
   name = 'Static Visualization (beta)';
 
   constructor(app) {

app/static/js/resource-lists/job-output-list.js

@@ -0,0 +1,137 @@
+nopaque.resource_lists.JobOutputList = class JobOutputList extends nopaque.resource_lists.ResourceList {
+  static htmlClass = 'job-output-list';
+
+  constructor(listContainerElement, options = {}) {
+    super(listContainerElement, options);
+    this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+    this.isInitialized = false;
+    this.userId = listContainerElement.dataset.userId;
+    this.jobOutput = listContainerElement.dataset.jobOutput;
+    this.jobIds = listContainerElement.dataset.jobIds;
+    if (this.userId === undefined) {return;}
+    app.subscribeUser(this.userId).then((response) => {
+      app.socket.on('PATCH', (patch) => {
+        if (this.isInitialized) {this.onPatch(patch);}
+      });
+    });
+    app.getUser(this.userId).then((user) => {
+      let jobIds = JSON.parse(this.jobIds.replace(/'/g, '"'));
+      let job_results = {};
+      for (let jobId of jobIds) {
+        for (let jobResult of Object.values(user.jobs[jobId].results)) {
+          if (jobResult.mimetype === 'application/pdf') {
+            job_results[jobResult.id] = jobResult;
+            job_results[jobResult.id].description = user.jobs[jobId].description;
+            job_results[jobResult.id].title = user.jobs[jobId].title;
+          }
+        }
+      }
+      this.add(Object.values(job_results));
+      this.isInitialized = true;
+    });
+  }
+
+  get item() {
+    return `
+      <tr class="list-item clickable hoverable">
+        <td><span class="title"></span></td>
+        <td><span class="description"></span></td>
+        <td><span class="filename"></span></td>
+        <td class="right-align">
+          <a class="list-action-trigger btn-flat waves-effect waves-light" data-list-action="add"><i class="material-icons">add</i></a>
+        </td>
+      </tr>
+    `.trim();
+  }
+
+  get valueNames() {
+    return [
+      {data: ['id']},
+      {data: ['creation-date']},
+      'title',
+      'description',
+      'filename'
+    ];
+  }
+
+  initListContainerElement() {
+    if (!this.listContainerElement.hasAttribute('id')) {
+      this.listContainerElement.id = nopaque.Utils.generateElementId('job-output-list-');
+    }
+    let listSearchElementId = nopaque.Utils.generateElementId(`${this.listContainerElement.id}-search-`);
+    this.listContainerElement.innerHTML = `
+      <div class="input-field">
+        <i class="material-icons prefix">search</i>
+        <input id="${listSearchElementId}" class="search" type="text"></input>
+        <label for="${listSearchElementId}">Search job output</label>
+      </div>
+      <table>
+        <thead>
+          <tr>
+            <th>Title</th>
+            <th>Description</th>
+            <th>Filename</th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody class="list"></tbody>
+      </table>
+      <ul class="pagination"></ul>
+    `;
+  }
+
+  mapResourceToValue(jobOutput) {
+    console.log(jobOutput);
+    return {
+      'id': jobOutput.id,
+      'creation-date': jobOutput.creationDate,
+      'title': jobOutput.title,
+      'description': jobOutput.description,
+      'filename': jobOutput.filename
+    };
+  }
+
+  sort() {
+    this.listjs.sort('title', {order: 'asc'});
+  }
+
+  onClick(event) {
+    let listItemElement = event.target.closest('.list-item[data-id]');
+    if (listItemElement === null) {return;}
+    let itemId = listItemElement.dataset.id;
+    let listActionElement = event.target.closest('.list-action-trigger[data-list-action]');
+    let listAction = listActionElement === null ? 'add' : listActionElement.dataset.listAction;
+    switch (listAction) {
+      case 'add': {
+        listActionElement.querySelector('i').textContent = 'done';
+        listActionElement.dataset.listAction = 'remove';
+        break;
+      }
+      case 'remove': {
+        listActionElement.querySelector('i').textContent = 'add';
+        listActionElement.dataset.listAction = 'add';
+        break;
+      }
+      default: {
+        break;
+      }
+    }
+  }
+
+  // onPatch(patch) {
+  //   let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)`);
+  //   let filteredPatch = patch.filter(operation => re.test(operation.path));
+  //   for (let operation of filteredPatch) {
+  //     switch(operation.op) {
+  //       case 'add': {
+  //         let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)$`);
+  //         if (re.test(operation.path)) {this.add(operation.value);}
+  //         break;
+  //       }
+  //       default: {
+  //         break;
+  //       }
+  //     }
+  //   }
+  // }
+};

app/templates/_base/_modals/_manual/03_dashboard.html.j2

@@ -8,7 +8,7 @@
       The <a href="{{ url_for('main.dashboard') }}">dashboard</a> provides a central overview of all resources assigned to the
       user. These are <a href="{{ url_for('main.dashboard', _anchor='corpora') }}">corpora</a> and created <a href="{{ url_for('main.dashboard', _anchor='jobs') }}">jobs</a>. Corpora are freely composable
       annotated text collections and jobs are the initiated file processing
-      procedures. Both the job and the corpus listings can be searched using
+      procedures. One can search for jobs as well as corpus listings using
       the search field displayed above them.
     </p>
   </div>
@@ -20,10 +20,10 @@
         <p>
           A corpus is a collection of texts that can be analyzed using the
           Corpus Analysis service. All texts must be in the verticalized text
-          file format, which can be obtained via the Natrual Language
-          Processing service. It contains, in addition to the actual text,
+          file format, which can be obtained via the Natural Language
+          Processing service. It contains, in addition to the text,
           further annotations that are searchable in combination with optional
-          addable metadata during your analysis. 
+          metadata that can be added during your analysis. 
         </p>
       </div>
     </div>

app/templates/_base/_modals/_manual/06_services.html.j2

@@ -35,13 +35,13 @@
 </p>
 
 <h4>Optical Character Recognition (OCR)</h4>
-<p>Comming soon...</p>
+<p>Coming soon...</p>
 
 <h4>Handwritten Text Recognition (HTR)</h4>
-<p>Comming soon...</p>
+<p>Coming soon...</p>
 
 <h4>Natural Language Processing (NLP)</h4>
-<p>Comming soon...</p>
+<p>Coming soon...</p>
 
 <h4>Corpus Analysis</h4>
 <p>

app/templates/_base/_modals/_manual/07_a_closer_look_at_the_corpus_analysis.html.j2

@@ -7,7 +7,7 @@
   <div class="col s12 m8">
     <p>
       To <a href="{{ url_for('corpora.create_corpus') }}">create a corpus</a>, you
-      can use the "New Corpus" button, which can be found on both, the Corpus
+      can use the "New Corpus" button, which can be found on both the Corpus
       Analysis Service page and the Dashboard below the corpus list. Fill in the input
       mask to Create a corpus. After you have completed the input mask, you will
       be automatically taken to the corpus overview page (which can be called up
@@ -43,5 +43,5 @@
   the way of how a token is displayed, by using the text style switch. The
   concordance module offers some more options regarding the context size of
   search results. If the context does not provide enough information you can
-  hop into the reader module by using the lupe icon next to a match.
+  hop into the reader module by using the magnifier icon next to a match.
 </p>

app/templates/_base/_modals/_manual/09_query_builder.html.j2

@@ -1,22 +1,30 @@
 <h3 class="manual-chapter-title">Query Builder Tutorial</h3>
-
-<p>The query builder helps you to make a query in the form of the Corpus Query 
-Language (CQL) to your text. You can use the CQL to filter out various types of 
-text parameters, for example, a specific word, a lemma, or you can set part-of-speech 
+<h4>Overview</h4>
+<p>The query builder can be accessed via "My Corpora" or "Corpus Analysis" in the sidebar options. 
+Select the desired corpus and click on the "Analyze" and then "Concordance"
+buttons to open the query builder.</p>
+<p>The query builder uses the Corpus Query Language (CQL) to help you make a query for analyzing your texts. 
+In this way, it is possible to filter out various types of text parameters, for 
+example, a specific word, a lemma, or you can set part-of-speech 
 tags (pos) that indicate the type of word you are looking for (a noun, an 
 adjective, etc.). In addition, you can also search for structural attributes, 
 or specify your query for a token (word, lemma, pos) via entity typing. And of 
-course everything can be combined. You can find examples for different queries 
-under the tab "Examples".</p>
-<p></p>
+course, the different text parameters can be combined.</p>
+<p>Tokens and structural attributes can be added by clicking on the "+" button
+(the "input marker") in the input field or the labeled buttons below it. Elements 
+added are shown as chips. These can be reorganized using drag and drop. The input 
+marker can also be moved in this way. Its position shows where new elements will be added. <br>
+A "translation" of your query into Corpus Query Language (CQL) is shown below.</p>
+<p>Advanced users can make direct use of the Corpus Query Language (CQL) by switching to "expert mode" via the toggle button.</p>
+<p>The entire input field can be cleared using the red trash icon on the right.</p>
 <br>
 
 <div style="border: 1px solid; padding-left: 20px; margin-right: 400px; margin-bottom: 40px;">
   <h5>Content</h5>
   <ol style="list-style-type:disc">
-    <li><a href="#add-new-token-tutorial">Add new token to your query</a></li>
-    <li><a href="#edit-options-tutorial">Options to edit your query</a></li>
-    <li><a href="#add-structural-attribute-tutorial">Add structural Attributes to your query</a></li>
+    <li><a href="#add-new-token-tutorial">Add a new token to your query</a></li>
+    <li><a href="#edit-options-tutorial">Options for editing your query</a></li>
+    <li><a href="#add-structural-attribute-tutorial">Add structural attributes to your query</a></li>
     <li><a href="#general-options-query-builder">General options</a></li>
   </ol>
 </div>
@@ -30,7 +38,7 @@ under the tab "Examples".</p>
   <p>If you are only looking for a specific token, you can click on the left 
   button and select the type of token you are looking for from the drop-down menu. 
   By default "Word" is selected. </p>
-
+  <br>
   <h5>Word and Lemma</h5>
   <p>If you want to search for a specific word or lemma and the respective 
   category is selected in the drop-down menu, you can type in the word or lemma 
@@ -46,13 +54,13 @@ under the tab "Examples".</p>
   "simple_pos" to search for different parts-of-speech. You can find an overview 
   of all tags under the "Tagsets" tab.</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/pos.gif') }}" alt="part-of-speech-tag explanation" width="100%;" style="margin-bottom:20px;">
+  <br>
 
   <h5>Empty Token</h5>
-  <p>Here you can search for an empty token. This selection should never stand 
-  alone and should always be extended with an incidence modifier or stand in a 
+  <p>Here you can search for a token with unspecified attributes (also called wildcard token). This 
+  selection should never stand alone and should always be extended with an incidence modifier or stand in a 
   larger query, because otherwise all possible tokens would be searched for and 
   the program would crash.</p>
-  <p></p>
   <br>
 </div>
 
@@ -61,8 +69,8 @@ under the tab "Examples".</p>
   <hr>
   <p></p>
   <br>
-  <h4 id="edit-options-tutorial">Options to edit your token</h4> 
-  <p>You have the possibility to extend or specify your searched token with 
+  <h4 id="edit-options-tutorial">Options for editing your query</h4> 
+  <p>You have the possibility to extend or specify the token you are searching for with 
   certain factors. For this the query builder offers some fixed options. You can 
   find more information about the options in the Corpus Query Language Tutorial.</p>
   <br>
@@ -76,7 +84,6 @@ under the tab "Examples".</p>
   variants are not limited, so you can manually enter more options in the same 
   format. "Option1" and "option2" must be replaced accordingly. </p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/option_group.gif') }}" alt="option group explanation" width="100%;" style="margin-bottom:20px;">
-  <p></p>
   <br>
 
   <h5>Incidence Modifiers</h5>
@@ -85,7 +92,7 @@ under the tab "Examples".</p>
   not at all or once: <br> 
   [word = "is"] [word="it"] [word="your"] [word="litte"]? [word = "dog"] <br>
   Here the word "little" should occur either once or not at all. With 
-  [word="dogs?"] the search is for "dog "or "dogs". </p>
+  [word="dogs?"] the search is for "dog "or "dogs".</p>
   <br>
 
   <h5>Ignore Case</h5>
@@ -101,7 +108,10 @@ under the tab "Examples".</p>
   this case. For this you can simply string them together: <br>
   [word="I"] [word="will" & simple_pos="VERB"] [word="go"].</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/or_and.gif') }}" alt="OR/AND explanation" width="100%;" style="margin-bottom:20px;">
-  <p></p>
+  <p>Tokens that have already been added can also be modified by clicking on the corresponding 
+  pen icon. Click on the "ignore case" box, for example, and the query builder will 
+  not differentiate between upper- and lower- case letters for that respective token. 
+  New conditions added apply to the most recent token information.</p>
   <br>
 </div>
 
@@ -120,26 +130,33 @@ under the tab "Examples".</p>
   This search can of course be specified if you search for particular tokens or 
   entities between the sentence tags (<s></s>). For example, you can search for
   sentences that contain only a noun, verb, and adjective. <br> 
-  After clicking on Sentence you will see a <div class="chip" style="background-color:#FD9720;">Sentence Start</div>.
-  When you are done with your query or the content 
-  between the Sentence tags, you have to click the Sentence button one more time 
-  to close it. The corresponding button is called 
-  <div class="chip" style="background-color:#FD9720;">Sentence End</div>.<br>
+  Click on Sentence to add the sentence chips: <div class="chip" style="background-color:#FD9720;">Sentence Start</div> 
+  and <div class="chip" style="background-color:#FD9720;">Sentence End</div>. 
+  These mark where the sentence starts and ends. Use drag-and-drop to place them accordingly. When 
+  the Sentence attribute is added, the input marker will automatically be 
+  moved between the sentence chips. Use drag-and-drop as needed to continue your query 
+  at a different position.
   <br>
 
   <h5>Entities</h5>
-  <p>With entities, i.e. units of meaning, you search for text sections that 
-  follow a certain code. For example, persons, dates, certain events. You can 
-  select the codes using the drop-down menus. You can find an explanation of 
-  the respective abbreviations under the tab "Tagsets". <br>
+  <p>With entities, i.e. units of meaning, you can search for text sections that 
+  contain more specific information, for example, persons, dates, or events. The 
+  codes for these categories can be selected using the drop-down menus. You can find an explanation of 
+  these abbreviations under the tab "Tagsets". <br>
   You can also search for unspecified entities by selecting "Add entity of any type".</p>
-  To close the entity query you started, you have to click the entity button one more time. This will make the <div class="chip" style="background-color:#A6E22D;">Entity End</div> element appear in your query.
+  Click on the Entity button to add the entity chips <div class="chip" style="background-color:#A6E22D;">Entity Type=</div> and <div class="chip" style="background-color:#A6E22D;">Entity End</div>.
+  <p>The entity type can be changed by clicking on the pen symbol on the chip. When 
+  the Entity attribute is added, the input marker will automatically be 
+  moved between the entity chips. Use drag-and-drop as needed to continue your query 
+  at a different position.</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/entity.gif') }}" alt="entity explanation" width="100%;" style="margin-bottom:20px;">
   <p></p>
   <br>
 
-  <h5>Meta Data</h5>
-  <p>With the meta data you can annotate your text and add specific conditions. 
+  <h5>Meta Data (currently unavailable)</h5>
+  <p>The meta data function is being worked on and cannot currently be used!
+  <br>
+  With the meta data you can annotate your text and add specific conditions. 
   You can select a category on the left and enter your desired value on the right. 
   The selected metadata will apply to your entire request and will be added at the end.</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/meta_data.gif') }}" alt="meta data explanation" width="100%;" style="margin-bottom:20px;">
@@ -155,14 +172,39 @@ under the tab "Examples".</p>
   <br>
   <h4 id="general-options-query-builder">General Options of the query builder</h4>
   <p>You have several options to edit your query after adding it to the preview.</p>
+  <br>
+
+  <h5>Editing the elements</h5>
+  <p>You can edit your query chips by clicking on the pen icon.</p>
+  <img src="{{ url_for('static', filename='images/manual/query_builder/editing_chips.gif') }}" alt="editing explanation" width="100%;" style="margin-bottom:20px;">
+  <br>
 
   <h5>Deleting the elements</h5>
   <p>You can delete the added elements from the query by clicking the X behind the respective content.</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/delete.gif') }}" alt="delete explanation" width="100%;" style="margin-bottom:20px;">
+  <br>
 
   <h5>Move the elements of your query</h5>
   <p>You can drag and drop elements to customize your query.</p>
   <img src="{{ url_for('static', filename='images/manual/query_builder/drag_and_drop.gif') }}" alt="Drag&Drop explanation" width="100%;" style="margin-bottom:20px;">
+  <br>
+
+  <h5>Setting an incidence modifier</h5>
+  <p>With the incidence modifier option, you can specify the amount of 
+  times a token should appear in your query. This is particularly relevant for empty
+  tokens (tokens with unspecified attributes). Click on a token (blue chip) and 
+  select the desired option from the list to add an incidence modifier. To 
+  close the list without adding anything, click on the token again.</p>
+  <img src="{{ url_for('static', filename='images/manual/query_builder/incidence_modifier.gif') }}" alt="incidence modifier explanation" width="100%;" style="margin-bottom:20px;">
+  <br>
+  
+  <h5>Switching between Query Builder and Expert mode</h5>
+  <p>To work with the plain Corpus Query Language instead of using the Query Builder, click on the "expert mode" 
+  switch. Your query can be entered into the input field. All elements previously added will be carried over 
+  into expert mode. Click on the switch again to switch back to the Query Builder if desired. All recognized elements 
+  will be parsed into chips; those not recognized will be deleted from the query.</p>
+  <img src="{{ url_for('static', filename='images/manual/query_builder/expert_mode.gif') }}" alt="expert mode explanation" width="100%;" style="margin-bottom:20px;">
+
 
 </div>
 

app/templates/_base/_modals/manual.html.j2

@@ -13,35 +13,35 @@
     </ul>
     <div id="manual-modal-introduction">
       <br>
-      {% include "main/_manual_modal/_01_introduction.html.j2" %}
+      {% include "_base/_modals/_manual/01_introduction.html.j2" %}
     </div>
     <div id="manual-modal-registration-and-log-in">
       <br>
-      {% include "main/_manual_modal/_02_registration_and_log_in.html.j2" %}
+      {% include "_base/_modals/_manual/02_registration_and_log_in.html.j2" %}
     </div>
     <div id="manual-modal-dashboard">
       <br>
-      {% include "main/_manual_modal/_03_dashboard.html.j2" %}
+      {% include "_base/_modals/_manual/03_dashboard.html.j2" %}
     </div>
     <div id="manual-modal-services">
       <br>
-      {% include "main/_manual_modal/_06_services.html.j2" %}
+      {% include "_base/_modals/_manual/06_services.html.j2" %}
     </div>
     <div id="manual-modal-a-closer-look-at-the-corpus-analysis">
       <br>
-      {% include "main/_manual_modal/_07_a_closer_look_at_the_corpus_analysis.html.j2" %}
+      {% include "_base/_modals/_manual/07_a_closer_look_at_the_corpus_analysis.html.j2" %}
     </div>
     <div id="manual-modal-cqp-query-language">
       <br>
-      {% include "main/_manual_modal/_08_cqp_query_language.html.j2" %}
+      {% include "_base/_modals/_manual/08_cqp_query_language.html.j2" %}
     </div>
     <div id="manual-modal-query-builder">
       <br>
-      {% include "main/_manual_modal/_09_query_builder.html.j2" %}
+      {% include "_base/_modals/_manual/09_query_builder.html.j2" %}
     </div>
     <div id="manual-modal-tagsets">
       <br>
-      {% include "main/_manual_modal/_10_tagsets.html.j2" %}
+      {% include "_base/_modals/_manual/10_tagsets.html.j2" %}
     </div>
   </div>
   <div class="modal-footer">

app/templates/_base/_modals/terms_of_use.html.j2

@@ -0,0 +1,32 @@
+<div id="terms-of-use-modal" class="modal modal-fixed-footer">
+  <div class="modal-content">
+    <div class="container">
+      <div class="row">
+        <div class="col s12">
+          <h1 id="title">Terms of use</h1>
+        </div>
+        <div class="col s12">
+          <div class="switch">
+            <label>
+              DE
+              <input type="checkbox" id="terms-of-use-modal-switch">
+              <span class="lever"></span>
+              EN
+            </label>
+          </div>
+          <br>
+        </div>
+        <div class="terms-of-use-modal-content hide">
+          {% include "main/terms_of_use_en.html.j2" %}
+        </div>
+        <div class="terms-of-use-modal-content">
+          {% include "main/terms_of_use_de.html.j2" %}
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <span style="margin-right:20px;">I have taken note of the new GTC and agree to their validity in the context of my further use.</span>
+    <a href="#!" class="modal-close waves-effect waves-green btn">Yes</a>
+  </div>
+</div>

app/templates/_base/footer.html.j2

@@ -27,7 +27,10 @@
       <div class="col s12 m3">
         <span>© 2020 Bielefeld University</span>
       </div>
-      <div class="col s12 m9 right-align">
+      <div class="col s12 m2">
+        <span class="right"><b>Version {{ config.NOPAQUE_VERSION }}</b></span>
+      </div>
+      <div class="col s12 m7 right-align">
         <a class="btn-small primary-variant-color waves-effect waves-light" href="{{ url_for('main.faq') }}"><i class="left material-icons">info_outline</i>Frequently Asked Questions</a>
         <a class="btn-small primary-variant-color waves-effect waves-light" href="mailto:{{ config.NOPAQUE_SERVICE_DESK }}"><i class="left material-icons">mail</i>Report an issue</a>
         <a class="btn-small primary-variant-color waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque" target="_blank"><i class="left material-icons">code</i>GitLab</a>

app/templates/_base/modals.html.j2

@@ -0,0 +1,5 @@
+{% include "_base/_modals/manual.html.j2" %}
+
+{% if current_user.is_authenticated and not current_user.terms_of_use_accepted %}
+{% include "_base/_modals/terms_of_use.html.j2" %}
+{% endif %}

app/templates/_base/navbar.html.j2

@@ -5,12 +5,20 @@
       <a href="#" data-target="sidenav" class="sidenav-trigger"><i class="material-icons">menu</i></a>
       {% endif %}
       <a href="{{ url_for('main.index') }}" class="brand-logo" style="height: 100%; overflow: hidden;">
-        <img class="hide-on-small-only" src="{{ url_for('static', filename='images/nopaque_-_logo_name_slogan.svg') }}" style="height: 128px; margin-top: -32px; margin-left: -32px;">
         <img class="hide-on-med-and-up" src="{{ url_for('static', filename='images/nopaque_-_logo.svg') }}" style="height: 128px; margin-top: -32px; margin-left: -32px;">
       </a>
       <ul class="right hide-on-med-and-down">
         <li><a href="{{ url_for('main.news') }}"><i class="material-icons left">email</i>News</a></li>
-        <li><a class="dropdown-trigger no-autoinit" data-target="nav-more-dropdown" href="#!" id="nav-more-dropdown-trigger"><i class="material-icons">more_vert</i></a></li>
+        <li>
+          <a class="dropdown-trigger no-autoinit" data-target="nav-more-dropdown" href="#!" id="nav-more-dropdown-trigger">
+            {% if current_user.is_authenticated %}
+            <img src="{{ url_for('users.user_avatar', user_id=current_user.id) }}" alt="avatar" class="circle left" style="height: 54px; padding:8px;">
+            {{ current_user.username }} ({{ current_user.email }})
+            {% else %}
+            <i class="material-icons left">more_vert</i>
+            {% endif %}
+          </a>
+        </li>
       </ul>
     </div>
     <div class="nav-content primary-variant-color">
@@ -22,17 +30,14 @@
         {% endif %}
         {%- endfor -%}
       </ul>
-      {# {% if current_user.is_authenticated %}
-      <a class="btn-floating btn-large halfway-fab modal-trigger pink tooltipped waves-effect waves-light" data-tooltip="Roadmap" href="#roadmap-modal"><i class="material-icons">explore</i></a>
-      {% endif %} #}
-      <a class="btn-floating btn-large halfway-fab modal-trigger pink tooltipped waves-effect waves-light" data-tooltip="Manual" href="#manual-modal"><i class="material-icons">help</i></a>
+      <a class="btn-floating btn-large halfway-fab modal-trigger pink tooltipped waves-effect waves-light" data-tooltip="Manual" href="#manual-modal"><i class="material-icons">school</i></a>
     </div>
   </nav>
 </div>
 
 <ul class="dropdown-content" id="nav-more-dropdown">
-  {# <li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons left">help</i>Manual</a></li> #}
   {% if current_user.is_authenticated %}
+  <li><a href="{{ url_for('users.user', user_id=current_user.id) }}"><i class="material-icons left">person</i>My Profile</a></li>
   <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>Settings</a></li>
   <li class="divider" tabindex="-1"></li>
   <li><a href="{{ url_for('auth.logout') }}">Log out</a></li>

app/templates/_base/scripts.html.j2

@@ -52,6 +52,7 @@
   'js/resource-lists/job-input-list.js',
   'js/resource-lists/job-list.js',
   'js/resource-lists/job-result-list.js',
+  'js/resource-lists/job-output-list.js',
   'js/resource-lists/public-corpus-list.js',
   'js/resource-lists/public-user-list.js',
   'js/resource-lists/spacy-nlp-pipeline-model-list.js',
@@ -76,28 +77,28 @@
 {%- assets
   filters='rjsmin',
   output='gen/corpus-analysis.%(version)s.js',
-  'js/CorpusAnalysis/index.js',
-  'js/CorpusAnalysis/cqi/index.js',
-  'js/CorpusAnalysis/cqi/constants.js',
-  'js/CorpusAnalysis/cqi/errors.js',
-  'js/CorpusAnalysis/cqi/status.js',
-  'js/CorpusAnalysis/cqi/api/index.js',
-  'js/CorpusAnalysis/cqi/api/client.js',
-  'js/CorpusAnalysis/cqi/models/index.js',
-  'js/CorpusAnalysis/cqi/models/resource.js',
-  'js/CorpusAnalysis/cqi/models/attributes.js',
-  'js/CorpusAnalysis/cqi/models/subcorpora.js',
-  'js/CorpusAnalysis/cqi/models/corpora.js',
-  'js/CorpusAnalysis/cqi/client.js',
-  'js/CorpusAnalysis/query-builder/index.js',
-  'js/CorpusAnalysis/query-builder/element-references.js',
-  'js/CorpusAnalysis/query-builder/general-query-builder-functions.js',
-  'js/CorpusAnalysis/query-builder/structural-attribute-builder-functions.js',
-  'js/CorpusAnalysis/query-builder/token-attribute-builder-functions.js',
-  'js/CorpusAnalysis/CorpusAnalysisApp.js',
-  'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
-  'js/CorpusAnalysis/CorpusAnalysisReader.js',
-  'js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js'
+  'js/corpus-analysis/index.js',
+  'js/corpus-analysis/cqi/index.js',
+  'js/corpus-analysis/cqi/constants.js',
+  'js/corpus-analysis/cqi/errors.js',
+  'js/corpus-analysis/cqi/status.js',
+  'js/corpus-analysis/cqi/api/index.js',
+  'js/corpus-analysis/cqi/api/client.js',
+  'js/corpus-analysis/cqi/models/index.js',
+  'js/corpus-analysis/cqi/models/resource.js',
+  'js/corpus-analysis/cqi/models/attributes.js',
+  'js/corpus-analysis/cqi/models/subcorpora.js',
+  'js/corpus-analysis/cqi/models/corpora.js',
+  'js/corpus-analysis/cqi/client.js',
+  'js/corpus-analysis/query-builder/index.js',
+  'js/corpus-analysis/query-builder/element-references.js',
+  'js/corpus-analysis/query-builder/query-builder.js',
+  'js/corpus-analysis/query-builder/structural-attribute-builder-functions.js',
+  'js/corpus-analysis/query-builder/token-attribute-builder-functions.js',
+  'js/corpus-analysis/app.js',
+  'js/corpus-analysis/concordance-extension.js',
+  'js/corpus-analysis/reader-extension.js',
+  'js/corpus-analysis/static-visualization-extension.js'
 %}
 <script src="{{ ASSET_URL }}"></script>
 {%- endassets %}
@@ -131,3 +132,15 @@
     app.flash(message, message);
   }
 </script>
+
+<script>
+  let languageModalSwitch = document.querySelector('#terms-of-use-modal-switch');
+  let termsOfUseModalContent = document.querySelectorAll('.terms-of-use-modal-content');
+  if (languageModalSwitch) {
+    languageModalSwitch.addEventListener('change', function() {
+      termsOfUseModalContent.forEach(content => {
+        content.classList.toggle('hide');
+      });
+    });
+  }
+</script>

app/templates/_base/sidenav.html.j2

@@ -1,42 +1,18 @@
 <ul class="sidenav sidenav-fixed" id="sidenav">
-  <li>
-    <div class="user-view" style="padding-top: 1px; padding-left: 20px !important; padding-right: 20px !important; height: 112px;">
-      <div class="background primary-color"></div>
-      <div class="row">
-        <div class="col s5">
-          <a href="{{ url_for('users.user', user_id=current_user.id) }}">
-            <img src="{{ url_for('users.user_avatar', user_id=current_user.id) }}" alt="user-image" class="circle responsive-img"  style="height:80%; margin-top: 22px;">
-          </a>
-        </div>
-        <div class="col s5" style="word-wrap: break-word; margin-left:-10px;">
-          <span class="white-text name">
-          {% if current_user.username|length > 18 %}
-            {{ current_user.username[:15] + '...' }}
-          {% else %}
-            {{ current_user.username }}
-          {% endif %}
-          </span>
-          <span class="white-text email" style="padding-top:5px;">
-          {% if current_user.email|length > 32 %}
-            {{ current_user.email[:29] + '...' }}
-          {% else %}
-            {{ current_user.email }}
-          {% endif %}
-          </span>
-        </div>
+  <li class="primary-color hide-on-small-only">
+      <div style="overflow: hidden; height: 64px; width: 250px;">
+        <a href="{{ url_for('main.index') }}">
+          <img class="hide-on-small-only" src="{{ url_for('static', filename='images/nopaque_-_logo_name_slogan.svg') }}" style="height: 128px; margin-top: -32px;">
+        </a>
       </div>
-    </div>
   </li>
-  {# <li class="primary-color">
-    <div style="overflow: hidden;height: 64px; width: 250px;">
-      <img class="hide-on-small-only" src="{{ url_for('static', filename='images/nopaque_-_logo_name_slogan.svg') }}" style="height: 128px; margin-top: -32px; margin-left: ;">
-    </div>
-  </li> #}
-  {# <li><a href="{{ url_for('main.index') }}">nopaque</a></li> #}
-  <li class="hide-on-large-only">
+  <li class="primary-variant-color center-align hide-on-small-only" style="padding-top: 8px; height:48px;">
+    <img src="{{ url_for('static', filename='images/nopaque_slogan_transparent.png') }}" style="width:85%">
+  </li>
+  <li class="hide-on-med-and-up"><a class="waves-effect" href="{{ url_for('main.index') }}"><i class="material-icons left">home</i>nopaque</a></li>
+  <li>
     <a class="waves-effect" href="{{ url_for('main.news') }}"><i class="material-icons left">email</i>News</a>
   </li>
-  {# <li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons">help</i>Manual</a></li> #}
   <li>
     <a class="waves-effect" class="waves-effect" href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a>
     <ul>
@@ -74,12 +50,13 @@
   <li>
     <a class="waves-effect" class="waves-effect" href="{{ url_for('main.social_area') }}"><i class="material-icons">rocket_launch</i>Social Area</a>
     <ul>
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-users') }}" style="padding-left: 47px;"><i class="material-icons">person</i>Public Users</a>
-  </li>
-  <li>
-    <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-corpora') }}" style="padding-left: 47px;"><i class="nopaque-icons">I</i>Public Corpora</a>
-  </li>
+      <li><a href="{{ url_for('users.user', user_id=current_user.id) }}" style="padding-left: 47px;"><i class="material-icons left">person</i>My Profile</a></li>
+      <li>
+        <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-users') }}" style="padding-left: 47px;"><i class="material-icons">group</i>Public Users</a>
+      </li>
+      <li>
+        <a class="waves-effect" href="{{ url_for('main.social_area', _anchor='public-corpora') }}" style="padding-left: 47px;"><i class="nopaque-icons">I</i>Public Corpora</a>
+      </li>
     </ul>
   </li>
   <li class="hide-on-large-only"><div class="divider"></div></li>