Merge branch 'daemon-rework'

.env.tpl

@@ -9,122 +9,134 @@
 # NOTE: Use `.` as <project-root-dir>
 # HOST_MQ_DIR=
 
-# Example: 999
+# Example: 1000
-# HINT: Use this bash command `getent group docker | cut -d: -f3`
+# HINT: Use this bash command `id -u`
-HOST_DOCKER_GID=
+HOST_UID=
 
 # Example: 1000
 # HINT: Use this bash command `id -g`
 HOST_GID=
 
-# DEFAULT: ./nopaqued.log
+# Example: 999
-# NOTES: Use `.` as <project-root-dir>,
+# HINT: Use this bash command `getent group docker | cut -d: -f3`
-#        This file must be present on container startup
+HOST_DOCKER_GID=
-# HOST_NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: ./nopaque.log
 # NOTES: Use `.` as <project-root-dir>,
 #        This file must be present on container startup
-# HOST_NOPAQUE_LOG_FILE=
+# HOST_LOG_FILE=
-
-# Example: 1000
-# HINT: Use this bash command `id -u`
-HOST_UID=
 
 
 ################################################################################
-# Cookies                                                                      #
+# Flask                                                                        #
+# https://flask.palletsprojects.com/en/1.1.x/config/                           #
 ################################################################################
-# CHOOSE ONE: False, True
+# CHOOSE ONE: http, https
-# DEFAULT: False
+# DEFAULT: http
-# HINT: Set to true if you redirect http to https
+# PREFERRED_URL_SCHEME=
-# NOPAQUE_REMEMBER_COOKIE_SECURE=
+
+# DEFAULT: hard to guess string
+# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
+# SECRET_KEY=
+
+# Example: nopaque.example.com/nopaque.example.com:5000
+# HINT: If your instance is publicly available on a different Port then 80/443,
+#       you will have to add this to the server name
+SERVER_NAME=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
 # HINT: Set to true if you redirect http to https
-# NOPAQUE_SESSION_COOKIE_SECURE=
+# SESSION_COOKIE_SECURE=
 
 
 ################################################################################
-# Database                                                                     #
+# Flask-Login                                                                  #
-# DATABASE_URI blueprint:                                                      #
+# https://flask-login.readthedocs.io/en/latest/                                #
-#   - dialect[+driver]://username:password@host[:port]/database                #
-#     - sqlite is not supported                                                #
-#     - values in square brackets are optional                                 #
 ################################################################################
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
+# CHOOSE ONE: False, True
-# NOPAQUE_DATABASE_URL=
+# DEFAULT: False
-
+# HINT: Set to true if you redirect http to https
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
+# REMEMBER_COOKIE_SECURE=
-# NOPAQUE_DEV_DATABASE_URL=
-
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
-# NOPAQUE_TEST_DATABASE_URL=
 
 
 ################################################################################
-# Email                                                                        #
+# Flask-Mail                                                                   #
+# https://pythonhosted.org/Flask-Mail/                                         #
 ################################################################################
 # EXAMPLE: nopaque Admin <nopaque@example.com>
-NOPAQUE_SMTP_DEFAULT_SENDER=
+MAIL_DEFAULT_SENDER=
 
-NOPAQUE_SMTP_PASSWORD=
+MAIL_PASSWORD=
 
 # EXAMPLE: smtp.example.com
-NOPAQUE_SMTP_SERVER=
+MAIL_SERVER=
 
 # EXAMPLE: 587
-NOPAQUE_SMTP_PORT=
+MAIL_PORT=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
-# NOPAQUE_SMTP_USE_SSL=
+# MAIL_USE_SSL=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
-# NOPAQUE_SMTP_USE_TLS=
+# MAIL_USE_TLS=
 
 # EXAMPLE: nopaque@example.com
-NOPAQUE_SMTP_USERNAME=
+MAIL_USERNAME=
 
 
 ################################################################################
-# General                                                                      #
+# Flask-SQLAlchemy                                                             #
+# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/                  #
 ################################################################################
+# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
+# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
+# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
+# SQLALCHEMY_DATABASE_URI=
+
+
+################################################################################
+# nopaque                                                                      #
+################################################################################
+# If an account is registered with this email adress gets automatically
+# assigned the administrator role.
 # EXAMPLE: admin.nopaque@example.com
-NOPAQUE_ADMIN_EMAIL_ADRESS=
+NOPAQUE_ADMIN=
 
 # DEFAULT: development
 # CHOOSE ONE: development, production, testing
 # NOPAQUE_CONFIG=
 
+# This email adress is used for the contact button in the nopaque footer. If
+# not set, no contact button is displayed.
 # DEFAULT: None
 # EXAMPLE: contact.nopaque@example.com
-# NOPAQUE_CONTACT_EMAIL_ADRESS=
+# NOPAQUE_CONTACT=
 
 # DEFAULT: /mnt/nopaque
-# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
+# NOTE: This must be a network share and it must be available on all Docker
+#       Swarm nodes
 # NOPAQUE_DATA_DIR=
 
-# DEFAULT: localhost
+# CHOOSE ONE: False, True
-# NOPAQUE_DOMAIN=
+# DEFAULT: True
+# NOPAQUE_DAEMON_ENABLED=
 
-# CHOOSE ONE: http, https
+# The hostname or IP address for the server to listen on.
-# DEFAULT: http
+# DEFAULT: 0.0.0.0
-# NOPAQUE_PROTOCOL=
+# NOTES: To use a domain locally, add any names that should route to the app to your hosts file.
+#        If nopaque is running in a Docker container, you propably want to use the default value.
+# NOPAQUE_HOST=
 
-# DEFAULT: hard to guess string
+# The port number for the server to listen on.
-# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
+# DEFAULT: 5000
-# NOPAQUE_SECRET_KEY=
+# NOTE: If nopaque is running in a Docker container, you propably want to use the default value.
+# NOPAQUE_PORT=
 
-
+# transport://[userid:password]@hostname[:port]/[virtual_host]
-################################################################################
+NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-# Logging                                                                      #
-################################################################################
-# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
-# NOTE: Use `.` as <nopaqued-root-dir>
-# NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: %Y-%m-%d %H:%M:%S
 # NOPAQUE_LOG_DATE_FORMAT=
@@ -140,37 +152,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
 # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
 # NOPAQUE_LOG_LEVEL=
 
-
-################################################################################
-# Message queue                                                                #
-# MESSAGE_QUEUE_URI blueprint:                                                 #
-#   - transport://[userid:password]@hostname[:port]/[virtual_host]             #
-#     - values in square brackets are optional                                 #
-################################################################################
-# DEFAULT: None
-# HINT: A message queue is not required when using a single server process
-# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-
-
-################################################################################
-# Proxy fix                                                                    #
-################################################################################
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-For
-# NOPAQUE_NUM_PROXIES_X_FOR=
+# NOPAQUE_PROXY_FIX_X_FOR=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Host
-# NOPAQUE_NUM_PROXIES_X_HOST=
+# NOPAQUE_PROXY_FIX_X_HOST=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Port
-# NOPAQUE_NUM_PROXIES_X_PORT=
+# NOPAQUE_PROXY_FIX_X_PORT=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Prefix
-# NOPAQUE_NUM_PROXIES_X_PREFIX=
+# NOPAQUE_PROXY_FIX_X_PREFIX=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Proto
-# NOPAQUE_NUM_PROXIES_X_PROTO=
+# NOPAQUE_PROXY_FIX_X_PROTO=

README.md

@@ -56,6 +56,12 @@ username@hostname:~$ docker-compose build
 ``` bash
 # Create log files
 touch nopaque.log nopaqued.log
-# For background execution add the -d flag and to scale the app, add --scale web=<NUM-INSTANCES>
+# For background execution add the -d flag
 username@hostname:~$ docker-compose up
+# To scale your app use
+username@hostname:~$ docker-compose -f docker-compose.yml \
+                                    -f docker-compose.override.yml
+                                    -f docker-compose.scale.yml
+                                    up
+                                    -d --no-recreate --scale nopaque=<NUM_INSTANCES>
 ```

daemon/.dockerignore

@@ -1,6 +0,0 @@
-# Docker related files
-Dockerfile
-.dockerignore
-
-# Packages
-__pycache__

daemon/Dockerfile

@@ -1,32 +0,0 @@
-FROM python:3.6.12-slim-buster
-
-
-LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
-
-
-ARG DOCKER_GID
-ARG GID
-ARG UID
-ENV LANG=C.UTF-8
-
-
-RUN apt-get update \
- && apt-get install --no-install-recommends --yes \
-      build-essential \
-      libpq-dev \
- && rm -r /var/lib/apt/lists/*
-
-
-RUN groupadd --gid ${DOCKER_GID} --system docker \
- && groupadd --gid ${GID} --system nopaqued \
- && useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaqued
-USER nopaqued
-WORKDIR /home/nopaqued
-
-
-COPY --chown=nopaqued:nopaqued [".", "."]
-RUN python -m venv venv \
- && venv/bin/pip install --requirement requirements.txt
-
-
-ENTRYPOINT ["./boot.sh"]

daemon/app/__init__.py

@@ -1,31 +0,0 @@
-from config import config
-from sqlalchemy import create_engine
-from sqlalchemy.orm import scoped_session, sessionmaker
-from time import sleep
-import docker
-import os
-
-
-configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
-configuration.init()
-docker_client = docker.from_env()
-engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
-Session = scoped_session(sessionmaker(bind=engine))
-
-
-def run():
-    from .tasks.check_corpora import check_corpora
-    check_corpora_thread = check_corpora()
-    from .tasks.check_jobs import check_jobs
-    check_jobs_thread = check_jobs()
-    from .tasks.notify import notify
-    notify_thread = notify()
-
-    while True:
-        if not check_corpora_thread.is_alive():
-            check_corpora_thread = check_corpora()
-        if not check_jobs_thread.is_alive():
-            check_jobs_thread = check_jobs()
-        if not notify_thread.is_alive():
-            notify_thread = notify()
-        sleep(3)

daemon/app/decorators.py

@@ -1,14 +0,0 @@
-from functools import wraps
-from threading import Thread
-
-
-def background(f):
-    '''
-    ' This decorator executes a function in a Thread.
-    '''
-    @wraps(f)
-    def wrapped(*args, **kwargs):
-        thread = Thread(target=f, args=args, kwargs=kwargs)
-        thread.start()
-        return thread
-    return wrapped

daemon/app/models.py

@@ -1,52 +0,0 @@
-from sqlalchemy.ext.automap import automap_base
-from sqlalchemy.orm import relationship
-from . import engine
-
-
-Base = automap_base()
-
-
-# Classes for database models
-class Corpus(Base):
-    __tablename__ = 'corpora'
-    files = relationship('CorpusFile', collection_class=set)
-
-
-class CorpusFile(Base):
-    __tablename__ = 'corpus_files'
-
-
-class Job(Base):
-    __tablename__ = 'jobs'
-    inputs = relationship('JobInput', collection_class=set)
-    results = relationship('JobResult', collection_class=set)
-    notification_data = relationship('NotificationData', collection_class=list)
-    notification_email_data = relationship('NotificationEmailData',
-                                           collection_class=list)
-
-
-class JobInput(Base):
-    __tablename__ = 'job_results'
-
-
-class JobResult(Base):
-    __tablename__ = 'job_results'
-
-
-class NotificationData(Base):
-    __tablename__ = 'notification_data'
-    job = relationship('Job', collection_class=set)
-
-
-class NotificationEmailData(Base):
-    __tablename__ = 'notification_email_data'
-    job = relationship('Job', collection_class=set)
-
-
-class User(Base):
-    __tablename__ = 'users'
-    jobs = relationship('Job', collection_class=set)
-    corpora = relationship('Corpus', collection_class=set)
-
-
-Base.prepare(engine, reflect=True)

daemon/app/tasks/check_corpora.py

@@ -1,140 +0,0 @@
-from .. import configuration as config
-from .. import docker_client, Session
-from ..decorators import background
-from ..models import Corpus
-import docker
-import logging
-import os
-import shutil
-
-
-@background
-def check_corpora():
-    session = Session()
-    corpora = session.query(Corpus).all()
-    for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
-        __create_build_corpus_service(corpus)
-    for corpus in filter(lambda corpus: (corpus.status == 'queued'
-                                         or corpus.status == 'running'),
-                         corpora):
-        __checkout_build_corpus_service(corpus)
-    for corpus in filter(lambda corpus: corpus.status == 'start analysis',
-                         corpora):
-        __create_cqpserver_container(corpus)
-    for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
-                         corpora):
-        __remove_cqpserver_container(corpus)
-    session.commit()
-    Session.remove()
-
-
-def __create_build_corpus_service(corpus):
-    corpus_dir = os.path.join(config.DATA_DIR,
-                              str(corpus.user_id),
-                              'corpora',
-                              str(corpus.id))
-    corpus_data_dir = os.path.join(corpus_dir, 'data')
-    corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
-    corpus_registry_dir = os.path.join(corpus_dir, 'registry')
-    if os.path.exists(corpus_data_dir):
-        shutil.rmtree(corpus_data_dir)
-    if os.path.exists(corpus_registry_dir):
-        shutil.rmtree(corpus_registry_dir)
-    os.mkdir(corpus_data_dir)
-    os.mkdir(corpus_registry_dir)
-    service_args = {'command': 'docker-entrypoint.sh build-corpus',
-                    'constraints': ['node.role==worker'],
-                    'labels': {'origin': 'nopaque',
-                               'type': 'corpus.prepare',
-                               'corpus_id': str(corpus.id)},
-                    'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
-                               corpus_data_dir + ':/corpora/data:rw',
-                               corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
-                    'name': 'build-corpus_{}'.format(corpus.id),
-                    'restart_policy': docker.types.RestartPolicy()}
-    service_image = \
-        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
-    try:
-        service = docker_client.services.get(service_args['name'])
-    except docker.errors.NotFound:
-        pass
-    except docker.errors.DockerException:
-        return
-    else:
-        service.remove()
-    try:
-        docker_client.services.create(service_image, **service_args)
-    except docker.errors.DockerException:
-        corpus.status = 'failed'
-    else:
-        corpus.status = 'queued'
-
-
-def __checkout_build_corpus_service(corpus):
-    service_name = 'build-corpus_{}'.format(corpus.id)
-    try:
-        service = docker_client.services.get(service_name)
-    except docker.errors.NotFound:
-        logging.error('__checkout_build_corpus_service({}):'.format(corpus.id)
-                      + ' The service does not exist.'
-                      + ' (stauts: {} -> failed)'.format(corpus.status))
-        corpus.status = 'failed'
-        return
-    except docker.errors.DockerException:
-        return
-    service_tasks = service.tasks()
-    if not service_tasks:
-        return
-    task_state = service_tasks[0].get('Status').get('State')
-    if corpus.status == 'queued' and task_state != 'pending':
-        corpus.status = 'running'
-    elif corpus.status == 'running' and task_state == 'complete':
-        service.remove()
-        corpus.status = 'prepared'
-    elif corpus.status == 'running' and task_state == 'failed':
-        service.remove()
-        corpus.status = task_state
-
-
-def __create_cqpserver_container(corpus):
-    corpus_dir = os.path.join(config.DATA_DIR,
-                              str(corpus.user_id),
-                              'corpora',
-                              str(corpus.id))
-    corpus_data_dir = os.path.join(corpus_dir, 'data')
-    corpus_registry_dir = os.path.join(corpus_dir, 'registry')
-    container_args = {'command': 'cqpserver',
-                      'detach': True,
-                      'volumes': [corpus_data_dir + ':/corpora/data:rw',
-                                  corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
-                      'name': 'cqpserver_{}'.format(corpus.id),
-                      'network': 'nopaque_default'}
-    container_image = \
-        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
-    try:
-        container = docker_client.containers.get(container_args['name'])
-    except docker.errors.NotFound:
-        pass
-    except docker.errors.DockerException:
-        return
-    else:
-        container.remove(force=True)
-    try:
-        docker_client.containers.run(container_image, **container_args)
-    except docker.errors.DockerException:
-        return
-    else:
-        corpus.status = 'analysing'
-
-
-def __remove_cqpserver_container(corpus):
-    container_name = 'cqpserver_{}'.format(corpus.id)
-    try:
-        container = docker_client.containers.get(container_name)
-    except docker.errors.NotFound:
-        pass
-    except docker.errors.DockerException:
-        return
-    else:
-        container.remove(force=True)
-    corpus.status = 'prepared'

daemon/app/tasks/check_jobs.py

@@ -1,147 +0,0 @@
-from datetime import datetime
-from .. import configuration as config
-from .. import docker_client, Session
-from ..decorators import background
-from ..models import Job, JobResult, NotificationData, NotificationEmailData
-import docker
-import logging
-import json
-import os
-
-
-@background
-def check_jobs():
-    session = Session()
-    jobs = session.query(Job).all()
-    for job in filter(lambda job: job.status == 'submitted', jobs):
-        __create_job_service(job)
-    for job in filter(lambda job: job.status == 'queued', jobs):
-        __checkout_job_service(job, session)
-        __add_notification_data(job, 'queued', session)
-    for job in filter(lambda job: job.status == 'running', jobs):
-        __checkout_job_service(job, session)
-        __add_notification_data(job, 'running', session)
-    for job in filter(lambda job: job.status == 'complete', jobs):
-        __add_notification_data(job, 'complete', session)
-    for job in filter(lambda job: job.status == 'failed', jobs):
-        __add_notification_data(job, 'failed', session)
-    for job in filter(lambda job: job.status == 'canceling', jobs):
-        __remove_job_service(job)
-    session.commit()
-    Session.remove()
-
-
-def __add_notification_data(job, notified_on_status, session):
-    # checks if user wants any notifications at all
-    if (job.user.setting_job_status_mail_notifications == 'none'):
-        return
-    # checks if user wants only notification on completed jobs
-    elif (job.user.setting_job_status_mail_notifications == 'end'
-          and notified_on_status != 'complete'):
-        return
-    else:
-        # check if a job already has associated NotificationData
-        notification_exists = len(job.notification_data)
-        # create notification_data for current job if there is none
-        if (notification_exists == 0):
-            notification_data = NotificationData(job_id=job.id)
-            session.add(notification_data)
-            # If no commit job will have no NotificationData
-            session.commit()
-        if (job.notification_data[0].notified_on != notified_on_status):
-            notification_email_data = NotificationEmailData(job_id=job.id)
-            notification_email_data.notify_status = notified_on_status
-            notification_email_data.creation_date = datetime.utcnow()
-            job.notification_data[0].notified_on = notified_on_status
-            session.add(notification_email_data)
-
-
-def __create_job_service(job):
-    job_dir = os.path.join(config.DATA_DIR,
-                           str(job.user_id),
-                           'jobs',
-                           str(job.id))
-    cmd = '{} -i /files -o /files/output'.format(job.service)
-    if job.service == 'file-setup':
-        cmd += ' -f {}'.format(job.secure_filename)
-    cmd += ' --log-dir /files'
-    cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
-    cmd += ' ' + ' '.join(json.loads(job.service_args))
-    service_args = {'command': cmd,
-                    'constraints': ['node.role==worker'],
-                    'labels': {'origin': 'nopaque',
-                               'type': 'service.{}'.format(job.service),
-                               'job_id': str(job.id)},
-                    'mounts': [job_dir + ':/files:rw'],
-                    'name': 'job_{}'.format(job.id),
-                    'resources': docker.types.Resources(
-                        cpu_reservation=job.n_cores * (10 ** 9),
-                        mem_reservation=job.mem_mb * (10 ** 6)),
-                    'restart_policy': docker.types.RestartPolicy()}
-    service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
-                     + job.service + ':' + job.service_version)
-    try:
-        service = docker_client.services.get(service_args['name'])
-    except docker.errors.NotFound:
-        pass
-    except docker.errors.DockerException:
-        return
-    else:
-        service.remove()
-    try:
-        docker_client.services.create(service_image, **service_args)
-    except docker.errors.DockerException:
-        job.status = 'failed'
-    else:
-        job.status = 'queued'
-
-
-def __checkout_job_service(job, session):
-    service_name = 'job_{}'.format(job.id)
-    try:
-        service = docker_client.services.get(service_name)
-    except docker.errors.NotFound:
-        logging.error('__checkout_job_service({}): '.format(job.id)
-                      + 'The service does not exist. '
-                      + '(status: {} -> failed)'.format(job.status))
-        job.status = 'failed'
-        return
-    except docker.errors.DockerException:
-        return
-    service_tasks = service.tasks()
-    if not service_tasks:
-        return
-    task_state = service_tasks[0].get('Status').get('State')
-    if job.status == 'queued' and task_state != 'pending':
-        job.status = 'running'
-    elif (job.status == 'running'
-          and (task_state == 'complete' or task_state == 'failed')):
-        service.remove()
-        job.end_date = datetime.utcnow()
-        job.status = task_state
-        if task_state == 'complete':
-            results_dir = os.path.join(config.DATA_DIR,
-                                       str(job.user_id),
-                                       'jobs',
-                                       str(job.id),
-                                       'output')
-            results = filter(lambda x: x.endswith('.zip'),
-                             os.listdir(results_dir))
-            for result in results:
-                job_result = JobResult(dir=results_dir,
-                                       filename=result,
-                                       job_id=job.id)
-                session.add(job_result)
-
-
-def __remove_job_service(job):
-    service_name = 'job_{}'.format(job.id)
-    try:
-        service = docker_client.services.get(service_name)
-    except docker.errors.NotFound:
-        job.status = 'canceled'
-    except docker.errors.DockerException:
-        return
-    else:
-        service.update(mounts=None)
-        service.remove()

daemon/app/tasks/libnotify/notification.py

@@ -1,28 +0,0 @@
-from email.message import EmailMessage
-
-
-class Notification(EmailMessage):
-    """docstring for Email."""
-
-    def set_notification_content(self,
-                                 subject_template,
-                                 subject_template_values_dict,
-                                 body_txt_template_path,
-                                 body_html_template_path,
-                                 body_template_values_dict):
-        # Create subject with subject_template_values_dict
-        self['subject'] = subject_template.format(
-            **subject_template_values_dict)
-        # Open template files and insert values from body_template_values_dict
-        with open(body_txt_template_path) as nfile:
-            self.body = nfile.read().format(**body_template_values_dict)
-        with open(body_html_template_path) as nfile:
-            self.html = nfile.read().format(**body_template_values_dict)
-        # Set txt of email
-        self.set_content(self.body)
-        # Set html alternative
-        self.add_alternative(self.html, subtype='html')
-
-    def set_addresses(self, sender, recipient):
-        self['From'] = sender
-        self['to'] = recipient

daemon/app/tasks/libnotify/service.py

@@ -1,16 +0,0 @@
-class NotificationService:
-    """This is a nopaque notifcation service object."""
-
-    def __init__(self, smtp):
-        # Bool to show if the mail server stoped sending mails due to exceeding
-        # its sending limit
-        self.mail_limit_exceeded = False
-        # Holds due to an error unsent email notifications
-        self.not_sent = {}
-        self.smtp = smtp
-
-    def send(self, email):
-        self.smtp.send_message(email)
-
-    def quit(self):
-        self.smtp.quit()

daemon/app/tasks/libnotify/templates/notification.html

@@ -1,15 +0,0 @@
-<html>
-  <body>
-    <p>Dear <b>{username}</b>,</p>
-
-    <p>The status of your Job/Corpus({id}) with the title <b>"{title}"</b> has changed!</p>
-    <p>It is now <b>{status}</b>!</p>
-    <p>Time of this status update was: <b>{time} UTC</b></p>
-
-    <p>You can access your Job/Corpus here: <a href="{url}">{url}</a>
-</p>
-
-    <p>Kind regards!<br>
-    Your nopaque team</p>
-  </body>
-</html>

daemon/app/tasks/libnotify/templates/notification.txt

@@ -1,10 +0,0 @@
-Dear {username},
-
-The status of your Job/Corpus({id}) with the title "{title}" has changed!
-It is now {status}!
-Time of this status update was: {time} UTC
-
-You can access your Job/Corpus here: {url}
-
-Kind regards!
-Your nopaque team

daemon/app/tasks/notify.py

@@ -1,111 +0,0 @@
-from sqlalchemy import asc
-from .libnotify.notification import Notification
-from .libnotify.service import NotificationService
-from .. import configuration as config
-from .. import Session
-from ..decorators import background
-from ..models import NotificationEmailData
-import logging
-import os
-import smtplib
-
-
-ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
-
-
-@background
-def notify():
-    session = Session()
-    if config.SMTP_USE_SSL:
-        smtp = smtplib.SMTP_SSL(host=config.SMTP_SERVER, port=config.SMTP_PORT)
-    else:
-        smtp = smtplib.SMTP(host=config.SMTP_SERVER, port=config.SMTP_PORT)
-    if config.SMTP_USE_TLS:
-        smtp.starttls()
-    try:
-        smtp.login(config.SMTP_USERNAME, config.SMTP_PASSWORD)
-    except smtplib.SMTPHeloError:
-        logging.warning('The server didn’t reply properly to the HELO '
-                        'greeting.')
-        return
-    except smtplib.SMTPAuthenticationError as e:
-        logging.warning('The server didn’t accept the username/password '
-                        'combination.')
-        logging.warning(e)
-        return
-    except smtplib.SMTPNotSupportedError:
-        logging.warning('The AUTH command is not supported by the server.')
-        return
-    except smtplib.SMTPException:
-        logging.warning('No suitable authentication method was found.')
-        return
-    notification_service = NotificationService(smtp)
-    # create notifications (content, recipient etc.)
-    notifications = __create_mail_notifications(notification_service, session)
-    # only login and send mails if there are any notifications
-    if (len(notifications) > 0):
-        # combine new and unsent notifications
-        notifications.update(notification_service.not_sent)
-        # send all notifications
-        __send_mail_notifications(notifications, notification_service)
-        # remove unsent notifications because they have been sent now
-        # but only if mail limit has not been exceeded
-        if (notification_service.mail_limit_exceeded is not True):
-            notification_service.not_sent = {}
-    smtp.quit()
-    Session.remove()
-
-
-# Email notification functions
-def __create_mail_notifications(notification_service, session):
-    notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all()  # noqa
-    notifications = {}
-    for data in notification_email_data:
-        notification = Notification()
-        notification.set_addresses(config.SMTP_DEFAULT_SENDER,
-                                   data.job.user.email)
-        subject_template = ('[nopaque] Status update for your Job/Corpora: '
-                            '{title}!')
-        subject_template_values_dict = {'title': data.job.title}
-        url = '{}://{}/{}/{}'.format(config.PROTOCOL,
-                                     config.DOMAIN,
-                                     'jobs',
-                                     data.job.id)
-        body_template_values_dict = {'username': data.job.user.username,
-                                     'id': data.job.id,
-                                     'title': data.job.title,
-                                     'status': data.notify_status,
-                                     'time': data.creation_date,
-                                     'url': url}
-        txt_tmplt = os.path.join(ROOT_DIR,
-                                 'libnotify/templates/notification.txt')
-        html_tmplt = os.path.join(ROOT_DIR,
-                                  'libnotify/templates/notification.html')
-        notification.set_notification_content(subject_template,
-                                              subject_template_values_dict,
-                                              txt_tmplt,
-                                              html_tmplt,
-                                              body_template_values_dict)
-        notifications[data.job.id] = notification
-        # Using a dictionary for notifications avoids sending multiple mails
-        # if the status of a job changes in a few seconds. The user will not
-        # get swamped with mails for queued, running and complete if those
-        # happen in in a few seconds. Only the last update will be sent.
-        # This depends on the sleep time interval though.
-        session.delete(data)
-        session.commit()
-    return notifications
-
-
-def __send_mail_notifications(notifications, notification_service):
-    for key, notification in notifications.items():
-        try:
-            notification_service.send(notification)
-            notification_service.mail_limit_exceeded = False
-        except Exception:
-            # Adds notifications to unsent if mail server exceded limit for
-            # consecutive mail sending
-            logging.warning('limit')
-            notification_service.not_sent[key] = notification
-            notification_service.mail_limit_exceeded = True
-            notification_service.not_sent.update(notifications)

daemon/boot.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-source venv/bin/activate
-python nopaqued.py

daemon/config.py

@@ -1,71 +0,0 @@
-import logging
-import os
-
-
-ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
-
-
-class Config:
-    ''' # Email # '''
-    SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
-    SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
-    SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
-    SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
-    SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
-    SMTP_USE_SSL = os.environ.get(
-        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
-    SMTP_USE_TLS = os.environ.get(
-        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
-
-    ''' # General # '''
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
-    DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
-    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
-    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
-
-    ''' # Logging # '''
-    LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     '%Y-%m-%d %H:%M:%S')
-    LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
-                              os.path.join(ROOT_DIR, 'nopaqued.log'))
-    LOG_FORMAT = os.environ.get(
-        'NOPAQUE_LOG_FORMAT',
-        '[%(asctime)s] %(levelname)s in '
-        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
-    )
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
-
-    @classmethod
-    def init(cls):
-        # Set up logging according to the corresponding (LOG_*) variables
-        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
-                            filename=cls.LOG_FILE,
-                            format=cls.LOG_FORMAT,
-                            level=cls.LOG_LEVEL)
-
-
-class DevelopmentConfig(Config):
-    ''' # Database # '''
-    SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_DEV_DATABASE_URL',
-        'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
-    )
-
-
-class ProductionConfig(Config):
-    ''' # Database # '''
-    SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_DATABASE_URL',
-        'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
-    )
-
-
-class TestingConfig(Config):
-    ''' # Database # '''
-    SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
-
-
-config = {'development': DevelopmentConfig,
-          'production': ProductionConfig,
-          'testing': TestingConfig}

daemon/nopaqued.py

@@ -1,13 +0,0 @@
-from dotenv import load_dotenv
-from app import run
-import os
-
-
-# Load environment variables
-DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
-if os.path.exists(DOTENV_FILE):
-    load_dotenv(DOTENV_FILE)
-
-
-if __name__ == '__main__':
-    run()

daemon/requirements.txt

@@ -1,4 +0,0 @@
-docker
-psycopg2
-python-dotenv
-SQLAlchemy

docker-compose.development.yml

@@ -13,11 +13,3 @@ services:
       - "./web/nopaque.py:/home/nopaque/nopaque.py"
       - "./web/requirements.txt:/home/nopaque/requirements.txt"
       - "./web/tests:/home/nopaque/tests"
-  nopaqued:
-    volumes:
-      # Mount code as volumes
-      - "./daemon/app:/home/nopaqued/app"
-      - "./daemon/boot.sh:/home/nopaqued/boot.sh"
-      - "./daemon/config.py:/home/nopaqued/config.py"
-      - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
-      - "./daemon/requirements.txt:/home/nopaqued/requirements.txt"

docker-compose.scale.yml

@@ -0,0 +1,6 @@
+version: "3.5"
+
+services:
+  nopaque:
+    environment:
+      - NOPAQUE_DAEMON_ENABLED=False

docker-compose.traefik.yml

@@ -18,13 +18,13 @@ services:
       - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
       - "traefik.http.routers.nopaque.entrypoints=web"
       - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
-      - "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)"
+      - "traefik.http.routers.nopaque.rule=Host(`${SERVER_NAME}`)"
       ### </http> ###
       ### <https> ###
       - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
       - "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
       - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
-      - "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)"
+      - "traefik.http.routers.nopaque-secure.rule=Host(`${SERVER_NAME}`)"
       - "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
       - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
       ### </https> ###

docker-compose.yml

@@ -1,9 +1,23 @@
 version: "3.5"
 
 services:
+  db:
+    env_file: db.env
+    image: postgres:11
+    restart: unless-stopped
+    volumes:
+      - "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
+
+  mq:
+    image: redis:6
+    restart: unless-stopped
+    volumes:
+      - "${HOST_MQ_DIR:-./mq}:/data"
+
   nopaque:
     build:
       args:
+        DOCKER_GID: ${HOST_DOCKER_GID}
         GID: ${HOST_GID}
         UID: ${HOST_UID}
       context: ./web
@@ -13,34 +27,7 @@ services:
     env_file: .env
     image: nopaque:latest
     restart: unless-stopped
-    volumes:
-      - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
-      - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
-  nopaqued:
-    build:
-      args:
-        DOCKER_GID: ${HOST_DOCKER_GID}
-        GID: ${HOST_GID}
-        UID: ${HOST_UID}
-      context: ./daemon
-    depends_on:
-      - db
-      - nopaque
-    env_file: .env
-    image: nopaqued:latest
-    restart: unless-stopped
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
-      - "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}"
+      - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
-  db:
-    env_file: db.env
-    image: postgres:11
-    restart: unless-stopped
-    volumes:
-      - "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
-  mq:
-    image: redis:6
-    restart: unless-stopped
-    volumes:
-      - "${HOST_MQ_DIR:-./mq}:/data"

web/.flaskenv

@@ -0,0 +1 @@
+FLASK_APP=nopaque.py

web/Dockerfile

@@ -1,12 +1,12 @@
-FROM python:3.6.12-slim-buster
+FROM python:3.9.0-slim-buster
 
 
 LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
 
 
+ARG DOCKER_GID
 ARG UID
 ARG GID
-ENV FLASK_APP=nopaque.py
 ENV LANG=C.UTF-8
 
 
@@ -17,12 +17,12 @@ RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
       libpq-dev \
-      wait-for-it \
  && rm -r /var/lib/apt/lists/*
 
 
-RUN groupadd --gid ${GID} --system nopaque \
+RUN groupadd --gid ${DOCKER_GID} --system docker \
- && useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
+ && groupadd --gid ${GID} --system nopaque \
+ && useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
 USER nopaque
 WORKDIR /home/nopaque
 

web/app/__init__.py

@@ -26,7 +26,7 @@ def create_app(config_name):
     mail.init_app(app)
     paranoid.init_app(app)
     socketio.init_app(
-        app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
+        app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
 
     with app.app_context():
         from . import events
@@ -38,6 +38,7 @@ def create_app(config_name):
         from .main import main as main_blueprint
         from .services import services as services_blueprint
         from .settings import settings as settings_blueprint
+
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
     app.register_blueprint(auth_blueprint, url_prefix='/auth')
     app.register_blueprint(corpora_blueprint, url_prefix='/corpora')

web/app/admin/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 admin = Blueprint('admin', __name__)
-from . import views  # noqa
+from . import views

web/app/admin/forms.py

@@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
         super().__init__(*args, user=user, **kwargs)
         self.role.choices = [(role.id, role.name)
                              for role in Role.query.order_by(Role.name).all()]
-        self.user = user

web/app/admin/views.py

@@ -1,5 +1,5 @@
 from flask import flash, redirect, render_template, url_for
-from flask_login import current_user, login_required
+from flask_login import login_required
 from . import admin
 from .forms import EditGeneralSettingsAdminForm
 from .. import db
@@ -8,17 +8,19 @@ from ..models import Role, User
 from ..settings import tasks as settings_tasks
 
 
+@admin.route('/')
+@login_required
+@admin_required
+def index():
+    return redirect(url_for('.users'))
+
+
 @admin.route('/users')
 @login_required
 @admin_required
 def users():
-    users = User.query.all()
+    # users = [user.to_dict() for user in User.query.all()]
-    users = [dict(username=u.username,
+    users = {user.id: user.to_dict() for user in User.query.all()}
-                  email=u.email,
-                  role_id=u.role_id,
-                  confirmed=u.confirmed,
-                  id=u.id)
-             for u in users]
     return render_template('admin/users.html.j2', title='Users', users=users)
 
 
@@ -35,15 +37,14 @@ def user(user_id):
 @admin_required
 def delete_user(user_id):
     settings_tasks.delete_user(user_id)
-    flash('User has been deleted!')
+    flash('User has been marked for deletion!')
     return redirect(url_for('.users'))
 
 
-@admin.route('/users/<int:user_id>/edit_general_settings',
+@admin.route('/users/<int:user_id>/edit', methods=['GET', 'POST'])  # noqa
-             methods=['GET', 'POST'])
 @login_required
 @admin_required
-def edit_general_settings(user_id):
+def edit_user(user_id):
     user = User.query.get_or_404(user_id)
     form = EditGeneralSettingsAdminForm(user=user)
     if form.validate_on_submit():
@@ -52,16 +53,13 @@ def edit_general_settings(user_id):
         user.username = form.username.data
         user.confirmed = form.confirmed.data
         user.role = Role.query.get(form.role.data)
-        db.session.add(user)
         db.session.commit()
-        flash('The profile has been updated.')
+        flash('Settings have been updated.')
-        return redirect(url_for('admin.edit_general_settings', user_id=user.id))
+        return redirect(url_for('.edit_user', user_id=user.id))
     form.confirmed.data = user.confirmed
     form.dark_mode.data = user.setting_dark_mode
     form.email.data = user.email
     form.role.data = user.role_id
     form.username.data = user.username
-    return render_template('admin/edit_general_settings.html.j2',
+    return render_template('admin/edit_user.html.j2', form=form,
-                           form=form,
+                           title='Edit user', user=user)
-                           title='General settings',
-                           user=user)

web/app/auth/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 auth = Blueprint('auth', __name__)
-from . import views  # noqa
+from . import views

web/app/auth/forms.py

@@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
     username = StringField(
         'Username',
         validators=[DataRequired(), Length(1, 64),
-                    Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+                    Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
                            message='Usernames must have only letters, numbers,'
                                    ' dots or underscores')]
     )

web/app/auth/views.py

@@ -1,5 +1,5 @@
-from flask import (current_app, flash, redirect, render_template, request,
+from datetime import datetime
-                   url_for)
+from flask import abort, flash, redirect, render_template, request, url_for
 from flask_login import current_user, login_user, login_required, logout_user
 from . import auth
 from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
 from .. import db
 from ..email import create_message, send
 from ..models import User
+import logging
 import os
-import shutil
 
 
 @auth.before_app_request
@@ -18,11 +18,12 @@ def before_request():
     unconfirmed view if user is unconfirmed.
     """
     if current_user.is_authenticated:
-        current_user.ping()
+        current_user.last_seen = datetime.utcnow()
-        if not current_user.confirmed \
+        db.session.commit()
-                and request.endpoint \
+        if (not current_user.confirmed
-                and request.blueprint != 'auth' \
+                and request.endpoint
-                and request.endpoint != 'static':
+                and request.blueprint != 'auth'
+                and request.endpoint != 'static'):
             return redirect(url_for('auth.unconfirmed'))
 
 
@@ -30,20 +31,19 @@ def before_request():
 def login():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    login_form = LoginForm(prefix='login-form')
+    form = LoginForm(prefix='login-form')
-    if login_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User.query.filter_by(username=login_form.user.data).first()
+        user = User.query.filter_by(username=form.user.data).first()
         if user is None:
-            user = User.query.filter_by(email=login_form.user.data).first()
+            user = User.query.filter_by(email=form.user.data.lower()).first()
-        if user is not None and user.verify_password(login_form.password.data):
+        if user is not None and user.verify_password(form.password.data):
-            login_user(user, login_form.remember_me.data)
+            login_user(user, form.remember_me.data)
             next = request.args.get('next')
             if next is None or not next.startswith('/'):
                 next = url_for('main.dashboard')
             return redirect(next)
         flash('Invalid email/username or password.')
-    return render_template('auth/login.html.j2', login_form=login_form,
+    return render_template('auth/login.html.j2', form=form, title='Log in')
-                           title='Log in')
 
 
 @auth.route('/logout')
@@ -58,26 +58,28 @@ def logout():
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    registration_form = RegistrationForm(prefix='registration-form')
+    form = RegistrationForm(prefix='registration-form')
-    if registration_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User(email=registration_form.email.data.lower(),
+        user = User(email=form.email.data.lower(),
-                    password=registration_form.password.data,
+                    password=form.password.data,
-                    username=registration_form.username.data)
+                    username=form.username.data)
         db.session.add(user)
         db.session.commit()
-        user_dir = os.path.join(current_app.config['DATA_DIR'],
+        try:
-                                str(user.id))
+            os.makedirs(user.path)
-        if os.path.exists(user_dir):
+        except OSError:
-            shutil.rmtree(user_dir)
+            logging.error('Make dir {} led to an OSError!'.format(user.path))
-        os.mkdir(user_dir)
+            db.session.delete(user)
+            db.session.commit()
+            abort(500)
+        else:
             token = user.generate_confirmation_token()
             msg = create_message(user.email, 'Confirm Your Account',
                                  'auth/email/confirm', token=token, user=user)
             send(msg)
             flash('A confirmation email has been sent to you by email.')
-        return redirect(url_for('auth.login'))
+            return redirect(url_for('.login'))
-    return render_template('auth/register.html.j2',
+    return render_template('auth/register.html.j2', form=form,
-                           registration_form=registration_form,
                            title='Register')
 
 
@@ -92,7 +94,7 @@ def confirm(token):
         return redirect(url_for('main.dashboard'))
     else:
         flash('The confirmation link is invalid or has expired.')
-        return redirect(url_for('auth.unconfirmed'))
+        return redirect(url_for('.unconfirmed'))
 
 
 @auth.route('/unconfirmed')
@@ -119,23 +121,18 @@ def resend_confirmation():
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    reset_password_request_form = ResetPasswordRequestForm(
+    form = ResetPasswordRequestForm(prefix='reset-password-request-form')
-        prefix='reset-password-request-form')
+    if form.validate_on_submit():
-    if reset_password_request_form.validate_on_submit():
+        user = User.query.filter_by(email=form.email.data.lower()).first()
-        submitted_email = reset_password_request_form.email.data
+        if user is not None:
-        user = User.query.filter_by(email=submitted_email.lower()).first()
-        if user:
             token = user.generate_reset_token()
             msg = create_message(user.email, 'Reset Your Password',
                                  'auth/email/reset_password', token=token,
                                  user=user)
             send(msg)
-        flash('An email with instructions to reset your password has been '
+        flash('An email with instructions to reset your password has been sent to you.')  # noqa
-              'sent to you.')
+        return redirect(url_for('.login'))
-        return redirect(url_for('auth.login'))
+    return render_template('auth/reset_password_request.html.j2', form=form,
-    return render_template(
-        'auth/reset_password_request.html.j2',
-        reset_password_request_form=reset_password_request_form,
                            title='Password Reset')
 
 
@@ -143,15 +140,13 @@ def reset_password_request():
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    reset_password_form = ResetPasswordForm(prefix='reset-password-form')
+    form = ResetPasswordForm(prefix='reset-password-form')
-    if reset_password_form.validate_on_submit():
+    if form.validate_on_submit():
-        if User.reset_password(token, reset_password_form.password.data):
+        if User.reset_password(token, form.password.data):
             db.session.commit()
             flash('Your password has been updated.')
-            return redirect(url_for('auth.login'))
+            return redirect(url_for('.login'))
         else:
             return redirect(url_for('main.index'))
-    return render_template('auth/reset_password.html.j2',
+    return render_template('auth/reset_password.html.j2', form=form,
-                           reset_password_form=reset_password_form,
+                           title='Password Reset', token=token)
-                           title='Password Reset',
-                           token=token)

web/app/corpora/events.py

@@ -24,27 +24,29 @@ corpus_analysis_sessions = {}
 corpus_analysis_clients = {}
 
 
-@socketio.on('corpus_create_zip')
+@socketio.on('export_corpus')
 @socketio_login_required
-def corpus_create_zip(corpus_id):
+def export_corpus(corpus_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
+    # TODO: This should not be get_or_404 here - Socket.IO != HTTP request
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        response = {'code': 404, 'msg': 'Not found'}
+        socketio.emit('export_corpus', response, room=request.sid)
+        return
+    if corpus.status not in ['prepared', 'start analysis', 'stop analysis']:
+        response = {'code': 412, 'msg': 'Precondition Failed'}
+        socketio.emit('export_corpus', response, room=request.sid)
+        return
     # delete old corpus archive if it exists/has been build before
-    if corpus.archive_file is not None:
+    if corpus.archive_file is not None and os.path.isfile(corpus.archive_file):
-        if (os.path.isfile(corpus.archive_file)):
         os.remove(corpus.archive_file)
-    root_dir = os.path.join(current_app.config['DATA_DIR'],
-                            str(current_user.id),
-                            'corpora')
-    base_dir = os.path.join(root_dir, str(corpus.id))
     zip_name = corpus.title
-    zip_path = os.path.join(root_dir, zip_name)
+    zip_path = os.path.join(current_user.path, 'corpora', zip_name)
-    corpus.archive_file = os.path.join(base_dir, zip_name) + '.zip'
+    corpus.archive_file = os.path.join(corpus.path, zip_name) + '.zip'
     db.session.commit()
-    shutil.make_archive(zip_path,
+    shutil.make_archive(zip_path, 'zip', corpus.path)
-                        'zip',
-                        base_dir)
     shutil.move(zip_path + '.zip', corpus.archive_file)
-    socketio.emit('corpus_zip_created', room=request.sid)
+    socketio.emit('export_corpus_' + str(corpus.id), room=request.sid)
 
 
 @socketio.on('corpus_analysis_init')

web/app/corpora/views.py

@@ -1,4 +1,4 @@
-from flask import (abort, current_app, flash, make_response, redirect, request,
+from flask import (abort, flash, make_response, redirect, request,
                    render_template, url_for, send_from_directory)
 from flask_login import current_user, login_required
 from . import corpora
@@ -11,6 +11,7 @@ from jsonschema import validate
 from .. import db
 from ..models import Corpus, CorpusFile, QueryResult
 import json
+import logging
 import os
 import shutil
 import glob
@@ -22,64 +23,58 @@ from .import_corpus import check_zip_contents
 @corpora.route('/add', methods=['GET', 'POST'])
 @login_required
 def add_corpus():
-    add_corpus_form = AddCorpusForm()
+    form = AddCorpusForm()
-    if add_corpus_form.validate_on_submit():
+    if form.validate_on_submit():
         corpus = Corpus(creator=current_user,
-                        description=add_corpus_form.description.data,
+                        description=form.description.data,
-                        status='unprepared', title=add_corpus_form.title.data)
+                        title=form.title.data)
         db.session.add(corpus)
         db.session.commit()
-        dir = os.path.join(current_app.config['DATA_DIR'],
-                           str(corpus.user_id), 'corpora', str(corpus.id))
         try:
-            os.makedirs(dir)
+            os.makedirs(corpus.path)
         except OSError:
-            flash('[ERROR]: Could not add corpus!', 'corpus')
+            logging.error('Make dir {} led to an OSError!'.format(corpus.path))
-            corpus.delete()
+            db.session.delete(corpus)
-        else:
+            db.session.commit()
-            url = url_for('corpora.corpus', corpus_id=corpus.id)
+            abort(500)
-            flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
+        flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
-                  'corpus')
+        return redirect(url_for('.corpus', corpus_id=corpus.id))
-            return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
+    return render_template('corpora/add_corpus.html.j2', form=form,
-    return render_template('corpora/add_corpus.html.j2',
-                           add_corpus_form=add_corpus_form,
                            title='Add corpus')
 
 
 @corpora.route('/import', methods=['GET', 'POST'])
 @login_required
 def import_corpus():
-    import_corpus_form = ImportCorpusForm()
+    form = ImportCorpusForm()
-    if import_corpus_form.is_submitted():
+    if form.is_submitted():
-        if not import_corpus_form.validate():
+        if not form.validate():
-            return make_response(import_corpus_form.errors, 400)
+            return make_response(form.errors, 400)
         corpus = Corpus(creator=current_user,
-                        description=import_corpus_form.description.data,
+                        description=form.description.data,
-                        status='unprepared',
+                        title=form.title.data)
-                        title=import_corpus_form.title.data)
         db.session.add(corpus)
         db.session.commit()
-        dir = os.path.join(current_app.config['DATA_DIR'],
-                           str(corpus.user_id), 'corpora', str(corpus.id))
         try:
-            os.makedirs(dir)
+            os.makedirs(corpus.path)
         except OSError:
-            flash('[ERROR]: Could not import corpus!', 'corpus')
+            logging.error('Make dir {} led to an OSError!'.format(corpus.path))
-            corpus.delete()
+            db.session.delete(corpus)
-        else:
+            db.session.commit()
+            flash('Internal Server Error', 'error')
+            return make_response(
+                {'redirect_url': url_for('.import_corpus')}, 500)
         # Upload zip
-            archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
+        archive_file = os.path.join(corpus.path, form.file.data.filename)
-                                        import_corpus_form.file.data.filename)
+        form.file.data.save(archive_file)
-            corpus_dir = os.path.dirname(archive_file)
-            import_corpus_form.file.data.save(archive_file)
         # Some checks to verify it is a valid exported corpus
         with ZipFile(archive_file, 'r') as zip:
             contents = zip.namelist()
         if set(check_zip_contents).issubset(contents):
             # Unzip
-                shutil.unpack_archive(archive_file, corpus_dir)
+            shutil.unpack_archive(archive_file, corpus.path)
             # Register vrt files to corpus
-                vrts = glob.glob(corpus_dir + '/*.vrt')
+            vrts = glob.glob(corpus.path + '/*.vrt')
             for file in vrts:
                 element_tree = ET.parse(file)
                 text_node = element_tree.find('text')
@@ -89,7 +84,6 @@ def import_corpus():
                     booktitle=text_node.get('booktitle',  'NULL'),
                     chapter=text_node.get('chapter',  'NULL'),
                     corpus=corpus,
-                        dir=dir,
                     editor=text_node.get('editor',  'NULL'),
                     filename=os.path.basename(file),
                     institution=text_node.get('institution',  'NULL'),
@@ -98,30 +92,23 @@ def import_corpus():
                     publisher=text_node.get('publisher',  'NULL'),
                     publishing_year=text_node.get('publishing_year', ''),
                     school=text_node.get('school',  'NULL'),
-                        title=text_node.get('title', 'NULL'))
+                    title=text_node.get('title', 'NULL')
+                )
                 db.session.add(corpus_file)
-                # finish import and got to imported corpus
+            # finish import and redirect to imported corpus
-                url = url_for('corpora.corpus', corpus_id=corpus.id)
             corpus.status = 'prepared'
             db.session.commit()
             os.remove(archive_file)
-                flash('[<a href="{}">{}</a>] imported'.format(url,
+            flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
-                                                              corpus.title),
-                      'corpus')
             return make_response(
-                    {'redirect_url': url_for('corpora.corpus',
+                {'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
-                                             corpus_id=corpus.id)},
-                    201)
         else:
             # If imported zip is not valid delete corpus and give feedback
-                corpus.delete()
+            flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error')  # noqa
-                db.session.commit()
+            tasks.delete_corpus(corpus.id)
-                flash('Imported corpus is not valid.', 'error')
             return make_response(
-                    {'redirect_url': url_for('corpora.import_corpus')},
+                {'redirect_url': url_for('.import_corpus')}, 201)
-                    201)
+    return render_template('corpora/import_corpus.html.j2', form=form,
-    return render_template('corpora/import_corpus.html.j2',
-                           import_corpus_form=import_corpus_form,
                            title='Import Corpus')
 
 
@@ -131,31 +118,22 @@ def corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
-    corpus_files = [dict(filename=corpus_file.filename,
+    corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
-                         author=corpus_file.author,
+    return render_template('corpora/corpus.html.j2', corpus=corpus,
-                         title=corpus_file.title,
+                           corpus_files=corpus_files, title='Corpus')
-                         publishing_year=corpus_file.publishing_year,
-                         corpus_id=corpus.id,
-                         id=corpus_file.id)
-                    for corpus_file in corpus.files]
-    return render_template('corpora/corpus.html.j2',
-                           corpus=corpus,
-                           corpus_files=corpus_files,
-                           title='Corpus')
 
 
-@corpora.route('/<int:corpus_id>/export')
+@corpora.route('/<int:corpus_id>/download')
 @login_required
-def export_corpus(corpus_id):
+def download_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
+    # TODO: Check what happens here
     dir = os.path.dirname(corpus.archive_file)
     filename = os.path.basename(corpus.archive_file)
-    return send_from_directory(directory=dir,
+    return send_from_directory(as_attachment=True, directory=dir,
-                               filename=filename,
+                               filename=filename, mimetype='zip')
-                               mimetype='zip',
-                               as_attachment=True)
 
 
 @corpora.route('/<int:corpus_id>/analyse')
@@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
     display_options_form = DisplayOptionsForm(
         prefix='display-options-form',
         result_context=request.args.get('context', 20),
-        results_per_page=request.args.get('results_per_page', 30))
+        results_per_page=request.args.get('results_per_page', 30)
+    )
     query_form = QueryForm(prefix='query-form',
                            query=request.args.get('query'))
     query_download_form = QueryDownloadForm(prefix='query-download-form')
@@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
     return render_template(
         'corpora/analyse_corpus.html.j2',
         corpus=corpus,
-        corpus_id=corpus_id,
         display_options_form=display_options_form,
+        inspect_display_options_form=inspect_display_options_form,
         query_form=query_form,
         query_download_form=query_download_form,
-        inspect_display_options_form=inspect_display_options_form,
+        title='Corpus analysis'
-        title='Corpus analysis')
+    )
 
 
 @corpora.route('/<int:corpus_id>/delete')
@@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
+    flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
     tasks.delete_corpus(corpus_id)
-    flash('Corpus deleted!', 'corpus')
     return redirect(url_for('main.dashboard'))
 
 
@@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
-    add_corpus_file_form = AddCorpusFileForm(corpus,
+    form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
-                                             prefix='add-corpus-file-form')
+    if form.is_submitted():
-    if add_corpus_file_form.is_submitted():
+        if not form.validate():
-        if not add_corpus_file_form.validate():
+            return make_response(form.errors, 400)
-            return make_response(add_corpus_file_form.errors, 400)
         # Save the file
-        dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
+        form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
-        add_corpus_file_form.file.data.save(
+        corpus_file = CorpusFile(address=form.address.data,
-            os.path.join(current_app.config['DATA_DIR'], dir,
+                                 author=form.author.data,
-                         add_corpus_file_form.file.data.filename))
+                                 booktitle=form.booktitle.data,
-        corpus_file = CorpusFile(
+                                 chapter=form.chapter.data,
-            address=add_corpus_file_form.address.data,
-            author=add_corpus_file_form.author.data,
-            booktitle=add_corpus_file_form.booktitle.data,
-            chapter=add_corpus_file_form.chapter.data,
                                  corpus=corpus,
-            dir=dir,
+                                 editor=form.editor.data,
-            editor=add_corpus_file_form.editor.data,
+                                 filename=form.file.data.filename,
-            filename=add_corpus_file_form.file.data.filename,
+                                 institution=form.institution.data,
-            institution=add_corpus_file_form.institution.data,
+                                 journal=form.journal.data,
-            journal=add_corpus_file_form.journal.data,
+                                 pages=form.pages.data,
-            pages=add_corpus_file_form.pages.data,
+                                 publisher=form.publisher.data,
-            publisher=add_corpus_file_form.publisher.data,
+                                 publishing_year=form.publishing_year.data,
-            publishing_year=add_corpus_file_form.publishing_year.data,
+                                 school=form.school.data,
-            school=add_corpus_file_form.school.data,
+                                 title=form.title.data)
-            title=add_corpus_file_form.title.data)
         db.session.add(corpus_file)
         corpus.status = 'unprepared'
         db.session.commit()
-        flash('Corpus file added!', 'corpus')
+        flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
-        return make_response(
+        return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)  # noqa
-            {'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
+    return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
-            201)
+                           form=form, title='Add corpus file')
-    return render_template('corpora/add_corpus_file.html.j2',
-                           corpus=corpus,
-                           add_corpus_file_form=add_corpus_file_form,
-                           title='Add corpus file')
 
 
 @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
+    flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus')  # noqa
     tasks.delete_corpus_file(corpus_file_id)
-    flash('Corpus file deleted!', 'corpus')
+    return redirect(url_for('.corpus', corpus_id=corpus_id))
-    return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
 
 
 @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       corpus_file.dir)
+                               directory=os.path.dirname(corpus_file.path),
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=corpus_file.filename)
 
 
@@ -275,47 +243,44 @@ def download_corpus_file(corpus_id, corpus_file_id):
 @login_required
 def corpus_file(corpus_id, corpus_file_id):
     corpus = Corpus.query.get_or_404(corpus_id)
-    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
+    if not (corpus.creator == current_user or current_user.is_administrator()):
-    if not corpus_file.corpus_id == corpus_id:
-        abort(404)
-    if not (corpus_file.corpus.creator == current_user
-            or current_user.is_administrator()):
         abort(403)
-    edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
+    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-    if edit_corpus_file_form.validate_on_submit():
+    if corpus_file.corpus != corpus:
-        corpus_file.address = edit_corpus_file_form.address.data
+        abort(404)
-        corpus_file.author = edit_corpus_file_form.author.data
+    form = EditCorpusFileForm(prefix='edit-corpus-file-form')
-        corpus_file.booktitle = edit_corpus_file_form.booktitle.data
+    if form.validate_on_submit():
-        corpus_file.chapter = edit_corpus_file_form.chapter.data
+        corpus_file.address = form.address.data
-        corpus_file.editor = edit_corpus_file_form.editor.data
+        corpus_file.author = form.author.data
-        corpus_file.institution = edit_corpus_file_form.institution.data
+        corpus_file.booktitle = form.booktitle.data
-        corpus_file.journal = edit_corpus_file_form.journal.data
+        corpus_file.chapter = form.chapter.data
-        corpus_file.pages = edit_corpus_file_form.pages.data
+        corpus_file.editor = form.editor.data
-        corpus_file.publisher = edit_corpus_file_form.publisher.data
+        corpus_file.institution = form.institution.data
-        corpus_file.publishing_year = \
+        corpus_file.journal = form.journal.data
-            edit_corpus_file_form.publishing_year.data
+        corpus_file.pages = form.pages.data
-        corpus_file.school = edit_corpus_file_form.school.data
+        corpus_file.publisher = form.publisher.data
-        corpus_file.title = edit_corpus_file_form.title.data
+        corpus_file.publishing_year = form.publishing_year.data
+        corpus_file.school = form.school.data
+        corpus_file.title = form.title.data
         corpus.status = 'unprepared'
         db.session.commit()
-        flash('Corpus file edited!', 'corpus')
+        flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus')  # noqa
-        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+        return redirect(url_for('.corpus', corpus_id=corpus_id))
     # If no form is submitted or valid, fill out fields with current values
-    edit_corpus_file_form.address.data = corpus_file.address
+    form.address.data = corpus_file.address
-    edit_corpus_file_form.author.data = corpus_file.author
+    form.author.data = corpus_file.author
-    edit_corpus_file_form.booktitle.data = corpus_file.booktitle
+    form.booktitle.data = corpus_file.booktitle
-    edit_corpus_file_form.chapter.data = corpus_file.chapter
+    form.chapter.data = corpus_file.chapter
-    edit_corpus_file_form.editor.data = corpus_file.editor
+    form.editor.data = corpus_file.editor
-    edit_corpus_file_form.institution.data = corpus_file.institution
+    form.institution.data = corpus_file.institution
-    edit_corpus_file_form.journal.data = corpus_file.journal
+    form.journal.data = corpus_file.journal
-    edit_corpus_file_form.pages.data = corpus_file.pages
+    form.pages.data = corpus_file.pages
-    edit_corpus_file_form.publisher.data = corpus_file.publisher
+    form.publisher.data = corpus_file.publisher
-    edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
+    form.publishing_year.data = corpus_file.publishing_year
-    edit_corpus_file_form.school.data = corpus_file.school
+    form.school.data = corpus_file.school
-    edit_corpus_file_form.title.data = corpus_file.title
+    form.title.data = corpus_file.title
-    return render_template('corpora/corpus_file.html.j2',
+    return render_template('corpora/corpus_file.html.j2', corpus=corpus,
-                           corpus_file=corpus_file, corpus=corpus,
+                           corpus_file=corpus_file, form=form,
-                           edit_corpus_file_form=edit_corpus_file_form,
                            title='Edit corpus file')
 
 
@@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
         abort(403)
     if corpus.files.all():
         tasks.build_corpus(corpus_id)
-        flash('Building Corpus...', 'corpus')
+        flash('Corpus "{}" has been marked to get build!'.format(corpus.title), 'corpus')  # noqa
     else:
-        flash('Can not build corpus, please add corpus file(s).', 'corpus')
+        flash('Can not build corpus "{}": No corpus file(s)!'.format(corpus.title), 'error')  # noqa
-    return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    return redirect(url_for('.corpus', corpus_id=corpus_id))
 
 
 # Following are view functions to add, view etc. exported results.
@@ -340,35 +305,29 @@ def add_query_result():
     '''
     View to import a result as a json file.
     '''
-    add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
+    form = AddQueryResultForm(prefix='add-query-result-form')
-    if add_query_result_form.is_submitted():
+    if form.is_submitted():
-        if not add_query_result_form.validate():
+        if not form.validate():
-            return make_response(add_query_result_form.errors, 400)
+            return make_response(form.errors, 400)
-        query_result = QueryResult(
+        query_result = QueryResult(creator=current_user,
-            creator=current_user,
+                                   description=form.description.data,
-            description=add_query_result_form.description.data,
+                                   filename=form.file.data.filename,
-            filename=add_query_result_form.file.data.filename,
+                                   title=form.title.data)
-            title=add_query_result_form.title.data
-        )
         db.session.add(query_result)
         db.session.commit()
-        # create paths to save the uploaded json file
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                        str(current_user.id),
-                                        'query_results',
-                                        str(query_result.id))
         try:
-            os.makedirs(query_result_dir)
+            os.makedirs(query_result.path)
-        except Exception:
+        except OSError:
+            logging.error('Make dir {} led to an OSError!'.format(query_result.path))  # noqa
             db.session.delete(query_result)
             db.session.commit()
             flash('Internal Server Error', 'error')
-            redirect_url = url_for('corpora.add_query_result')
+            return make_response(
-            return make_response({'redirect_url': redirect_url}, 500)
+                {'redirect_url': url_for('.add_query_result')}, 500)
         # save the uploaded file
-        query_result_file_path = os.path.join(query_result_dir,
+        query_result_file_path = os.path.join(query_result.path,
                                               query_result.filename)
-        add_query_result_form.file.data.save(query_result_file_path)
+        form.file.data.save(query_result_file_path)
         # parse json from file
         with open(query_result_file_path, 'r') as file:
             query_result_file_content = json.load(file)
@@ -381,19 +340,16 @@ def add_query_result():
         except Exception:
             tasks.delete_query_result(query_result.id)
             flash('Uploaded file is invalid', 'result')
-            redirect_url = url_for('corpora.add_query_result')
+            return make_response(
-            return make_response({'redirect_url': redirect_url}, 201)
+                {'redirect_url': url_for('.add_query_result')}, 201)
         query_result_file_content.pop('matches')
         query_result_file_content.pop('cpos_lookup')
         query_result.query_metadata = query_result_file_content
         db.session.commit()
         flash('Query result added!', 'result')
-        redirect_url = url_for('corpora.query_result',
+        return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201)  # noqa
-                               query_result_id=query_result.id)
-        return make_response({'redirect_url': redirect_url}, 201)
     return render_template('corpora/query_results/add_query_result.html.j2',
-                           add_query_result_form=add_query_result_form,
+                           form=form, title='Add query result')
-                           title='Add query result')
 
 
 @corpora.route('/result/<int:query_result_id>')
@@ -404,8 +360,7 @@ def query_result(query_result_id):
             or current_user.is_administrator()):
         abort(403)
     return render_template('corpora/query_results/query_result.html.j2',
-                           query_result=query_result,
+                           query_result=query_result, title='Query result')
-                           title='Query result')
 
 
 @corpora.route('/result/<int:query_result_id>/inspect')
@@ -427,14 +382,7 @@ def inspect_query_result(query_result_id):
     inspect_display_options_form = InspectDisplayOptionsForm(
         prefix='inspect-display-options-form'
     )
-    query_result_file_path = os.path.join(
+    with open(query_result.path, 'r') as query_result_file:
-        current_app.config['DATA_DIR'],
-        str(current_user.id),
-        'query_results',
-        str(query_result.id),
-        query_result.filename
-    )
-    with open(query_result_file_path, 'r') as query_result_file:
         query_result_file_content = json.load(query_result_file)
     return render_template('corpora/query_results/inspect.html.j2',
                            query_result=query_result,
@@ -452,8 +400,8 @@ def delete_query_result(query_result_id):
     if not (query_result.creator == current_user
             or current_user.is_administrator()):
         abort(403)
+    flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result')  # noqa
     tasks.delete_query_result(query_result_id)
-    flash('Query result deleted!', 'result')
     return redirect(url_for('services.service', service="corpus_analysis"))
 
 
@@ -464,10 +412,6 @@ def download_query_result(query_result_id):
     if not (query_result.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                    str(current_user.id),
-                                    'query_results',
-                                    str(query_result.id))
     return send_from_directory(as_attachment=True,
-                               directory=query_result_dir,
+                               directory=os.path.dirname(query_result.path),
                                filename=query_result.filename)

web/app/decorators.py

@@ -38,7 +38,7 @@ def socketio_admin_required(f):
         if current_user.is_administrator:
             return f(*args, **kwargs)
         else:
-            response = {'code': 401, 'desc': 'Unauthorized'}
+            response = {'code': 401, 'msg': 'Unauthorized'}
             socketio.emit(request.event['message'], response, room=request.sid)
     return wrapped
 
@@ -49,6 +49,6 @@ def socketio_login_required(f):
         if current_user.is_authenticated:
             return f(*args, **kwargs)
         else:
-            response = {'code': 401, 'desc': 'Unauthorized'}
+            response = {'code': 401, 'msg': 'Unauthorized'}
             socketio.emit(request.event['message'], response, room=request.sid)
     return wrapped

web/app/email.py

@@ -1,11 +1,11 @@
-from flask import render_template
+from flask import current_app, render_template
 from flask_mail import Message
 from . import mail
 from .decorators import background
 
 
 def create_message(recipient, subject, template, **kwargs):
-    msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
+    msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient])  # noqa
     msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
     msg.html = render_template('{}.html.j2'.format(template), **kwargs)
     return msg

web/app/events.py

@@ -33,38 +33,24 @@ def disconnect():
     connected_sessions.remove(request.sid)
 
 
-@socketio.on('user_data_stream_init')
+@socketio.on('start_user_session')
 @socketio_login_required
-def user_data_stream_init():
+def start_user_session(user_id):
-    socketio.start_background_task(user_data_stream,
+    if not (current_user.id == user_id or current_user.is_administrator):
+        return
+    socketio.start_background_task(user_session,
                                    current_app._get_current_object(),
-                                   current_user.id, request.sid)
+                                   user_id, request.sid)
 
 
-@socketio.on('foreign_user_data_stream_init')
+def user_session(app, user_id, session_id):
-@socketio_login_required
-@socketio_admin_required
-def foreign_user_data_stream_init(user_id):
-    socketio.start_background_task(user_data_stream,
-                                   current_app._get_current_object(),
-                                   user_id, request.sid, foreign=True)
-
-
-def user_data_stream(app, user_id, session_id, foreign=False):
     '''
-    ' Sends initial corpus and job lists to the client. Afterwards it checks
+    ' Sends initial user data to the client. Afterwards it checks every 3s if
-    ' every 3 seconds if changes to the initial values appeared. If changes are
+    ' changes to the initial values appeared. If changes are detected, a
-    ' detected, a RFC 6902 compliant JSON patch gets send.
+    ' RFC 6902 compliant JSON patch gets send.
-    '
-    ' NOTE: The initial values are send as a init events.
-    '       The JSON patches are send as update events.
     '''
-    if foreign:
+    init_event = 'user_{}_init'.format(user_id)
-        init_event = 'foreign_user_data_stream_init'
+    patch_event = 'user_{}_patch'.format(user_id)
-        update_event = 'foreign_user_data_stream_update'
-    else:
-        init_event = 'user_data_stream_init'
-        update_event = 'user_data_stream_update'
     with app.app_context():
         # Gather current values from database.
         user = User.query.get(user_id)
@@ -80,7 +66,7 @@ def user_data_stream(app, user_id, session_id, foreign=False):
                                                        new_user_dict)
             # In case there are patches, send them to the client.
             if user_patch:
-                socketio.emit(update_event, user_patch.to_string(),
+                socketio.emit(patch_event, user_patch.to_string(),
                               room=session_id)
             # Set new values as references for the next iteration.
             user_dict = new_user_dict

web/app/jobs/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 jobs = Blueprint('jobs', __name__)
-from . import views  # noqa
+from . import views

web/app/jobs/views.py

@@ -1,4 +1,4 @@
-from flask import (abort, current_app, flash, redirect, render_template,
+from flask import (abort, flash, redirect, render_template,
                    send_from_directory, url_for)
 from flask_login import current_user, login_required
 from . import jobs
@@ -14,13 +14,8 @@ def job(job_id):
     job = Job.query.get_or_404(job_id)
     if not (job.creator == current_user or current_user.is_administrator()):
         abort(403)
-    job_inputs = [dict(filename=input.filename,
+    job_inputs = [job_input.to_dict() for job_input in job.inputs]
-                       id=input.id,
+    return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
-                       job_id=job.id)
-                  for input in job.inputs]
-    return render_template('jobs/job.html.j2',
-                           job=job,
-                           job_inputs=job_inputs,
                            title='Job')
 
 
@@ -31,22 +26,19 @@ def delete_job(job_id):
     if not (job.creator == current_user or current_user.is_administrator()):
         abort(403)
     tasks.delete_job(job_id)
-    flash('Job has been deleted!', 'job')
+    flash('Job has been marked for deletion!', 'job')
     return redirect(url_for('main.dashboard'))
 
 
 @jobs.route('/<int:job_id>/inputs/<int:job_input_id>/download')
 @login_required
 def download_job_input(job_id, job_input_id):
-    job_input = JobInput.query.get_or_404(job_input_id)
+    job_input = JobInput.query.filter(JobInput.job_id == job_id, JobInput.id == job_input_id).first_or_404()  # noqa
-    if not job_input.job_id == job_id:
-        abort(404)
     if not (job_input.job.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       job_input.dir)
+                               directory=os.path.dirname(job_input.path),
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=job_input.filename)
 
 
@@ -56,23 +48,20 @@ def download_job_input(job_id, job_input_id):
 def restart(job_id):
     job = Job.query.get_or_404(job_id)
     if job.status != 'failed':
-        flash('Could not restart job: status is not "failed"', 'error')
+        flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error')  # noqa
     else:
         tasks.restart_job(job_id)
-        flash('Job has been restarted!', 'job')
+        flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job')  # noqa
-    return redirect(url_for('jobs.job', job_id=job_id))
+    return redirect(url_for('.job', job_id=job_id))
 
 
 @jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
 @login_required
 def download_job_result(job_id, job_result_id):
-    job_result = JobResult.query.get_or_404(job_result_id)
+    job_result = JobResult.query.filter(JobResult.job_id == job_id, JobResult.id == job_result_id).first_or_404()  # noqa
-    if not job_result.job_id == job_id:
-        abort(404)
     if not (job_result.job.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       job_result.dir)
+                               directory=os.path.dirname(job_result.path),
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=job_result.filename)

web/app/main/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 main = Blueprint('main', __name__)
-from . import views  # noqa
+from . import views

web/app/main/views.py

@@ -7,17 +7,16 @@ from ..models import User
 
 @main.route('/', methods=['GET', 'POST'])
 def index():
-    login_form = LoginForm(prefix='login-form')
+    form = LoginForm(prefix='login-form')
-    if login_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User.query.filter_by(username=login_form.user.data).first()
+        user = User.query.filter_by(username=form.user.data).first()
         if user is None:
-            user = User.query.filter_by(email=login_form.user.data).first()
+            user = User.query.filter_by(email=form.user.data.lower()).first()
-        if user is not None and user.verify_password(login_form.password.data):
+        if user is not None and user.verify_password(form.password.data):
-            login_user(user, login_form.remember_me.data)
+            login_user(user, form.remember_me.data)
-            return redirect(url_for('main.dashboard'))
+            return redirect(url_for('.dashboard'))
         flash('Invalid email/username or password.')
-    return render_template('main/index.html.j2', login_form=login_form,
+    return render_template('main/index.html.j2', form=form, title='nopaque')
-                           title='nopaque')
 
 
 @main.route('/about_and_faq')
@@ -31,7 +30,6 @@ def dashboard():
     return render_template('main/dashboard.html.j2', title='Dashboard')
 
 
-
 @main.route('/news')
 def news():
     return render_template('main/news.html.j2', title='News')
@@ -40,12 +38,9 @@ def news():
 @main.route('/privacy_policy')
 def privacy_policy():
     return render_template('main/privacy_policy.html.j2',
-                           title=('Information on the processing of personal'
+                           title='Privacy statement (GDPR)')
-                                  ' data for the nopaque platform (GDPR)'))
 
 
 @main.route('/terms_of_use')
 def terms_of_use():
-    return render_template('main/terms_of_use.html.j2',
+    return render_template('main/terms_of_use.html.j2', title='Terms of Use')
-                           title='General Terms of Use of the platform '
-                                 'nopaque')

web/app/models.py

@@ -1,12 +1,12 @@
 from datetime import datetime
-from flask import current_app
+from flask import current_app, url_for
 from flask_login import UserMixin, AnonymousUserMixin
 from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer
 from time import sleep
 from werkzeug.security import generate_password_hash, check_password_hash
-from werkzeug.utils import secure_filename
 import xml.etree.ElementTree as ET
 from . import db, login_manager
+import logging
 import os
 import shutil
 
@@ -35,7 +35,7 @@ class Role(db.Model):
     # Fields
     default = db.Column(db.Boolean, default=False, index=True)
     name = db.Column(db.String(64), unique=True)
-    permissions = db.Column(db.BigInteger)
+    permissions = db.Column(db.Integer)
     # Relationships
     users = db.relationship('User', backref='role', lazy='dynamic')
 
@@ -54,7 +54,7 @@ class Role(db.Model):
         '''
         String representation of the Role. For human readability.
         '''
-        return '<Role {role_name}>'.format(role_name=self.name)
+        return '<Role {}>'.format(self.name)
 
     def add_permission(self, perm):
         '''
@@ -138,6 +138,19 @@ class User(UserMixin, db.Model):
                                     cascade='save-update, merge, delete',
                                     lazy='dynamic')
 
+    @property
+    def path(self):
+        return os.path.join(current_app.config['NOPAQUE_DATA_DIR'],
+                            str(self.id))
+
+    @property
+    def password(self):
+        raise AttributeError('password is not a readable attribute')
+
+    @password.setter
+    def password(self, password):
+        self.password_hash = generate_password_hash(password)
+
     def to_dict(self):
         return {'id': self.id,
                 'role_id': self.role_id,
@@ -145,28 +158,29 @@ class User(UserMixin, db.Model):
                 'email': self.email,
                 'last_seen': self.last_seen.timestamp(),
                 'member_since': self.member_since.timestamp(),
-                'username': self.username,
                 'settings': {'dark_mode': self.setting_dark_mode,
                              'job_status_mail_notifications':
                                  self.setting_job_status_mail_notifications,
                              'job_status_site_notifications':
                                  self.setting_job_status_site_notifications},
+                'username': self.username,
                 'corpora': {corpus.id: corpus.to_dict()
                             for corpus in self.corpora},
                 'jobs': {job.id: job.to_dict() for job in self.jobs},
                 'query_results': {query_result.id: query_result.to_dict()
-                                  for query_result in self.query_results}}
+                                  for query_result in self.query_results},
+                'role': self.role.to_dict()}
 
     def __repr__(self):
         '''
         String representation of the User. For human readability.
         '''
-        return '<User {username}>'.format(username=self.username)
+        return '<User {}>'.format(self.username)
 
     def __init__(self, **kwargs):
         super(User, self).__init__(**kwargs)
         if self.role is None:
-            if self.email == current_app.config['ADMIN_EMAIL_ADRESS']:
+            if self.email == current_app.config['NOPAQUE_ADMIN']:
                 self.role = Role.query.filter_by(name='Administrator').first()
             if self.role is None:
                 self.role = Role.query.filter_by(default=True).first()
@@ -219,14 +233,6 @@ class User(UserMixin, db.Model):
         db.session.add(user)
         return True
 
-    @property
-    def password(self):
-        raise AttributeError('password is not a readable attribute')
-
-    @password.setter
-    def password(self, password):
-        self.password_hash = generate_password_hash(password)
-
     def verify_password(self, password):
         return check_password_hash(self.password_hash, password)
 
@@ -243,17 +249,11 @@ class User(UserMixin, db.Model):
         '''
         return self.can(Permission.ADMIN)
 
-    def ping(self):
-        self.last_seen = datetime.utcnow()
-        db.session.add(self)
-
     def delete(self):
         '''
         Delete the user and its corpora and jobs from database and filesystem.
         '''
-        user_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                str(self.id))
-        shutil.rmtree(user_dir, ignore_errors=True)
         db.session.delete(self)
 
 
@@ -279,17 +279,32 @@ class JobInput(db.Model):
     # Foreign keys
     job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
     # Fields
-    dir = db.Column(db.String(255))
     filename = db.Column(db.String(255))
 
+    @property
+    def download_url(self):
+        return url_for('jobs.download_job_input', job_id=self.job_id,
+                       job_input_id=self.id)
+
+    @property
+    def path(self):
+        return os.path.join(self.job.path, self.filename)
+
+    @property
+    def url(self):
+        return url_for('jobs.job', job_id=self.job_id,
+                       _anchor='job-{}-input-{}'.format(self.job_id, self.id))
+
     def __repr__(self):
         '''
         String representation of the JobInput. For human readability.
         '''
-        return '<JobInput {filename}>'.format(filename=self.filename)
+        return '<JobInput {}>'.format(self.filename)
 
     def to_dict(self):
-        return {'id': self.id,
+        return {'download_url': self.download_url,
+                'url': self.url,
+                'id': self.id,
                 'job_id': self.job_id,
                 'filename': self.filename}
 
@@ -304,17 +319,32 @@ class JobResult(db.Model):
     # Foreign keys
     job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
     # Fields
-    dir = db.Column(db.String(255))
     filename = db.Column(db.String(255))
 
+    @property
+    def download_url(self):
+        return url_for('jobs.download_job_result', job_id=self.job_id,
+                       job_result_id=self.id)
+
+    @property
+    def path(self):
+        return os.path.join(self.job.path, 'output', self.filename)
+
+    @property
+    def url(self):
+        return url_for('jobs.job', job_id=self.job_id,
+                       _anchor='job-{}-result-{}'.format(self.job_id, self.id))
+
     def __repr__(self):
         '''
         String representation of the JobResult. For human readability.
         '''
-        return '<JobResult {filename}>'.format(filename=self.filename)
+        return '<JobResult {}>'.format(self.filename)
 
     def to_dict(self):
-        return {'id': self.id,
+        return {'download_url': self.download_url,
+                'url': self.url,
+                'id': self.id,
                 'job_id': self.job_id,
                 'filename': self.filename}
 
@@ -334,7 +364,6 @@ class Job(db.Model):
     end_date = db.Column(db.DateTime())
     mem_mb = db.Column(db.Integer)
     n_cores = db.Column(db.Integer)
-    secure_filename = db.Column(db.String(32))
     service = db.Column(db.String(64))
     '''
     ' Service specific arguments as string list.
@@ -349,25 +378,20 @@ class Job(db.Model):
                              cascade='save-update, merge, delete')
     results = db.relationship('JobResult', backref='job', lazy='dynamic',
                               cascade='save-update, merge, delete')
-    notification_data = db.relationship('NotificationData',
+
-                                        cascade='save-update, merge, delete',
+    @property
-                                        uselist=False,
+    def path(self):
-                                        back_populates='job')  # One-to-One relationship
+        return os.path.join(self.creator.path, 'jobs', str(self.id))
-    notification_email_data = db.relationship('NotificationEmailData',
+
-                                              cascade='save-update, merge, delete',
+    @property
-                                              back_populates='job')
+    def url(self):
+        return url_for('jobs.job', job_id=self.id)
 
     def __repr__(self):
         '''
         String representation of the Job. For human readability.
         '''
-        return '<Job {job_title}>'.format(job_title=self.title)
+        return '<Job {}>'.format(self.title)
-
-    def create_secure_filename(self):
-        '''
-        Takes the job.title string nad cratesa a secure filename from this.
-        '''
-        self.secure_filename = secure_filename(self.title)
 
     def delete(self):
         '''
@@ -383,11 +407,7 @@ class Job(db.Model):
                     db.session.commit()
                 sleep(1)
                 db.session.refresh(self)
-        job_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                               str(self.user_id),
-                               'jobs',
-                               str(self.id))
-        shutil.rmtree(job_dir, ignore_errors=True)
         db.session.delete(self)
 
     def restart(self):
@@ -397,89 +417,27 @@ class Job(db.Model):
 
         if self.status != 'failed':
             raise Exception('Could not restart job: status is not "failed"')
-        job_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
-                               str(self.user_id),
+        shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True)  # noqa
-                               'jobs',
-                               str(self.id))
-        shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
-        shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
         self.end_date = None
         self.status = 'submitted'
 
     def to_dict(self):
-        return {'id': self.id,
+        return {'url': self.url,
+                'id': self.id,
                 'user_id': self.user_id,
                 'creation_date': self.creation_date.timestamp(),
                 'description': self.description,
                 'end_date': (self.end_date.timestamp() if self.end_date else
                              None),
-                'inputs': {input.id: input.to_dict() for input in self.inputs},
-                'mem_mb': self.mem_mb,
-                'n_cores': self.n_cores,
-                'results': {result.id: result.to_dict()
-                            for result in self.results},
                 'service': self.service,
                 'service_args': self.service_args,
                 'service_version': self.service_version,
                 'status': self.status,
-                'title': self.title}
+                'title': self.title,
-
+                'inputs': {input.id: input.to_dict() for input in self.inputs},
-
+                'results': {result.id: result.to_dict()
-class NotificationData(db.Model):
+                            for result in self.results}}
-    '''
-    Class to define notification data used for sending a notification mail with
-    nopaque_notify.
-    '''
-    __tablename__ = 'notification_data'
-    # Primary key
-    id = db.Column(db.Integer, primary_key=True)
-    # Foreign Key
-    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
-    # relationships
-    job = db.relationship('Job', back_populates='notification_data')
-    # Fields
-    notified_on = db.Column(db.String(16), default=None)
-
-    def __repr__(self):
-        '''
-        String representation of the NotificationData. For human readability.
-        '''
-        return '<NotificationData {id}>'.format(id=self.id)
-
-    def to_dict(self):
-        return {'id': self.id,
-                'job_id': self.job_id,
-                'job': self.job,
-                'notified': self.notified}
-
-
-class NotificationEmailData(db.Model):
-    '''
-    Class to define data that will be used to send a corresponding Notification
-    via email.
-    '''
-    __tablename__ = 'notification_email_data'
-    # Primary Key
-    id = db.Column(db.Integer, primary_key=True)
-    # Foreign Key
-    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
-    # relationships
-    job = db.relationship('Job', back_populates='notification_email_data')
-    notify_status = db.Column(db.String(16), default=None)
-    creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
-
-    def __repr__(self):
-        '''
-        String representation of the NotificationEmailData. For human readability.
-        '''
-        return '<NotificationData {id}>'.format(id=self.id)
-
-    def to_dict(self):
-        return {'id': self.id,
-                'job_id': self.job_id,
-                'job': self.job,
-                'notify_status': self.notify_status,
-                'creation_date': self.creation_date}
 
 
 class CorpusFile(db.Model):
@@ -496,7 +454,6 @@ class CorpusFile(db.Model):
     author = db.Column(db.String(255))
     booktitle = db.Column(db.String(255))
     chapter = db.Column(db.String(255))
-    dir = db.Column(db.String(255))
     editor = db.Column(db.String(255))
     filename = db.Column(db.String(255))
     institution = db.Column(db.String(255))
@@ -507,21 +464,33 @@ class CorpusFile(db.Model):
     school = db.Column(db.String(255))
     title = db.Column(db.String(255))
 
+    @property
+    def download_url(self):
+        return url_for('corpora.download_corpus_file',
+                       corpus_id=self.corpus_id, corpus_file_id=self.id)
+
+    @property
+    def path(self):
+        return os.path.join(self.corpus.path, self.filename)
+
+    @property
+    def url(self):
+        return url_for('corpora.corpus_file', corpus_id=self.corpus_id,
+                       corpus_file_id=self.id)
+
     def delete(self):
-        corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
-                                        str(self.corpus.user_id),
-                                        'corpora',
-                                        str(self.corpus_id),
-                                        self.filename)
         try:
-            os.remove(corpus_file_path)
+            os.remove(self.path)
         except OSError:
+            logging.error('Removing {} led to an OSError!'.format(self.path))
             pass
         db.session.delete(self)
         self.corpus.status = 'unprepared'
 
     def to_dict(self):
-        return {'id': self.id,
+        return {'download_url': self.download_url,
+                'url': self.url,
+                'id': self.id,
                 'corpus_id': self.corpus_id,
                 'address': self.address,
                 'author': self.author,
@@ -553,37 +522,48 @@ class Corpus(db.Model):
     description = db.Column(db.String(255))
     last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
     max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
-    status = db.Column(db.String(16))
+    status = db.Column(db.String(16), default='unprepared')
     title = db.Column(db.String(32))
     archive_file = db.Column(db.String(255))
     # Relationships
     files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
                             cascade='save-update, merge, delete')
 
+    @property
+    def analysis_url(self):
+        return url_for('corpora.analyse_corpus', corpus_id=self.id)
+
+    @property
+    def path(self):
+        return os.path.join(self.creator.path, 'corpora', str(self.id))
+
+    @property
+    def url(self):
+        return url_for('corpora.corpus', corpus_id=self.id)
+
     def to_dict(self):
-        return {'id': self.id,
+        return {'analysis_url': self.analysis_url,
+                'url': self.url,
+                'id': self.id,
                 'user_id': self.user_id,
                 'creation_date': self.creation_date.timestamp(),
+                'current_nr_of_tokens': self.current_nr_of_tokens,
                 'description': self.description,
                 'status': self.status,
                 'last_edited_date': self.last_edited_date.timestamp(),
+                'max_nr_of_tokens': self.max_nr_of_tokens,
                 'title': self.title,
                 'files': {file.id: file.to_dict() for file in self.files}}
 
     def build(self):
-        corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+        output_dir = os.path.join(self.path, 'merged')
-                                  str(self.user_id),
-                                  'corpora',
-                                  str(self.id))
-        output_dir = os.path.join(corpus_dir, 'merged')
         shutil.rmtree(output_dir, ignore_errors=True)
         os.mkdir(output_dir)
         master_element_tree = ET.ElementTree(
             ET.fromstring('<corpus>\n</corpus>')
         )
         for corpus_file in self.files:
-            corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
+            element_tree = ET.parse(corpus_file.path)
-            element_tree = ET.parse(corpus_file_path)
             text_node = element_tree.find('text')
             text_node.set('address', corpus_file.address or "NULL")
             text_node.set('author', corpus_file.author)
@@ -597,7 +577,7 @@ class Corpus(db.Model):
             text_node.set('publishing_year', str(corpus_file.publishing_year))
             text_node.set('school', corpus_file.school or "NULL")
             text_node.set('title', corpus_file.title)
-            element_tree.write(corpus_file_path)
+            element_tree.write(corpus_file.path)
             master_element_tree.getroot().insert(1, text_node)
         output_file = os.path.join(output_dir, 'corpus.vrt')
         master_element_tree.write(output_file,
@@ -607,18 +587,14 @@ class Corpus(db.Model):
         self.status = 'submitted'
 
     def delete(self):
-        corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                  str(self.user_id),
-                                  'corpora',
-                                  str(self.id))
-        shutil.rmtree(corpus_dir, ignore_errors=True)
         db.session.delete(self)
 
     def __repr__(self):
         '''
         String representation of the corpus. For human readability.
         '''
-        return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
+        return '<Corpus {}>'.format(self.title)
 
 
 class QueryResult(db.Model):
@@ -636,25 +612,39 @@ class QueryResult(db.Model):
     query_metadata = db.Column(db.JSON())
     title = db.Column(db.String(32))
 
+    @property
+    def download_url(self):
+        return url_for('corpora.download_query_result',
+                       query_result_id=self.id)
+
+    @property
+    def path(self):
+        return os.path.join(
+            self.creator.path, 'query_results', str(self.id), self.filename)
+
+    @property
+    def url(self):
+        return url_for('corpora.query_result', query_result_id=self.id)
+
     def delete(self):
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                        str(self.user_id),
-                                        'query_results',
-                                        str(self.id))
-        shutil.rmtree(query_result_dir, ignore_errors=True)
         db.session.delete(self)
 
     def to_dict(self):
-        return {'id': self.id,
+        return {'download_url': self.download_url,
+                'url': self.url,
+                'id': self.id,
                 'user_id': self.user_id,
+                'corpus_title': self.query_metadata['corpus_name'],
                 'description': self.description,
                 'filename': self.filename,
+                'query': self.query_metadata['query'],
                 'query_metadata': self.query_metadata,
                 'title': self.title}
 
     def __repr__(self):
         '''
-        String representation of the CorpusAnalysisResult. For human readability.
+        String representation of the QueryResult. For human readability.
         '''
         return '<QueryResult {}>'.format(self.title)
 

web/app/query_results/views.py

@@ -1,150 +0,0 @@
-from . import query_results
-from . import tasks
-from .. import db
-from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
-from ..models import QueryResult
-from .forms import AddQueryResultForm
-from flask import (abort, current_app, flash, make_response, redirect,
-                   render_template, request, send_from_directory, url_for)
-from flask_login import current_user, login_required
-import json
-import os
-from jsonschema import validate
-
-
-@query_results.route('/add', methods=['GET', 'POST'])
-@login_required
-def add_query_result():
-    '''
-    View to import a result as a json file.
-    '''
-    add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
-    if add_query_result_form.is_submitted():
-        if not add_query_result_form.validate():
-            return make_response(add_query_result_form.errors, 400)
-        query_result = QueryResult(
-            creator=current_user,
-            description=add_query_result_form.description.data,
-            filename=add_query_result_form.file.data.filename,
-            title=add_query_result_form.title.data
-        )
-        db.session.add(query_result)
-        db.session.commit()
-        # create paths to save the uploaded json file
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                        str(current_user.id),
-                                        'query_results',
-                                        str(query_result.id))
-        try:
-            os.makedirs(query_result_dir)
-        except Exception:
-            db.session.delete(query_result)
-            db.session.commit()
-            flash('Internal Server Error', 'error')
-            redirect_url = url_for('query_results.add_query_result')
-            return make_response({'redirect_url': redirect_url}, 500)
-        # save the uploaded file
-        query_result_file_path = os.path.join(query_result_dir,
-                                              query_result.filename)
-        add_query_result_form.file.data.save(query_result_file_path)
-        # parse json from file
-        with open(query_result_file_path, 'r') as file:
-            query_result_file_content = json.load(file)
-        # parse json schema
-        with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file:  # noqa
-            schema = json.load(file)
-        try:
-            # validate imported json file
-            validate(instance=query_result_file_content, schema=schema)
-        except Exception:
-            tasks.delete_query_result(query_result.id)
-            flash('Uploaded file is invalid', 'result')
-            redirect_url = url_for('query_results.add_query_result')
-            return make_response({'redirect_url': redirect_url}, 201)
-        query_result_file_content.pop('matches')
-        query_result_file_content.pop('cpos_lookup')
-        query_result.query_metadata = query_result_file_content
-        db.session.commit()
-        flash('Query result added!', 'result')
-        redirect_url = url_for('query_results.query_result',
-                               query_result_id=query_result.id)
-        return make_response({'redirect_url': redirect_url}, 201)
-    return render_template('corpora/query_results/add_query_result.html.j2',
-                           add_query_result_form=add_query_result_form,
-                           title='Add query result')
-
-
-@query_results.route('/<int:query_result_id>')
-@login_required
-def query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    return render_template('corpora/query_results/query_result.html.j2',
-                           query_result=query_result,
-                           title='Query result')
-
-
-@query_results.route('/<int:query_result_id>/inspect')
-@login_required
-def inspect_query_result(query_result_id):
-    '''
-    View to inspect imported result file in a corpus analysis like interface
-    '''
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    query_metadata = query_result.query_metadata
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    display_options_form = DisplayOptionsForm(
-        prefix='display-options-form',
-        results_per_page=request.args.get('results_per_page', 30),
-        result_context=request.args.get('context', 20)
-    )
-    inspect_display_options_form = InspectDisplayOptionsForm(
-        prefix='inspect-display-options-form'
-    )
-    query_result_file_path = os.path.join(
-        current_app.config['DATA_DIR'],
-        str(current_user.id),
-        'query_results',
-        str(query_result.id),
-        query_result.filename
-    )
-    with open(query_result_file_path, 'r') as query_result_file:
-        query_result_file_content = json.load(query_result_file)
-    return render_template('corpora/query_results/inspect.html.j2',
-                           display_options_form=display_options_form,
-                           inspect_display_options_form=inspect_display_options_form,
-                           query_result_file_content=query_result_file_content,
-                           query_metadata=query_metadata,
-                           title='Inspect query result')
-
-
-@query_results.route('/<int:query_result_id>/delete')
-@login_required
-def delete_query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    tasks.delete_query_result(query_result_id)
-    flash('Query result deleted!', 'result')
-    return redirect(url_for('services.service', service="corpus_analysis"))
-
-
-@query_results.route('/<int:query_result_id>/download')
-@login_required
-def download_query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                    str(current_user.id),
-                                    'query_results',
-                                    str(query_result.id))
-    return send_from_directory(as_attachment=True,
-                               directory=query_result_dir,
-                               filename=query_result.filename)

web/app/services/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 services = Blueprint('services', __name__)
-from . import views  # noqa
+from . import views

web/app/services/views.py

@@ -1,5 +1,4 @@
-from flask import (abort, current_app, flash, make_response, render_template,
+from flask import abort, flash, make_response, render_template, url_for
-                   url_for)
 from flask_login import current_user, login_required
 from werkzeug.utils import secure_filename
 from . import services
@@ -7,19 +6,20 @@ from .. import db
 from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
 from ..models import Job, JobInput
 import json
+import logging
 import os
 
 
 SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
             'file-setup': {'name': 'File setup',
                            'resources': {'mem_mb': 4096, 'n_cores': 4},
-                           'add_job_form': AddFileSetupJobForm},
+                           'form': AddFileSetupJobForm},
             'nlp': {'name': 'Natural Language Processing',
                     'resources': {'mem_mb': 4096, 'n_cores': 2},
-                    'add_job_form': AddNLPJobForm},
+                    'form': AddNLPJobForm},
             'ocr': {'name': 'Optical Character Recognition',
                     'resources': {'mem_mb': 8192, 'n_cores': 4},
-                    'add_job_form': AddOCRJobForm}}
+                    'form': AddOCRJobForm}}
 
 
 @services.route('/<service>', methods=['GET', 'POST'])
@@ -30,54 +30,47 @@ def service(service):
     if service == 'corpus_analysis':
         return render_template('services/{}.html.j2'.format(service),
                                title=SERVICES[service]['name'])
-    add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
+    form = SERVICES[service]['form'](prefix='add-job-form')
-    if add_job_form.is_submitted():
+    if form.is_submitted():
-        if not add_job_form.validate():
+        if not form.validate():
-            return make_response(add_job_form.errors, 400)
+            return make_response(form.errors, 400)
         service_args = []
         if service == 'nlp':
-            service_args.append('-l {}'.format(add_job_form.language.data))
+            service_args.append('-l {}'.format(form.language.data))
-            if add_job_form.check_encoding.data:
+            if form.check_encoding.data:
                 service_args.append('--check-encoding')
         if service == 'ocr':
-            service_args.append('-l {}'.format(add_job_form.language.data))
+            service_args.append('-l {}'.format(form.language.data))
-            if add_job_form.binarization.data:
+            if form.binarization.data:
                 service_args.append('--binarize')
         job = Job(creator=current_user,
-                  description=add_job_form.description.data,
+                  description=form.description.data,
                   mem_mb=SERVICES[service]['resources']['mem_mb'],
                   n_cores=SERVICES[service]['resources']['n_cores'],
                   service=service, service_args=json.dumps(service_args),
-                  service_version=add_job_form.version.data,
+                  service_version=form.version.data,
-                  status='preparing', title=add_job_form.title.data)
+                  status='preparing', title=form.title.data)
-        if job.service != 'corpus_analysis':
-            job.create_secure_filename()
         db.session.add(job)
         db.session.commit()
-        relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
-        absolut_dir = os.path.join(current_app.config['DATA_DIR'],
-                                   relative_dir)
         try:
-            os.makedirs(absolut_dir)
+            os.makedirs(job.path)
         except OSError:
-            job.delete()
+            logging.error('Make dir {} led to an OSError!'.format(job.path))
-            flash('Internal Server Error', 'job')
+            db.session.delete(job)
-            return make_response({'redirect_url': url_for('services.service',
+            db.session.commit()
-                                                          service=service)},
+            flash('Internal Server Error', 'error')
-                                 500)
+            return make_response(
+                {'redirect_url': url_for('.service', service=service)}, 500)
         else:
-            for file in add_job_form.files.data:
+            for file in form.files.data:
                 filename = secure_filename(file.filename)
-                file.save(os.path.join(absolut_dir, filename))
+                job_input = JobInput(filename=filename, job=job)
-                job_input = JobInput(dir=relative_dir, filename=filename,
+                file.save(job_input.path)
-                                     job=job)
                 db.session.add(job_input)
             job.status = 'submitted'
             db.session.commit()
-            url = url_for('jobs.job', job_id=job.id)
+            flash('Job "{}" added'.format(job.title), 'job')
-            flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
             return make_response(
                 {'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
     return render_template('services/{}.html.j2'.format(service),
-                           title=SERVICES[service]['name'],
+                           form=form, title=SERVICES[service]['name'])
-                           add_job_form=add_job_form)

web/app/settings/forms.py

@@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
         'Benutzername',
         validators=[DataRequired(),
                     Length(1, 64),
-                    Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+                    Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
                            message='Usernames must have only letters, numbers,'
                                    ' dots or underscores')]
     )

web/app/settings/views.py

@@ -1,13 +1,9 @@
-from flask import current_app, flash, redirect, render_template, url_for
+from flask import flash, redirect, render_template, url_for
 from flask_login import current_user, login_required, logout_user
 from . import settings, tasks
 from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
                     EditNotificationSettingsForm)
 from .. import db
-from ..decorators import admin_required
-from ..models import Role, User
-import os
-import uuid
 
 
 @settings.route('/')
@@ -26,8 +22,7 @@ def change_password():
         flash('Your password has been updated.')
         return redirect(url_for('.change_password'))
     return render_template('settings/change_password.html.j2',
-                           form=form,
+                           form=form, title='Change password')
-                           title='Change password')
 
 
 @settings.route('/edit_general_settings', methods=['GET', 'POST'])
@@ -40,12 +35,12 @@ def edit_general_settings():
         current_user.username = form.username.data
         db.session.commit()
         flash('Your changes have been saved.')
+        return redirect(url_for('.edit_general_settings'))
     form.dark_mode.data = current_user.setting_dark_mode
     form.email.data = current_user.email
     form.username.data = current_user.username
     return render_template('settings/edit_general_settings.html.j2',
-                           form=form,
+                           form=form, title='General settings')
-                           title='General settings')
 
 
 @settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@@ -59,13 +54,13 @@ def edit_notification_settings():
             form.job_status_site_notifications.data
         db.session.commit()
         flash('Your changes have been saved.')
+        return redirect(url_for('.edit_notification_settings'))
     form.job_status_mail_notifications.data = \
         current_user.setting_job_status_mail_notifications
     form.job_status_site_notifications.data = \
         current_user.setting_job_status_site_notifications
     return render_template('settings/edit_notification_settings.html.j2',
-                           form=form,
+                           form=form, title='Notification settings')
-                           title='Notification settings')
 
 
 @settings.route('/delete')
@@ -76,5 +71,5 @@ def delete():
     """
     tasks.delete_user(current_user.id)
     logout_user()
-    flash('Your account has been deleted!')
+    flash('Your account has been marked for deletion!')
     return redirect(url_for('main.index'))

web/app/static/css/nopaque.css

@@ -8,6 +8,10 @@ main {
   margin-top: 48px;
 }
 
+table.ressource-list tr {
+  cursor: pointer;
+}
+
 .parallax-container .parallax {
   z-index: auto;
 }

web/app/static/js/modules/corpus_analysis/client/Client.js

@@ -136,7 +136,7 @@ class Client {
       tmp_first_cpos.push(results.data.matches[dataIndex].c[0]);
       tmp_last_cpos.push(results.data.matches[dataIndex].c[1]);
     }
-    nopaque.socket.emit('corpus_analysis_get_match_with_full_context',
+    this.socket.emit('corpus_analysis_get_match_with_full_context',
                         {type: resultsType,
                          data_indexes: dataIndexes,
                          first_cpos: tmp_first_cpos,

web/app/static/js/nopaque.js

@@ -1,96 +1,138 @@
+class AppClient {
+  constructor(currentUserId) {
+    this.socket = io({transports: ['websocket']});
+    this.users = {};
+    this.users.self = this.loadUser(currentUserId);
+  }
+
+  loadUser(userId) {
+    let user = new User();
+    this.users[userId] = user;
+    this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
+    this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
+    this.socket.emit('start_user_session', userId);
+    return user;
+  }
+}
+
+
+class User {
+  constructor() {
+    this.data = undefined;
+    this.eventListeners = {
+      corporaInit: [],
+      corporaPatch: [],
+      jobsInit: [],
+      jobsPatch: [],
+      queryResultsInit: [],
+      queryResultsPatch: []
+    };
+  }
+
+  init(data) {
+    this.data = data;
+
+    let listener;
+    for (listener of this.eventListeners.corporaInit) {
+      listener(this.data.corpora);
+    }
+    for (listener of this.eventListeners.jobsInit) {
+      listener(this.data.jobs);
+    }
+    for (listener of this.eventListeners.queryResultsInit) {
+      listener(this.data.query_results);
+    }
+  }
+
+  patch(patch) {
+    this.data = jsonpatch.apply_patch(this.data, patch);
+
+    let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
+    let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
+    let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
+
+    for (let listener of this.eventListeners.corporaPatch) {
+      if (corporaPatch.length > 0) {listener(corporaPatch);}
+    }
+    for (let listener of this.eventListeners.jobsPatch) {
+      if (jobsPatch.length > 0) {listener(jobsPatch);}
+    }
+    for (let listener of this.eventListeners.queryResultsPatch) {
+      if (queryResultsPatch.length > 0) {listener(queryResultsPatch);}
+    }
+
+    for (let operation of jobsPatch) {
+      if (operation.op !== 'replace') {continue;}
+      // Matches the only path that should be handled here: /jobs/{jobId}/status
+      if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
+        let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
+        if (this.data.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
+        nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, "job");
+      }
+    }
+  }
+
+  addEventListener(type, listener) {
+    switch (type) {
+      case 'corporaInit':
+        this.eventListeners.corporaInit.push(listener);
+        if (this.data !== undefined) {listener(this.data.corpora);}
+        break;
+      case 'corporaPatch':
+        this.eventListeners.corporaPatch.push(listener);
+        break;
+      case 'jobsInit':
+        this.eventListeners.jobsInit.push(listener);
+        if (this.data !== undefined) {listener(this.data.jobs);}
+        break;
+      case 'jobsPatch':
+        this.eventListeners.jobsPatch.push(listener);
+        break;
+      case 'queryResultsInit':
+        this.eventListeners.queryResultsInit.push(listener);
+        if (this.data !== undefined) {listener(this.data.query_results);}
+        break;
+      case 'queryResultsPatch':
+        this.eventListeners.queryResultsPatch.push(listener);
+        break;
+      default:
+        console.error(`Unknown event type: ${type}`);
+    }
+  }
+}
+
+
 /*
  * The nopaque object is used as a namespace for nopaque specific functions and
  * variables.
  */
 var nopaque = {};
 
-// User data
+nopaque.flash = function(message, category) {
-nopaque.user = {};
+  let toast;
-nopaque.user.settings = {};
+  let toastActionElement;
-nopaque.user.settings.darkMode = undefined;
-nopaque.corporaSubscribers = [];
-nopaque.jobsSubscribers = [];
-nopaque.queryResultsSubscribers = [];
 
-// Foreign user (user inspected with admin credentials) data
+  switch (category) {
-nopaque.foreignUser = {};
+    case "corpus":
-nopaque.foreignUser.isAuthenticated = undefined;
+      message = `<i class="left material-icons">book</i>${message}`;
-nopaque.foreignUser.settings = {};
+      break;
-nopaque.foreignUser.settings.darkMode = undefined;
+    case "error":
-nopaque.foreignCorporaSubscribers = [];
+      message = `<i class="left material-icons red-text">error</i>${message}`;
-nopaque.foreignJobsSubscribers = [];
+      break;
-nopaque.foreignQueryResultsSubscribers = [];
+    case "job":
+      message = `<i class="left material-icons">work</i>${message}`;
+      break;
+    default:
+      message = `<i class="left material-icons">notifications</i>${message}`;
+  }
 
-// nopaque functions
+  toast = M.toast({html: `<span>${message}</span>
-nopaque.socket = io({transports: ['websocket']});
+                          <button data-action="close" class="btn-flat toast-action white-text">
-// Add event handlers
+                            <i class="material-icons">close</i>
-nopaque.socket.on("user_data_stream_init", function(msg) {
+                          </button>`});
-  nopaque.user = JSON.parse(msg);
+  toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
-  for (let subscriber of nopaque.corporaSubscribers) {
+  toastActionElement.addEventListener('click', () => {toast.dismiss();});
-    subscriber._init(nopaque.user.corpora);
+};
-  }
-  for (let subscriber of nopaque.jobsSubscribers) {
-    subscriber._init(nopaque.user.jobs);
-  }
-  for (let subscriber of nopaque.queryResultsSubscribers) {
-    subscriber._init(nopaque.user.query_results);
-  }
-});
-
-nopaque.socket.on("user_data_stream_update", function(msg) {
-  var patch;
-
-  patch = JSON.parse(msg);
-  nopaque.user = jsonpatch.apply_patch(nopaque.user, patch);
-  corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
-  jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
-  query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
-  for (let subscriber of nopaque.corporaSubscribers) {
-    subscriber._update(corpora_patch);
-  }
-  for (let subscriber of nopaque.jobsSubscribers) {
-    subscriber._update(jobs_patch);
-  }
-  for (let subscriber of nopaque.queryResultsSubscribers) {
-    subscriber._update(query_results_patch);
-  }
-  if (["all", "end"].includes(nopaque.user.settings.job_status_site_notifications)) {
-    for (operation of jobs_patch) {
-      /* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
-      pathArray = operation.path.split("/").slice(2);
-      if (operation.op === "replace" && pathArray[1] === "status") {
-        if (nopaque.user.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
-        nopaque.flash(`[<a href="/jobs/${pathArray[0]}">${nopaque.user.jobs[pathArray[0]].title}</a>] New status: ${operation.value}`, "job");
-      }
-    }
-  }
-});
-
-nopaque.socket.on("foreign_user_data_stream_init", function(msg) {
-  nopaque.foreignUser = JSON.parse(msg);
-  for (let subscriber of nopaque.foreignCorporaSubscribers) {
-    subscriber._init(nopaque.foreignUser.corpora);
-  }
-  for (let subscriber of nopaque.foreignJobsSubscribers) {
-    subscriber._init(nopaque.foreignUser.jobs);
-  }
-  for (let subscriber of nopaque.foreignQueryResultsSubscribers) {
-    subscriber._init(nopaque.foreignUser.query_results);
-  }
-});
-
-nopaque.socket.on("foreign_user_data_stream_update", function(msg) {
-  var patch;
-
-  patch = JSON.parse(msg);
-  nopaque.foreignUser = jsonpatch.apply_patch(nopaque.foreignUser, patch);
-  corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
-  jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
-  query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
-  for (let subscriber of nopaque.foreignCorporaSubscribers) {subscriber._update(corpora_patch);}
-  for (let subscriber of nopaque.foreignJobsSubscribers) {subscriber._update(jobs_patch);}
-  for (let subscriber of nopaque.foreignQueryResultsSubscribers) {subscriber._update(query_results_patch);}
-});
 
 nopaque.Forms = {};
 nopaque.Forms.init = function() {
@@ -163,30 +205,3 @@ nopaque.Forms.init = function() {
     }
   }
 }
-
-
-nopaque.flash = function(message, category) {
-  let toast;
-  let toastActionElement;
-
-  switch (category) {
-    case "corpus":
-      message = `<i class="left material-icons">book</i>${message}`;
-      break;
-    case "error":
-      message = `<i class="left material-icons red-text">error</i>${message}`;
-      break;
-    case "job":
-      message = `<i class="left material-icons">work</i>${message}`;
-      break;
-    default:
-      message = `<i class="left material-icons">notifications</i>${message}`;
-  }
-
-  toast = M.toast({html: `<span>${message}</span>
-                          <button data-action="close" class="btn-flat toast-action white-text">
-                            <i class="material-icons">close</i>
-                          </button>`});
-  toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
-  toastActionElement.addEventListener('click', () => {toast.dismiss();});
-}

web/app/static/js/nopaque.lists.js

@@ -1,432 +1,258 @@
-class RessourceList extends List {
+class RessourceList {
-  constructor(idOrElement, subscriberList, type, options) {
+  /* A wrapper class for the list.js list.
-    if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) {
+   * This class is not meant to be used directly, instead it should be used as
-      throw "Unknown Type!";
+   * a template for concrete ressource list implementations.
+   */
+  constructor(listElement, options = {}) {
+    if (listElement.dataset.userId) {
+      if (listElement.dataset.userId in nopaque.appClient.users) {
+        this.user = nopaque.appClient.users[listElement.dataset.userId];
+      } else {
+        console.error(`User not found: ${listElement.dataset.userId}`);
+        return;
+      }
+    } else {
+      this.user = nopaque.appClient.users.self;
+    }
+    this.list = new List(listElement, {...RessourceList.options, ...options});
+    this.valueNames = ['id'];
+    for (let element of this.list.valueNames) {
+      switch (typeof element) {
+        case 'object':
+          if (element.hasOwnProperty('name')) {this.valueNames.push(element.name);}
+          break;
+        case 'string':
+          this.valueNames.push(element);
+          break;
+        default:
+          console.error(`Unknown value name definition: ${element}`);
+      }
     }
-    super(idOrElement, {...RessourceList.options['common'],
-                        ...RessourceList.options[type],
-                        ...(options ? options : {})});
-    if (subscriberList) {subscriberList.push(this);}
-    this.type = type;
   }
 
-
+  init(ressources) {
-  _init(ressources) {
+    this.list.clear();
-    this.clear();
+    this.add(Object.values(ressources));
-    this._add(Object.values(ressources));
+    this.list.sort('id', {order: 'desc'});
-    this.sort("creation_date", {order: "desc"});
   }
 
+  patch(patch) {
+    /*
+     * It's not possible to generalize a patch Handler for all type of
+     * ressources. So this method is meant to be an interface.
+     */
+    console.error('patch method not implemented!');
+  }
 
-  _update(patch) {
+  add(values) {
-    let item, pathArray;
+    let ressources = Array.isArray(values) ? values : [values];
+    // Discard ressource values, that are not defined to be used in the list.
+    ressources = ressources.map(ressource => {
+      let cleanedRessource = {};
+      for (let [valueName, value] of Object.entries(ressource)) {
+        if (this.valueNames.includes(valueName)) {cleanedRessource[valueName] = value;}
+      }
+      return cleanedRessource;
+    });
+    // Set a callback function ('() => {return;}') to force List.js perform the
+    // add method asynchronous: https://listjs.com/api/#add
+    this.list.add(ressources, () => {return;});
+  }
 
+  remove(id) {
+    this.list.remove('id', id);
+  }
+
+  replace(id, valueName, newValue) {
+    if (this.valueNames.includes(valueName)) {
+      let item = this.list.get('id', id)[0];
+      item.values({[valueName]: newValue});
+    }
+  }
+}
+RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};
+
+
+class CorpusList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...CorpusList.options, ...options});
+    this.user.addEventListener('corporaInit', corpora => this.init(corpora));
+    this.user.addEventListener('corporaPatch', patch => this.patch(patch));
+    listElement.addEventListener('click', (event) => {this.onclick(event)});
+  }
+
+  onclick(event) {
+    let corpusId = event.target.closest('tr').dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'analyse':
+        window.location.href = nopaque.user.corpora[corpusId].analysis_url;
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm corpus deletion</h4>
+                                   <p>Do you really want to delete the corpus <b>${nopaque.user.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${nopaque.user.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'view':
+        // TODO: handle unprepared corpora
+        window.location.href = nopaque.user.corpora[corpusId].url;
+        break;
+      default:
+        console.error(`Unknown action: ${action}`);
+        break;
+    }
+  }
+
+  patch(patch) {
     for (let operation of patch) {
-      /* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
-      pathArray = operation.path.split("/").slice(2);
       switch(operation.op) {
-        case "add":
+        case 'add':
-          if (pathArray.includes("results")) {break;}
+          // Matches the only paths that should be handled here: /corpora/{corpusId}
-          this._add([operation.value]);
+          if (/^\/corpora\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
           break;
-        case "remove":
+        case 'remove':
-          this.remove("id", pathArray[0]);
+          // See case 'add' ;)
+          if (/^\/corpora\/(\d+)$/.test(operation.path)) {
+            let [match, id] = operation.path.match(/^\/corpora\/(\d+)$/);
+            this.remove(corpusId);
+          }
           break;
-        case "replace":
+        case 'replace':
-          item = this.get("id", pathArray[0])[0];
+          // Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
-          switch(pathArray[1]) {
+          if (/^\/corpora\/(\d+)\/(status|description|title)$/.test(operation.path)) {
-            case "status":
+            let [match, id, valueName] = operation.path.match(/^\/corpora\/(\d+)\/(status|description|title)$/);
-              item.values({status: operation.value,
+            this.replace(id, valueName, operation.value);
-                           "analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
+          }
           break;
         default:
           break;
       }
-        default:
-          break;
     }
   }
 }
-
+CorpusList.options = {
-  _add(values, callback) {
+  item: `<tr>
-    this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback);
+           <td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
-    // Initialize modal and tooltipped elements in list
+           <td><b class="title"></b><br><i class="description"></i></td>
-    M.AutoInit(this.listContainer);
+           <td><span class="badge new status" data-badge-caption=""></span></td>
-  }
+           <td class="right-align">
-}
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
-
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
-
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
-
+           </td>
-
+         </tr>`,
-RessourceList.dataMappers = {
+  valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
-  // A data mapper describes entitys rendered per row. One key value pair holds
-  // the data to be rendered in the list.js table. Key has to correspond
-  // with the ValueNames defined below in RessourceList.options ValueNames.
-  // Links are declared with double ticks(") around them. The key for links
-  // have to correspond with the class of an <a> element in the
-  // RessourceList.options item blueprint.
-
-  /* ### Corpus mapper ### */
-  Corpus: corpus => ({
-    creation_date: corpus.creation_date,
-    description: corpus.description,
-    id: corpus.id,
-    link: `/corpora/${corpus.id}`,
-    status: corpus.status,
-    title: corpus.title,
-    title1: corpus.title,
-    "analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
-    "delete-link": `/corpora/${corpus.id}/delete`,
-    "delete-modal": `delete-corpus-${corpus.id}-modal`,
-    "delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
-  }),
-  /* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
-  CorpusFile: corpus_file => ({
-    author: corpus_file.author,
-    filename: corpus_file.filename,
-    link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
-    title: corpus_file.title,
-    title1: corpus_file.title,
-    "delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
-    "delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
-    "delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
-    "download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
-  }),
-  /* ### Job mapper ### */
-  Job: job => ({
-    creation_date: job.creation_date,
-    description: job.description,
-    id: job.id,
-    link: `/jobs/${job.id}`,
-    service: job.service,
-    status: job.status,
-    title: job.title,
-    title1: job.title,
-    "delete-link": `/jobs/${job.id}/delete`,
-    "delete-modal": `delete-job-${job.id}-modal`,
-    "delete-modal-trigger": `delete-job-${job.id}-modal`,
-  }),
-  /* ### JobInput mapper ### */
-  JobInput: job_input => ({
-    filename: job_input.filename,
-    id: job_input.job_id,
-    "download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
-  }),
-  /* ### QueryResult mapper ### */
-  QueryResult: query_result => ({
-    corpus_name: query_result.query_metadata.corpus_name,
-    description: query_result.description,
-    id: query_result.id,
-    link: `/corpora/result/${query_result.id}`,
-    query: query_result.query_metadata.query,
-    title: query_result.title,
-    "delete-link": `/corpora/result/${query_result.id}/delete`,
-    "delete-modal": `delete-query-result-${query_result.id}-modal`,
-    "delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
-    "inspect-link": `/corpora/result/${query_result.id}/inspect`,
-  }),
-  /* ### User mapper ### */
-  User: user => ({
-    confirmed: user.confirmed,
-    email: user.email,
-    id: user.id,
-    link: `users/${user.id}`,
-    role_id: user.role_id,
-    username: user.username,
-    username2: user.username,
-    "delete-link": `/admin/users/${user.id}/delete`,
-    "delete-modal": `delete-user-${user.id}-modal`,
-    "delete-modal-trigger": `delete-user-${user.id}-modal`,
-  }),
 };
 
 
-RessourceList.options = {
+class JobList extends RessourceList {
-  // common list.js options for 5 rows per page etc.
+  constructor(listElement, options = {}) {
-  common: {
+    super(listElement, {...JobList.options, ...options});
-    page: 5,
+    this.user.addEventListener('jobsInit', jobs => this.init(jobs));
-    pagination: [
+    this.user.addEventListener('jobsPatch', patch => this.patch(patch));
-      {
+    listElement.addEventListener('click', (event) => {this.onclick(event)});
-        name: "paginationTop",
+  }
-        paginationClass: "paginationTop",
+
-        innerWindow: 4,
+  onclick(event) {
-        outerWindow: 1
+    let jobId = event.target.closest('tr').dataset.id;
-      },
+    let actionButtonElement = event.target.closest('.action-button');
-      {
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
-        paginationClass: "paginationBottom",
+    switch (action) {
-        innerWindow: 4,
+      case 'delete':
-        outerWindow: 1,
+        let deleteModalHTML = `<div class="modal">
-      },
-    ],
-  },
-  // extended list.js options for 10 rows per page etc.
-  extended: {
-    page: 10,
-    pagination: [
-      {
-        name: "paginationTop",
-        paginationClass: "paginationTop",
-        innerWindow: 8,
-        outerWindow: 1
-      },
-      {
-        paginationClass: "paginationBottom",
-        innerWindow: 8,
-        outerWindow: 1,
-      },
-    ],
-  },
-  /* Type specific List.js options. Usually only "item" and "valueNames" gets
-   * defined here but it is possible to define other List.js options.
-   * item: https://listjs.com/api/#item
-   * valueNames: https://listjs.com/api/#valueNames
-   */
-  Corpus: {
-    item: `<tr>
-             <td>
-               <a class="btn-floating disabled">
-                 <i class="material-icons service">book</i>
-               </a>
-             </td>
-             <td>
-               <b class="title"></b><br>
-               <i class="description"></i>
-             </td>
-             <td>
-               <span class="badge new status" data-badge-caption=""></span>
-             </td>
-             <td>
-               <div class="right-align">
-                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
-                   <i class="material-icons">delete</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
-                   <i class="material-icons">edit</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
-                   <i class="material-icons">search</i>
-                 </a>
-               </div>
-               <div class="modal delete-modal">
-                 <div class="modal-content">
-                   <h4>Confirm corpus deletion</h4>
-                   <p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
-                 </div>
-                 <div class="modal-footer">
-                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
-                 </div>
-               </div>
-             </td>
-           </tr>`,
-    valueNames: [
-      "creation_date",
-      "description",
-      "title",
-      "title1",
-      {data: ["id"]},
-      {name: "analyse-link", attr: "href"},
-      {name: "delete-link", attr: "href"},
-      {name: "delete-modal-trigger", attr: "data-target"},
-      {name: "delete-modal", attr: "id"},
-      {name: "link", attr: "href"},
-      {name: "status", attr: "data-status"},
-    ]
-  },
-  CorpusFile: {
-    item: `<tr>
-             <td class="filename" style="word-break: break-word;"></td>
-             <td class="author" style="word-break: break-word;"></td>
-             <td class="title" style="word-break: break-word;"></td>
-             <td>
-               <div class="right-align">
-                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
-                   <i class="material-icons">delete</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
-                   <i class="material-icons">file_download</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
-                   <i class="material-icons">edit</i>
-                 </a>
-               </div>
-               <div class="modal delete-modal">
-                 <div class="modal-content">
-                   <h4>Confirm corpus file deletion</h4>
-                   <p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
-                 </div>
-                 <div class="modal-footer">
-                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
-                 </div>
-               </div>
-             </td>
-           </tr>`,
-    valueNames: [
-      "author",
-      "filename",
-      "title",
-      "title1",
-      {name: "delete-link", attr: "href"},
-      {name: "delete-modal-trigger", attr: "data-target"},
-      {name: "delete-modal", attr: "id"},
-      {name: "download-link", attr: "href"},
-      {name: "link", attr: "href"},
-    ],
-  },
-  Job: {
-    item: `<tr>
-             <td>
-               <a class="btn-floating disabled">
-                 <i class="material-icons service"></i>
-               </a>
-             </td>
-             <td>
-               <b class="title"></b><br>
-               <i class="description"></i>
-             </td>
-             <td>
-               <span class="badge new status" data-badge-caption=""></span>
-             </td>
-             <td>
-               <div class="right-align">
-                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
-                   <i class="material-icons">delete</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
-                   <i class="material-icons">send</i>
-                 </a>
-               </div>
-               <div class="modal delete-modal">
                                  <div class="modal-content">
                                    <h4>Confirm job deletion</h4>
-                   <p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
+                                   <p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
                                  </div>
                                  <div class="modal-footer">
                                    <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
                                  </div>
-               </div>
+                               </div>`;
-             </td>
+        let deleteModalParentElement = document.querySelector('main');
-           </tr>`,
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
-    valueNames: [
+        let deleteModalElement = deleteModalParentElement.lastChild;
-      "creation_date",
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
-      "description",
+        deleteModal.open();
-      "title",
+        break;
-      "title1",
+      case 'view':
-      {data: ["id"]},
+        window.location.href = this.user.data.jobs[jobId].url;
-      {name: "delete-link", attr: "href"},
+        break;
-      {name: "delete-modal-trigger", attr: "data-target"},
+      default:
-      {name: "delete-modal", attr: "id"},
+        console.error(`Unknown action: "${action}"`);
-      {name: "link", attr: "href"},
+        break;
-      {name: "service", attr: "data-service"},
+    }
-      {name: "status", attr: "data-status"},
+  }
-    ],
+
-  },
+  patch(patch) {
-  JobInput: {
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /jobs/{jobId}
+          if (/^\/jobs\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
+          break;
+        case 'remove':
+          // See case add ;)
+          if (/^\/jobs\/(\d+)$/.test(operation.path)) {
+            let [match, id] = operation.path.match(/^\/jobs\/(\d+)$/);
+            this.remove(jobId);
+          }
+          break;
+        case 'replace':
+          // Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
+          if (/^\/jobs\/(\d+)\/(service|status|description|title)$/.test(operation.path)) {
+            let [match, id, valueName] = operation.path.match(/^\/jobs\/(\d+)\/(service|status|description|title)$/);
+            this.replace(id, valueName, operation.value);
+          }
+          break;
+        default:
+          break;
+      }
+    }
+  }
+}
+JobList.options = {
   item: `<tr>
-              <td class="filename"></td>
+           <td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
+           <td><b class="title"></b><br><i class="description"></i></td>
+           <td><span class="badge new status" data-badge-caption=""></span></td>
            <td class="right-align">
-                <a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
-                  <i class="material-icons">file_download</i>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
-                </a>
            </td>
          </tr>`,
-    valueNames: [
+  valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
-      "filename",
-      "id",
-      {name: "download-link", attr: "href"},
-    ],
-  },
-  QueryResult: {
-    item: `<tr>
-             <td>
-               <b class="title"></b><br>
-               <i class="description"></i><br>
-             </td>
-             <td>
-               <span class="corpus_name"></span><br>
-               <span class="query"></span>
-             </td>
-             <td>
-               <div class="right-align">
-                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
-                   <i class="material-icons">delete</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
-                   <i class="material-icons">info</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
-                   <i class="material-icons">search</i>
-                 </a>
-               </div>
-               <div class="modal delete-modal">
-                 <div class="modal-content">
-                   <h4>Confirm query result deletion</h4>
-                   <p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
-                 </div>
-                 <div class="modal-footer">
-                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
-                 </div>
-               </div>
-             </td>
-           </tr>`,
-    valueNames: [
-      "corpus_name",
-      "description",
-      "query",
-      "title",
-      "title2",
-      {data: ["id"]},
-      {name: "delete-link", attr: "href"},
-      {name: "delete-modal-trigger", attr: "data-target"},
-      {name: "delete-modal", attr: "id"},
-      {name: "inspect-link", attr: "href"},
-      {name: "link", attr: "href"},
-    ],
-  },
-  User: {
-    item: `<tr>
-             <td class="username"></td>
-             <td class="email"></td>
-             <td class="role_id"></td>
-             <td class="confirmed"></td>
-             <td class="id"></td>
-             <td>
-               <div class="right-align">
-                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
-                   <i class="material-icons">delete</i>
-                 </a>
-                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
-                   <i class="material-icons">send</i>
-                 </a>
-               </div>
-               <div class="modal delete-modal">
-                 <div class="modal-content">
-                   <h4>Confirm corpus deletion</h4>
-                   <p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
-                 </div>
-                 <div class="modal-footer">
-                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
-                 </div>
-               </div>
-             </td>
-           </tr>`,
-    valueNames: [
-      "username",
-      "username2",
-      "email",
-      "role_id",
-      "confirmed",
-      "id",
-      {name: "link", attr: "href"},
-      {name: "delete-link", attr: "href"},
-      {name: "delete-modal-trigger", attr: "data-target"},
-      {name: "delete-modal", attr: "id"},
-    ],
-  },
 };
 
-export { RessourceList, };
+
+class QueryResultList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...QueryResultList.options, ...options});
+    this.user.addEventListener('queryResultsInit', queryResults => this.init(queryResults));
+    this.user.addEventListener('queryResultsPatch', patch => this.init(patch));
+  }
+}
+QueryResultList.options = {
+  item: `<tr>
+           <td><b class="title"></b><br><i class="description"></i><br></td>
+           <td><span class="corpus_title"></span><br><span class="query"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
+};

web/app/static/js/nopaque.lists.js.bak

@@ -0,0 +1,420 @@
+class RessourceList extends List {
+  constructor(idOrElement, subscriberList, type, options) {
+    if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) {
+      throw "Unknown Type!";
+    }
+    super(idOrElement, {...RessourceList.options['common'],
+                        ...RessourceList.options[type],
+                        ...(options ? options : {})});
+    if (subscriberList) {subscriberList.push(this);}
+    this.type = type;
+  }
+
+
+  _init(ressources) {
+    this.clear();
+    this._add(Object.values(ressources));
+    this.sort("id", {order: "desc"});
+  }
+
+
+  _update(patch) {
+    let item, pathArray;
+
+    for (let operation of patch) {
+      /* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
+      pathArray = operation.path.split("/").slice(2);
+      switch(operation.op) {
+        case "add":
+          if (pathArray.includes("results")) {break;}
+          this._add([operation.value]);
+          break;
+        case "remove":
+          this.remove("id", pathArray[0]);
+          break;
+        case "replace":
+          item = this.get("id", pathArray[0])[0];
+          switch(pathArray[1]) {
+            case "status":
+              item.values({status: operation.value,
+                           "analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
+              break;
+            default:
+              break;
+          }
+        default:
+          break;
+      }
+    }
+  }
+
+  _add(values, callback) {
+    this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback);
+    // Initialize modal and tooltipped elements in list
+    M.AutoInit(this.listContainer);
+  }
+}
+
+
+
+
+RessourceList.dataMappers = {
+  // A data mapper describes entitys rendered per row. One key value pair holds
+  // the data to be rendered in the list.js table. Key has to correspond
+  // with the ValueNames defined below in RessourceList.options ValueNames.
+  // Links are declared with double ticks(") around them. The key for links
+  // have to correspond with the class of an <a> element in the
+  // RessourceList.options item blueprint.
+
+  /* ### Corpus mapper ### */
+  Corpus: corpus => ({
+    creation_date: corpus.creation_date,
+    description: corpus.description,
+    id: corpus.id,
+    link: `/corpora/${corpus.id}`,
+    status: corpus.status,
+    title: corpus.title,
+    title1: corpus.title,
+    "analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
+    "delete-link": `/corpora/${corpus.id}/delete`,
+    "delete-modal": `delete-corpus-${corpus.id}-modal`,
+    "delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
+  }),
+  /* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
+  CorpusFile: corpus_file => ({
+    author: corpus_file.author,
+    filename: corpus_file.filename,
+    id: corpus_file.id,
+    link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
+    "publishing-year": corpus_file.publishing_year,
+    title: corpus_file.title,
+    title1: corpus_file.title,
+    "delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
+    "delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
+    "delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
+    "download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
+  }),
+  /* ### Job mapper ### */
+  Job: job => ({
+    creation_date: job.creation_date,
+    description: job.description,
+    id: job.id,
+    link: `/jobs/${job.id}`,
+    service: job.service.name,
+    status: job.status,
+    title: job.title,
+    title1: job.title,
+    "delete-link": `/jobs/${job.id}/delete`,
+    "delete-modal": `delete-job-${job.id}-modal`,
+    "delete-modal-trigger": `delete-job-${job.id}-modal`,
+  }),
+  /* ### JobInput mapper ### */
+  JobInput: job_input => ({
+    filename: job_input.filename,
+    id: job_input.job_id,
+    "download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
+  }),
+  /* ### QueryResult mapper ### */
+  QueryResult: query_result => ({
+    corpus_name: query_result.query_metadata.corpus_name,
+    description: query_result.description,
+    id: query_result.id,
+    link: `/corpora/result/${query_result.id}`,
+    query: query_result.query_metadata.query,
+    title: query_result.title,
+    "delete-link": `/corpora/result/${query_result.id}/delete`,
+    "delete-modal": `delete-query-result-${query_result.id}-modal`,
+    "delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
+    "inspect-link": `/corpora/result/${query_result.id}/inspect`,
+  }),
+  /* ### User mapper ### */
+  User: user => ({
+    confirmed: user.confirmed,
+    email: user.email,
+    id: user.id,
+    link: `users/${user.id}`,
+    role: user.role.name,
+    username: user.username,
+    username2: user.username,
+    "delete-link": `/admin/users/${user.id}/delete`,
+    "delete-modal": `delete-user-${user.id}-modal`,
+    "delete-modal-trigger": `delete-user-${user.id}-modal`,
+  }),
+};
+
+
+RessourceList.options = {
+  // common list.js options for 5 rows per page etc.
+  common: {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]},
+  // extended list.js options for 10 rows per page etc.
+  extended: {
+    page: 10,
+    pagination: [
+      {
+        name: "paginationTop",
+        paginationClass: "paginationTop",
+        innerWindow: 8,
+        outerWindow: 1
+      },
+      {
+        paginationClass: "paginationBottom",
+        innerWindow: 8,
+        outerWindow: 1,
+      },
+    ],
+  },
+  /* Type specific List.js options. Usually only "item" and "valueNames" gets
+   * defined here but it is possible to define other List.js options.
+   * item: https://listjs.com/api/#item
+   * valueNames: https://listjs.com/api/#valueNames
+   */
+  Corpus: {
+    item: `<tr>
+             <td>
+               <a class="btn-floating disabled">
+                 <i class="material-icons service">book</i>
+               </a>
+             </td>
+             <td>
+               <b class="title"></b><br>
+               <i class="description"></i>
+             </td>
+             <td>
+               <span class="badge new status" data-badge-caption=""></span>
+             </td>
+             <td>
+               <div class="right-align">
+                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
+                   <i class="material-icons">delete</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
+                   <i class="material-icons">edit</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
+                   <i class="material-icons">search</i>
+                 </a>
+               </div>
+               <div class="modal delete-modal">
+                 <div class="modal-content">
+                   <h4>Confirm corpus deletion</h4>
+                   <p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
+                 </div>
+                 <div class="modal-footer">
+                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                 </div>
+               </div>
+             </td>
+           </tr>`,
+    valueNames: [
+      "creation_date",
+      "description",
+      "title",
+      "title1",
+      {data: ["id"]},
+      {name: "analyse-link", attr: "href"},
+      {name: "delete-link", attr: "href"},
+      {name: "delete-modal-trigger", attr: "data-target"},
+      {name: "delete-modal", attr: "id"},
+      {name: "link", attr: "href"},
+      {name: "status", attr: "data-status"},
+    ]
+  },
+  CorpusFile: {
+    item: `<tr>
+             <td class="filename" style="word-break: break-word;"></td>
+             <td class="author" style="word-break: break-word;"></td>
+             <td class="title" style="word-break: break-word;"></td>
+             <td class="publishing-year" style="word-break: break-word;"></td>
+             <td>
+               <div class="right-align">
+                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
+                   <i class="material-icons">delete</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
+                   <i class="material-icons">file_download</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
+                   <i class="material-icons">edit</i>
+                 </a>
+               </div>
+               <div class="modal delete-modal">
+                 <div class="modal-content">
+                   <h4>Confirm corpus file deletion</h4>
+                   <p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
+                 </div>
+                 <div class="modal-footer">
+                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                 </div>
+               </div>
+             </td>
+           </tr>`,
+    valueNames: [
+      "author",
+      "filename",
+      "publishing-year",
+      "title",
+      "title1",
+      {data: ["id"]},
+      {name: "delete-link", attr: "href"},
+      {name: "delete-modal-trigger", attr: "data-target"},
+      {name: "delete-modal", attr: "id"},
+      {name: "download-link", attr: "href"},
+      {name: "link", attr: "href"},
+    ],
+  },
+  Job: {
+    item: `<tr>
+             <td>
+               <a class="btn-floating disabled">
+                 <i class="material-icons service"></i>
+               </a>
+             </td>
+             <td>
+               <b class="title"></b><br>
+               <i class="description"></i>
+             </td>
+             <td>
+               <span class="badge new status" data-badge-caption=""></span>
+             </td>
+             <td>
+               <div class="right-align">
+                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
+                   <i class="material-icons">delete</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
+                   <i class="material-icons">send</i>
+                 </a>
+               </div>
+               <div class="modal delete-modal">
+                 <div class="modal-content">
+                   <h4>Confirm job deletion</h4>
+                   <p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
+                 </div>
+                 <div class="modal-footer">
+                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                 </div>
+               </div>
+             </td>
+           </tr>`,
+    valueNames: [
+      "creation_date",
+      "description",
+      "title",
+      "title1",
+      {data: ["id"]},
+      {name: "delete-link", attr: "href"},
+      {name: "delete-modal-trigger", attr: "data-target"},
+      {name: "delete-modal", attr: "id"},
+      {name: "link", attr: "href"},
+      {name: "service", attr: "data-service"},
+      {name: "status", attr: "data-status"},
+    ],
+  },
+  JobInput: {
+    item : `<tr>
+              <td class="filename"></td>
+              <td class="right-align">
+                <a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
+                  <i class="material-icons">file_download</i>
+                </a>
+              </td>
+            </tr>`,
+    valueNames: [
+      "filename",
+      "id",
+      {name: "download-link", attr: "href"},
+    ],
+  },
+  QueryResult: {
+    item: `<tr>
+             <td>
+               <b class="title"></b><br>
+               <i class="description"></i><br>
+             </td>
+             <td>
+               <span class="corpus_name"></span><br>
+               <span class="query"></span>
+             </td>
+             <td>
+               <div class="right-align">
+                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
+                   <i class="material-icons">delete</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
+                   <i class="material-icons">info</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
+                   <i class="material-icons">search</i>
+                 </a>
+               </div>
+               <div class="modal delete-modal">
+                 <div class="modal-content">
+                   <h4>Confirm query result deletion</h4>
+                   <p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
+                 </div>
+                 <div class="modal-footer">
+                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                 </div>
+               </div>
+             </td>
+           </tr>`,
+    valueNames: [
+      "corpus_name",
+      "description",
+      "query",
+      "title",
+      "title2",
+      {data: ["id"]},
+      {name: "delete-link", attr: "href"},
+      {name: "delete-modal-trigger", attr: "data-target"},
+      {name: "delete-modal", attr: "id"},
+      {name: "inspect-link", attr: "href"},
+      {name: "link", attr: "href"},
+    ],
+  },
+  User: {
+    item: `<tr>
+             <td class="id"></td>
+             <td class="username"></td>
+             <td class="email"></td>
+             <td class="role"></td>
+             <td>
+               <div class="right-align">
+                 <a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
+                   <i class="material-icons">delete</i>
+                 </a>
+                 <a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
+                   <i class="material-icons">send</i>
+                 </a>
+               </div>
+               <div class="modal delete-modal">
+                 <div class="modal-content">
+                   <h4>Confirm corpus deletion</h4>
+                   <p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
+                 </div>
+                 <div class="modal-footer">
+                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                   <a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
+                 </div>
+               </div>
+             </td>
+           </tr>`,
+    valueNames: [
+      "username",
+      "username2",
+      "email",
+      "role",
+      "id",
+      {name: "link", attr: "href"},
+      {name: "delete-link", attr: "href"},
+      {name: "delete-modal-trigger", attr: "data-target"},
+      {name: "delete-modal", attr: "id"},
+    ],
+  },
+};
+
+export { RessourceList, };

web/app/static/js/nopaque/displays/CorpusDisplay.js

@@ -0,0 +1,104 @@
+class CorpusDisplay extends RessourceDisplay {
+  constructor(displayElement) {
+    super(displayElement);
+    this.corpus = undefined;
+    this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.corpusId);
+  }
+
+  init(corpus) {
+    this.corpus = corpus;
+    for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.addEventListener('click', () => this.requestCorpusExport());}
+    nopaque.appClient.socket.on(`export_corpus_${this.corpus.id}`, () => this.downloadCorpus());
+    this.setCreationDate(this.corpus.creation_date);
+    this.setDescription(this.corpus.description);
+    this.setLastEditedDate(this.corpus.last_edited_date);
+    this.setStatus(this.corpus.status);
+    this.setTitle(this.corpus.title);
+    this.setTokenRatio(this.corpus.current_nr_of_tokens, this.corpus.max_nr_of_tokens);
+  }
+
+  patch(patch) {
+    let re;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'replace':
+          // Matches: /jobs/{this.job.id}/status
+          re = new RegExp('^/corpora/' + this.corpus.id + '/last_edited_date');
+          if (re.test(operation.path)) {this.setLastEditedDate(operation.value); break;}
+          // Matches: /jobs/{this.job.id}/status
+          re = new RegExp('^/corpora/' + this.corpus.id + '/status$');
+          if (re.test(operation.path)) {this.setStatus(operation.value); break;}
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  requestCorpusExport() {
+    nopaque.appClient.socket.emit('export_corpus', this.corpus.id);
+    nopaque.flash('Preparing your corpus export...', 'corpus');
+    for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', true);}
+  }
+
+  downloadCorpus() {
+    nopaque.flash('Corpus export is done. Your corpus download is ready!', 'corpus');
+    for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', false);}
+    // Little trick to call the download view after ziping has finished
+    let fakeBtn = document.createElement('a');
+    fakeBtn.href = `/corpora/${this.corpus.id}/download`;
+    fakeBtn.click();
+  }
+
+  setTitle(title) {
+    for (let element of this.displayElement.querySelectorAll('.corpus-title')) {this.setElement(element, title);}
+  }
+
+  setTokenRatio(currentNrOfTokens, maxNrOfTokens) {
+    let tokenRatio = `${currentNrOfTokens}/${maxNrOfTokens}`;
+    for (let element of this.displayElement.querySelectorAll('.corpus-token-ratio')) {this.setElement(element, tokenRatio);}
+  }
+
+  setDescription(description) {
+    for (let element of this.displayElement.querySelectorAll('.corpus-description')) {this.setElement(element, description);}
+  }
+
+  setStatus(status) {
+    for (let element of this.displayElement.querySelectorAll('.analyse-corpus-trigger')) {
+      if (['analysing', 'prepared', 'start analysis'].includes(status)) {
+        element.classList.remove('disabled');
+      } else {
+        element.classList.add('disabled');
+      }
+    }
+    for (let element of this.displayElement.querySelectorAll('.build-corpus-trigger')) {
+      if (status === 'unprepared' && Object.values(this.corpus.files).length > 0) {
+        element.classList.remove('disabled');
+      } else {
+        element.classList.add('disabled');
+      }
+    }
+    for (let element of this.displayElement.querySelectorAll('.corpus-status')) {this.setElement(element, status);}
+    for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {
+      exportCorpusTriggerElement.classList.toggle('disabled', !['prepared', 'start analysis', 'stop analysis'].includes(status));
+    }
+    for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
+    for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
+      if (['submitted', 'queued', 'running', 'canceling', 'start analysis', 'stop analysis'].includes(status)) {
+        element.classList.remove('hide');
+      } else {
+        element.classList.add('hide');
+      }
+    }
+  }
+
+  setCreationDate(creationDateTimestamp) {
+    let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
+    for (let element of this.displayElement.querySelectorAll('.corpus-creation-date')) {this.setElement(element, creationDate);}
+  }
+
+  setLastEditedDate(endDateTimestamp) {
+    let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
+    for (let element of this.displayElement.querySelectorAll('.corpus-end-date')) {this.setElement(element, endDate);}
+  }
+}

web/app/static/js/nopaque/displays/JobDisplay.js

@@ -0,0 +1,88 @@
+class JobDisplay extends RessourceDisplay {
+  constructor(displayElement) {
+    super(displayElement);
+    this.job = undefined;
+    this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.jobId);
+  }
+
+  init(job) {
+    this.job = job;
+    this.setCreationDate(this.job.creation_date);
+    this.setEndDate(this.job.creation_date);
+    this.setDescription(this.job.description);
+    this.setService(this.job.service);
+    this.setServiceArgs(this.job.service_args);
+    this.setServiceVersion(this.job.service_version);
+    this.setStatus(this.job.status);
+    this.setTitle(this.job.title);
+  }
+
+  patch(patch) {
+    let re;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'replace':
+          // Matches: /jobs/{this.job.id}/status
+          re = new RegExp('^/jobs/' + this.job.id + '/end_date');
+          if (re.test(operation.path)) {this.setEndDate(operation.value); break;}
+          // Matches: /jobs/{this.job.id}/status
+          re = new RegExp('^/jobs/' + this.job.id + '/status$');
+          if (re.test(operation.path)) {this.setStatus(operation.value); break;}
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  setTitle(title) {
+    for (let element of this.displayElement.querySelectorAll('.job-title')) {this.setElement(element, title);}
+  }
+
+  setDescription(description) {
+    for (let element of this.displayElement.querySelectorAll('.job-description')) {this.setElement(element, description);}
+  }
+
+  setStatus(status) {
+    for (let element of this.displayElement.querySelectorAll('.job-status')) {
+      this.setElement(element, status);
+    }
+    for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
+    for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
+      if (['complete', 'failed'].includes(status)) {
+        element.classList.add('hide');
+      } else {
+        element.classList.remove('hide');
+      }
+    }
+    for (let element of this.displayElement.querySelectorAll('.restart-job-trigger')) {
+      if (['complete', 'failed'].includes(status)) {
+        element.classList.remove('hide');
+      } else {
+        element.classList.add('hide');
+      }
+    }
+  }
+
+  setCreationDate(creationDateTimestamp) {
+    let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
+    for (let element of this.displayElement.querySelectorAll('.job-creation-date')) {this.setElement(element, creationDate);}
+  }
+
+  setEndDate(endDateTimestamp) {
+    let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
+    for (let element of this.displayElement.querySelectorAll('.job-end-date')) {this.setElement(element, endDate);}
+  }
+
+  setService(service) {
+    for (let element of this.displayElement.querySelectorAll('.job-service')) {this.setElement(element, service);}
+  }
+
+  setServiceArgs(serviceArgs) {
+    for (let element of this.displayElement.querySelectorAll('.job-service-args')) {this.setElement(element, serviceArgs);}
+  }
+
+  setServiceVersion(serviceVersion) {
+    for (let element of this.displayElement.querySelectorAll('.job-service-version')) {this.setElement(element, serviceVersion);}
+  }
+}

web/app/static/js/nopaque/displays/RessourceDisplay.js

@@ -0,0 +1,45 @@
+class RessourceDisplay {
+  constructor(displayElement) {
+    if (displayElement.dataset.userId) {
+      if (displayElement.dataset.userId in nopaque.appClient.users) {
+        this.user = nopaque.appClient.users[displayElement.dataset.userId];
+      } else {
+        console.error(`User not found: ${displayElement.dataset.userId}`);
+        return;
+      }
+    } else {
+      this.user = nopaque.appClient.users.self;
+    }
+    this.displayElement = displayElement;
+  }
+
+  eventHandler(eventType, payload) {
+    switch (eventType) {
+      case 'init':
+        this.init(payload);
+        break;
+      case 'patch':
+        this.patch(payload);
+        break;
+      default:
+        console.log(`Unknown event type: ${eventType}`);
+        break;
+    }
+  }
+
+  init() {console.error('init method not implemented!');}
+
+  patch() {console.error('patch method not implemented!');}
+
+  setElement(element, value) {
+    switch (element.tagName) {
+      case 'INPUT':
+        element.value = value;
+        M.updateTextFields();
+        break;
+      default:
+        element.innerText = value;
+        break;
+    }
+  }
+}

web/app/static/js/nopaque/lists/CorpusFileList.js

@@ -0,0 +1,98 @@
+class CorpusFileList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...CorpusFileList.options, ...options});
+    this.corpus = undefined;
+    this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.corpusId);
+  }
+
+  init(corpus) {
+    this.corpus = corpus;
+    super.init(this.corpus.files);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let corpusFileId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    if (actionButtonElement === null) {return;}
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm corpus deletion</h4>
+                                   <p>Do you really want to delete the corpus file <b>${this.corpus.files[corpusFileId].filename}</b>? It will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${this.corpus.files[corpusFileId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'download':
+        window.location.href = this.corpus.files[corpusFileId].download_url;
+        break;
+      case 'view':
+        if (corpusFileId !== '-1') {window.location.href = this.corpus.files[corpusFileId].url;}
+        break;
+      default:
+        console.error(`Unknown action: "${action}"`);
+        break;
+    }
+  }
+
+  patch(patch) {
+    let id, match, re, valueName;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /corpora/{this.corpus.id}/files/{corpusFileId}
+          re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        case 'remove':
+          // See case add ;)
+          re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
+          if (re.test(operation.path)) {
+            [match, id] = operation.path.match(re);
+            this.remove(id);
+          }
+          break;
+        case 'replace':
+          // Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
+          re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)/(author|filename|publishing_year|title)$');
+          if (re.test(operation.path)) {
+            [match, id, valueName] = operation.path.match(re);
+            this.replace(id, valueName, operation.value);
+          }
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  preprocessRessource(corpusFile) {
+    return {id: corpusFile.id, author: corpusFile.author, filename: corpusFile.filename, publishing_year: corpusFile.publishing_year, title: corpusFile.title};
+  }
+}
+CorpusFileList.options = {
+  item: `<tr>
+           <td><span class="filename"></span></td>
+           <td><span class="author"></span></td>
+           <td><span class="title"></span></td>
+           <td><span class="publishing_year"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'author', 'filename', 'publishing_year', 'title']
+};

web/app/static/js/nopaque/lists/CorpusList.js

@@ -0,0 +1,95 @@
+class CorpusList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...CorpusList.options, ...options});
+    this.corpora = undefined;
+    this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
+  }
+
+  init(corpora) {
+    this.corpora = corpora;
+    super.init(corpora);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let corpusId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm corpus deletion</h4>
+                                   <p>Do you really want to delete the corpus <b>${this.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${this.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'view':
+        if (corpusId !== '-1') {window.location.href = this.corpora[corpusId].url;}
+        break;
+      default:
+        console.error(`Unknown action: ${action}`);
+        break;
+    }
+  }
+
+  patch(patch) {
+    let id, match, re, valueName;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /corpora/{corpusId}
+          re = /^\/corpora\/(\d+)$/;
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        case 'remove':
+          // See case 'add' ;)
+          re = /^\/corpora\/(\d+)$/;
+          if (re.test(operation.path)) {
+            [match, id] = operation.path.match(re);
+            this.remove(id);
+          }
+          break;
+        case 'replace':
+          // Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
+          re = /^\/corpora\/(\d+)\/(status|description|title)$/;
+          if (re.test(operation.path)) {
+            [match, id, valueName] = operation.path.match(re);
+            this.replace(id, valueName, operation.value);
+          }
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  preprocessRessource(corpus) {
+    return {id: corpus.id,
+            status: corpus.status,
+            description: corpus.description,
+            title: corpus.title};
+  }
+}
+CorpusList.options = {
+  item: `<tr>
+           <td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
+           <td><b class="title"></b><br><i class="description"></i></td>
+           <td><span class="badge new status" data-badge-caption=""></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
+};

web/app/static/js/nopaque/lists/JobInputList.js

@@ -0,0 +1,42 @@
+class JobInputList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...JobInputList.options, ...options});
+    this.job = undefined;
+    this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
+  }
+
+  init(job) {
+    this.job = job;
+    super.init(this.job.inputs);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let jobInputId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    if (actionButtonElement === null) {return;}
+    let action = actionButtonElement.dataset.action;
+    switch (action) {
+      case 'download':
+        window.location.href = this.job.inputs[jobInputId].download_url;
+        break;
+      default:
+        console.error(`Unknown action: "${action}"`);
+        break;
+    }
+  }
+
+  preprocessRessource(jobInput) {
+    return {id: jobInput.id, filename: jobInput.filename};
+  }
+}
+JobInputList.options = {
+  item: `<tr>
+           <td><span class="filename"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'filename']
+};

web/app/static/js/nopaque/lists/JobList.js

@@ -0,0 +1,96 @@
+class JobList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...JobList.options, ...options});
+    this.jobs = undefined;
+    this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
+  }
+
+  init(jobs) {
+    this.jobs = jobs;
+    super.init(jobs);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let jobId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm job deletion</h4>
+                                   <p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'view':
+        if (jobId !== '-1') {window.location.href = this.user.data.jobs[jobId].url;}
+        break;
+      default:
+        console.error(`Unknown action: "${action}"`);
+        break;
+    }
+  }
+
+  patch(patch) {
+    let id, match, re, valueName;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /jobs/{jobId}
+          re = /^\/jobs\/(\d+)$/;
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        case 'remove':
+          // See case add ;)
+          re = /^\/jobs\/(\d+)$/;
+          if (re.test(operation.path)) {
+            [match, id] = operation.path.match(re);
+            this.remove(id);
+          }
+          break;
+        case 'replace':
+          // Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
+          re = /^\/jobs\/(\d+)\/(service|status|description|title)$/;
+          if (re.test(operation.path)) {
+            [match, id, valueName] = operation.path.match(re);
+            this.replace(id, valueName, operation.value);
+          }
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  preprocessRessource(job) {
+    return {id: job.id,
+            service: job.service,
+            status: job.status,
+            description: job.description,
+            title: job.title};
+  }
+}
+JobList.options = {
+  item: `<tr>
+           <td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
+           <td><b class="title"></b><br><i class="description"></i></td>
+           <td><span class="badge new status" data-badge-caption=""></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
+};

web/app/static/js/nopaque/lists/JobResultList.js

@@ -0,0 +1,72 @@
+class JobResultList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...JobResultList.options, ...options});
+    this.job = undefined;
+    this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
+  }
+
+  init(job) {
+    this.job = job;
+    super.init(this.job.results);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let jobResultId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    if (actionButtonElement === null) {return;}
+    let action = actionButtonElement.dataset.action;
+    switch (action) {
+      case 'download':
+        window.location.href = this.job.results[jobResultId].download_url;
+        break;
+      default:
+        console.error(`Unknown action: "${action}"`);
+        break;
+    }
+  }
+
+  patch(patch) {
+    let re;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /jobs/{this.job.id}/results/{jobResultId}
+          re = new RegExp('^/jobs/' + this.job.id + '/results/(\\d+)$');
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  preprocessRessource(jobResult) {
+    let description;
+    if (jobResult.filename.endsWith('.pdf.zip')) {
+      description = 'PDF files with text layer';
+    } else if (jobResult.filename.endsWith('.txt.zip')) {
+      description = 'Raw text files';
+    } else if (jobResult.filename.endsWith('.vrt.zip')) {
+      description = 'VRT compliant files including the NLP data';
+    } else if (jobResult.filename.endsWith('.xml.zip')) {
+      description = 'TEI compliant files';
+    } else if (jobResult.filename.endsWith('.poco.zip')) {
+      description = 'HOCR and image files for post correction (PoCo)';
+    } else {
+      description = 'All result files created during this job';
+    }
+    return {id: jobResult.id, description: description, filename: jobResult.filename};
+  }
+}
+JobResultList.options = {
+  item: `<tr>
+           <td><span class="description"></span></td>
+           <td><span class="filename"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'description', 'filename']
+};

web/app/static/js/nopaque/lists/QueryResultList.js

@@ -0,0 +1,95 @@
+class QueryResultList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...QueryResultList.options, ...options});
+    this.queryResults = undefined;
+    this.user.eventListeners.queryResult.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
+  }
+
+  init(queryResults) {
+    this.queryResults = queryResults;
+    super.init(queryResults);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let queryResultId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm query result deletion</h4>
+                                   <p>Do you really want to delete the query result <b>${this.user.data.query_results[queryResultId].title}</b>? It will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="${this.user.data.query_results[queryResultId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'view':
+        if (queryResultId !== '-1') {window.location.href = this.user.data.query_results[queryResultId].url;}
+        break;
+      default:
+        console.error(`Unknown action: "${action}"`);
+        break;
+    }
+  }
+
+  patch(patch) {
+    let id, match, re, valueName;
+    for (let operation of patch) {
+      switch(operation.op) {
+        case 'add':
+          // Matches the only paths that should be handled here: /jobs/{jobId}
+          re = /^\/query_results\/(\d+)$/;
+          if (re.test(operation.path)) {this.add(operation.value);}
+          break;
+        case 'remove':
+          // See case add ;)
+          re = /^\/query_results\/(\d+)$/;
+          if (re.test(operation.path)) {
+            [match, id] = operation.path.match(re);
+            this.remove(id);
+          }
+          break;
+        case 'replace':
+          // Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
+          re = /^\/query_results\/(\d+)\/(corpus_title|description|query|title)$/;
+          if (re.test(operation.path)) {
+            [match, id, valueName] = operation.path.match(re);
+            this.replace(id, valueName, operation.value);
+          }
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  preprocessRessource(queryResult) {
+    return {id: queryResult.id,
+            corpus_title: queryResult.corpus_title,
+            description: queryResult.description,
+            query: queryResult.query,
+            title: queryResult.title};
+  }
+}
+QueryResultList.options = {
+  item: `<tr>
+           <td><b class="title"></b><br><i class="description"></i><br></td>
+           <td><span class="corpus_title"></span><br><span class="query"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
+};

web/app/static/js/nopaque/lists/RessourceList.js

@@ -0,0 +1,98 @@
+class RessourceList {
+  /* A wrapper class for the list.js list.
+   * This class is not meant to be used directly, instead it should be used as
+   * a base class for concrete ressource list implementations.
+   */
+  constructor(listElement, options = {}) {
+    if (listElement.dataset.userId) {
+      if (listElement.dataset.userId in nopaque.appClient.users) {
+        this.user = nopaque.appClient.users[listElement.dataset.userId];
+      } else {
+        console.error(`User not found: ${listElement.dataset.userId}`);
+        return;
+      }
+    } else {
+      this.user = nopaque.appClient.users.self;
+    }
+    this.list = new List(listElement, {...RessourceList.options, ...options});
+    this.list.list.innerHTML = `<tr>
+                                  <td class="row" colspan="100%">
+                                    <div class="col s12">&nbsp;</div>
+                                    <div class="col s3 m2 xl1">
+                                      <div class="preloader-wrapper active">
+                                        <div class="spinner-layer spinner-green-only">
+                                          <div class="circle-clipper left">
+                                            <div class="circle"></div>
+                                          </div>
+                                          <div class="gap-patch">
+                                            <div class="circle"></div>
+                                          </div>
+                                          <div class="circle-clipper right">
+                                            <div class="circle"></div>
+                                          </div>
+                                        </div>
+                                      </div>
+                                    </div>
+                                    <div class="col s9 m6 xl5">
+                                      <span class="card-title">Waiting for data...</span>
+                                      <p>This list is not initialized yet.</p>
+                                    </div>
+                                  </td>
+                                </tr>`;
+    if (typeof this.onclick === 'function') {this.list.list.addEventListener('click', event => this.onclick(event));}
+  }
+
+  eventHandler(eventType, payload) {
+    switch (eventType) {
+      case 'init':
+        this.init(payload);
+        break;
+      case 'patch':
+        this.patch(payload);
+        break;
+      default:
+        console.error(`Unknown event type: ${eventType}`);
+        break;
+    }
+  }
+
+  init(ressources) {
+    this.list.clear();
+    this.add(Object.values(ressources));
+    this.list.sort('id', {order: 'desc'});
+    let emptyListElementHTML = `<tr class="show-if-only-child" data-id="-1">
+                                  <td colspan="100%">
+                                    <span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
+                                    <p>No ressource available.</p>
+                                  </td>
+                                </tr>`;
+    this.list.list.insertAdjacentHTML('afterbegin', emptyListElementHTML);
+  }
+
+  patch(patch) {
+    /*
+     * It's not possible to generalize a patch Handler for all type of
+     * ressources. So this method is meant to be an interface.
+     */
+    console.error('patch method not implemented!');
+  }
+
+  add(values) {
+    let ressources = Array.isArray(values) ? values : [values];
+    if (typeof this.preprocessRessource === 'function') {
+      ressources = ressources.map(ressource => this.preprocessRessource(ressource));
+    }
+    // Set a callback function ('() => {return;}') to force List.js perform the
+    // add method asynchronous: https://listjs.com/api/#add
+    this.list.add(ressources, () => {return;});
+  }
+
+  remove(id) {
+    this.list.remove('id', id);
+  }
+
+  replace(id, valueName, newValue) {
+      this.list.get('id', id)[0].values({[valueName]: newValue});
+  }
+}
+RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};

web/app/static/js/nopaque/lists/Userlist.js

@@ -0,0 +1,71 @@
+class UserList extends RessourceList {
+  constructor(listElement, options = {}) {
+    super(listElement, {...UserList.options, ...options});
+    users = undefined;
+  }
+
+  init(users) {
+    this.users = users;
+    super.init(users);
+  }
+
+  onclick(event) {
+    let ressourceElement = event.target.closest('tr');
+    if (ressourceElement === null) {return;}
+    let userId = ressourceElement.dataset.id;
+    let actionButtonElement = event.target.closest('.action-button');
+    let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
+    switch (action) {
+      case 'delete':
+        let deleteModalHTML = `<div class="modal">
+                                 <div class="modal-content">
+                                   <h4>Confirm user deletion</h4>
+                                   <p>Do you really want to delete the corpus <b>${this.users[userId].username}</b>? All files will be permanently deleted!</p>
+                                 </div>
+                                 <div class="modal-footer">
+                                   <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+                                   <a class="btn modal-close red waves-effect waves-light" href="/admin/users/${userId}/delete"><i class="material-icons left">delete</i>Delete</a>
+                                 </div>
+                               </div>`;
+        let deleteModalParentElement = document.querySelector('main');
+        deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
+        let deleteModalElement = deleteModalParentElement.lastChild;
+        let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
+        deleteModal.open();
+        break;
+      case 'edit':
+        window.location.href = `/admin/users/${userId}/edit`;
+        break;
+      case 'view':
+        if (userId !== '-1') {window.location.href = `/admin/users/${userId}`;}
+        break;
+      default:
+        console.error(`Unknown action: ${action}`);
+        break;
+    }
+  }
+
+  preprocessRessource(user) {
+    return {id: user.id,
+            id_: user.id,
+            username: user.username,
+            email: user.email,
+            last_seen: new Date(user.last_seen * 1000).toLocaleString("en-US"),
+            role: user.role.name};
+  }
+}
+UserList.options = {
+  item: `<tr>
+           <td><span class="id_"></span></td>
+           <td><span class="username"></span></td>
+           <td><span class="email"></span></td>
+           <td><span class="last_seen"></span></td>
+           <td><span class="role"></span></td>
+           <td class="right-align">
+             <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="edit" data-position="top" data-tooltip="Edit"><i class="material-icons">edit</i></a>
+             <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+           </td>
+         </tr>`,
+  valueNames: [{data: ['id']}, 'id_', 'username', 'email', 'last_seen', 'role']
+};

web/app/static/js/nopaque/main.js

@@ -0,0 +1,235 @@
+class AppClient {
+  constructor(currentUserId) {
+    this.socket = io({transports: ['websocket']});
+    this.users = {};
+    this.users.self = this.loadUser(currentUserId);
+  }
+
+  loadUser(userId) {
+    if (userId in this.users) {return this.users[userId];}
+    let user = new User();
+    this.users[userId] = user;
+    this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
+    this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
+    this.socket.emit('start_user_session', userId);
+    return user;
+  }
+}
+
+
+class User {
+  constructor() {
+    this.data = undefined;
+    this.eventListeners = {
+      corpus: {
+        addEventListener(listener, corpusId='*') {
+          if (corpusId in this) {this[corpusId].push(listener);} else {this[corpusId] = [listener];}
+        }
+      },
+      job: {
+        addEventListener(listener, jobId='*') {
+          if (jobId in this) {this[jobId].push(listener);} else {this[jobId] = [listener];}
+        }
+      },
+      queryResult: {
+        addEventListener(listener, queryResultId='*') {
+          if (queryResultId in this) {this[queryResultId].push(listener);} else {this[queryResultId] = [listener];}
+        }
+      }
+    };
+  }
+
+  init(data) {
+    this.data = data;
+
+    for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
+      if (corpusId === '*') {
+        for (let eventListener of eventListeners) {eventListener('init', this.data.corpora);}
+      } else {
+        if (corpusId in this.data.corpora) {
+          for (let eventListener of eventListeners) {eventListener('init', this.data.corpora[corpusId]);}
+        }
+      }
+    }
+
+    for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
+      if (jobId === '*') {
+        for (let eventListener of eventListeners) {eventListener('init', this.data.jobs);}
+      } else {
+        if (jobId in this.data.jobs) {
+          for (let eventListener of eventListeners) {eventListener('init', this.data.jobs[jobId]);}
+        }
+      }
+    }
+
+    for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
+      if (queryResultId === '*') {
+        for (let eventListener of eventListeners) {eventListener('init', this.data.query_results);}
+      } else {
+        if (queryResultId in this.data.query_results) {
+          for (let eventListener of eventListeners) {eventListener('init', this.data.query_results[queryResultId]);}
+        }
+      }
+    }
+  }
+
+  patch(patch) {
+    this.data = jsonpatch.apply_patch(this.data, patch);
+
+    let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
+    if (corporaPatch.length > 0) {
+      for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
+        if (corpusId === '*') {
+          for (let eventListener of eventListeners) {eventListener('patch', corporaPatch);}
+        } else {
+          let corpusPatch = corporaPatch.filter(operation => operation.path.startsWith(`/corpora/${corpusId}`));
+          if (corpusPatch.length > 0) {
+            for (let eventListener of eventListeners) {eventListener('patch', corpusPatch);}
+          }
+        }
+      }
+    }
+
+    let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
+    if (jobsPatch.length > 0) {
+      for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
+        if (jobId === '*') {
+          for (let eventListener of eventListeners) {eventListener('patch', jobsPatch);}
+        } else {
+          let jobPatch = jobsPatch.filter(operation => operation.path.startsWith(`/jobs/${jobId}`));
+          if (jobPatch.length > 0) {
+            for (let eventListener of eventListeners) {eventListener('patch', jobPatch);}
+          }
+        }
+      }
+    }
+
+    let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
+    if (queryResultsPatch.length > 0) {
+      for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
+        if (queryResultId === '*') {
+          for (let eventListener of eventListeners) {eventListener('patch', queryResultsPatch);}
+        } else {
+          let queryResultPatch = queryResultsPatch.filter(operation => operation.path.startsWith(`/query_results/${queryResultId}`));
+          if (queryResultPatch.length > 0) {
+            for (let eventListener of eventListeners) {eventListener('patch', queryResultPatch);}
+          }
+        }
+      }
+    }
+
+    for (let operation of jobsPatch) {
+      if (operation.op !== 'replace') {continue;}
+      // Matches the only path that should be handled here: /jobs/{jobId}/status
+      if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
+        let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
+        if (this.data.settings.job_status_site_notifications === "end" && !['complete', 'failed'].includes(operation.value)) {continue;}
+        nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, 'job');
+      }
+    }
+  }
+}
+
+
+/*
+ * The nopaque object is used as a namespace for nopaque specific functions and
+ * variables.
+ */
+var nopaque = {};
+
+nopaque.flash = function(message, category) {
+  let toast;
+  let toastActionElement;
+
+  switch (category) {
+    case "corpus":
+      message = `<i class="left material-icons">book</i>${message}`;
+      break;
+    case "error":
+      message = `<i class="left material-icons red-text">error</i>${message}`;
+      break;
+    case "job":
+      message = `<i class="left material-icons">work</i>${message}`;
+      break;
+    default:
+      message = `<i class="left material-icons">notifications</i>${message}`;
+  }
+
+  toast = M.toast({html: `<span>${message}</span>
+                          <button data-action="close" class="btn-flat toast-action white-text">
+                            <i class="material-icons">close</i>
+                          </button>`});
+  toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
+  toastActionElement.addEventListener('click', () => {toast.dismiss();});
+};
+
+nopaque.Forms = {};
+nopaque.Forms.init = function() {
+  var abortRequestElement, parentElement, progressElement, progressModal,
+      progressModalElement, request, submitElement;
+
+  for (let form of document.querySelectorAll(".nopaque-submit-form")) {
+    submitElement = form.querySelector('button[type="submit"]');
+    submitElement.addEventListener("click", function() {
+      for (let selectElement of form.querySelectorAll('select')) {
+        if (selectElement.value === "") {
+          parentElement = selectElement.closest(".input-field");
+          parentElement.querySelector(".select-dropdown").classList.add("invalid");
+          for (let helperTextElement of parentElement.querySelectorAll(".helper-text")) {
+            helperTextElement.remove();
+          }
+          parentElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">Please select an option.</span>`);
+        }
+      }
+    });
+
+    request = new XMLHttpRequest();
+    if (form.dataset.hasOwnProperty("progressModal")) {
+      progressModalElement = document.getElementById(form.dataset.progressModal);
+      progressModal = M.Modal.getInstance(progressModalElement);
+      progressModal.options.dismissible = false;
+      abortRequestElement = progressModalElement.querySelector(".abort-request");
+      abortRequestElement.addEventListener("click", function() {request.abort();});
+      progressElement = progressModalElement.querySelector(".determinate");
+    }
+    form.addEventListener("submit", function(event) {
+      event.preventDefault();
+      var formData;
+
+      formData = new FormData(form);
+      // Initialize progress modal
+      if (progressModalElement) {
+        progressElement.style.width = "0%";
+        progressModal.open();
+      }
+      request.open("POST", window.location.href);
+      request.send(formData);
+    });
+    request.addEventListener("load", function(event) {
+      var fieldElement;
+
+      if (request.status === 201) {
+        window.location.href = JSON.parse(this.responseText).redirect_url;
+      }
+      if (request.status === 400) {
+        for (let [field, errors] of Object.entries(JSON.parse(this.responseText))) {
+          fieldElement = form.querySelector(`input[name$="${field}"]`).closest(".input-field");
+          for (let error of errors) {
+            fieldElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">${error}</span>`);
+          }
+        }
+        if (progressModalElement) {
+          progressModal.close();
+        }
+      }
+      if (request.status === 500) {
+        location.reload();
+      }
+    });
+    if (progressModalElement) {
+      request.upload.addEventListener("progress", function(event) {
+        progressElement.style.width = Math.floor(100 * event.loaded / event.total).toString() + "%";
+      });
+    }
+  }
+}

web/app/tasks/__init__.py

@@ -0,0 +1,31 @@
+from .. import db
+from ..models import Corpus, Job
+import docker
+
+
+docker_client = docker.from_env()
+from . import corpus_utils, job_utils  # noqa
+
+
+def check_corpora():
+    corpora = Corpus.query.all()
+    for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
+        corpus_utils.create_build_corpus_service(corpus)
+    for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora):  # noqa
+        corpus_utils.checkout_build_corpus_service(corpus)
+    for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora):  # noqa
+        corpus_utils.create_cqpserver_container(corpus)
+    for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora):  # noqa
+        corpus_utils.remove_cqpserver_container(corpus)
+    db.session.commit()
+
+
+def check_jobs():
+    jobs = Job.query.all()
+    for job in filter(lambda job: job.status == 'submitted', jobs):
+        job_utils.create_job_service(job)
+    for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
+        job_utils.checkout_job_service(job)
+    for job in filter(lambda job: job.status == 'canceling', jobs):
+        job_utils.remove_job_service(job)
+    db.session.commit()

web/app/tasks/corpus_utils.py

@@ -0,0 +1,174 @@
+from . import docker_client
+import docker
+import logging
+import os
+import shutil
+
+
+def create_build_corpus_service(corpus):
+    corpus_data_dir = os.path.join(corpus.path, 'data')
+    shutil.rmtree(corpus_data_dir, ignore_errors=True)
+    os.mkdir(corpus_data_dir)
+    corpus_registry_dir = os.path.join(corpus.path, 'registry')
+    shutil.rmtree(corpus_registry_dir, ignore_errors=True)
+    os.mkdir(corpus_registry_dir)
+    corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
+    service_kwargs = {
+        'command': 'docker-entrypoint.sh build-corpus',
+        'constraints': ['node.role==worker'],
+        'labels': {'origin': 'nopaque',
+                   'type': 'corpus.build',
+                   'corpus_id': str(corpus.id)},
+        'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
+                   corpus_data_dir + ':/corpora/data:rw',
+                   corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
+        'name': 'build-corpus_{}'.format(corpus.id),
+        'restart_policy': docker.types.RestartPolicy()
+    }
+    service_image = \
+        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
+    try:
+        docker_client.services.create(service_image, **service_kwargs)
+    except docker.errors.APIError as e:
+        logging.error(
+            'Create "{}" service raised '.format(service_kwargs['name'])
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+    else:
+        corpus.status = 'queued'
+
+
+def checkout_build_corpus_service(corpus):
+    service_name = 'build-corpus_{}'.format(corpus.id)
+    try:
+        service = docker_client.services.get(service_name)
+    except docker.errors.NotFound:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.NotFound" The service does not exist. '
+            + '(corpus.status: {} -> failed)'.format(corpus.status)
+        )
+        corpus.status = 'failed'
+    except docker.errors.APIError as e:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+    except docker.errors.InvalidVersion:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.InvalidVersion" One of the arguments is '
+            + 'not supported with the current API version.'
+        )
+    else:
+        service_tasks = service.tasks()
+        if not service_tasks:
+            return
+        task_state = service_tasks[0].get('Status').get('State')
+        if corpus.status == 'queued' and task_state != 'pending':
+            corpus.status = 'running'
+        elif (corpus.status == 'running'
+              and task_state in ['complete', 'failed']):
+            try:
+                service.remove()
+            except docker.errors.APIError as e:
+                logging.error(
+                    'Remove "{}" service raised '.format(service_name)
+                    + '"docker.errors.APIError" The server returned an error. '
+                    + 'Details: {}'.format(e)
+                )
+                return
+            else:
+                corpus.status = 'prepared' if task_state == 'complete' \
+                                else 'failed'
+
+
+def create_cqpserver_container(corpus):
+    corpus_data_dir = os.path.join(corpus.path, 'data')
+    corpus_registry_dir = os.path.join(corpus.path, 'registry')
+    container_kwargs = {
+        'command': 'cqpserver',
+        'detach': True,
+        'volumes': [corpus_data_dir + ':/corpora/data:rw',
+                    corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
+        'name': 'cqpserver_{}'.format(corpus.id),
+        'network': 'nopaque_default'
+    }
+    container_image = \
+        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
+    # Check if a cqpserver container already exists. If this is the case,
+    # remove it and create a new one
+    try:
+        container = docker_client.containers.get(container_kwargs['name'])
+    except docker.errors.NotFound:
+        pass
+    except docker.errors.APIError as e:
+        logging.error(
+            'Get "{}" container raised '.format(container_kwargs['name'])
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+        return
+    else:
+        try:
+            container.remove(force=True)
+        except docker.errors.APIError as e:
+            logging.error(
+                'Remove "{}" container raised '.format(container_kwargs['name'])  # noqa
+                + '"docker.errors.APIError" The server returned an error. '
+                + 'Details: {}'.format(e)
+            )
+            return
+    try:
+        docker_client.containers.run(container_image, **container_kwargs)
+    except docker.errors.ContainerError:
+        # This case should not occur, because detach is True.
+        logging.error(
+            'Run "{}" container raised '.format(container_kwargs['name'])
+            + '"docker.errors.ContainerError" The container exits with a '
+            + 'non-zero exit code and detach is False.'
+        )
+        corpus.status = 'failed'
+    except docker.errors.ImageNotFound:
+        logging.error(
+            'Run "{}" container raised '.format(container_kwargs['name'])
+            + '"docker.errors.ImageNotFound" The specified image does not '
+            + 'exist.'
+        )
+        corpus.status = 'failed'
+    except docker.errors.APIError as e:
+        logging.error(
+            'Run "{}" container raised '.format(container_kwargs['name'])
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+    else:
+        corpus.status = 'analysing'
+
+
+def remove_cqpserver_container(corpus):
+    container_name = 'cqpserver_{}'.format(corpus.id)
+    try:
+        container = docker_client.containers.get(container_name)
+    except docker.errors.NotFound:
+        pass
+    except docker.errors.APIError as e:
+        logging.error(
+            'Get "{}" container raised '.format(container_name)
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+        return
+    else:
+        try:
+            container.remove(force=True)
+        except docker.errors.APIError as e:
+            logging.error(
+                'Remove "{}" container raised '.format(container_name)
+                + '"docker.errors.APIError" The server returned an error. '
+                + 'Details: {}'.format(e)
+            )
+            return
+    corpus.status = 'prepared'

web/app/tasks/job_utils.py

@@ -0,0 +1,152 @@
+from datetime import datetime
+from werkzeug.utils import secure_filename
+from . import docker_client
+from .. import db, mail
+from ..email import create_message
+from ..models import JobResult
+import docker
+import logging
+import json
+import os
+
+
+def create_job_service(job):
+    cmd = '{} -i /files -o /files/output'.format(job.service)
+    if job.service == 'file-setup':
+        cmd += ' -f {}'.format(secure_filename(job.title))
+    cmd += ' --log-dir /files'
+    cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
+    cmd += ' ' + ' '.join(json.loads(job.service_args))
+    service_kwargs = {'command': cmd,
+                      'constraints': ['node.role==worker'],
+                      'labels': {'origin': 'nopaque',
+                                 'type': 'service.{}'.format(job.service),
+                                 'job_id': str(job.id)},
+                      'mounts': [job.path + ':/files:rw'],
+                      'name': 'job_{}'.format(job.id),
+                      'resources': docker.types.Resources(
+                          cpu_reservation=job.n_cores * (10 ** 9),
+                          mem_reservation=job.mem_mb * (10 ** 6)
+                      ),
+                      'restart_policy': docker.types.RestartPolicy()}
+    service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
+        job.service, job.service_version)
+    try:
+        docker_client.services.create(service_image, **service_kwargs)
+    except docker.errors.APIError as e:
+        logging.error(
+            'Create "{}" service raised '.format(service_kwargs['name'])
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+        return
+    else:
+        job.status = 'queued'
+    finally:
+        send_notification(job)
+
+
+def checkout_job_service(job):
+    service_name = 'job_{}'.format(job.id)
+    try:
+        service = docker_client.services.get(service_name)
+    except docker.errors.NotFound:
+        logging.error('Get "{}" service raised '.format(service_name)
+                      + '"docker.errors.NotFound" The service does not exist. '
+                      + '(job.status: {} -> failed)'.format(job.status))
+        job.status = 'failed'
+    except docker.errors.APIError as e:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+        return
+    except docker.errors.InvalidVersion:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.InvalidVersion" One of the arguments is '
+            + 'not supported with the current API version.'
+        )
+        return
+    else:
+        service_tasks = service.tasks()
+        if not service_tasks:
+            return
+        task_state = service_tasks[0].get('Status').get('State')
+        if job.status == 'queued' and task_state != 'pending':
+            job.status = 'running'
+        elif job.status == 'running' and task_state in ['complete', 'failed']:
+            try:
+                service.remove()
+            except docker.errors.APIError as e:
+                logging.error(
+                    'Remove "{}" service raised '.format(service_name)
+                    + '"docker.errors.APIError" The server returned an error. '
+                    + 'Details: {}'.format(e)
+                )
+                return
+            else:
+                if task_state == 'complete':
+                    job_results_dir = os.path.join(job.path, 'output')
+                    job_results = filter(lambda x: x.endswith('.zip'),
+                                         os.listdir(job_results_dir))
+                    for job_result in job_results:
+                        job_result = JobResult(filename=job_result, job=job)
+                        db.session.add(job_result)
+                job.end_date = datetime.utcnow()
+                job.status = task_state
+    finally:
+        send_notification(job)
+
+
+def remove_job_service(job):
+    service_name = 'job_{}'.format(job.id)
+    try:
+        service = docker_client.services.get(service_name)
+    except docker.errors.NotFound:
+        job.status = 'canceled'
+    except docker.errors.APIError as e:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.APIError" The server returned an error. '
+            + 'Details: {}'.format(e)
+        )
+        return
+    except docker.errors.InvalidVersion:
+        logging.error(
+            'Get "{}" service raised '.format(service_name)
+            + '"docker.errors.InvalidVersion" One of the arguments is '
+            + 'not supported with the current API version.'
+        )
+        return
+    else:
+        try:
+            service.update(mounts=None)
+        except docker.errors.APIError as e:
+            logging.error(
+                'Update "{}" service raised '.format(service_name)
+                + '"docker.errors.APIError" The server returned an error. '
+                + 'Details: {}'.format(e)
+            )
+            return
+        try:
+            service.remove()
+        except docker.errors.APIError as e:
+            logging.error(
+                'Remove "{}" service raised '.format(service_name)
+                + '"docker.errors.APIError" The server returned an error. '
+                + 'Details: {}'.format(e)
+            )
+
+
+def send_notification(job):
+    if job.creator.setting_job_status_mail_notifications == 'none':
+        return
+    if (job.creator.setting_job_status_mail_notifications == 'end'
+            and job.status not in ['complete', 'failed']):
+        return
+    msg = create_message(job.creator.email,
+                         'Status update for your Job "{}"'.format(job.title),
+                         'tasks/email/notification', job=job)
+    mail.send(msg)

web/app/templates/admin/_breadcrumbs.html.j2

@@ -0,0 +1,19 @@
+<ul class="tabs tabs-transparent">
+  <li class="tab"><a href="{{ url_for('main.index') }}" target="_self"><i class="material-icons">home</i></a></li>
+  <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
+  <li class="tab"><a href="{{ url_for('.index') }}" target="_self">Administration</a></li>
+  <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
+  {% if request.path == url_for('.users') %}
+  <li class="tab"><a class="active" href="{{ url_for('.users') }}" target="_self">Users</a></li>
+  {% elif request.path == url_for('.user', user_id=user.id) %}
+  <li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
+  <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
+  <li class="tab"><a class="active" href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
+  {% elif request.path == url_for('.edit_user', user_id=user.id) %}
+  <li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
+  <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
+  <li class="tab"><a href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
+  <li class="tab disabled"><i class="material-icons">navigate_next</i></li>
+  <li class="tab"><a class="active" href="{{ url_for('.edit_user', user_id=user.id) }}" target="_self">Edit</a></li>
+  {% endif %}
+</ul>

web/app/templates/admin/edit_user.html.j2

@@ -1,6 +1,10 @@
 {% extends "nopaque.html.j2" %}
 {% import 'materialize/wtf.html.j2' as wtf %}
 
+{% block nav_content %}
+{% include 'admin/_breadcrumbs.html.j2' %}
+{% endblock nav_content %}
+
 {% block page_content %}
 <div class="container">
   <div class="row">

web/app/templates/admin/user.html.j2

@@ -1,5 +1,9 @@
 {% extends "nopaque.html.j2" %}
 
+{% block nav_content %}
+{% include 'admin/_breadcrumbs.html.j2' %}
+{% endblock nav_content %}
+
 {% block page_content %}
 <div class="container">
   <div class="row">
@@ -30,23 +34,22 @@
           </ul>
         </div>
         <div class="card-action right-align">
-          <a href="{{ url_for('.edit_general_settings', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a>
+          <a href="{{ url_for('.edit_user', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a>
           <a data-target="delete-user-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a>
         </div>
       </div>
     </div>
 
-<div class="col s12 l6">
+<div class="col s12 l6" id="corpora" data-user-id="{{ user.id }}">
   <h3>Corpora</h3>
   <div class="card">
-    <div class="card-content" id="corpora">
+    <div class="card-content">
       <div class="input-field">
         <i class="material-icons prefix">search</i>
         <input id="search-corpus" class="search" type="search"></input>
         <label for="search-corpus">Search corpus</label>
       </div>
-      <ul class="pagination paginationTop"></ul>
+      <table class="highlight ressource-list">
-      <table class="highlight">
         <thead>
           <tr>
             <th></th>
@@ -60,22 +63,21 @@
         </thead>
         <tbody class="list"></tbody>
       </table>
-      <ul class="pagination paginationBottom"></ul>
+      <ul class="pagination"></ul>
     </div>
   </div>
 </div>
 
-<div class="col s12 l6">
+<div class="col s12 l6" id="jobs" data-user-id="{{ user.id }}">
   <h3>Jobs</h3>
   <div class="card">
-    <div class="card-content" id="jobs">
+    <div class="card-content">
       <div class="input-field">
         <i class="material-icons prefix">search</i>
         <input id="search-job" class="search" type="search"></input>
         <label for="search-job">Search job</label>
       </div>
-      <ul class="pagination paginationTop"></ul>
+      <table class="highlight ressource-list">
-      <table class="highlight">
         <thead>
           <tr>
             <th><span class="sort" data-sort="service">Service</span></th>
@@ -89,7 +91,7 @@
         </thead>
         <tbody class="list"></tbody>
       </table>
-      <ul class="pagination paginationBottom"></ul>
+      <ul class="pagination"></ul>
     </div>
   </div>
 </div>
@@ -111,10 +113,9 @@
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-  import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}';
+  nopaque.appClient.loadUser({{ user.id }});
-  let corpusList = new RessourceList("corpora", nopaque.foreignCorporaSubscribers, "Corpus");
+  let corpusList = new CorpusList(document.querySelector('#corpora'));
-  let jobList = new RessourceList("jobs", nopaque.foreignJobsSubscribers, "Job");
+  let jobList = new JobList(document.querySelector('#jobs'));
-  nopaque.socket.emit("foreign_user_data_stream_init", {{ user.id }});
 </script>
 {% endblock scripts %}

web/app/templates/admin/users.html.j2

@@ -1,5 +1,9 @@
 {% extends "nopaque.html.j2" %}
 
+{% block nav_content %}
+{% include 'admin/_breadcrumbs.html.j2' %}
+{% endblock nav_content %}
+
 {% block page_content %}
 <div class="container">
   <div class="row">
@@ -7,30 +11,28 @@
       <h1 id="title">{{ title }}</h1>
     </div>
 
-    <div class="col s12">
+    <div class="col s12" id="users">
       <div class="card">
-        <div class="card-content" id="users">
+        <div class="card-content">
           <div class="input-field">
             <i class="material-icons prefix">search</i>
             <input id="search-user" class="search" type="text"></input>
             <label for="search-user">Search user</label>
           </div>
-          <ul class="pagination paginationTop"></ul>
+          <table class="highlight ressource-list">
-          <table class="highlight responsive-table">
             <thead>
               <tr>
+                <th class="sort" data-sort="id">Id</th>
                 <th class="sort" data-sort="username">Username</th>
                 <th class="sort" data-sort="email">Email</th>
-                <th class="sort" data-sort="role_id">Role</th>
+                <th class="sort" data-sort="last_seen">Last seen</th>
-                <th class="sort" data-sort="confirmed">Confirmed Status</th>
+                <th class="sort" data-sort="role">Role</th>
-                <th class="sort" data-sort="id">Id</th>
+                <th></th>
-                <th>{# Actions #}</th>
               </tr>
             </thead>
-            <tbody class="list">
+            <tbody class="list"></tbody>
-            </tbody>
           </table>
-          <ul class="pagination paginationBottom"></ul>
+          <ul class="pagination"></ul>
         </div>
       </div>
     </div>
@@ -40,9 +42,8 @@
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-  import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}';
+  let userList = new UserList(document.querySelector('#users'), {page: 10});
-  let userList = new RessourceList('users', null, "User", RessourceList.options.extended);
+  userList.init({{ users|tojson }});
-  userList._add({{ users|tojson}});
 </script>
 {% endblock scripts %}

web/app/templates/auth/login.html.j2

@@ -35,20 +35,20 @@
       <div class="card medium">
         <form method="POST">
           <div class="card-content">
-            {{ login_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(login_form.user, material_icon='person') }}
+            {{ wtf.render_field(form.user, material_icon='person') }}
-            {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password, material_icon='vpn_key') }}
             <div class="row" style="margin-bottom: 0;">
               <div class="col s6 left-align">
                 <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
               </div>
               <div class="col s6 right-align">
-                {{ wtf.render_field(login_form.remember_me) }}
+                {{ wtf.render_field(form.remember_me) }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(login_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/register.html.j2

@@ -34,14 +34,14 @@
       <div class="card medium">
         <form method="POST">
           <div class="card-content">
-            {{ registration_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
+            {{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
-            {{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
-            {{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
-            {{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
+            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(registration_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/reset_password.html.j2

@@ -20,12 +20,12 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ reset_password_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(reset_password_form.password, data_length='128') }}
+            {{ wtf.render_field(form.password, data_length='128') }}
-            {{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
+            {{ wtf.render_field(form.password_confirmation, data_length='128') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/reset_password_request.html.j2

@@ -20,11 +20,11 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ reset_password_request_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
+            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/add_corpus.html.j2

@@ -27,18 +27,18 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ add_corpus_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/add_corpus_file.html.j2

@@ -27,24 +27,24 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ add_corpus_file_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
+                {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
+                {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
+                {{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
         <br>
@@ -52,7 +52,7 @@
           <li>
             <div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
             <div class="collapsible-body">
-              {% for field in add_corpus_file_form
+              {% for field in form
                  if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
               {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
               {% endfor %}

web/app/templates/corpora/analyse_corpus.html.j2

@@ -155,9 +155,9 @@ import {
  */
 document.addEventListener("DOMContentLoaded", () => {
   // Initialize the client for server client communication in dynamic mode
-  let corpusId = {{ corpus_id }}
+  let corpusId = {{ corpus.id }}
   const client = new Client({'corpusId': corpusId,
-                             'socket': nopaque.socket,
+                             'socket': nopaque.appClient.socket,
                              'logging': true,
                              'dynamicMode': true});
   /**

web/app/templates/corpora/corpus.html.j2

@@ -2,25 +2,26 @@
 {% from '_colors.html.j2' import colors %}
 
 {% set scheme_primary_color = colors.corpus_analysis_darken %}
-{% set scheme_secondary_color = colors.corpus_analysis %}
+{% set scheme_secondary_color = colors.corpus_analysis_lighten %}
+{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
 
 {% block nav_content %}
 {% include 'corpora/_breadcrumbs.html.j2' %}
 {% endblock nav_content %}
 
-{% block main_attribs %} class="corpus-analysis-color lighten"{% endblock main_attribs %}
-
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12">
+    <div class="col s12" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}" id="corpus-display">
-      <h1 id="title">{{ corpus.title }}</h1>
+      <div class="row">
-      <p id="description">{{ corpus.description }}</p>
+        <div class="col s8 m9 l10">
+          <h1 id="title"><span class="corpus-title"></span></h1>
         </div>
-
+        <div class="col s4 m3 l2 right-align">
-    <div class="col s12 m4">
+          <p>&nbsp;</p>
-      <span class="chip status white-text hide" id="status"></span>
+          <p>&nbsp;</p>
-      <div class="active preloader-wrapper small hide status-spinner" id="progress-indicator">
+          <span class="chip status white-text"></span>
+          <div class="active preloader-wrapper small status-spinner">
             <div class="spinner-layer spinner-blue-only">
               <div class="circle-clipper left">
                 <div class="circle"></div>
@@ -34,75 +35,82 @@
             </div>
           </div>
         </div>
+      </div>
 
-    <div class="col s12 m8">
       <div class="card">
-        <div class="card-content">
+        <div class="card-content" style="border-top: 10px solid {{ scheme_primary_color }}">
-          <span class="card-title">Chronometrics</span>
           <div class="row">
+            <div class="col s12">
+              <div class="input-field">
+                <input class="corpus-description" disabled id="corpus-description" type="text">
+                <label for="corpus-description">Description</label>
+              </div>
+            </div>
+
             <div class="col s12 m6">
               <div class="input-field">
-                <input disabled value="{{ corpus.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
+                <input class="corpus-creation-date validate" disabled id="corpus-creation-date" type="text">
-                <label for="creation-date">Creation date</label>
+                <label for="corpus-creation-date">Creation date</label>
               </div>
             </div>
+
             <div class="col s12 m6">
               <div class="input-field">
-                <input disabled value="{{ corpus.last_edited_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="last_edited_date" type="text" class="validate">
+                <input class="corpus-last-edited-date validate" disabled id="corpus-last-edited-date" type="text">
-                <label for="creation-date">Last edited</label>
+                <label for="corpus-last-edited-date">Last edited</label>
               </div>
             </div>
+
             <div class="col s12 m6">
               <div class="input-field">
-                <input disabled value="{{ corpus.current_nr_of_tokens }} / {{ corpus.max_nr_of_tokens }}" id="nr_of_tokens" type="text" class="validate">
+                <input class="corpus-token-ratio validate" disabled id="corpus-token-ratio" type="text">
-                <label for="creation-date">Nr. of tokens used
+                <label for="corpus-token-ratio">Nr. of tokens used <sup><i class="material-icons tooltipped tiny" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i></sup></label>
-                  <i class="material-icons tooltipped" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i>
-                </label>
               </div>
             </div>
           </div>
         </div>
         <div class="card-action right-align">
-          <a href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="analyze"><i class="material-icons left">search</i>Analyze</a>
+          <a class="analyse-corpus-trigger btn disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
-          <a href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="build"><i class="material-icons left">build</i>Build</a>
+          <a class="btn build-corpus-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">build</i>Build</a>
-          <a class="btn hide waves-effect waves-light download" id="corpus_create_zip"><i class="material-icons left">import_export</i>Export Corpus</a>
+          <a class="btn disabled export-corpus-trigger waves-effect waves-light"><i class="material-icons left">import_export</i>Export</a>
-          <a data-target="delete-corpus-modal" class="btn modal-trigger red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
+          <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-corpus-modal"><i class="material-icons left">delete</i>Delete</a>
+        </div>
+      </div>
+
+      <div id="delete-corpus-modal" class="modal">
+        <div class="modal-content">
+          <h4>Confirm corpus deletion</h4>
+          <p>Do you really want to delete the corpus <span class="corpus-title"></span>? All files will be permanently deleted!</p>
+        </div>
+        <div class="modal-footer">
+          <a class="btn modal-close waves-effect waves-light" href="#!">Cancel</a>
+          <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">delete</i>Delete</a>
         </div>
       </div>
     </div>
 
-    <div class="col s12"></div>
+    <div class="col s12" id="corpus-files" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}">
-
-    <div class="col s12">
       <div class="card">
-        <div class="card-content" id="corpus-files" style="overflow: hidden;">
+        <div class="card-content">
           <span class="card-title" id="files">Corpus files</span>
           <div class="input-field">
             <i class="material-icons prefix">search</i>
-            <input id="search-results" class="search" type="search"></input>
+            <input class="search" id="search-corpus-files" type="search"></input>
-            <label for="search-results">Search results</label>
+            <label for="search-corpus-files">Search corpus files</label>
           </div>
-          <ul class="pagination paginationTop"></ul>
           <table class="highlight responsive-table">
             <thead>
               <tr>
                 <th class="sort" data-sort="filename">Filename</th>
                 <th class="sort" data-sort="author">Author</th>
                 <th class="sort" data-sort="title">Title</th>
-                <th>{# Actions #}</th>
+                <th class="sort" data-sort="publishing-year">Publishing year</th>
+                <th></th>
               </tr>
             </thead>
-            <tbody class="list">
+            <tbody class="list"></tbody>
-              {% if corpus_files|length == 0 %}
-              <tr class="show-if-only-child">
-                <td colspan="5">
-                  <span class="card-title"><i class="material-icons left">book</i>Nothing here...</span>
-                  <p>Corpus is empty. Add texts using the option below.</p>
-                </td>
-              </tr>
-              {% endif %}
           </table>
-          <ul class="pagination paginationBottom"></ul>
+          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
           <a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
@@ -111,140 +119,13 @@
     </div>
   </div>
 </div>
-
-<!-- Modals -->
-<div id="delete-corpus-modal" class="modal">
-  <div class="modal-content">
-    <h4>Confirm corpus deletion</h4>
-    <p>Do you really want to delete the corpus {{corpus.title}}? All files will be permanently deleted!</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-    <a href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}" class="btn modal-close red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
-  </div>
-</div>
 {% endblock page_content %}
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-import {
+  nopaque.appClient.loadUser({{ corpus.creator.id }});
-  RessourceList
+  let corpusDisplay = new CorpusDisplay(document.querySelector('#corpus-display'));
-} from '../../static/js/nopaque.lists.js';
+  let corpusFileList = new CorpusFileList(document.querySelector('#corpus-files'));
-
-class InformationUpdater {
-  constructor(corpusId, foreignCorpusFlag) {
-    this.corpusId = corpusId;
-    this.foreignCorpusFlag = foreignCorpusFlag;
-
-    if (this.foreignCorpusFlag) {
-      nopaque.foreignCorporaSubscribers.push(this);
-    } else {
-      nopaque.corporaSubscribers.push(this);
-    }
-  }
-
-  _init() {
-    let corpus;
-
-    corpus = (this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId]
-                                     : nopaque.user.corpora[this.corpusId]);
-
-    // Status
-    this.setStatus(corpus.status);
-  }
-
-  _update(patch) {
-    let pathArray;
-
-    for (let operation of patch) {
-      /* "/corpora/{corpusId}/valueName" -> ["{corpusId}", ...] */
-      pathArray = operation.path.split("/").slice(2);
-      if (pathArray[0] != this.corpusId) {continue;}
-      switch(operation.op) {
-        case "add":
-          location.reload();
-          break;
-        case "delete":
-          location.reload();
-          break;
-        case "replace":
-          if (pathArray[1] === "status") {
-            this.setStatus(operation.value);
-          }
-          break;
-        default:
-          break;
-      }
-    }
-  }
-
-  setStatus(status) {
-    let analyzeElement, buildElement, numFiles, progressIndicatorElement, statusElement;
-
-    numFiles = Object.keys((this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId] : nopaque.user.corpora[this.corpusId]).files).length;
-
-    progressIndicatorElement = document.getElementById("progress-indicator");
-    if (["queued", "running", "start analysis", "stop analysis"].includes(status)) {
-      progressIndicatorElement.classList.remove("hide");
-    } else {
-      progressIndicatorElement.classList.add("hide");
-    }
-
-    statusElement = document.getElementById("status");
-    statusElement.dataset.status = status;
-    statusElement.classList.remove("hide");
-
-    analyzeElement = document.getElementById("analyze");
-    if (["analysing", "prepared", "start analysis"].includes(status)) {
-      analyzeElement.classList.remove("disabled", "hide");
-    } else {
-      analyzeElement.classList.add("disabled", "hide");
-    }
-
-    buildElement = document.getElementById("build");
-    if (status === "unprepared" && numFiles > 0) {
-      buildElement.classList.remove("disabled", "hide");
-    } else {
-      buildElement.classList.add("disabled", "hide");
-    }
-
-    let downloadBtn = document.querySelector('#corpus_create_zip');
-    if (status === "prepared") {
-      downloadBtn.classList.toggle('hide', false);
-    } else {
-      downloadBtn.classList.toggle('hide', true);
-    }
-  }
-}
-
-{% if corpus.creator == current_user %}
-var informationUpdater = new InformationUpdater({{ corpus.id }}, false);
-{% else %}
-var informationUpdater = new InformationUpdater({{ corpus.id }}, true);
-nopaque.socket.emit("foreign_user_data_stream_init", {{ corpus.user_id }});
-{% endif %}
-
-let corpusFilesList = new RessourceList("corpus-files", null, "CorpusFile");
-corpusFilesList._add({{ corpus_files|tojson|safe }});
-
-// Events to handle full corpus download
-let downloadBtn = document.querySelector('#corpus_create_zip');
-downloadBtn.addEventListener('click', () => {
-  nopaque.flash('Compressing your corpus', 'corpus')
-  nopaque.socket.emit('corpus_create_zip', {{ corpus.id }});
-  downloadBtn.classList.toggle('disabled', true);
-});
-document.addEventListener('DOMContentLoaded', () => {
-  nopaque.socket.on('corpus_zip_created', () => {
-    nopaque.flash('Downloading your corpus', 'corpus');
-    downloadBtn.classList.toggle('disabled', false);
-    // Little trick to call the download view after ziping has finished
-    let fakeBtn = document.createElement('a');
-    fakeBtn.href = '{{ url_for('corpora.export_corpus',
-                               corpus_id=corpus.id) }}';
-    fakeBtn.click();
-  });
-});
 </script>
 {% endblock scripts %}

web/app/templates/corpora/corpus_file.html.j2

@@ -20,23 +20,23 @@
 
     <div class="col s12">
       <form method="POST">
-        {{ edit_corpus_file_form.hidden_tag() }}
+        {{ form.hidden_tag() }}
         <div class="card">
           <div class="card-content">
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
+                {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
+                {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
         <br>
@@ -44,7 +44,7 @@
           <li>
             <div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
             <div class="collapsible-body">
-              {% for field in edit_corpus_file_form
+              {% for field in form
                  if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
               {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
               {% endfor %}

web/app/templates/corpora/import_corpus.html.j2

@@ -27,23 +27,23 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ import_corpus_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
             </div>
             <div class="row">
               <div class="col s12">
-                {{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
+                {{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/query_results/add_query_result.html.j2

@@ -27,21 +27,21 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ add_query_result_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
+                {{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
       </form>

web/app/templates/jobs/job.html.j2

@@ -2,43 +2,34 @@
 {% from '_colors.html.j2' import colors %}
 
 {% if job.service == 'file-setup' %}
-  {% set border_color = colors.file_setup_darken %}
+  {% set scheme_primary_color = colors.file_setup_darken %}
-  {% set main_class = 'file-setup-color lighten' %}
+  {% set scheme_secondary_color = colors.file_setup_lighten %}
-  {% set scheme_color = colors.file_setup_darken %}
 {% elif job.service == 'nlp' %}
-  {% set border_color = colors.nlp_darken %}
+  {% set scheme_primary_color = colors.nlp_darken %}
-  {% set main_class = 'nlp-color lighten' %}
+  {% set scheme_secondary_color = colors.nlp_lighten %}
-  {% set scheme_color = colors.nlp_darken %}
 {% elif job.service == 'ocr' %}
-  {% set border_color = colors.ocr_darken %}
+  {% set scheme_primary_color = colors.ocr_darken %}
-  {% set main_class = 'ocr-color lighten' %}
+  {% set scheme_secondary_color = colors.ocr_lighten %}
-  {% set scheme_color = colors.ocr_darken %}
 {% endif %}
+{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
 
 {% block nav_content %}
 {% include 'jobs/_breadcrumbs.html.j2' %}
 {% endblock nav_content %}
 
-{% block main_attribs %} class="{{ main_class }}"{% endblock main_attribs %}
-
 {% block page_content %}
 <div class="container">
   <div class="row">
-    <div class="col s12">
+    <div class="col s12" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}" id="job-display">
-      <h1>[{{ job.service }}] {{ job.title }}</h1>
-    </div>
-
-    <div class="col s12">
-      <div class="card" style="border-top: 10px solid {{border_color}}">
-        <div class="card-content">
       <div class="row">
         <div class="col s8 m9 l10">
-              <span class="card-title title">{{ job.title }}</span>
+          <h1 id="title">[<span class="job-service"></span>] <span class="job-title"></span></h1>
         </div>
-
         <div class="col s4 m3 l2 right-align">
+          <p>&nbsp;</p>
+          <p>&nbsp;</p>
           <span class="chip status white-text"></span>
-              <div class="active preloader-wrapper small status-spinner" id="progress-indicator">
+          <div class="active preloader-wrapper small status-spinner">
             <div class="spinner-layer spinner-blue-only">
               <div class="circle-clipper left">
                 <div class="circle"></div>
@@ -52,87 +43,114 @@
             </div>
           </div>
         </div>
+      </div>
 
+      <div class="card" style="border-top: 10px solid {{ scheme_primary_color }}">
+        <div class="card-content">
+          <div class="row">
             <div class="col s12">
-              <p class="description">{{ job.description }}</p>
-            </div>
-
-            <div class="col s12">&nbsp;</div>
-
-            <div class="col s12 m6">
               <div class="input-field">
-                <input disabled id="creation-date" type="text" value="{{ job.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}">
+                <input class="job-description" disabled id="job-description" type="text">
-                <label for="creation-date">Creation date</label>
+                <label for="job-description">Description</label>
               </div>
             </div>
 
             <div class="col s12 m6">
               <div class="input-field">
-                <input class="end-date" disabled id="end-date" type="text" value="">
+                <input class="job-creation-date" disabled id="job-creation-date" type="text">
-                <label for="end-date">End date</label>
+                <label for="job-creation-date">Creation date</label>
+              </div>
+            </div>
+
+            <div class="col s12 m6">
+              <div class="input-field">
+                <input class="job-end-date" disabled id="job-end-date" type="text">
+                <label for="job-end-date">End date</label>
               </div>
             </div>
 
             <div class="col s12 m4">
               <div class="input-field">
-                <input disabled id="service" type="text" value="{{ job.service }}">
+                <input class="job-service" disabled id="job-service" type="text">
-                <label for="service">Service</label>
+                <label for="job-service">Service</label>
               </div>
             </div>
 
             <div class="col s12 m4">
               <div class="input-field">
-                <input disabled id="service-args" type="text" value="{{ job.service_args|e }}">
+                <input class="job-service-args" disabled id="job-service-args" type="text">
-                <label for="service-args">Service arguments</label>
+                <label for="job-service-args">Service arguments</label>
               </div>
             </div>
 
             <div class="col s12 m4">
               <div class="input-field">
-                <input disabled id="service-version" type="text" value="{{ job.service_version }}">
+                <input class="job-service-version" disabled id="job-service-version" type="text">
-                <label for="service-version">Service version</label>
+                <label for="job-service-version">Service version</label>
               </div>
             </div>
           </div>
         </div>
         <div class="card-action right-align">
-          {% if current_user.is_administrator() and job.status == 'failed' %}
+          {% if current_user.is_administrator()  %}
-          <a href="{{ url_for('jobs.restart', job_id=job.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">repeat</i>Restart</a>
+          <a class="btn hide modal-trigger restart-job-trigger waves-effect waves-light" data-target="restart-job-modal"><i class="material-icons left">repeat</i>Restart</a>
           {% endif %}
           <!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> -->
-          <a data-target="delete-job-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a>
+          <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-job-modal"><i class="material-icons left">delete</i>Delete</a>
-        </div>
         </div>
       </div>
 
-    <div class="col s12">
+      <div id="delete-job-modal" class="modal">
+        <div class="modal-content">
+          <h4>Confirm deletion</h4>
+          <p>Do you really want to delete the job <span class="job-title"></span>? All associated files will be permanently deleted.</p>
+        </div>
+        <div class="modal-footer">
+          <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+          <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
+        </div>
+      </div>
+
+      {% if current_user.is_administrator() %}
+      <div id="restart-job-modal" class="modal">
+        <div class="modal-content">
+          <h4>Confirm restart</h4>
+          <p>Do you really want to restart the job <span class="job-title"></span>? All log and result files will be permanently deleted.</p>
+        </div>
+        <div class="modal-footer">
+          <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
+          <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.restart', job_id=job.id) }}"><i class="material-icons left">restart</i>Restart</a>
+        </div>
+      </div>
+      {% endif %}
+    </div>
+
+    <div class="col s12" id="job-inputs" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
       <div class="card">
-        <div class="card-content" id="inputs">
+        <div class="card-content">
           <div class="row">
             <div class="col s12 m2">
               <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
               <p>Original input files.</p>
             </div>
             <div class="col s12 m10">
-              <ul class="pagination paginationTop"></ul>
               <table class="highlight responsive-table">
                 <thead>
                   <tr>
                     <th class="sort" data-sort="filename">Filename</th>
-                    <th>{# Actions #}</th>
+                    <th></th>
                   </tr>
                 </thead>
-                <tbody class="list">
+                <tbody class="list"></tbody>
-                </tbody>
               </table>
-              <ul class="pagination paginationBottom"></ul>
+              <ul class="pagination"></ul>
             </div>
           </div>
         </div>
       </div>
     </div>
 
-    <div class="col s12">
+    <div class="col s12" id="job-results" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
       <div class="card">
         <div class="card-content">
           <div class="row">
@@ -144,24 +162,14 @@
               <table class="highlight responsive-table">
                 <thead>
                   <tr>
-                    <th>Result Type</th>
+                    <th>Description</th>
-                    <th>Archive Name</th>
+                    <th>Filename</th>
-                    <th>{# Actions #}</th>
+                    <th></th>
                   </tr>
                 </thead>
-                <tbody class="results">
+                <tbody class="list"></tbody>
-                  <tr class="show-if-only-child">
-                    <td colspan="3">
-                      <span class="card-title">
-                        <i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...
-                      </span>
-                      <p>
-                        No results available (yet). Is the job already completed?
-                      </p>
-                    </td>
-                  </tr>
-                </tbody>
               </table>
+              <ul class="pagination"></ul>
             </div>
           </div>
         </div>
@@ -169,158 +177,14 @@
     </div>
   </div>
 </div>
-
-<!-- Modals -->
-<div id="delete-job-modal" class="modal">
-  <div class="modal-content">
-    <h4>Confirm deletion</h4>
-    <p>Do you really want to delete the job {{ job.title }}? All associated files will be permanently deleted.</p>
-  </div>
-  <div class="modal-footer">
-    <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
-    <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
-  </div>
-</div>
 {% endblock page_content %}
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-import {RessourceList} from '../../static/js/nopaque.lists.js';
+  nopaque.appClient.loadUser({{ job.creator.id }});
-class InformationUpdater {
+  let jobDisplay = new JobDisplay(document.querySelector('#job-display'));
-  constructor(jobId, foreignJobFlag) {
+  let jobInputList = new JobInputList(document.querySelector('#job-inputs'));
-    this.jobId = jobId;
+  let jobResultList = new JobResultList(document.querySelector('#job-results'));
-    this.foreignJobFlag = foreignJobFlag;
-
-    if (this.foreignJobFlag) {
-      nopaque.foreignJobsSubscribers.push(this);
-    } else {
-      nopaque.jobsSubscribers.push(this);
-    }
-  }
-
-  _init() {
-    let job;
-
-    job = (this.foreignJobFlag ? nopaque.foreignUser.jobs[this.jobId]
-                               : nopaque.user.jobs[this.jobId]);
-    // Results
-    this.addResults(job.results);
-    // End date
-    this.setEndDate(job.end_date);
-    // Status
-    this.setStatus(job.status);
-  }
-
-  _update(patch) {
-    let pathArray;
-
-    for (let operation of patch) {
-      /* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
-      pathArray = operation.path.split("/").slice(2);
-      if (pathArray[0] != this.jobId) {continue;}
-      switch(operation.op) {
-        case "add":
-          if (pathArray[1] === "results") {
-            this.addResults([operation.value]);
-          }
-          break;
-        case "delete":
-          location.reload();
-          break;
-        case "replace":
-          if (pathArray[1] === "end_date") {
-            this.setEndDate(operation.value);
-          } else if (pathArray[1] === "status") {
-            this.setStatus(operation.value);
-          }
-          break;
-        default:
-          break;
-      }
-    }
-  }
-
-  addResults(results) {
-    let resultsArray, resultsElements, resultsHTML, resultType;
-    resultsArray = Object.values(results);
-    resultsArray.sort(function (a, b) {
-      if (a.filename < b.filename) {return -1;}
-      if (a.filename > b.filename) {return 1;}
-      return 0;
-    });
-    resultsHTML = ``;
-    for (let result of resultsArray) {
-      if (result.filename.endsWith(".pdf.zip")) {
-        resultType = "PDF file with text layer";
-      } else if (result.filename.endsWith(".txt.zip")) {
-        resultType = "Raw text files";
-      } else if (result.filename.endsWith(".vrt.zip")) {
-        resultType = "VRT(XML dialect) files holding the NLP data";
-      } else if (result.filename.endsWith(".xml.zip")) {
-        resultType = "XML files";
-      } else if (result.filename.endsWith(".poco.zip")) {
-        resultType = "HCOR und image files needed for Post correction(PoCo)";
-      } else {
-        resultType = "All result files created during this job";
-      }
-      resultsHTML += `
-                      <tr>
-                        <td>${resultType}</td>
-                        <td>${result.filename}</td>
-                        <td class="right-align">
-                          <a class="btn-floating tooltipped waves-effect waves-light"
-                             download href="/jobs/${result.job_id}/results/${result.id}/download"
-                             data-position="top"
-                             data-tooltip="Download">
-                            <i class="material-icons">file_download</i>
-                          </a>
-                        </td>
-                      </tr>
-                     `;
-    };
-    resultsHTML += `
-                    </tbody>
-                    </table>
-                   `;
-    resultsElements = document.querySelectorAll(".results");
-    for (let resultsElement of resultsElements) {
-      resultsElement.innerHTML += resultsHTML;
-    }
-  }
-
-  setEndDate(timestamp) {
-    let endDate;
-
-    if (timestamp === null) {
-      endDate = "N.a.";
-    } else {
-      endDate = new Date(timestamp * 1000).toLocaleString("en-US");
-    }
-    document.getElementById("end-date").value = endDate;
-    M.updateTextFields();
-  }
-
-  setStatus(status) {
-    let progressIndicator, statusElements;
-    if (status === "complete" || status === "failed") {
-      progressIndicator = document.getElementById("progress-indicator");
-      progressIndicator.classList.add("hide");
-    }
-    statusElements = document.querySelectorAll(".status");
-    for (let statusElement of statusElements) {
-      statusElement.dataset.status = status;
-    }
-  }
-}
-
-{% if job.creator == current_user %}
-var informationUpdater = new InformationUpdater({{ job.id }}, false);
-{% else %}
-var informationUpdater = new InformationUpdater({{ job.id }}, true);
-nopaque.socket.emit("foreign_user_data_stream_init", {{ job.user_id }});
-{% endif %}
-let jobInputsList = new RessourceList("inputs", null, "JobInput");
-jobInputsList._add({{ job_inputs|tojson|safe }});
 </script>
 {% endblock scripts %}

web/app/templates/main/dashboard.html.j2

@@ -29,8 +29,7 @@
                 <input id="search-corpus" class="search" type="search"></input>
                 <label for="search-corpus">Search corpus</label>
               </div>
-              <ul class="pagination paginationTop"></ul>
+              <table class="highlight ressource-list">
-              <table class="highlight">
                 <thead>
                   <tr>
                     <th></th>
@@ -44,7 +43,7 @@
                 </thead>
                 <tbody class="list"></tbody>
               </table>
-              <ul class="pagination paginationBottom"></ul>
+              <ul class="pagination"></ul>
             </div>
             <div class="card-action right-align">
               <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@@ -60,8 +59,7 @@
                 <input id="search-query-results" class="search" type="search"></input>
                 <label for="search-query-results">Search query result</label>
               </div>
-              <ul class="pagination paginationTop"></ul>
+              <table class="highlight ressource-list">
-              <table class="highlight responsive-table">
                 <thead>
                   <tr>
                     <th>
@@ -72,7 +70,7 @@
                       <span class="sort" data-sort="corpus">Corpus</span> and<br>
                       <span class="sort" data-sort="query">Query</span>
                     </th>
-                    <th>{# Actions #}</th>
+                    <th></th>
                   </tr>
                 </thead>
                 <tbody class="list">
@@ -84,7 +82,7 @@
                   </tr>
                 </tbody>
               </table>
-              <ul class="pagination paginationBottom"></ul>
+              <ul class="pagination"></ul>
             </div>
             <div class="card-action right-align">
               <a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@@ -104,8 +102,7 @@
             <input id="search-job" class="search" type="search"></input>
             <label for="search-job">Search job</label>
           </div>
-          <ul class="pagination paginationTop"></ul>
+          <table class="highlight ressource-list">
-          <table class="highlight">
             <thead>
               <tr>
                 <th><span class="sort" data-sort="service">Service</span></th>
@@ -119,12 +116,13 @@
             </thead>
             <tbody class="list"></tbody>
           </table>
-          <ul class="pagination paginationBottom"></ul>
+          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
           <p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p>
         </div>
       </div>
+
       <div id="new-job-modal" class="modal">
         <div class="modal-content">
           <h4>Select a service</h4>
@@ -178,10 +176,9 @@
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-  import {RessourceList} from '../../static/js/nopaque.lists.js';
+  let corpusList = new CorpusList(document.querySelector('#corpora'));
-  let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus");
+  let jobList = new JobList(document.querySelector('#jobs'));
-  let jobList = new RessourceList("jobs", nopaque.jobsSubscribers, "Job");
+  let queryResultList = new QueryResultList(document.querySelector('#query-results'));
-  let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
 </script>
 {% endblock scripts %}

web/app/templates/main/index.html.j2

@@ -159,20 +159,20 @@
             <form method="POST">
               <div class="card-content">
                 <span class="card-title">Log in</span>
-                {{ login_form.hidden_tag() }}
+                {{ form.hidden_tag() }}
-                {{ wtf.render_field(login_form.user, material_icon='person') }}
+                {{ wtf.render_field(form.user, material_icon='person') }}
-                {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+                {{ wtf.render_field(form.password, material_icon='vpn_key') }}
                 <div class="row" style="margin-bottom: 0;">
                   <div class="col s6 left-align">
                     <a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
                   </div>
                   <div class="col s6 right-align">
-                    {{ wtf.render_field(login_form.remember_me) }}
+                    {{ wtf.render_field(form.remember_me) }}
                   </div>
                 </div>
               </div>
               <div class="card-action right-align">
-                {{ wtf.render_field(login_form.submit, material_icon='send') }}
+                {{ wtf.render_field(form.submit, material_icon='send') }}
               </div>
             </form>
           </div>

web/app/templates/modals/analysis_init.html.j2

@@ -5,7 +5,7 @@
     <p>If the loading takes to long or an error occured,
       <a onclick="window.location.reload()" href="#">click here</a>
       to refresh your session or
-      <a href="{{ url_for('corpora.corpus', corpus_id=corpus_id) }}">go back</a>!
+      <a href="{{ url_for('corpora.corpus', corpus_id=corpus.id) }}">go back</a>!
     </p>
     <div id="analysis-init-progress" class="progress">
       <div class="indeterminate"></div>

web/app/templates/nopaque.html.j2

@@ -150,7 +150,7 @@
   {% if current_user.is_administrator() %}
   <li><div class="divider"></div></li>
   <li><a class="subheader">Administration</a></li>
-  <li><a href="{{ url_for('admin.users') }}"><i class="material-icons">build</i>Administration tools</a></li>
+  <li><a href="{{ url_for('admin.index') }}"><i class="material-icons">build</i>Administration</a></li>
   {% endif %}
 </ul>
 {% endblock sidenav %}
@@ -231,9 +231,9 @@
       </div>
       <div class="col s12 m9 right-align">
         <a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
-        {% if config.CONTACT_EMAIL_ADRESS %}
+        {% if config.NOPAQUE_CONTACT %}
-        <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
+        <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
-        <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
+        <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
         {% endif %}
         <a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
       </div>
@@ -244,28 +244,39 @@
 
 {% block scripts %}
 {{ super() }}
+{% if current_user.setting_dark_mode %}
 <script src="{{ url_for('static', filename='js/darkreader.js') }}"></script>
+<script>
+  DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
+</script>
+{% endif %}
 <script src="{{ url_for('static', filename='js/jsonpatch.min.js') }}"></script>
 <script src="{{ url_for('static', filename='js/list.min.js') }}"></script>
-<script src="{{ url_for('static', filename='js/socket.io.slim.js') }}"></script>
+<script src="{{ url_for('static', filename='js/socket.io.min.js') }}"></script>
-<script src="{{ url_for('static', filename='js/nopaque.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/main.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/displays/RessourceDisplay.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/displays/CorpusDisplay.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/displays/JobDisplay.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/RessourceList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusFileList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/JobList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/JobInputList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/JobResultList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/QueryResultList.js') }}"></script>
+<script src="{{ url_for('static', filename='js/nopaque/lists/UserList.js') }}"></script>
 <script>
-  {% if current_user.setting_dark_mode %}
-  DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
-  {% endif %}
   // Disable all option elements with no value
-  for (let optionElement of document.querySelectorAll('option[value=""]')) {
+  for (let optionElement of document.querySelectorAll('option[value=""]')) {optionElement.disabled = true;}
-    optionElement.disabled = true;
-  }
   M.AutoInit();
   M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="email"], input[data-length][type="password"], input[data-length][type="text"], textarea[data-length]'));
   M.Dropdown.init(document.querySelectorAll('#nav-more-dropdown-trigger'), {alignment: 'right', constrainWidth: false, coverTrigger: false});
   nopaque.Forms.init();
+  for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {nopaque.flash(flashedMessage[1], flashedMessage[0]);}
+</script>
+<script>
   {% if current_user.is_authenticated %}
-  nopaque.socket.emit('user_data_stream_init');
+  nopaque.appClient = new AppClient({{ current_user.id }});
   {% endif %}
-  for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {
-    nopaque.flash(flashedMessage[1], flashedMessage[0]);
-  }
 </script>
 {% endblock scripts %}

web/app/templates/services/corpus_analysis.html.j2

@@ -29,17 +29,16 @@
       <p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p>
     </div>
 
-    <div class="col s12">
+    <div class="col s12" id="corpora">
       <h2>My Corpora</h2>
       <div class="card">
-        <div class="card-content" id="corpora">
+        <div class="card-content">
           <div class="input-field">
             <i class="material-icons prefix">search</i>
             <input id="search-corpus" class="search" type="search"></input>
             <label for="search-corpus">Search corpus</label>
           </div>
-          <ul class="pagination paginationTop"></ul>
+          <table class="highlight ressource-list">
-          <table>
             <thead>
               <tr>
                 <th></th>
@@ -53,7 +52,7 @@
             </thead>
             <tbody class="list"></tbody>
           </table>
-          <ul class="pagination paginationBottom"></ul>
+          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
           <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@@ -62,17 +61,16 @@
       </div>
     </div>
 
-    <div class="col s12">
+    <div class="col s12" id="query-results">
       <h2>My query results</h2>
       <div class="card">
-        <div class="card-content" id="query-results">
+        <div class="card-content">
           <div class="input-field">
             <i class="material-icons prefix">search</i>
             <input id="search-query-results" class="search" type="search"></input>
             <label for="search-query-results">Search query result</label>
           </div>
-          <ul class="pagination paginationTop"></ul>
+          <table class="highlight ressource-list">
-          <table class="highlight responsive-table">
             <thead>
               <tr>
                 <th>
@@ -83,19 +81,12 @@
                   <span class="sort" data-sort="corpus">Corpus</span> and<br>
                   <span class="sort" data-sort="query">Query</span>
                 </th>
-                <th>{# Actions #}</th>
+                <th></th>
               </tr>
             </thead>
-            <tbody class="list">
+            <tbody class="list"></tbody>
-              <tr class="show-if-only-child">
-                <td colspan="5">
-                  <span class="card-title"><i class="material-icons left">folder</i>Nothing here...</span>
-                  <p>No query results yet imported.</p>
-                </td>
-              </tr>
-            </tbody>
           </table>
-          <ul class="pagination paginationBottom"></ul>
+          <ul class="pagination"></ul>
         </div>
         <div class="card-action right-align">
           <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@@ -108,9 +99,8 @@
 
 {% block scripts %}
 {{ super() }}
-<script type="module">
+<script>
-  import {RessourceList} from '../../static/js/nopaque.lists.js';
+  let corpusList = new CorpusList(document.querySelector('#corpora'));
-  let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus");
+  let queryResultList = new QueryResultList(document.querySelector('#query-results'));
-  let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
 </script>
 {% endblock scripts %}

web/app/templates/services/file-setup.html.j2

@@ -48,24 +48,24 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
+                {{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
               </div>
               <div class="col s12 hide">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/services/nlp.html.j2

@@ -66,34 +66,34 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12 l5">
-                {{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
+                {{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
               </div>
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+                {{ wtf.render_field(form.language, material_icon='language') }}
               </div>
               <div class="col s12 l3">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
               <div class="col s12">
                 <span class="card-title">Preprocessing</span>
               </div>
               <div class="col s9">
-                <p>{{ add_job_form.check_encoding.label.text }}</p>
+                <p>{{ form.check_encoding.label.text }}</p>
                 <p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
               </div>
               <div class="col s3 right-align">
                 <div class="switch">
                   <label>
-                    {{ add_job_form.check_encoding() }}
+                    {{ form.check_encoding() }}
                     <span class="lever"></span>
                   </label>
                 </div>
@@ -107,7 +107,7 @@
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/services/ocr.html.j2

@@ -48,34 +48,34 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12 l5">
-                {{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
+                {{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
               </div>
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+                {{ wtf.render_field(form.language, material_icon='language') }}
               </div>
               <div class="col s12 l3">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
               <div class="col s12">
                 <span class="card-title">Preprocessing</span>
               </div>
               <div class="col s9">
-                <p>{{ add_job_form.binarization.label.text }}</p>
+                <p>{{ form.binarization.label.text }}</p>
                 <p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
               </div>
               <div class="col s3 right-align">
                 <div class="switch">
                   <label>
-                    {{ add_job_form.binarization() }}
+                    {{ form.binarization() }}
                     <span class="lever"></span>
                   </label>
                 </div>
@@ -134,7 +134,7 @@
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
+            {{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/tasks/email/notification.html.j2

@@ -0,0 +1,8 @@
+<p>Dear <b>{{ job.creator.username }}</b>,</p>
+
+<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
+<p>It is now <b>{{ job.status }}</b>!</p>
+
+<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
+
+<p>Kind regards!<br>Your nopaque team</p>

web/app/templates/tasks/email/notification.txt.j2

@@ -0,0 +1,9 @@
+Dear {{ job.creator.username }},
+
+The status of your Job "{{ job.title }}" has changed!
+It is now {{ job.status }}!
+
+You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
+
+Kind regards!
+Your nopaque team

web/boot.sh

@@ -1,20 +1,26 @@
 #!/bin/bash
+
 source venv/bin/activate
-export FLASK_APP=nopaque.py
+
-if [[ "$#" -eq 0 ]]; then
+if [[ "${NOPAQUE_DAEMON_ENABLED:-True}" == "True" ]]; then
+    echo "INFO  Starting nopaque daemon process..."
+    ./nopaque-daemon.sh &
+fi
+
+if [[ "${#}" -eq 0 ]]; then
     while true; do
         flask deploy
-        if [[ "$?" == "0" ]]; then
+        if [[ "${?}" == "0" ]]; then
             break
         fi
-        echo Deploy command failed, retrying in 5 secs...
+        echo "Deploy command failed, retrying in 5 secs..."
         sleep 5
     done
     python nopaque.py
-elif [[ "$1" == "flask" ]]; then
+elif [[ "${1}" == "flask" ]]; then
     exec ${@:1}
 else
-    echo "$0 [COMMAND]"
+    echo "${0} [COMMAND]"
     echo ""
     echo "nopaque startup script"
     echo ""