More exception handling. Remove unused database models. New common view structure!

.env.tpl

@@ -9,128 +9,116 @@
 # NOTE: Use `.` as <project-root-dir>
 # HOST_MQ_DIR=
 
-# Example: 999
+# Example: 1000
-# HINT: Use this bash command `getent group docker | cut -d: -f3`
+# HINT: Use this bash command `id -u`
-HOST_DOCKER_GID=
+HOST_UID=
 
 # Example: 1000
 # HINT: Use this bash command `id -g`
 HOST_GID=
 
-# DEFAULT: ./nopaqued.log
+# Example: 999
-# NOTES: Use `.` as <project-root-dir>,
+# HINT: Use this bash command `getent group docker | cut -d: -f3`
-#        This file must be present on container startup
+HOST_DOCKER_GID=
-# HOST_NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: ./nopaque.log
 # NOTES: Use `.` as <project-root-dir>,
 #        This file must be present on container startup
-# HOST_NOPAQUE_LOG_FILE=
+# HOST_LOG_FILE=
-
-# Example: 1000
-# HINT: Use this bash command `id -u`
-HOST_UID=
 
 
 ################################################################################
-# Cookies                                                                      #
+# Flask                                                                        #
+# https://flask.palletsprojects.com/en/1.1.x/config/                           #
 ################################################################################
-# CHOOSE ONE: False, True
+# DEFAULT: hard to guess string
-# DEFAULT: False
+# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
-# HINT: Set to true if you redirect http to https
+# SECRET_KEY=
-# NOPAQUE_REMEMBER_COOKIE_SECURE=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
 # HINT: Set to true if you redirect http to https
-# NOPAQUE_SESSION_COOKIE_SECURE=
+# SESSION_COOKIE_SECURE=
 
 
 ################################################################################
-# Database                                                                     #
+# Flask-Login                                                                  #
-# DATABASE_URI blueprint:                                                      #
+# https://flask-login.readthedocs.io/en/latest/                                #
-#   - dialect[+driver]://username:password@host[:port]/database                #
-#     - sqlite is not supported                                                #
-#     - values in square brackets are optional                                 #
 ################################################################################
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
+# CHOOSE ONE: False, True
-# NOPAQUE_DATABASE_URL=
+# DEFAULT: False
-
+# HINT: Set to true if you redirect http to https
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
+# REMEMBER_COOKIE_SECURE=
-# NOPAQUE_DEV_DATABASE_URL=
-
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
-# NOPAQUE_TEST_DATABASE_URL=
 
 
 ################################################################################
-# Email                                                                        #
+# Flask-Mail                                                                   #
+# https://pythonhosted.org/Flask-Mail/                                         #
 ################################################################################
 # EXAMPLE: nopaque Admin <nopaque@example.com>
-NOPAQUE_SMTP_DEFAULT_SENDER=
+MAIL_DEFAULT_SENDER=
 
-NOPAQUE_SMTP_PASSWORD=
+MAIL_PASSWORD=
 
 # EXAMPLE: smtp.example.com
-NOPAQUE_SMTP_SERVER=
+MAIL_SERVER=
 
 # EXAMPLE: 587
-NOPAQUE_SMTP_PORT=
+MAIL_PORT=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
-# NOPAQUE_SMTP_USE_SSL=
+# MAIL_USE_SSL=
 
 # CHOOSE ONE: False, True
 # DEFAULT: False
-# NOPAQUE_SMTP_USE_TLS=
+# MAIL_USE_TLS=
 
 # EXAMPLE: nopaque@example.com
-NOPAQUE_SMTP_USERNAME=
+MAIL_USERNAME=
 
 
 ################################################################################
-# General                                                                      #
+# Flask-SQLAlchemy                                                             #
+# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/                  #
 ################################################################################
+# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
+# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
+# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
+# SQLALCHEMY_DATABASE_URI=
+
+
+################################################################################
+# nopaque                                                                      #
+################################################################################
+# If an account is registered with this email adress gets automatically
+# assigned the administrator role.
 # EXAMPLE: admin.nopaque@example.com
-NOPAQUE_ADMIN_EMAIL_ADRESS=
+NOPAQUE_ADMIN=
 
 # DEFAULT: development
 # CHOOSE ONE: development, production, testing
 # NOPAQUE_CONFIG=
 
+# This email adress is used for the contact button in the nopaque footer. If
+# not set, no contact button is displayed.
 # DEFAULT: None
 # EXAMPLE: contact.nopaque@example.com
-# NOPAQUE_CONTACT_EMAIL_ADRESS=
+# NOPAQUE_CONTACT=
 
 # DEFAULT: /mnt/nopaque
-# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
+# NOTE: This must be a network share and it must be available on all Docker
+#       Swarm nodes
 # NOPAQUE_DATA_DIR=
 
-# DEFAULT: localhost
-# NOPAQUE_DOMAIN=
-
 # DEFAULT: 0.0.0.0
 # NOPAQUE_HOST=
 
 # DEFAULT: 5000
 # NOPAQUE_PORT=
 
-# CHOOSE ONE: http, https
+# transport://[userid:password]@hostname[:port]/[virtual_host]
-# DEFAULT: http
+NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-# NOPAQUE_PROTOCOL=
-
-# DEFAULT: hard to guess string
-# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
-# NOPAQUE_SECRET_KEY=
-
-
-################################################################################
-# Logging                                                                      #
-################################################################################
-# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
-# NOTE: Use `.` as <nopaqued-root-dir>
-# NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: %Y-%m-%d %H:%M:%S
 # NOPAQUE_LOG_DATE_FORMAT=
@@ -146,37 +134,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
 # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
 # NOPAQUE_LOG_LEVEL=
 
-
-################################################################################
-# Message queue                                                                #
-# MESSAGE_QUEUE_URI blueprint:                                                 #
-#   - transport://[userid:password]@hostname[:port]/[virtual_host]             #
-#     - values in square brackets are optional                                 #
-################################################################################
-# DEFAULT: None
-# HINT: A message queue is not required when using a single server process
-# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-
-
-################################################################################
-# Proxy fix                                                                    #
-################################################################################
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-For
-# NOPAQUE_NUM_PROXIES_X_FOR=
+# NOPAQUE_PROXY_FIX_X_FOR=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Host
-# NOPAQUE_NUM_PROXIES_X_HOST=
+# NOPAQUE_PROXY_FIX_X_HOST=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Port
-# NOPAQUE_NUM_PROXIES_X_PORT=
+# NOPAQUE_PROXY_FIX_X_PORT=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Prefix
-# NOPAQUE_NUM_PROXIES_X_PREFIX=
+# NOPAQUE_PROXY_FIX_X_PREFIX=
 
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-Proto
-# NOPAQUE_NUM_PROXIES_X_PROTO=
+# NOPAQUE_PROXY_FIX_X_PROTO=

docker-compose.yml

@@ -28,5 +28,6 @@ services:
     image: nopaque:development
     restart: unless-stopped
     volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
       - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
       - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"

web/Dockerfile

@@ -21,8 +21,9 @@ RUN apt-get update \
  && rm -r /var/lib/apt/lists/*
 
 
-RUN groupadd --gid ${GID} --system nopaque \
+RUN groupadd --gid ${DOCKER_GID} --system docker \
- && useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
+ && groupadd --gid ${GID} --system nopaque \
+ && useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
 USER nopaque
 WORKDIR /home/nopaque
 

web/app/__init__.py

@@ -26,7 +26,7 @@ def create_app(config_name):
     mail.init_app(app)
     paranoid.init_app(app)
     socketio.init_app(
-        app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
+        app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
 
     with app.app_context():
         from . import events

web/app/admin/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 admin = Blueprint('admin', __name__)
-from . import views  # noqa
+from . import views

web/app/admin/forms.py

@@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
         super().__init__(*args, user=user, **kwargs)
         self.role.choices = [(role.id, role.name)
                              for role in Role.query.order_by(Role.name).all()]
-        self.user = user

web/app/admin/views.py

@@ -29,12 +29,11 @@ def user(user_id):
 @admin_required
 def delete_user(user_id):
     settings_tasks.delete_user(user_id)
-    flash('User has been deleted!')
+    flash('User has been marked for deletion!')
     return redirect(url_for('.users'))
 
 
-@admin.route('/users/<int:user_id>/edit_general_settings',
+@admin.route('/users/<int:user_id>/edit_general_settings', methods=['GET', 'POST'])  # noqa
-             methods=['GET', 'POST'])
 @login_required
 @admin_required
 def edit_general_settings(user_id):
@@ -46,16 +45,13 @@ def edit_general_settings(user_id):
         user.username = form.username.data
         user.confirmed = form.confirmed.data
         user.role = Role.query.get(form.role.data)
-        db.session.add(user)
         db.session.commit()
-        flash('The profile has been updated.')
+        flash('Settings have been updated.')
-        return redirect(url_for('admin.edit_general_settings', user_id=user.id))
+        return redirect(url_for('.edit_general_settings', user_id=user.id))
     form.confirmed.data = user.confirmed
     form.dark_mode.data = user.setting_dark_mode
     form.email.data = user.email
     form.role.data = user.role_id
     form.username.data = user.username
     return render_template('admin/edit_general_settings.html.j2',
-                           form=form,
+                           form=form, title='General settings', user=user)
-                           title='General settings',
-                           user=user)

web/app/auth/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 auth = Blueprint('auth', __name__)
-from . import views  # noqa
+from . import views

web/app/auth/forms.py

@@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
     username = StringField(
         'Username',
         validators=[DataRequired(), Length(1, 64),
-                    Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+                    Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
                            message='Usernames must have only letters, numbers,'
                                    ' dots or underscores')]
     )

web/app/auth/views.py

@@ -1,5 +1,5 @@
-from flask import (current_app, flash, redirect, render_template, request,
+from datetime import datetime
-                   url_for)
+from flask import abort, flash, redirect, render_template, request, url_for
 from flask_login import current_user, login_user, login_required, logout_user
 from . import auth
 from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
 from .. import db
 from ..email import create_message, send
 from ..models import User
+import logging
 import os
-import shutil
 
 
 @auth.before_app_request
@@ -18,11 +18,12 @@ def before_request():
     unconfirmed view if user is unconfirmed.
     """
     if current_user.is_authenticated:
-        current_user.ping()
+        current_user.last_seen = datetime.utcnow()
-        if not current_user.confirmed \
+        db.session.commit()
-                and request.endpoint \
+        if (not current_user.confirmed
-                and request.blueprint != 'auth' \
+                and request.endpoint
-                and request.endpoint != 'static':
+                and request.blueprint != 'auth'
+                and request.endpoint != 'static'):
             return redirect(url_for('auth.unconfirmed'))
 
 
@@ -30,20 +31,19 @@ def before_request():
 def login():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    login_form = LoginForm(prefix='login-form')
+    form = LoginForm(prefix='login-form')
-    if login_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User.query.filter_by(username=login_form.user.data).first()
+        user = User.query.filter_by(username=form.user.data).first()
         if user is None:
-            user = User.query.filter_by(email=login_form.user.data).first()
+            user = User.query.filter_by(email=form.user.data.lower()).first()
-        if user is not None and user.verify_password(login_form.password.data):
+        if user is not None and user.verify_password(form.password.data):
-            login_user(user, login_form.remember_me.data)
+            login_user(user, form.remember_me.data)
             next = request.args.get('next')
             if next is None or not next.startswith('/'):
                 next = url_for('main.dashboard')
             return redirect(next)
         flash('Invalid email/username or password.')
-    return render_template('auth/login.html.j2', login_form=login_form,
+    return render_template('auth/login.html.j2', form=form, title='Log in')
-                           title='Log in')
 
 
 @auth.route('/logout')
@@ -58,26 +58,28 @@ def logout():
 def register():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    registration_form = RegistrationForm(prefix='registration-form')
+    form = RegistrationForm(prefix='registration-form')
-    if registration_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User(email=registration_form.email.data.lower(),
+        user = User(email=form.email.data.lower(),
-                    password=registration_form.password.data,
+                    password=form.password.data,
-                    username=registration_form.username.data)
+                    username=form.username.data)
         db.session.add(user)
         db.session.commit()
-        user_dir = os.path.join(current_app.config['DATA_DIR'],
+        try:
-                                str(user.id))
+            os.makedirs(user.path)
-        if os.path.exists(user_dir):
+        except OSError:
-            shutil.rmtree(user_dir)
+            logging.error('Make dir {} led to an OSError!'.format(user.path))
-        os.mkdir(user_dir)
+            db.session.delete(user)
-        token = user.generate_confirmation_token()
+            db.session.commit()
-        msg = create_message(user.email, 'Confirm Your Account',
+            abort(500)
-                             'auth/email/confirm', token=token, user=user)
+        else:
-        send(msg)
+            token = user.generate_confirmation_token()
-        flash('A confirmation email has been sent to you by email.')
+            msg = create_message(user.email, 'Confirm Your Account',
-        return redirect(url_for('auth.login'))
+                                 'auth/email/confirm', token=token, user=user)
-    return render_template('auth/register.html.j2',
+            send(msg)
-                           registration_form=registration_form,
+            flash('A confirmation email has been sent to you by email.')
+            return redirect(url_for('.login'))
+    return render_template('auth/register.html.j2', form=form,
                            title='Register')
 
 
@@ -92,7 +94,7 @@ def confirm(token):
         return redirect(url_for('main.dashboard'))
     else:
         flash('The confirmation link is invalid or has expired.')
-        return redirect(url_for('auth.unconfirmed'))
+        return redirect(url_for('.unconfirmed'))
 
 
 @auth.route('/unconfirmed')
@@ -119,39 +121,32 @@ def resend_confirmation():
 def reset_password_request():
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    reset_password_request_form = ResetPasswordRequestForm(
+    form = ResetPasswordRequestForm(prefix='reset-password-request-form')
-        prefix='reset-password-request-form')
+    if form.validate_on_submit():
-    if reset_password_request_form.validate_on_submit():
+        user = User.query.filter_by(email=form.email.data.lower()).first()
-        submitted_email = reset_password_request_form.email.data
+        if user is not None:
-        user = User.query.filter_by(email=submitted_email.lower()).first()
-        if user:
             token = user.generate_reset_token()
             msg = create_message(user.email, 'Reset Your Password',
                                  'auth/email/reset_password', token=token,
                                  user=user)
             send(msg)
-        flash('An email with instructions to reset your password has been '
+        flash('An email with instructions to reset your password has been sent to you.')  # noqa
-              'sent to you.')
+        return redirect(url_for('.login'))
-        return redirect(url_for('auth.login'))
+    return render_template('auth/reset_password_request.html.j2', form=form,
-    return render_template(
+                           title='Password Reset')
-        'auth/reset_password_request.html.j2',
-        reset_password_request_form=reset_password_request_form,
-        title='Password Reset')
 
 
 @auth.route('/reset/<token>', methods=['GET', 'POST'])
 def reset_password(token):
     if current_user.is_authenticated:
         return redirect(url_for('main.dashboard'))
-    reset_password_form = ResetPasswordForm(prefix='reset-password-form')
+    form = ResetPasswordForm(prefix='reset-password-form')
-    if reset_password_form.validate_on_submit():
+    if form.validate_on_submit():
-        if User.reset_password(token, reset_password_form.password.data):
+        if User.reset_password(token, form.password.data):
             db.session.commit()
             flash('Your password has been updated.')
-            return redirect(url_for('auth.login'))
+            return redirect(url_for('.login'))
         else:
             return redirect(url_for('main.index'))
-    return render_template('auth/reset_password.html.j2',
+    return render_template('auth/reset_password.html.j2', form=form,
-                           reset_password_form=reset_password_form,
+                           title='Password Reset', token=token)
-                           title='Password Reset',
-                           token=token)

web/app/corpora/views.py

@@ -1,4 +1,4 @@
-from flask import (abort, current_app, flash, make_response, redirect, request,
+from flask import (abort, flash, make_response, redirect, request,
                    render_template, url_for, send_from_directory)
 from flask_login import current_user, login_required
 from . import corpora
@@ -11,6 +11,7 @@ from jsonschema import validate
 from .. import db
 from ..models import Corpus, CorpusFile, QueryResult
 import json
+import logging
 import os
 import shutil
 import glob
@@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
 @corpora.route('/add', methods=['GET', 'POST'])
 @login_required
 def add_corpus():
-    add_corpus_form = AddCorpusForm()
+    form = AddCorpusForm()
-    if add_corpus_form.validate_on_submit():
+    if form.validate_on_submit():
         corpus = Corpus(creator=current_user,
-                        description=add_corpus_form.description.data,
+                        description=form.description.data,
-                        status='unprepared', title=add_corpus_form.title.data)
+                        title=form.title.data)
         db.session.add(corpus)
         db.session.commit()
-        dir = os.path.join(current_app.config['DATA_DIR'],
-                           str(corpus.user_id), 'corpora', str(corpus.id))
         try:
-            os.makedirs(dir)
+            os.makedirs(corpus.path)
         except OSError:
-            flash('[ERROR]: Could not add corpus!', 'corpus')
+            logging.error('Make dir {} led to an OSError!'.format(corpus.path))
-            corpus.delete()
+            db.session.delete(corpus)
-        else:
+            db.session.commit()
-            url = url_for('corpora.corpus', corpus_id=corpus.id)
+            abort(500)
-            flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
+        flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
-                  'corpus')
+        return redirect(url_for('.corpus', corpus_id=corpus.id))
-            return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
+    return render_template('corpora/add_corpus.html.j2', form=form,
-    return render_template('corpora/add_corpus.html.j2',
-                           add_corpus_form=add_corpus_form,
                            title='Add corpus')
 
 
 @corpora.route('/import', methods=['GET', 'POST'])
 @login_required
 def import_corpus():
-    import_corpus_form = ImportCorpusForm()
+    form = ImportCorpusForm()
-    if import_corpus_form.is_submitted():
+    if form.is_submitted():
-        if not import_corpus_form.validate():
+        if not form.validate():
-            return make_response(import_corpus_form.errors, 400)
+            return make_response(form.errors, 400)
         corpus = Corpus(creator=current_user,
-                        description=import_corpus_form.description.data,
+                        description=form.description.data,
-                        status='unprepared',
+                        title=form.title.data)
-                        title=import_corpus_form.title.data)
         db.session.add(corpus)
         db.session.commit()
-        dir = os.path.join(current_app.config['DATA_DIR'],
-                           str(corpus.user_id), 'corpora', str(corpus.id))
         try:
-            os.makedirs(dir)
+            os.makedirs(corpus.path)
         except OSError:
-            flash('[ERROR]: Could not import corpus!', 'corpus')
+            logging.error('Make dir {} led to an OSError!'.format(corpus.path))
-            corpus.delete()
+            db.session.delete(corpus)
+            db.session.commit()
+            flash('Internal Server Error', 'error')
+            return make_response(
+                {'redirect_url': url_for('.import_corpus')}, 500)
+        # Upload zip
+        archive_file = os.path.join(corpus.path, form.file.data.filename)
+        form.file.data.save(archive_file)
+        # Some checks to verify it is a valid exported corpus
+        with ZipFile(archive_file, 'r') as zip:
+            contents = zip.namelist()
+        if set(check_zip_contents).issubset(contents):
+            # Unzip
+            shutil.unpack_archive(archive_file, corpus.path)
+            # Register vrt files to corpus
+            vrts = glob.glob(corpus.path + '/*.vrt')
+            for file in vrts:
+                element_tree = ET.parse(file)
+                text_node = element_tree.find('text')
+                corpus_file = CorpusFile(
+                    address=text_node.get('address',  'NULL'),
+                    author=text_node.get('author', 'NULL'),
+                    booktitle=text_node.get('booktitle',  'NULL'),
+                    chapter=text_node.get('chapter',  'NULL'),
+                    corpus=corpus,
+                    editor=text_node.get('editor',  'NULL'),
+                    filename=os.path.basename(file),
+                    institution=text_node.get('institution',  'NULL'),
+                    journal=text_node.get('journal',  'NULL'),
+                    pages=text_node.get('pages',  'NULL'),
+                    publisher=text_node.get('publisher',  'NULL'),
+                    publishing_year=text_node.get('publishing_year', ''),
+                    school=text_node.get('school',  'NULL'),
+                    title=text_node.get('title', 'NULL')
+                )
+                db.session.add(corpus_file)
+            # finish import and redirect to imported corpus
+            corpus.status = 'prepared'
+            db.session.commit()
+            os.remove(archive_file)
+            flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
+            return make_response(
+                {'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
         else:
-            # Upload zip
+            # If imported zip is not valid delete corpus and give feedback
-            archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
+            flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error')  # noqa
-                                        import_corpus_form.file.data.filename)
+            tasks.delete_corpus(corpus.id)
-            corpus_dir = os.path.dirname(archive_file)
+            return make_response(
-            import_corpus_form.file.data.save(archive_file)
+                {'redirect_url': url_for('.import_corpus')}, 201)
-            # Some checks to verify it is a valid exported corpus
+    return render_template('corpora/import_corpus.html.j2', form=form,
-            with ZipFile(archive_file, 'r') as zip:
-                contents = zip.namelist()
-            if set(check_zip_contents).issubset(contents):
-                # Unzip
-                shutil.unpack_archive(archive_file, corpus_dir)
-                # Register vrt files to corpus
-                vrts = glob.glob(corpus_dir + '/*.vrt')
-                for file in vrts:
-                    element_tree = ET.parse(file)
-                    text_node = element_tree.find('text')
-                    corpus_file = CorpusFile(
-                        address=text_node.get('address',  'NULL'),
-                        author=text_node.get('author', 'NULL'),
-                        booktitle=text_node.get('booktitle',  'NULL'),
-                        chapter=text_node.get('chapter',  'NULL'),
-                        corpus=corpus,
-                        dir=dir,
-                        editor=text_node.get('editor',  'NULL'),
-                        filename=os.path.basename(file),
-                        institution=text_node.get('institution',  'NULL'),
-                        journal=text_node.get('journal',  'NULL'),
-                        pages=text_node.get('pages',  'NULL'),
-                        publisher=text_node.get('publisher',  'NULL'),
-                        publishing_year=text_node.get('publishing_year', ''),
-                        school=text_node.get('school',  'NULL'),
-                        title=text_node.get('title', 'NULL'))
-                    db.session.add(corpus_file)
-                # finish import and got to imported corpus
-                url = url_for('corpora.corpus', corpus_id=corpus.id)
-                corpus.status = 'prepared'
-                db.session.commit()
-                os.remove(archive_file)
-                flash('[<a href="{}">{}</a>] imported'.format(url,
-                                                              corpus.title),
-                      'corpus')
-                return make_response(
-                    {'redirect_url': url_for('corpora.corpus',
-                                             corpus_id=corpus.id)},
-                    201)
-            else:
-                # If imported zip is not valid delete corpus and give feedback
-                corpus.delete()
-                db.session.commit()
-                flash('Imported corpus is not valid.', 'error')
-                return make_response(
-                    {'redirect_url': url_for('corpora.import_corpus')},
-                    201)
-    return render_template('corpora/import_corpus.html.j2',
-                           import_corpus_form=import_corpus_form,
                            title='Import Corpus')
 
 
@@ -131,17 +118,9 @@ def corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
-    corpus_files = [dict(filename=corpus_file.filename,
+    corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
-                         author=corpus_file.author,
+    return render_template('corpora/corpus.html.j2', corpus=corpus,
-                         title=corpus_file.title,
+                           corpus_files=corpus_files, title='Corpus')
-                         publishing_year=corpus_file.publishing_year,
-                         corpus_id=corpus.id,
-                         id=corpus_file.id)
-                    for corpus_file in corpus.files]
-    return render_template('corpora/corpus.html.j2',
-                           corpus=corpus,
-                           corpus_files=corpus_files,
-                           title='Corpus')
 
 
 @corpora.route('/<int:corpus_id>/export')
@@ -150,12 +129,11 @@ def export_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
+    # TODO: Check what happens here
     dir = os.path.dirname(corpus.archive_file)
     filename = os.path.basename(corpus.archive_file)
-    return send_from_directory(directory=dir,
+    return send_from_directory(as_attachment=True, directory=dir,
-                               filename=filename,
+                               filename=filename, mimetype='zip')
-                               mimetype='zip',
-                               as_attachment=True)
 
 
 @corpora.route('/<int:corpus_id>/analyse')
@@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
     display_options_form = DisplayOptionsForm(
         prefix='display-options-form',
         result_context=request.args.get('context', 20),
-        results_per_page=request.args.get('results_per_page', 30))
+        results_per_page=request.args.get('results_per_page', 30)
+    )
     query_form = QueryForm(prefix='query-form',
                            query=request.args.get('query'))
     query_download_form = QueryDownloadForm(prefix='query-download-form')
@@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
     return render_template(
         'corpora/analyse_corpus.html.j2',
         corpus=corpus,
-        corpus_id=corpus_id,
         display_options_form=display_options_form,
+        inspect_display_options_form=inspect_display_options_form,
         query_form=query_form,
         query_download_form=query_download_form,
-        inspect_display_options_form=inspect_display_options_form,
+        title='Corpus analysis'
-        title='Corpus analysis')
+    )
 
 
 @corpora.route('/<int:corpus_id>/delete')
@@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
+    flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
     tasks.delete_corpus(corpus_id)
-    flash('Corpus deleted!', 'corpus')
     return redirect(url_for('main.dashboard'))
 
 
@@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
-    add_corpus_file_form = AddCorpusFileForm(corpus,
+    form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
-                                             prefix='add-corpus-file-form')
+    if form.is_submitted():
-    if add_corpus_file_form.is_submitted():
+        if not form.validate():
-        if not add_corpus_file_form.validate():
+            return make_response(form.errors, 400)
-            return make_response(add_corpus_file_form.errors, 400)
         # Save the file
-        dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
+        form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
-        add_corpus_file_form.file.data.save(
+        corpus_file = CorpusFile(address=form.address.data,
-            os.path.join(current_app.config['DATA_DIR'], dir,
+                                 author=form.author.data,
-                         add_corpus_file_form.file.data.filename))
+                                 booktitle=form.booktitle.data,
-        corpus_file = CorpusFile(
+                                 chapter=form.chapter.data,
-            address=add_corpus_file_form.address.data,
+                                 corpus=corpus,
-            author=add_corpus_file_form.author.data,
+                                 editor=form.editor.data,
-            booktitle=add_corpus_file_form.booktitle.data,
+                                 filename=form.file.data.filename,
-            chapter=add_corpus_file_form.chapter.data,
+                                 institution=form.institution.data,
-            corpus=corpus,
+                                 journal=form.journal.data,
-            dir=dir,
+                                 pages=form.pages.data,
-            editor=add_corpus_file_form.editor.data,
+                                 publisher=form.publisher.data,
-            filename=add_corpus_file_form.file.data.filename,
+                                 publishing_year=form.publishing_year.data,
-            institution=add_corpus_file_form.institution.data,
+                                 school=form.school.data,
-            journal=add_corpus_file_form.journal.data,
+                                 title=form.title.data)
-            pages=add_corpus_file_form.pages.data,
-            publisher=add_corpus_file_form.publisher.data,
-            publishing_year=add_corpus_file_form.publishing_year.data,
-            school=add_corpus_file_form.school.data,
-            title=add_corpus_file_form.title.data)
         db.session.add(corpus_file)
         corpus.status = 'unprepared'
         db.session.commit()
-        flash('Corpus file added!', 'corpus')
+        flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
-        return make_response(
+        return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)  # noqa
-            {'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
+    return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
-            201)
+                           form=form, title='Add corpus file')
-    return render_template('corpora/add_corpus_file.html.j2',
-                           corpus=corpus,
-                           add_corpus_file_form=add_corpus_file_form,
-                           title='Add corpus file')
 
 
 @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
+    flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus')  # noqa
     tasks.delete_corpus_file(corpus_file_id)
-    flash('Corpus file deleted!', 'corpus')
+    return redirect(url_for('.corpus', corpus_id=corpus_id))
-    return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
 
 
 @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       corpus_file.dir)
+                               directory=corpus_file.corpus.path,
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=corpus_file.filename)
 
 
@@ -274,48 +242,45 @@ def download_corpus_file(corpus_id, corpus_file_id):
                methods=['GET', 'POST'])
 @login_required
 def corpus_file(corpus_id, corpus_file_id):
-    corpus = Corpus.query.get_or_404(corpus_id)
     corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-    if not corpus_file.corpus_id == corpus_id:
+    if corpus_file.corpus_id != corpus_id:
         abort(404)
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
+    form = EditCorpusFileForm(prefix='edit-corpus-file-form')
-    if edit_corpus_file_form.validate_on_submit():
+    if form.validate_on_submit():
-        corpus_file.address = edit_corpus_file_form.address.data
+        corpus_file.address = form.address.data
-        corpus_file.author = edit_corpus_file_form.author.data
+        corpus_file.author = form.author.data
-        corpus_file.booktitle = edit_corpus_file_form.booktitle.data
+        corpus_file.booktitle = form.booktitle.data
-        corpus_file.chapter = edit_corpus_file_form.chapter.data
+        corpus_file.chapter = form.chapter.data
-        corpus_file.editor = edit_corpus_file_form.editor.data
+        corpus_file.editor = form.editor.data
-        corpus_file.institution = edit_corpus_file_form.institution.data
+        corpus_file.institution = form.institution.data
-        corpus_file.journal = edit_corpus_file_form.journal.data
+        corpus_file.journal = form.journal.data
-        corpus_file.pages = edit_corpus_file_form.pages.data
+        corpus_file.pages = form.pages.data
-        corpus_file.publisher = edit_corpus_file_form.publisher.data
+        corpus_file.publisher = form.publisher.data
-        corpus_file.publishing_year = \
+        corpus_file.publishing_year = form.publishing_year.data
-            edit_corpus_file_form.publishing_year.data
+        corpus_file.school = form.school.data
-        corpus_file.school = edit_corpus_file_form.school.data
+        corpus_file.title = form.title.data
-        corpus_file.title = edit_corpus_file_form.title.data
         corpus.status = 'unprepared'
         db.session.commit()
-        flash('Corpus file edited!', 'corpus')
+        flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus')  # noqa
-        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+        return redirect(url_for('.corpus', corpus_id=corpus_id))
     # If no form is submitted or valid, fill out fields with current values
-    edit_corpus_file_form.address.data = corpus_file.address
+    form.address.data = corpus_file.address
-    edit_corpus_file_form.author.data = corpus_file.author
+    form.author.data = corpus_file.author
-    edit_corpus_file_form.booktitle.data = corpus_file.booktitle
+    form.booktitle.data = corpus_file.booktitle
-    edit_corpus_file_form.chapter.data = corpus_file.chapter
+    form.chapter.data = corpus_file.chapter
-    edit_corpus_file_form.editor.data = corpus_file.editor
+    form.editor.data = corpus_file.editor
-    edit_corpus_file_form.institution.data = corpus_file.institution
+    form.institution.data = corpus_file.institution
-    edit_corpus_file_form.journal.data = corpus_file.journal
+    form.journal.data = corpus_file.journal
-    edit_corpus_file_form.pages.data = corpus_file.pages
+    form.pages.data = corpus_file.pages
-    edit_corpus_file_form.publisher.data = corpus_file.publisher
+    form.publisher.data = corpus_file.publisher
-    edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
+    form.publishing_year.data = corpus_file.publishing_year
-    edit_corpus_file_form.school.data = corpus_file.school
+    form.school.data = corpus_file.school
-    edit_corpus_file_form.title.data = corpus_file.title
+    form.title.data = corpus_file.title
-    return render_template('corpora/corpus_file.html.j2',
+    return render_template('corpora/corpus_file.html.j2', corpus=corpus,
-                           corpus_file=corpus_file, corpus=corpus,
+                           corpus_file=corpus_file, form=form,
-                           edit_corpus_file_form=edit_corpus_file_form,
                            title='Edit corpus file')
 
 
@@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
         abort(403)
     if corpus.files.all():
         tasks.build_corpus(corpus_id)
-        flash('Building Corpus...', 'corpus')
+        flash('Corpus "{}" has been marked to get build!', 'corpus')
     else:
-        flash('Can not build corpus, please add corpus file(s).', 'corpus')
+        flash('Can not build corpus "{}": No corpus file(s)!', 'error')
-    return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    return redirect(url_for('.corpus', corpus_id=corpus_id))
 
 
 # Following are view functions to add, view etc. exported results.
@@ -340,35 +305,29 @@ def add_query_result():
     '''
     View to import a result as a json file.
     '''
-    add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
+    form = AddQueryResultForm(prefix='add-query-result-form')
-    if add_query_result_form.is_submitted():
+    if form.is_submitted():
-        if not add_query_result_form.validate():
+        if not form.validate():
-            return make_response(add_query_result_form.errors, 400)
+            return make_response(form.errors, 400)
-        query_result = QueryResult(
+        query_result = QueryResult(creator=current_user,
-            creator=current_user,
+                                   description=form.description.data,
-            description=add_query_result_form.description.data,
+                                   filename=form.file.data.filename,
-            filename=add_query_result_form.file.data.filename,
+                                   title=form.title.data)
-            title=add_query_result_form.title.data
-        )
         db.session.add(query_result)
         db.session.commit()
-        # create paths to save the uploaded json file
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                        str(current_user.id),
-                                        'query_results',
-                                        str(query_result.id))
         try:
-            os.makedirs(query_result_dir)
+            os.makedirs(query_result.path)
-        except Exception:
+        except OSError:
+            logging.error('Make dir {} led to an OSError!'.format(query_result.path))  # noqa
             db.session.delete(query_result)
             db.session.commit()
             flash('Internal Server Error', 'error')
-            redirect_url = url_for('corpora.add_query_result')
+            return make_response(
-            return make_response({'redirect_url': redirect_url}, 500)
+                {'redirect_url': url_for('.add_query_result')}, 500)
         # save the uploaded file
-        query_result_file_path = os.path.join(query_result_dir,
+        query_result_file_path = os.path.join(query_result.path,
                                               query_result.filename)
-        add_query_result_form.file.data.save(query_result_file_path)
+        form.file.data.save(query_result_file_path)
         # parse json from file
         with open(query_result_file_path, 'r') as file:
             query_result_file_content = json.load(file)
@@ -381,19 +340,16 @@ def add_query_result():
         except Exception:
             tasks.delete_query_result(query_result.id)
             flash('Uploaded file is invalid', 'result')
-            redirect_url = url_for('corpora.add_query_result')
+            return make_response(
-            return make_response({'redirect_url': redirect_url}, 201)
+                {'redirect_url': url_for('.add_query_result')}, 201)
         query_result_file_content.pop('matches')
         query_result_file_content.pop('cpos_lookup')
         query_result.query_metadata = query_result_file_content
         db.session.commit()
         flash('Query result added!', 'result')
-        redirect_url = url_for('corpora.query_result',
+        return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201)  # noqa
-                               query_result_id=query_result.id)
-        return make_response({'redirect_url': redirect_url}, 201)
     return render_template('corpora/query_results/add_query_result.html.j2',
-                           add_query_result_form=add_query_result_form,
+                           form=form, title='Add query result')
-                           title='Add query result')
 
 
 @corpora.route('/result/<int:query_result_id>')
@@ -404,8 +360,7 @@ def query_result(query_result_id):
             or current_user.is_administrator()):
         abort(403)
     return render_template('corpora/query_results/query_result.html.j2',
-                           query_result=query_result,
+                           query_result=query_result, title='Query result')
-                           title='Query result')
 
 
 @corpora.route('/result/<int:query_result_id>/inspect')
@@ -427,13 +382,7 @@ def inspect_query_result(query_result_id):
     inspect_display_options_form = InspectDisplayOptionsForm(
         prefix='inspect-display-options-form'
     )
-    query_result_file_path = os.path.join(
+    query_result_file_path = os.path.join(query_result.path, query_result.filename)  # noqa
-        current_app.config['DATA_DIR'],
-        str(current_user.id),
-        'query_results',
-        str(query_result.id),
-        query_result.filename
-    )
     with open(query_result_file_path, 'r') as query_result_file:
         query_result_file_content = json.load(query_result_file)
     return render_template('corpora/query_results/inspect.html.j2',
@@ -452,8 +401,8 @@ def delete_query_result(query_result_id):
     if not (query_result.creator == current_user
             or current_user.is_administrator()):
         abort(403)
+    flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result')  # noqa
     tasks.delete_query_result(query_result_id)
-    flash('Query result deleted!', 'result')
     return redirect(url_for('services.service', service="corpus_analysis"))
 
 
@@ -464,10 +413,5 @@ def download_query_result(query_result_id):
     if not (query_result.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    query_result_dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True, directory=query_result.path,
-                                    str(current_user.id),
-                                    'query_results',
-                                    str(query_result.id))
-    return send_from_directory(as_attachment=True,
-                               directory=query_result_dir,
                                filename=query_result.filename)

web/app/email.py

@@ -1,11 +1,11 @@
-from flask import render_template
+from flask import current_app, render_template
 from flask_mail import Message
 from . import mail
 from .decorators import background
 
 
 def create_message(recipient, subject, template, **kwargs):
-    msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
+    msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient])  # noqa
     msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
     msg.html = render_template('{}.html.j2'.format(template), **kwargs)
     return msg

web/app/jobs/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 jobs = Blueprint('jobs', __name__)
-from . import views  # noqa
+from . import views

web/app/jobs/views.py

@@ -1,11 +1,10 @@
-from flask import (abort, current_app, flash, redirect, render_template,
+from flask import (abort, flash, redirect, render_template,
                    send_from_directory, url_for)
 from flask_login import current_user, login_required
 from . import jobs
 from . import tasks
 from ..decorators import admin_required
 from ..models import Job, JobInput, JobResult
-import os
 
 
 @jobs.route('/<int:job_id>')
@@ -14,13 +13,8 @@ def job(job_id):
     job = Job.query.get_or_404(job_id)
     if not (job.creator == current_user or current_user.is_administrator()):
         abort(403)
-    job_inputs = [dict(filename=input.filename,
+    job_inputs = [job_input.to_dict() for job_input in job.inputs]
-                       id=input.id,
+    return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
-                       job_id=job.id)
-                  for input in job.inputs]
-    return render_template('jobs/job.html.j2',
-                           job=job,
-                           job_inputs=job_inputs,
                            title='Job')
 
 
@@ -31,7 +25,7 @@ def delete_job(job_id):
     if not (job.creator == current_user or current_user.is_administrator()):
         abort(403)
     tasks.delete_job(job_id)
-    flash('Job has been deleted!', 'job')
+    flash('Job has been marked for deletion!', 'job')
     return redirect(url_for('main.dashboard'))
 
 
@@ -44,9 +38,8 @@ def download_job_input(job_id, job_input_id):
     if not (job_input.job.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       job_input.dir)
+                               directory=job_input.job.path,
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=job_input.filename)
 
 
@@ -56,11 +49,11 @@ def download_job_input(job_id, job_input_id):
 def restart(job_id):
     job = Job.query.get_or_404(job_id)
     if job.status != 'failed':
-        flash('Could not restart job: status is not "failed"', 'error')
+        flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error')  # noqa
     else:
         tasks.restart_job(job_id)
-        flash('Job has been restarted!', 'job')
+        flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job')  # noqa
-    return redirect(url_for('jobs.job', job_id=job_id))
+    return redirect(url_for('.job', job_id=job_id))
 
 
 @jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@@ -72,7 +65,6 @@ def download_job_result(job_id, job_result_id):
     if not (job_result.job.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    dir = os.path.join(current_app.config['DATA_DIR'],
+    return send_from_directory(as_attachment=True,
-                       job_result.dir)
+                               directory=job_result.job.path,
-    return send_from_directory(as_attachment=True, directory=dir,
                                filename=job_result.filename)

web/app/main/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 main = Blueprint('main', __name__)
-from . import views  # noqa
+from . import views

web/app/main/views.py

@@ -7,17 +7,16 @@ from ..models import User
 
 @main.route('/', methods=['GET', 'POST'])
 def index():
-    login_form = LoginForm(prefix='login-form')
+    form = LoginForm(prefix='login-form')
-    if login_form.validate_on_submit():
+    if form.validate_on_submit():
-        user = User.query.filter_by(username=login_form.user.data).first()
+        user = User.query.filter_by(username=form.user.data).first()
         if user is None:
-            user = User.query.filter_by(email=login_form.user.data).first()
+            user = User.query.filter_by(email=form.user.data.lower()).first()
-        if user is not None and user.verify_password(login_form.password.data):
+        if user is not None and user.verify_password(form.password.data):
-            login_user(user, login_form.remember_me.data)
+            login_user(user, form.remember_me.data)
-            return redirect(url_for('main.dashboard'))
+            return redirect(url_for('.dashboard'))
         flash('Invalid email/username or password.')
-    return render_template('main/index.html.j2', login_form=login_form,
+    return render_template('main/index.html.j2', form=form, title='nopaque')
-                           title='nopaque')
 
 
 @main.route('/about_and_faq')
@@ -31,7 +30,6 @@ def dashboard():
     return render_template('main/dashboard.html.j2', title='Dashboard')
 
 
-
 @main.route('/news')
 def news():
     return render_template('main/news.html.j2', title='News')
@@ -40,12 +38,9 @@ def news():
 @main.route('/privacy_policy')
 def privacy_policy():
     return render_template('main/privacy_policy.html.j2',
-                           title=('Information on the processing of personal'
+                           title='Privacy statement (GDPR)')
-                                  ' data for the nopaque platform (GDPR)'))
 
 
 @main.route('/terms_of_use')
 def terms_of_use():
-    return render_template('main/terms_of_use.html.j2',
+    return render_template('main/terms_of_use.html.j2', title='Terms of Use')
-                           title='General Terms of Use of the platform '
-                                 'nopaque')

web/app/models.py

@@ -7,6 +7,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
 from werkzeug.utils import secure_filename
 import xml.etree.ElementTree as ET
 from . import db, login_manager
+import logging
 import os
 import shutil
 
@@ -54,7 +55,7 @@ class Role(db.Model):
         '''
         String representation of the Role. For human readability.
         '''
-        return '<Role {role_name}>'.format(role_name=self.name)
+        return '<Role {}>'.format(self.name)
 
     def add_permission(self, perm):
         '''
@@ -138,6 +139,18 @@ class User(UserMixin, db.Model):
                                     cascade='save-update, merge, delete',
                                     lazy='dynamic')
 
+    @property
+    def path(self):
+        return os.path.join(current_app.config['NOPAQUE_DATA_DIR'], str(self.id))
+
+    @property
+    def password(self):
+        raise AttributeError('password is not a readable attribute')
+
+    @password.setter
+    def password(self, password):
+        self.password_hash = generate_password_hash(password)
+
     def to_dict(self):
         return {'id': self.id,
                 'role_id': self.role_id,
@@ -162,7 +175,7 @@ class User(UserMixin, db.Model):
         '''
         String representation of the User. For human readability.
         '''
-        return '<User {username}>'.format(username=self.username)
+        return '<User {}>'.format(self.username)
 
     def __init__(self, **kwargs):
         super(User, self).__init__(**kwargs)
@@ -220,14 +233,6 @@ class User(UserMixin, db.Model):
         db.session.add(user)
         return True
 
-    @property
-    def password(self):
-        raise AttributeError('password is not a readable attribute')
-
-    @password.setter
-    def password(self, password):
-        self.password_hash = generate_password_hash(password)
-
     def verify_password(self, password):
         return check_password_hash(self.password_hash, password)
 
@@ -244,17 +249,11 @@ class User(UserMixin, db.Model):
         '''
         return self.can(Permission.ADMIN)
 
-    def ping(self):
-        self.last_seen = datetime.utcnow()
-        db.session.add(self)
-
     def delete(self):
         '''
         Delete the user and its corpora and jobs from database and filesystem.
         '''
-        user_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                str(self.id))
-        shutil.rmtree(user_dir, ignore_errors=True)
         db.session.delete(self)
 
 
@@ -280,14 +279,17 @@ class JobInput(db.Model):
     # Foreign keys
     job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
     # Fields
-    dir = db.Column(db.String(255))
     filename = db.Column(db.String(255))
 
+    @property
+    def path(self):
+        return os.path.join(self.job.path, self.filename)
+
     def __repr__(self):
         '''
         String representation of the JobInput. For human readability.
         '''
-        return '<JobInput {filename}>'.format(filename=self.filename)
+        return '<JobInput {}>'.format(self.filename)
 
     def to_dict(self):
         return {'id': self.id,
@@ -305,14 +307,17 @@ class JobResult(db.Model):
     # Foreign keys
     job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
     # Fields
-    dir = db.Column(db.String(255))
     filename = db.Column(db.String(255))
 
+    @property
+    def path(self):
+        return os.path.join(self.job.path, self.filename)
+
     def __repr__(self):
         '''
         String representation of the JobResult. For human readability.
         '''
-        return '<JobResult {filename}>'.format(filename=self.filename)
+        return '<JobResult {}>'.format(self.filename)
 
     def to_dict(self):
         return {'id': self.id,
@@ -351,19 +356,16 @@ class Job(db.Model):
                              cascade='save-update, merge, delete')
     results = db.relationship('JobResult', backref='job', lazy='dynamic',
                               cascade='save-update, merge, delete')
-    notification_data = db.relationship('NotificationData',
+
-                                        cascade='save-update, merge, delete',
+    @property
-                                        uselist=False,
+    def path(self):
-                                        back_populates='job')  # One-to-One relationship
+        return os.path.join(self.creator.path, 'jobs', str(self.id))
-    notification_email_data = db.relationship('NotificationEmailData',
-                                              cascade='save-update, merge, delete',
-                                              back_populates='job')
 
     def __repr__(self):
         '''
         String representation of the Job. For human readability.
         '''
-        return '<Job {job_title}>'.format(job_title=self.title)
+        return '<Job {}>'.format(self.title)
 
     def create_secure_filename(self):
         '''
@@ -385,11 +387,7 @@ class Job(db.Model):
                     db.session.commit()
                 sleep(1)
                 db.session.refresh(self)
-        job_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                               str(self.user_id),
-                               'jobs',
-                               str(self.id))
-        shutil.rmtree(job_dir, ignore_errors=True)
         db.session.delete(self)
 
     def restart(self):
@@ -399,12 +397,8 @@ class Job(db.Model):
 
         if self.status != 'failed':
             raise Exception('Could not restart job: status is not "failed"')
-        job_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
-                               str(self.user_id),
+        shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True)  # noqa
-                               'jobs',
-                               str(self.id))
-        shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
-        shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
         self.end_date = None
         self.status = 'submitted'
 
@@ -425,63 +419,6 @@ class Job(db.Model):
                             for result in self.results}}
 
 
-class NotificationData(db.Model):
-    '''
-    Class to define notification data used for sending a notification mail with
-    nopaque_notify.
-    '''
-    __tablename__ = 'notification_data'
-    # Primary key
-    id = db.Column(db.Integer, primary_key=True)
-    # Foreign Key
-    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
-    # relationships
-    job = db.relationship('Job', back_populates='notification_data')
-    # Fields
-    notified_on = db.Column(db.String(16), default=None)
-
-    def __repr__(self):
-        '''
-        String representation of the NotificationData. For human readability.
-        '''
-        return '<NotificationData {id}>'.format(id=self.id)
-
-    def to_dict(self):
-        return {'id': self.id,
-                'job_id': self.job_id,
-                'job': self.job,
-                'notified': self.notified}
-
-
-class NotificationEmailData(db.Model):
-    '''
-    Class to define data that will be used to send a corresponding Notification
-    via email.
-    '''
-    __tablename__ = 'notification_email_data'
-    # Primary Key
-    id = db.Column(db.Integer, primary_key=True)
-    # Foreign Key
-    job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
-    # relationships
-    job = db.relationship('Job', back_populates='notification_email_data')
-    notify_status = db.Column(db.String(16), default=None)
-    creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
-
-    def __repr__(self):
-        '''
-        String representation of the NotificationEmailData. For human readability.
-        '''
-        return '<NotificationData {id}>'.format(id=self.id)
-
-    def to_dict(self):
-        return {'id': self.id,
-                'job_id': self.job_id,
-                'job': self.job,
-                'notify_status': self.notify_status,
-                'creation_date': self.creation_date}
-
-
 class CorpusFile(db.Model):
     '''
     Class to define Files.
@@ -496,7 +433,6 @@ class CorpusFile(db.Model):
     author = db.Column(db.String(255))
     booktitle = db.Column(db.String(255))
     chapter = db.Column(db.String(255))
-    dir = db.Column(db.String(255))
     editor = db.Column(db.String(255))
     filename = db.Column(db.String(255))
     institution = db.Column(db.String(255))
@@ -507,15 +443,15 @@ class CorpusFile(db.Model):
     school = db.Column(db.String(255))
     title = db.Column(db.String(255))
 
+    @property
+    def path(self):
+        return os.path.join(self.corpus.path, self.filename)
+
     def delete(self):
-        corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
-                                        str(self.corpus.user_id),
-                                        'corpora',
-                                        str(self.corpus_id),
-                                        self.filename)
         try:
-            os.remove(corpus_file_path)
+            os.remove(self.path)
         except OSError:
+            logging.error('Removing {} led to an OSError!'.format(self.path))
             pass
         db.session.delete(self)
         self.corpus.status = 'unprepared'
@@ -553,13 +489,17 @@ class Corpus(db.Model):
     description = db.Column(db.String(255))
     last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
     max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
-    status = db.Column(db.String(16))
+    status = db.Column(db.String(16), default='unprepared')
     title = db.Column(db.String(32))
     archive_file = db.Column(db.String(255))
     # Relationships
     files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
                             cascade='save-update, merge, delete')
 
+    @property
+    def path(self):
+        return os.path.join(self.creator.path, 'corpora', str(self.id))
+
     def to_dict(self):
         return {'id': self.id,
                 'user_id': self.user_id,
@@ -571,19 +511,14 @@ class Corpus(db.Model):
                 'files': {file.id: file.to_dict() for file in self.files}}
 
     def build(self):
-        corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+        output_dir = os.path.join(self.path, 'merged')
-                                  str(self.user_id),
-                                  'corpora',
-                                  str(self.id))
-        output_dir = os.path.join(corpus_dir, 'merged')
         shutil.rmtree(output_dir, ignore_errors=True)
         os.mkdir(output_dir)
         master_element_tree = ET.ElementTree(
             ET.fromstring('<corpus>\n</corpus>')
         )
         for corpus_file in self.files:
-            corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
+            element_tree = ET.parse(corpus_file.path)
-            element_tree = ET.parse(corpus_file_path)
             text_node = element_tree.find('text')
             text_node.set('address', corpus_file.address or "NULL")
             text_node.set('author', corpus_file.author)
@@ -597,7 +532,7 @@ class Corpus(db.Model):
             text_node.set('publishing_year', str(corpus_file.publishing_year))
             text_node.set('school', corpus_file.school or "NULL")
             text_node.set('title', corpus_file.title)
-            element_tree.write(corpus_file_path)
+            element_tree.write(corpus_file.path)
             master_element_tree.getroot().insert(1, text_node)
         output_file = os.path.join(output_dir, 'corpus.vrt')
         master_element_tree.write(output_file,
@@ -607,18 +542,14 @@ class Corpus(db.Model):
         self.status = 'submitted'
 
     def delete(self):
-        corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                  str(self.user_id),
-                                  'corpora',
-                                  str(self.id))
-        shutil.rmtree(corpus_dir, ignore_errors=True)
         db.session.delete(self)
 
     def __repr__(self):
         '''
         String representation of the corpus. For human readability.
         '''
-        return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
+        return '<Corpus {}>'.format(self.title)
 
 
 class QueryResult(db.Model):
@@ -636,12 +567,12 @@ class QueryResult(db.Model):
     query_metadata = db.Column(db.JSON())
     title = db.Column(db.String(32))
 
+    @property
+    def path(self):
+        return os.path.join(self.creator.path, 'query_results', str(self.id))
+
     def delete(self):
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
+        shutil.rmtree(self.path, ignore_errors=True)
-                                        str(self.user_id),
-                                        'query_results',
-                                        str(self.id))
-        shutil.rmtree(query_result_dir, ignore_errors=True)
         db.session.delete(self)
 
     def to_dict(self):
@@ -654,7 +585,7 @@ class QueryResult(db.Model):
 
     def __repr__(self):
         '''
-        String representation of the CorpusAnalysisResult. For human readability.
+        String representation of the QueryResult. For human readability.
         '''
         return '<QueryResult {}>'.format(self.title)
 

web/app/query_results/views.py

@@ -1,150 +0,0 @@
-from . import query_results
-from . import tasks
-from .. import db
-from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
-from ..models import QueryResult
-from .forms import AddQueryResultForm
-from flask import (abort, current_app, flash, make_response, redirect,
-                   render_template, request, send_from_directory, url_for)
-from flask_login import current_user, login_required
-import json
-import os
-from jsonschema import validate
-
-
-@query_results.route('/add', methods=['GET', 'POST'])
-@login_required
-def add_query_result():
-    '''
-    View to import a result as a json file.
-    '''
-    add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
-    if add_query_result_form.is_submitted():
-        if not add_query_result_form.validate():
-            return make_response(add_query_result_form.errors, 400)
-        query_result = QueryResult(
-            creator=current_user,
-            description=add_query_result_form.description.data,
-            filename=add_query_result_form.file.data.filename,
-            title=add_query_result_form.title.data
-        )
-        db.session.add(query_result)
-        db.session.commit()
-        # create paths to save the uploaded json file
-        query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                        str(current_user.id),
-                                        'query_results',
-                                        str(query_result.id))
-        try:
-            os.makedirs(query_result_dir)
-        except Exception:
-            db.session.delete(query_result)
-            db.session.commit()
-            flash('Internal Server Error', 'error')
-            redirect_url = url_for('query_results.add_query_result')
-            return make_response({'redirect_url': redirect_url}, 500)
-        # save the uploaded file
-        query_result_file_path = os.path.join(query_result_dir,
-                                              query_result.filename)
-        add_query_result_form.file.data.save(query_result_file_path)
-        # parse json from file
-        with open(query_result_file_path, 'r') as file:
-            query_result_file_content = json.load(file)
-        # parse json schema
-        with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file:  # noqa
-            schema = json.load(file)
-        try:
-            # validate imported json file
-            validate(instance=query_result_file_content, schema=schema)
-        except Exception:
-            tasks.delete_query_result(query_result.id)
-            flash('Uploaded file is invalid', 'result')
-            redirect_url = url_for('query_results.add_query_result')
-            return make_response({'redirect_url': redirect_url}, 201)
-        query_result_file_content.pop('matches')
-        query_result_file_content.pop('cpos_lookup')
-        query_result.query_metadata = query_result_file_content
-        db.session.commit()
-        flash('Query result added!', 'result')
-        redirect_url = url_for('query_results.query_result',
-                               query_result_id=query_result.id)
-        return make_response({'redirect_url': redirect_url}, 201)
-    return render_template('corpora/query_results/add_query_result.html.j2',
-                           add_query_result_form=add_query_result_form,
-                           title='Add query result')
-
-
-@query_results.route('/<int:query_result_id>')
-@login_required
-def query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    return render_template('corpora/query_results/query_result.html.j2',
-                           query_result=query_result,
-                           title='Query result')
-
-
-@query_results.route('/<int:query_result_id>/inspect')
-@login_required
-def inspect_query_result(query_result_id):
-    '''
-    View to inspect imported result file in a corpus analysis like interface
-    '''
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    query_metadata = query_result.query_metadata
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    display_options_form = DisplayOptionsForm(
-        prefix='display-options-form',
-        results_per_page=request.args.get('results_per_page', 30),
-        result_context=request.args.get('context', 20)
-    )
-    inspect_display_options_form = InspectDisplayOptionsForm(
-        prefix='inspect-display-options-form'
-    )
-    query_result_file_path = os.path.join(
-        current_app.config['DATA_DIR'],
-        str(current_user.id),
-        'query_results',
-        str(query_result.id),
-        query_result.filename
-    )
-    with open(query_result_file_path, 'r') as query_result_file:
-        query_result_file_content = json.load(query_result_file)
-    return render_template('corpora/query_results/inspect.html.j2',
-                           display_options_form=display_options_form,
-                           inspect_display_options_form=inspect_display_options_form,
-                           query_result_file_content=query_result_file_content,
-                           query_metadata=query_metadata,
-                           title='Inspect query result')
-
-
-@query_results.route('/<int:query_result_id>/delete')
-@login_required
-def delete_query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    tasks.delete_query_result(query_result_id)
-    flash('Query result deleted!', 'result')
-    return redirect(url_for('services.service', service="corpus_analysis"))
-
-
-@query_results.route('/<int:query_result_id>/download')
-@login_required
-def download_query_result(query_result_id):
-    query_result = QueryResult.query.get_or_404(query_result_id)
-    if not (query_result.creator == current_user
-            or current_user.is_administrator()):
-        abort(403)
-    query_result_dir = os.path.join(current_app.config['DATA_DIR'],
-                                    str(current_user.id),
-                                    'query_results',
-                                    str(query_result.id))
-    return send_from_directory(as_attachment=True,
-                               directory=query_result_dir,
-                               filename=query_result.filename)

web/app/services/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 services = Blueprint('services', __name__)
-from . import views  # noqa
+from . import views

web/app/services/views.py

@@ -1,5 +1,4 @@
-from flask import (abort, current_app, flash, make_response, render_template,
+from flask import abort, flash, make_response, render_template, url_for
-                   url_for)
 from flask_login import current_user, login_required
 from werkzeug.utils import secure_filename
 from . import services
@@ -7,19 +6,20 @@ from .. import db
 from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
 from ..models import Job, JobInput
 import json
+import logging
 import os
 
 
 SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
             'file-setup': {'name': 'File setup',
                            'resources': {'mem_mb': 4096, 'n_cores': 4},
-                           'add_job_form': AddFileSetupJobForm},
+                           'form': AddFileSetupJobForm},
             'nlp': {'name': 'Natural Language Processing',
                     'resources': {'mem_mb': 4096, 'n_cores': 2},
-                    'add_job_form': AddNLPJobForm},
+                    'form': AddNLPJobForm},
             'ocr': {'name': 'Optical Character Recognition',
                     'resources': {'mem_mb': 8192, 'n_cores': 4},
-                    'add_job_form': AddOCRJobForm}}
+                    'form': AddOCRJobForm}}
 
 
 @services.route('/<service>', methods=['GET', 'POST'])
@@ -30,54 +30,49 @@ def service(service):
     if service == 'corpus_analysis':
         return render_template('services/{}.html.j2'.format(service),
                                title=SERVICES[service]['name'])
-    add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
+    form = SERVICES[service]['form'](prefix='add-job-form')
-    if add_job_form.is_submitted():
+    if form.is_submitted():
-        if not add_job_form.validate():
+        if not form.validate():
-            return make_response(add_job_form.errors, 400)
+            return make_response(form.errors, 400)
         service_args = []
         if service == 'nlp':
-            service_args.append('-l {}'.format(add_job_form.language.data))
+            service_args.append('-l {}'.format(form.language.data))
-            if add_job_form.check_encoding.data:
+            if form.check_encoding.data:
                 service_args.append('--check-encoding')
         if service == 'ocr':
-            service_args.append('-l {}'.format(add_job_form.language.data))
+            service_args.append('-l {}'.format(form.language.data))
-            if add_job_form.binarization.data:
+            if form.binarization.data:
                 service_args.append('--binarize')
         job = Job(creator=current_user,
-                  description=add_job_form.description.data,
+                  description=form.description.data,
                   mem_mb=SERVICES[service]['resources']['mem_mb'],
                   n_cores=SERVICES[service]['resources']['n_cores'],
                   service=service, service_args=json.dumps(service_args),
-                  service_version=add_job_form.version.data,
+                  service_version=form.version.data,
-                  status='preparing', title=add_job_form.title.data)
+                  status='preparing', title=form.title.data)
         if job.service != 'corpus_analysis':
             job.create_secure_filename()
         db.session.add(job)
         db.session.commit()
-        relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
-        absolut_dir = os.path.join(current_app.config['DATA_DIR'],
-                                   relative_dir)
         try:
-            os.makedirs(absolut_dir)
+            os.makedirs(job.path)
         except OSError:
-            job.delete()
+            logging.error('Make dir {} led to an OSError!'.format(job.path))
-            flash('Internal Server Error', 'job')
+            db.session.delete(job)
-            return make_response({'redirect_url': url_for('services.service',
+            db.session.commit()
-                                                          service=service)},
+            flash('Internal Server Error', 'error')
-                                 500)
+            return make_response(
+                {'redirect_url': url_for('.service', service=service)}, 500)
         else:
-            for file in add_job_form.files.data:
+            for file in form.files.data:
                 filename = secure_filename(file.filename)
-                file.save(os.path.join(absolut_dir, filename))
+                job_input = JobInput(dir=job.path, filename=filename, job=job)
-                job_input = JobInput(dir=relative_dir, filename=filename,
+                file.save(job_input.path)
-                                     job=job)
                 db.session.add(job_input)
             job.status = 'submitted'
             db.session.commit()
-            url = url_for('jobs.job', job_id=job.id)
+            flash('Job "{}" added'.format(job.title), 'job')
-            flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
             return make_response(
                 {'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
     return render_template('services/{}.html.j2'.format(service),
-                           title=SERVICES[service]['name'],
+                           form=form, title=SERVICES[service]['name'])
-                           add_job_form=add_job_form)

web/app/settings/forms.py

@@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
         'Benutzername',
         validators=[DataRequired(),
                     Length(1, 64),
-                    Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+                    Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
                            message='Usernames must have only letters, numbers,'
                                    ' dots or underscores')]
     )

web/app/settings/views.py

@@ -1,13 +1,9 @@
-from flask import current_app, flash, redirect, render_template, url_for
+from flask import flash, redirect, render_template, url_for
 from flask_login import current_user, login_required, logout_user
 from . import settings, tasks
 from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
                     EditNotificationSettingsForm)
 from .. import db
-from ..decorators import admin_required
-from ..models import Role, User
-import os
-import uuid
 
 
 @settings.route('/')
@@ -26,8 +22,7 @@ def change_password():
         flash('Your password has been updated.')
         return redirect(url_for('.change_password'))
     return render_template('settings/change_password.html.j2',
-                           form=form,
+                           form=form, title='Change password')
-                           title='Change password')
 
 
 @settings.route('/edit_general_settings', methods=['GET', 'POST'])
@@ -40,12 +35,12 @@ def edit_general_settings():
         current_user.username = form.username.data
         db.session.commit()
         flash('Your changes have been saved.')
+        return redirect(url_for('.edit_general_settings'))
     form.dark_mode.data = current_user.setting_dark_mode
     form.email.data = current_user.email
     form.username.data = current_user.username
     return render_template('settings/edit_general_settings.html.j2',
-                           form=form,
+                           form=form, title='General settings')
-                           title='General settings')
 
 
 @settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@@ -59,13 +54,13 @@ def edit_notification_settings():
             form.job_status_site_notifications.data
         db.session.commit()
         flash('Your changes have been saved.')
+        return redirect(url_for('.edit_notification_settings'))
     form.job_status_mail_notifications.data = \
         current_user.setting_job_status_mail_notifications
     form.job_status_site_notifications.data = \
         current_user.setting_job_status_site_notifications
     return render_template('settings/edit_notification_settings.html.j2',
-                           form=form,
+                           form=form, title='Notification settings')
-                           title='Notification settings')
 
 
 @settings.route('/delete')
@@ -76,5 +71,5 @@ def delete():
     """
     tasks.delete_user(current_user.id)
     logout_user()
-    flash('Your account has been deleted!')
+    flash('Your account has been marked for deletion!')
     return redirect(url_for('main.index'))

web/app/tasks/__init__.py

@@ -11,15 +11,11 @@ def check_corpora():
     corpora = Corpus.query.all()
     for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
         corpus_utils.create_build_corpus_service(corpus)
-    for corpus in filter(lambda corpus: (corpus.status == 'queued'
+    for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora):  # noqa
-                                         or corpus.status == 'running'),
-                         corpora):
         corpus_utils.checkout_build_corpus_service(corpus)
-    for corpus in filter(lambda corpus: corpus.status == 'start analysis',
+    for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora):  # noqa
-                         corpora):
         corpus_utils.create_cqpserver_container(corpus)
-    for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
+    for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora):  # noqa
-                         corpora):
         corpus_utils.remove_cqpserver_container(corpus)
     db.session.commit()
 
@@ -28,8 +24,6 @@ def check_jobs():
     jobs = Job.query.all()
     for job in filter(lambda job: job.status == 'submitted', jobs):
         job_utils.create_job_service(job)
-    for job in filter(lambda job: job.status == 'queued', jobs):
+    for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
-        job_utils.checkout_job_service(job)
-    for job in filter(lambda job: job.status == 'running', jobs):
         job_utils.checkout_job_service(job)
     db.session.commit()

web/app/tasks/corpus_utils.py

@@ -1,4 +1,3 @@
-from flask import current_app
 from . import docker_client
 import docker
 import logging
@@ -7,20 +6,14 @@ import shutil
 
 
 def create_build_corpus_service(corpus):
-    corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+    corpus_data_dir = os.path.join(corpus.path, 'data')
-                              str(corpus.user_id),
+    shutil.rmtree(corpus_data_dir, ignore_errors=True)
-                              'corpora',
-                              str(corpus.id))
-    corpus_data_dir = os.path.join(corpus_dir, 'data')
-    corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
-    corpus_registry_dir = os.path.join(corpus_dir, 'registry')
-    if os.path.exists(corpus_data_dir):
-        shutil.rmtree(corpus_data_dir)
-    if os.path.exists(corpus_registry_dir):
-        shutil.rmtree(corpus_registry_dir)
     os.mkdir(corpus_data_dir)
+    corpus_registry_dir = os.path.join(corpus.path, 'registry')
+    shutil.rmtree(corpus_registry_dir, ignore_errors=True)
     os.mkdir(corpus_registry_dir)
-    service_args = {
+    corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
+    service_kwargs = {
         'command': 'docker-entrypoint.sh build-corpus',
         'constraints': ['node.role==worker'],
         'labels': {'origin': 'nopaque',
@@ -32,30 +25,34 @@ def create_build_corpus_service(corpus):
         'name': 'build-corpus_{}'.format(corpus.id),
         'restart_policy': docker.types.RestartPolicy()
     }
-    service_image = \
+    service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'  # noqa
-        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
     try:
-        docker_client.services.create(service_image, **service_args)
+        docker_client.services.create(service_image, **service_kwargs)
     except docker.errors.APIError as e:
-        logging.error('create_build_corpus_service({}): '.format(corpus.id)
+        logging.error('Create "{}" service raised '.format(service_kwargs['name'])  # noqa
-                      + '{} (status: {} -> failed)'.format(e, corpus.status))
+                      + '[docker-APIError] The server returned an error. '
-        corpus.status = 'failed'
+                      + 'Details: {}'.format(e))
     else:
         corpus.status = 'queued'
-    finally:
-        # TODO: send email
-        pass
 
 
 def checkout_build_corpus_service(corpus):
     service_name = 'build-corpus_{}'.format(corpus.id)
     try:
         service = docker_client.services.get(service_name)
-    except docker.errors.NotFound as e:
+    except docker.errors.NotFound:
-        logging.error('checkout_build_corpus_service({}):'.format(corpus.id)
+        logging.error('Get "{}" service raised '.format(service_name)
-                      + ' {} (stauts: {} -> failed)'.format(e, corpus.status))
+                      + '[docker-NotFound] The service does not exist. '
+                      + '(corpus.status: {} -> failed)'.format(corpus.status))
         corpus.status = 'failed'
-    # TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
+    except docker.errors.APIError as e:
+        logging.error('Get "{}" service raised '.format(service_name)
+                      + '[docker-APIError] The server returned an error. '
+                      + 'Details: {}'.format(e))
+    except docker.errors.InvalidVersion:
+        logging.error('Get "{}" service raised '.format(service_name)
+                      + '[docker-InvalidVersion] One of the arguments is '
+                      + 'not supported with the current API version.')
     else:
         service_tasks = service.tasks()
         if not service_tasks:
@@ -63,25 +60,23 @@ def checkout_build_corpus_service(corpus):
         task_state = service_tasks[0].get('Status').get('State')
         if corpus.status == 'queued' and task_state != 'pending':
             corpus.status = 'running'
-        elif corpus.status == 'running' and task_state == 'complete':
+        elif corpus.status == 'running' and task_state in ['complete', 'failed']:  # noqa
-            service.remove()
+            try:
-            corpus.status = 'prepared'
+                service.remove()
-        elif corpus.status == 'running' and task_state == 'failed':
+            except docker.errors.APIError as e:
-            service.remove()
+                logging.error('Remove "{}" service raised '.format(service_name)  # noqa
-            corpus.status = task_state
+                              + '[docker-APIError] The server returned an error. '  # noqa
-    finally:
+                              + 'Details: {}'.format(e))
-        # TODO: send email
+                return
-        pass
+            else:
+                corpus.status = 'prepared' if task_state == 'complete' \
+                                else 'failed'
 
 
 def create_cqpserver_container(corpus):
-    corpus_dir = os.path.join(current_app.config['DATA_DIR'],
+    corpus_data_dir = os.path.join(corpus.path, 'data')
-                              str(corpus.user_id),
+    corpus_registry_dir = os.path.join(corpus.path, 'registry')
-                              'corpora',
+    container_kwargs = {
-                              str(corpus.id))
-    corpus_data_dir = os.path.join(corpus_dir, 'data')
-    corpus_registry_dir = os.path.join(corpus_dir, 'registry')
-    container_args = {
         'command': 'cqpserver',
         'detach': True,
         'volumes': [corpus_data_dir + ':/corpora/data:rw',
@@ -89,20 +84,43 @@ def create_cqpserver_container(corpus):
         'name': 'cqpserver_{}'.format(corpus.id),
         'network': 'nopaque_default'
     }
-    container_image = \
+    container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'  # noqa
-        'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
+    # Check if a cqpserver container already exists. If this is the case,
+    # remove it and create a new one
     try:
-        container = docker_client.containers.get(container_args['name'])
+        container = docker_client.containers.get(container_kwargs['name'])
     except docker.errors.NotFound:
         pass
-    except docker.errors.DockerException:
+    except docker.errors.APIError as e:
+        logging.error('Get "{}" container raised '.format(container_kwargs['name'])
+                      + '[docker-APIError] The server returned an error. '
+                      + 'Details: {}'.format(e))
         return
     else:
-        container.remove(force=True)
+        try:
+            container.remove(force=True)
+        except docker.errors.APIError as e:
+            logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
+                          + '[docker-APIError] The server returned an error. '
+                          + 'Details: {}'.format(e))
+            return
     try:
-        docker_client.containers.run(container_image, **container_args)
+        docker_client.containers.run(container_image, **container_kwargs)
-    except docker.errors.DockerException:
+    except docker.errors.ContainerError:
-        return
+        # This case should not occur, because detach is True.
+        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+                      + '[docker-ContainerError] The container exits with a '
+                      + 'non-zero exit code and detach is False.')
+        corpus.status = 'failed'
+    except docker.errors.ImageNotFound:
+        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+                      + '[docker-ImageNotFound] The specified image does not '
+                      + 'exist.')
+        corpus.status = 'failed'
+    except docker.errors.APIError as e:
+        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+                      + '[docker-APIError] The server returned an error. '
+                      + 'Details: {}'.format(e))
     else:
         corpus.status = 'analysing'
 
@@ -113,8 +131,17 @@ def remove_cqpserver_container(corpus):
         container = docker_client.containers.get(container_name)
     except docker.errors.NotFound:
         pass
-    except docker.errors.DockerException:
+    except docker.errors.APIError as e:
+        logging.error('Get "{}" container raised '.format(container_name)
+                      + '[docker-APIError] The server returned an error. '
+                      + 'Details: {}'.format(e))
         return
     else:
-        container.remove(force=True)
+        try:
+            container.remove(force=True)
+        except docker.errors.APIError as e:
+            logging.error('Remove "{}" container raised '.format(container_name)
+                          + '[docker-APIError] The server returned an error. '
+                          + 'Details: {}'.format(e))
+            return
     corpus.status = 'prepared'

web/app/tasks/job_utils.py

@@ -1,7 +1,7 @@
 from datetime import datetime
-from flask import current_app
 from . import docker_client
 from .. import db
+from ..email import create_message, send
 from ..models import JobResult
 import docker
 import logging
@@ -10,51 +10,60 @@ import os
 
 
 def create_job_service(job):
-    job_dir = os.path.join(current_app.config['DATA_DIR'],
-                           str(job.user_id),
-                           'jobs',
-                           str(job.id))
     cmd = '{} -i /files -o /files/output'.format(job.service)
     if job.service == 'file-setup':
         cmd += ' -f {}'.format(job.secure_filename)
     cmd += ' --log-dir /files'
     cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
     cmd += ' ' + ' '.join(json.loads(job.service_args))
-    service_args = {'command': cmd,
+    service_kwargs = {'command': cmd,
-                    'constraints': ['node.role==worker'],
+                      'constraints': ['node.role==worker'],
-                    'labels': {'origin': 'nopaque',
+                      'labels': {'origin': 'nopaque',
-                               'type': 'service.{}'.format(job.service),
+                                 'type': 'service.{}'.format(job.service),
-                               'job_id': str(job.id)},
+                                 'job_id': str(job.id)},
-                    'mounts': [job_dir + ':/files:rw'],
+                      'mounts': [job.path + ':/files:rw'],
-                    'name': 'job_{}'.format(job.id),
+                      'name': 'job_{}'.format(job.id),
-                    'resources': docker.types.Resources(
+                      'resources': docker.types.Resources(
-                        cpu_reservation=job.n_cores * (10 ** 9),
+                          cpu_reservation=job.n_cores * (10 ** 9),
-                        mem_reservation=job.mem_mb * (10 ** 6)),
+                          mem_reservation=job.mem_mb * (10 ** 6)
-                    'restart_policy': docker.types.RestartPolicy()}
+                      ),
+                      'restart_policy': docker.types.RestartPolicy()}
     service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
                      + job.service + ':' + job.service_version)
     try:
-        docker_client.services.create(service_image, **service_args)
+        docker_client.services.create(service_image, **service_kwargs)
     except docker.errors.APIError as e:
-        logging.error('create_job_service({}): {} '.format(job.id, e)
+        logging.error('Create "{}" service raised '.format(service_kwargs['name'])  # noqa
-                      + '(status: {} -> failed)'.format(job.status))
+                      + '[docker-APIError] The server returned an error. '
-        job.status = 'failed'
+                      + 'Details: {}'.format(e))
     else:
         job.status = 'queued'
-    finally:
+        msg = create_message(
-        # TODO: send email
+            job.creator.email,
-        pass
+            'Status update for your Job "{}"'.format(job.title),
+            'tasks/email/notification',
+            job=job
+        )
+        send(msg)
 
 
 def checkout_job_service(job):
     service_name = 'job_{}'.format(job.id)
     try:
         service = docker_client.services.get(service_name)
-    except docker.errors.NotFound as e:
+    except docker.errors.NotFound:
-        logging.error('checkout_job_service({}): {} '.format(job.id, e)
+        logging.error('Get "{}" service raised '.format(service_name)
-                      + '(status: {} -> submitted)'.format(job.status))
+                      + '[docker-NotFound] The service does not exist. '
-        job.status = 'submitted'
+                      + '(job.status: {} -> failed)'.format(job.status))
-    # TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
+        job.status = 'failed'
+    except docker.errors.APIError as e:
+        logging.error('Get "{}" service raised '.format(service_name)
+                      + '[docker-APIError] The server returned an error. '
+                      + 'Details: {}'.format(e))
+    except docker.errors.InvalidVersion:
+        logging.error('Get "{}" service raised '.format(service_name)
+                      + '[docker-InvalidVersion] One of the arguments is '
+                      + 'not supported with the current API version.')
     else:
         service_tasks = service.tasks()
         if not service_tasks:
@@ -62,22 +71,16 @@ def checkout_job_service(job):
         task_state = service_tasks[0].get('Status').get('State')
         if job.status == 'queued' and task_state != 'pending':
             job.status = 'running'
-        elif job.status == 'queued' and task_state == 'complete':
+        elif job.status == 'running' and task_state == 'complete':
             service.remove()
             job.end_date = datetime.utcnow()
             job.status = task_state
             if task_state == 'complete':
-                results_dir = os.path.join(current_app.config['DATA_DIR'],
+                job_results_dir = os.path.join(job.path, 'output')
-                                           str(job.user_id),
+                job_results = filter(lambda x: x.endswith('.zip'),
-                                           'jobs',
+                                     os.listdir(job_results_dir))
-                                           str(job.id),
+                for job_result in job_results:
-                                           'output')
+                    job_result = JobResult(filename=job_result, job=job)
-                results = filter(lambda x: x.endswith('.zip'),
-                                 os.listdir(results_dir))
-                for result in results:
-                    job_result = JobResult(dir=results_dir,
-                                           filename=result,
-                                           job_id=job.id)
                     db.session.add(job_result)
         elif job.status == 'running' and task_state == 'failed':
             service.remove()
@@ -85,6 +88,13 @@ def checkout_job_service(job):
             job.status = task_state
     finally:
         # TODO: send email
+        msg = create_message(
+            job.creator.email,
+            '[nopaque] Status update for your Job "{}"'.format(job.title),
+            'tasks/email/notification',
+            job=job
+        )
+        send(msg)
         pass
 
 

web/app/templates/auth/login.html.j2

@@ -35,20 +35,20 @@
       <div class="card medium">
         <form method="POST">
           <div class="card-content">
-            {{ login_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(login_form.user, material_icon='person') }}
+            {{ wtf.render_field(form.user, material_icon='person') }}
-            {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password, material_icon='vpn_key') }}
             <div class="row" style="margin-bottom: 0;">
               <div class="col s6 left-align">
                 <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
               </div>
               <div class="col s6 right-align">
-                {{ wtf.render_field(login_form.remember_me) }}
+                {{ wtf.render_field(form.remember_me) }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(login_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/register.html.j2

@@ -34,14 +34,14 @@
       <div class="card medium">
         <form method="POST">
           <div class="card-content">
-            {{ registration_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
+            {{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
-            {{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
-            {{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
+            {{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
-            {{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
+            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(registration_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/reset_password.html.j2

@@ -20,12 +20,12 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ reset_password_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(reset_password_form.password, data_length='128') }}
+            {{ wtf.render_field(form.password, data_length='128') }}
-            {{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
+            {{ wtf.render_field(form.password_confirmation, data_length='128') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/auth/reset_password_request.html.j2

@@ -20,11 +20,11 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ reset_password_request_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
-            {{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
+            {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/add_corpus.html.j2

@@ -27,18 +27,18 @@
       <div class="card">
         <form method="POST">
           <div class="card-content">
-            {{ add_corpus_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/add_corpus_file.html.j2

@@ -27,24 +27,24 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ add_corpus_file_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
+                {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
+                {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
+                {{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
         <br>
@@ -52,7 +52,7 @@
           <li>
             <div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
             <div class="collapsible-body">
-              {% for field in add_corpus_file_form
+              {% for field in form
                  if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
               {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
               {% endfor %}

web/app/templates/corpora/analyse_corpus.html.j2

@@ -155,7 +155,7 @@ import {
  */
 document.addEventListener("DOMContentLoaded", () => {
   // Initialize the client for server client communication in dynamic mode
-  let corpusId = {{ corpus_id }}
+  let corpusId = {{ corpus.id }}
   const client = new Client({'corpusId': corpusId,
                              'socket': nopaque.socket,
                              'logging': true,

web/app/templates/corpora/corpus_file.html.j2

@@ -20,23 +20,23 @@
 
     <div class="col s12">
       <form method="POST">
-        {{ edit_corpus_file_form.hidden_tag() }}
+        {{ form.hidden_tag() }}
         <div class="card">
           <div class="card-content">
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
+                {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
               </div>
               <div class="col s12 m4">
-                {{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
+                {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
         <br>
@@ -44,7 +44,7 @@
           <li>
             <div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
             <div class="collapsible-body">
-              {% for field in edit_corpus_file_form
+              {% for field in form
                  if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
               {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
               {% endfor %}

web/app/templates/corpora/import_corpus.html.j2

@@ -1,4 +1,4 @@
-{% extends "nopaque.html.j2" %}
+  {% extends "nopaque.html.j2" %}
 {% from '_colors.html.j2' import colors %}
 {% import 'materialize/wtf.html.j2' as wtf %}
 
@@ -27,23 +27,23 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ import_corpus_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
             </div>
             <div class="row">
               <div class="col s12">
-                {{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
+                {{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/corpora/query_results/add_query_result.html.j2

@@ -27,21 +27,21 @@
       <form class="nopaque-submit-form" data-progress-modal="progress-modal">
         <div class="card">
           <div class="card-content">
-            {{ add_query_result_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 m4">
-                {{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 m8">
-                {{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
+                {{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </div>
       </form>

web/app/templates/main/index.html.j2

@@ -159,20 +159,20 @@
             <form method="POST">
               <div class="card-content">
                 <span class="card-title">Log in</span>
-                {{ login_form.hidden_tag() }}
+                {{ form.hidden_tag() }}
-                {{ wtf.render_field(login_form.user, material_icon='person') }}
+                {{ wtf.render_field(form.user, material_icon='person') }}
-                {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+                {{ wtf.render_field(form.password, material_icon='vpn_key') }}
                 <div class="row" style="margin-bottom: 0;">
                   <div class="col s6 left-align">
                     <a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
                   </div>
                   <div class="col s6 right-align">
-                    {{ wtf.render_field(login_form.remember_me) }}
+                    {{ wtf.render_field(form.remember_me) }}
                   </div>
                 </div>
               </div>
               <div class="card-action right-align">
-                {{ wtf.render_field(login_form.submit, material_icon='send') }}
+                {{ wtf.render_field(form.submit, material_icon='send') }}
               </div>
             </form>
           </div>

web/app/templates/nopaque.html.j2

@@ -231,9 +231,9 @@
       </div>
       <div class="col s12 m9 right-align">
         <a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
-        {% if config.CONTACT_EMAIL_ADRESS %}
+        {% if config.NOPAQUE_CONTACT %}
-        <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
+        <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
-        <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
+        <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
         {% endif %}
         <a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
       </div>

web/app/templates/services/file-setup.html.j2

@@ -48,24 +48,24 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12">
-                {{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
+                {{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
               </div>
               <div class="col s12 hide">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/services/nlp.html.j2

@@ -66,34 +66,34 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12 l5">
-                {{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
+                {{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
               </div>
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+                {{ wtf.render_field(form.language, material_icon='language') }}
               </div>
               <div class="col s12 l3">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
               <div class="col s12">
                 <span class="card-title">Preprocessing</span>
               </div>
               <div class="col s9">
-                <p>{{ add_job_form.check_encoding.label.text }}</p>
+                <p>{{ form.check_encoding.label.text }}</p>
                 <p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
               </div>
               <div class="col s3 right-align">
                 <div class="switch">
                   <label>
-                    {{ add_job_form.check_encoding() }}
+                    {{ form.check_encoding() }}
                     <span class="lever"></span>
                   </label>
                 </div>
@@ -107,7 +107,7 @@
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+            {{ wtf.render_field(form.submit, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/services/ocr.html.j2

@@ -48,34 +48,34 @@
       <div class="card">
         <form class="nopaque-submit-form" data-progress-modal="progress-modal">
           <div class="card-content">
-            {{ add_job_form.hidden_tag() }}
+            {{ form.hidden_tag() }}
             <div class="row">
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+                {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
               </div>
               <div class="col s12 l8">
-                {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+                {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
               </div>
               <div class="col s12 l5">
-                {{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
+                {{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
               </div>
               <div class="col s12 l4">
-                {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+                {{ wtf.render_field(form.language, material_icon='language') }}
               </div>
               <div class="col s12 l3">
-                {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+                {{ wtf.render_field(form.version, material_icon='apps') }}
               </div>
               <div class="col s12">
                 <span class="card-title">Preprocessing</span>
               </div>
               <div class="col s9">
-                <p>{{ add_job_form.binarization.label.text }}</p>
+                <p>{{ form.binarization.label.text }}</p>
                 <p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
               </div>
               <div class="col s3 right-align">
                 <div class="switch">
                   <label>
-                    {{ add_job_form.binarization() }}
+                    {{ form.binarization() }}
                     <span class="lever"></span>
                   </label>
                 </div>
@@ -134,7 +134,7 @@
             </div>
           </div>
           <div class="card-action right-align">
-            {{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
+            {{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
           </div>
         </form>
       </div>

web/app/templates/tasks/email/notification.html.j2

@@ -1,9 +1,8 @@
-<p>Dear <b>{{ user.username }}</b>,</p>
+<p>Dear <b>{{ job.creator.username }}</b>,</p>
 
-<p>The status of your Job/Corpus({{ job.id }}) with the title <b>"{{ job.title }}"</b> has changed!</p>
+<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
 <p>It is now <b>{{ job.status }}</b>!</p>
-<p>Time of this status update was: <b>{time} UTC</b></p>
 
-<p>You can access your Job/Corpus here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
+<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
 
 <p>Kind regards!<br>Your nopaque team</p>

web/app/templates/tasks/email/notification.txt.j2

@@ -1,10 +1,9 @@
-Dear {{ user.username }},
+Dear {{ job.creator.username }},
 
-The status of your Job/Corpus({{ job.id }}) with the title "{{ job.title }}" has changed!
+The status of your Job "{{ job.title }}" has changed!
 It is now {{ job.status }}!
-Time of this status update was: {time} UTC
 
-You can access your Job/Corpus here: {{ url_for('jobs.job', job_id=job.id) }}
+You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
 
 Kind regards!
 Your nopaque team

web/boot.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 source venv/bin/activate
+
 while true; do
     flask deploy
     if [[ "$?" == "0" ]]; then

web/config.py

@@ -7,103 +7,96 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
 
 
 class Config:
-    ''' # Cookies # '''
+    ''' # Flask # '''
-    REMEMBER_COOKIE_HTTPONLY = True
+    SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
-    REMEMBER_COOKIE_SECURE = os.environ.get(
+    SESSION_COOKIE_SECURE = \
-        'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
+        os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
-    SESSION_COOKIE_SECURE = os.environ.get(
-        'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
 
-    ''' # Database # '''
+    ''' # Flask-Login # '''
+    REMEMBER_COOKIE_HTTPONLY = True
+    REMEMBER_COOKIE_SECURE = \
+        os.environ.get('REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
+
+    ''' # Flask-Mail # '''
+    MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
+    MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
+    MAIL_PORT = int(os.environ.get('MAIL_PORT'))
+    MAIL_SERVER = os.environ.get('MAIL_SERVER')
+    MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
+    MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', 'false').lower() == 'true'
+    MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'false').lower() == 'true'
+
+    ''' # Flask-SQLAlchemy # '''
     SQLALCHEMY_RECORD_QUERIES = True
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-    ''' # Email # '''
+    ''' # nopaque # '''
-    MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
+    NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
-    MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
+    NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
-    MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
+    NOPAQUE_DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
-    MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
+    NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
-    MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
+    NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
-    MAIL_USE_SSL = os.environ.get(
+        os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
-        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
+    NOPAQUE_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
-    MAIL_USE_TLS = os.environ.get(
-        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
-
-    ''' # General # '''
-    ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
-    ALLOWED_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
-    CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
-    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
-
-    ''' # Logging # '''
-    LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     '%Y-%m-%d %H:%M:%S')
-    LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
-                              os.path.join(ROOT_DIR, 'nopaque.log'))
-    LOG_FORMAT = os.environ.get(
-        'NOPAQUE_LOG_FORMAT',
-        '[%(asctime)s] %(levelname)s in '
-        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
-    )
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
-
-    ''' # Message queue # '''
-    SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
-        'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
-
-    ''' # Proxy fix # '''
-    PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
-    PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
-    PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
-    PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
-    PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
 
     @classmethod
     def init_app(cls, app):
-        # Set up logging according to the corresponding (LOG_*) variables
+        # Set up logging according to the corresponding (NOPAQUE_LOG_*)
-        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
+        # environment variables
-                            filename=cls.LOG_FILE,
+        basic_config_kwargs = {
-                            format=cls.LOG_FORMAT,
+            'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                            level=cls.LOG_LEVEL)
+                                      '%Y-%m-%d %H:%M:%S'),
+            'filename': os.environ.get('NOPAQUE_LOG_FILE',
+                                       os.path.join(ROOT_DIR, 'nopaque.log')),
+            'format': os.environ.get(
+                'NOPAQUE_LOG_FORMAT',
+                '[%(asctime)s] %(levelname)s in '
+                '%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
+                '%(message)s'
+            ),
+            'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
+        }
+        logging.basicConfig(**basic_config_kwargs)
         # Set up and apply the ProxyFix middleware according to the
-        # corresponding (PROXY_FIX_*) variables
+        # corresponding (NOPAQUE_PROXY_FIX_*) environment variables
-        app.wsgi_app = ProxyFix(app.wsgi_app,
+        proxy_fix_kwargs = {
-                                x_for=cls.PROXY_FIX_X_FOR,
+            'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
-                                x_host=cls.PROXY_FIX_X_HOST,
+            'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
-                                x_port=cls.PROXY_FIX_X_PORT,
+            'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
-                                x_prefix=cls.PROXY_FIX_X_PREFIX,
+            'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
-                                x_proto=cls.PROXY_FIX_X_PROTO)
+            'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+        }
+        app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
 
 
 class DevelopmentConfig(Config):
-    ''' # Database # '''
+    ''' # Flask # '''
+    DEBUG = True
+
+    ''' # Flask-SQLAlchemy # '''
     SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_DEV_DATABASE_URL',
+        'SQLALCHEMY_DATABASE_URI',
         'postgresql://nopaque:nopaque@db/nopaque_dev'
     )
 
-    ''' # General # '''
-    DEBUG = True
-
 
 class ProductionConfig(Config):
-    ''' # Database # '''
+    ''' # Flask-SQLAlchemy # '''
     SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
+        'SQLALCHEMY_DATABASE_URI', 'postgresql://nopaque:nopaque@db/nopaque')
 
 
 class TestingConfig(Config):
-    ''' # Database # '''
+    ''' # Flask # '''
-    SQLALCHEMY_DATABASE_URI = os.environ.get(
-        'NOPAQUE_TEST_DATABASE_URL',
-        'postgresql://nopaque:nopaque@db/nopaque_test'
-    )
-
-    ''' # General # '''
     TESTING = True
     WTF_CSRF_ENABLED = False
 
+    ''' # Flask-SQLAlchemy # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'SQLALCHEMY_DATABASE_URI',
+        'postgresql://nopaque:nopaque@db/nopaque_test'
+    )
+
 
 config = {'development': DevelopmentConfig,
           'production': ProductionConfig,

web/nopaque.py

@@ -17,8 +17,7 @@ if os.path.exists(DOTENV_FILE):
 
 from app import create_app, db, socketio  # noqa
 from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
-                        NotificationData, NotificationEmailData, QueryResult,
+                        QueryResult, Role, User)  # noqa
-                        Role, User)  # noqa
 from flask_migrate import Migrate, upgrade  # noqa
 
 
@@ -34,8 +33,6 @@ def make_shell_context():
             'Job': Job,
             'JobInput': JobInput,
             'JobResult': JobResult,
-            'NotificationData': NotificationData,
-            'NotificationEmailData': NotificationEmailData,
             'QueryResult': QueryResult,
             'Role': Role,
             'User': User}
@@ -53,9 +50,9 @@ def deploy():
 
 @app.cli.command()
 def tasks():
-    from app.tasks import process_corpora, process_jobs
+    from app.tasks import check_corpora, check_jobs
-    process_corpora()
+    check_corpora()
-    process_jobs()
+    check_jobs()
 
 
 @app.cli.command()