sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-15 01:05:42 +00:00
Code Issues Projects Releases Wiki Activity

More exception handling. Remove unused database models. New common view structure!

Browse Source
This commit is contained in:
Patrick Jentsch 2020-11-13 10:01:51 +01:00
parent cb9da5c7dd
commit 5a06a6b241
45 changed files with 692 additions and 1005 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
.env.tpl
View File

@ -9,128 +9,116 @@
# NOTE: Use `.` as <project-root-dir> # NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR= # HOST_MQ_DIR=
# Example: 999 # Example: 1000
# HINT: Use this bash command `getent group docker | cut -d: -f3` # HINT: Use this bash command `id -u`
HOST_DOCKER_GID= HOST_UID=
# Example: 1000 # Example: 1000
# HINT: Use this bash command `id -g` # HINT: Use this bash command `id -g`
HOST_GID= HOST_GID=
# DEFAULT: ./nopaqued.log # Example: 999
# NOTES: Use `.` as <project-root-dir>, # HINT: Use this bash command `getent group docker | cut -d: -f3`
# This file must be present on container startup HOST_DOCKER_GID=
# HOST_NOPAQUE_DAEMON_LOG_FILE=
# DEFAULT: ./nopaque.log # DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>, # NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup # This file must be present on container startup
# HOST_NOPAQUE_LOG_FILE= # HOST_LOG_FILE=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
################################################################################ ################################################################################
# Cookies # # Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################ ################################################################################
# CHOOSE ONE: False, True # DEFAULT: hard to guess string
# DEFAULT: False # HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# HINT: Set to true if you redirect http to https # SECRET_KEY=
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# HINT: Set to true if you redirect http to https # HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE= # SESSION_COOKIE_SECURE=
################################################################################ ################################################################################
# Database # # Flask-Login #
# DATABASE_URI blueprint: # # https://flask-login.readthedocs.io/en/latest/ #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
################################################################################ ################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque # CHOOSE ONE: False, True
# NOPAQUE_DATABASE_URL= # DEFAULT: False
# HINT: Set to true if you redirect http to https
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev # REMEMBER_COOKIE_SECURE=
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
################################################################################ ################################################################################
# Email # # Flask-Mail #
# https://pythonhosted.org/Flask-Mail/ #
################################################################################ ################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com> # EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER= MAIL_DEFAULT_SENDER=
NOPAQUE_SMTP_PASSWORD= MAIL_PASSWORD=
# EXAMPLE: smtp.example.com # EXAMPLE: smtp.example.com
NOPAQUE_SMTP_SERVER= MAIL_SERVER=
# EXAMPLE: 587 # EXAMPLE: 587
NOPAQUE_SMTP_PORT= MAIL_PORT=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# NOPAQUE_SMTP_USE_SSL= # MAIL_USE_SSL=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# NOPAQUE_SMTP_USE_TLS= # MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com # EXAMPLE: nopaque@example.com
NOPAQUE_SMTP_USERNAME= MAIL_USERNAME=
################################################################################ ################################################################################
# General # # Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################ ################################################################################
# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
# SQLALCHEMY_DATABASE_URI=
################################################################################
# nopaque #
################################################################################
# If an account is registered with this email adress gets automatically
# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com # EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS= NOPAQUE_ADMIN=
# DEFAULT: development # DEFAULT: development
# CHOOSE ONE: development, production, testing # CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG= # NOPAQUE_CONFIG=
# This email adress is used for the contact button in the nopaque footer. If
# not set, no contact button is displayed.
# DEFAULT: None # DEFAULT: None
# EXAMPLE: contact.nopaque@example.com # EXAMPLE: contact.nopaque@example.com
# NOPAQUE_CONTACT_EMAIL_ADRESS= # NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque # DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes # NOTE: This must be a network share and it must be available on all Docker
# Swarm nodes
# NOPAQUE_DATA_DIR= # NOPAQUE_DATA_DIR=
# DEFAULT: localhost
# NOPAQUE_DOMAIN=
# DEFAULT: 0.0.0.0 # DEFAULT: 0.0.0.0
# NOPAQUE_HOST= # NOPAQUE_HOST=
# DEFAULT: 5000 # DEFAULT: 5000
# NOPAQUE_PORT= # NOPAQUE_PORT=
# CHOOSE ONE: http, https # transport://[userid:password]@hostname[:port]/[virtual_host]
# DEFAULT: http NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# NOPAQUE_PROTOCOL=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# NOPAQUE_SECRET_KEY=
################################################################################
# Logging #
################################################################################
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# DEFAULT: %Y-%m-%d %H:%M:%S # DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT= # NOPAQUE_LOG_DATE_FORMAT=
@ -146,37 +134,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL= # NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-For # Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR= # NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Host # Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST= # NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Port # Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT= # NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix # Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX= # NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto # Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO= # NOPAQUE_PROXY_FIX_X_PROTO=

1
docker-compose.yml
View File

@ -28,5 +28,6 @@ services:
image: nopaque:development image: nopaque:development
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}" - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"

5
web/Dockerfile
View File

@ -21,8 +21,9 @@ RUN apt-get update \
&& rm -r /var/lib/apt/lists/* && rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \ RUN groupadd --gid ${DOCKER_GID} --system docker \
&& useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque && groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque USER nopaque
WORKDIR /home/nopaque WORKDIR /home/nopaque

2
web/app/__init__.py
View File

@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app) mail.init_app(app)
paranoid.init_app(app) paranoid.init_app(app)
socketio.init_app( socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI) app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context(): with app.app_context():
from . import events from . import events

2
web/app/admin/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__) admin = Blueprint('admin', __name__)
from . import views # noqa from . import views

1
web/app/admin/forms.py
View File

@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs) super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name) self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()] for role in Role.query.order_by(Role.name).all()]
self.user = user

14
web/app/admin/views.py
View File

@ -29,12 +29,11 @@ def user(user_id):
@admin_required @admin_required
def delete_user(user_id): def delete_user(user_id):
settings_tasks.delete_user(user_id) settings_tasks.delete_user(user_id)
flash('User has been deleted!') flash('User has been marked for deletion!')
return redirect(url_for('.users')) return redirect(url_for('.users'))
@admin.route('/users/<int:user_id>/edit_general_settings', @admin.route('/users/<int:user_id>/edit_general_settings', methods=['GET', 'POST']) # noqa
methods=['GET', 'POST'])
@login_required @login_required
@admin_required @admin_required
def edit_general_settings(user_id): def edit_general_settings(user_id):
@ -46,16 +45,13 @@ def edit_general_settings(user_id):
user.username = form.username.data user.username = form.username.data
user.confirmed = form.confirmed.data user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data) user.role = Role.query.get(form.role.data)
db.session.add(user)
db.session.commit() db.session.commit()
flash('The profile has been updated.') flash('Settings have been updated.')
return redirect(url_for('admin.edit_general_settings', user_id=user.id)) return redirect(url_for('.edit_general_settings', user_id=user.id))
form.confirmed.data = user.confirmed form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email form.email.data = user.email
form.role.data = user.role_id form.role.data = user.role_id
form.username.data = user.username form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2', return render_template('admin/edit_general_settings.html.j2',
form=form, form=form, title='General settings', user=user)
title='General settings',
user=user)

2
web/app/auth/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__) auth = Blueprint('auth', __name__)
from . import views # noqa from . import views

2
web/app/auth/forms.py
View File

@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField( username = StringField(
'Username', 'Username',
validators=[DataRequired(), Length(1, 64), validators=[DataRequired(), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'], Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,' message='Usernames must have only letters, numbers,'
' dots or underscores')] ' dots or underscores')]
) )

95
web/app/auth/views.py
View File

@ -1,5 +1,5 @@
from flask import (current_app, flash, redirect, render_template, request, from datetime import datetime
url_for) from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user from flask_login import current_user, login_user, login_required, logout_user
from . import auth from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm, from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db from .. import db
from ..email import create_message, send from ..email import create_message, send
from ..models import User from ..models import User
import logging
import os import os
import shutil
@auth.before_app_request @auth.before_app_request
@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed. unconfirmed view if user is unconfirmed.
""" """
if current_user.is_authenticated: if current_user.is_authenticated:
current_user.ping() current_user.last_seen = datetime.utcnow()
if not current_user.confirmed \ db.session.commit()
and request.endpoint \ if (not current_user.confirmed
and request.blueprint != 'auth' \ and request.endpoint
and request.endpoint != 'static': and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed')) return redirect(url_for('auth.unconfirmed'))
@ -30,20 +31,19 @@ def before_request():
def login(): def login():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
login_form = LoginForm(prefix='login-form') form = LoginForm(prefix='login-form')
if login_form.validate_on_submit(): if form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first() user = User.query.filter_by(username=form.user.data).first()
if user is None: if user is None:
user = User.query.filter_by(email=login_form.user.data).first() user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(login_form.password.data): if user is not None and user.verify_password(form.password.data):
login_user(user, login_form.remember_me.data) login_user(user, form.remember_me.data)
next = request.args.get('next') next = request.args.get('next')
if next is None or not next.startswith('/'): if next is None or not next.startswith('/'):
next = url_for('main.dashboard') next = url_for('main.dashboard')
return redirect(next) return redirect(next)
flash('Invalid email/username or password.') flash('Invalid email/username or password.')
return render_template('auth/login.html.j2', login_form=login_form, return render_template('auth/login.html.j2', form=form, title='Log in')
title='Log in')
@auth.route('/logout') @auth.route('/logout')
@ -58,26 +58,28 @@ def logout():
def register(): def register():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
registration_form = RegistrationForm(prefix='registration-form') form = RegistrationForm(prefix='registration-form')
if registration_form.validate_on_submit(): if form.validate_on_submit():
user = User(email=registration_form.email.data.lower(), user = User(email=form.email.data.lower(),
password=registration_form.password.data, password=form.password.data,
username=registration_form.username.data) username=form.username.data)
db.session.add(user) db.session.add(user)
db.session.commit() db.session.commit()
user_dir = os.path.join(current_app.config['DATA_DIR'], try:
str(user.id)) os.makedirs(user.path)
if os.path.exists(user_dir): except OSError:
shutil.rmtree(user_dir) logging.error('Make dir {} led to an OSError!'.format(user.path))
os.mkdir(user_dir) db.session.delete(user)
db.session.commit()
abort(500)
else:
token = user.generate_confirmation_token() token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account', msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user) 'auth/email/confirm', token=token, user=user)
send(msg) send(msg)
flash('A confirmation email has been sent to you by email.') flash('A confirmation email has been sent to you by email.')
return redirect(url_for('auth.login')) return redirect(url_for('.login'))
return render_template('auth/register.html.j2', return render_template('auth/register.html.j2', form=form,
registration_form=registration_form,
title='Register') title='Register')
@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
else: else:
flash('The confirmation link is invalid or has expired.') flash('The confirmation link is invalid or has expired.')
return redirect(url_for('auth.unconfirmed')) return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed') @auth.route('/unconfirmed')
@ -119,23 +121,18 @@ def resend_confirmation():
def reset_password_request(): def reset_password_request():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
reset_password_request_form = ResetPasswordRequestForm( form = ResetPasswordRequestForm(prefix='reset-password-request-form')
prefix='reset-password-request-form') if form.validate_on_submit():
if reset_password_request_form.validate_on_submit(): user = User.query.filter_by(email=form.email.data.lower()).first()
submitted_email = reset_password_request_form.email.data if user is not None:
user = User.query.filter_by(email=submitted_email.lower()).first()
if user:
token = user.generate_reset_token() token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password', msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token, 'auth/email/reset_password', token=token,
user=user) user=user)
send(msg) send(msg)
flash('An email with instructions to reset your password has been ' flash('An email with instructions to reset your password has been sent to you.') # noqa
'sent to you.') return redirect(url_for('.login'))
return redirect(url_for('auth.login')) return render_template('auth/reset_password_request.html.j2', form=form,
return render_template(
'auth/reset_password_request.html.j2',
reset_password_request_form=reset_password_request_form,
title='Password Reset') title='Password Reset')
@ -143,15 +140,13 @@ def reset_password_request():
def reset_password(token): def reset_password(token):
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
reset_password_form = ResetPasswordForm(prefix='reset-password-form') form = ResetPasswordForm(prefix='reset-password-form')
if reset_password_form.validate_on_submit(): if form.validate_on_submit():
if User.reset_password(token, reset_password_form.password.data): if User.reset_password(token, form.password.data):
db.session.commit() db.session.commit()
flash('Your password has been updated.') flash('Your password has been updated.')
return redirect(url_for('auth.login')) return redirect(url_for('.login'))
else: else:
return redirect(url_for('main.index')) return redirect(url_for('main.index'))
return render_template('auth/reset_password.html.j2', return render_template('auth/reset_password.html.j2', form=form,
reset_password_form=reset_password_form, title='Password Reset', token=token)
title='Password Reset',
token=token)

326
web/app/corpora/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, make_response, redirect, request, from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory) render_template, url_for, send_from_directory)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from . import corpora from . import corpora
@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db from .. import db
from ..models import Corpus, CorpusFile, QueryResult from ..models import Corpus, CorpusFile, QueryResult
import json import json
import logging
import os import os
import shutil import shutil
import glob import glob
@ -22,64 +23,58 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST']) @corpora.route('/add', methods=['GET', 'POST'])
@login_required @login_required
def add_corpus(): def add_corpus():
add_corpus_form = AddCorpusForm() form = AddCorpusForm()
if add_corpus_form.validate_on_submit(): if form.validate_on_submit():
corpus = Corpus(creator=current_user, corpus = Corpus(creator=current_user,
description=add_corpus_form.description.data, description=form.description.data,
status='unprepared', title=add_corpus_form.title.data) title=form.title.data)
db.session.add(corpus) db.session.add(corpus)
db.session.commit() db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try: try:
os.makedirs(dir) os.makedirs(corpus.path)
except OSError: except OSError:
flash('[ERROR]: Could not add corpus!', 'corpus') logging.error('Make dir {} led to an OSError!'.format(corpus.path))
corpus.delete() db.session.delete(corpus)
else: db.session.commit()
url = url_for('corpora.corpus', corpus_id=corpus.id) abort(500)
flash('[<a href="{}">{}</a>] added'.format(url, corpus.title), flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
'corpus') return redirect(url_for('.corpus', corpus_id=corpus.id))
return redirect(url_for('corpora.corpus', corpus_id=corpus.id)) return render_template('corpora/add_corpus.html.j2', form=form,
return render_template('corpora/add_corpus.html.j2',
add_corpus_form=add_corpus_form,
title='Add corpus') title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST']) @corpora.route('/import', methods=['GET', 'POST'])
@login_required @login_required
def import_corpus(): def import_corpus():
import_corpus_form = ImportCorpusForm() form = ImportCorpusForm()
if import_corpus_form.is_submitted(): if form.is_submitted():
if not import_corpus_form.validate(): if not form.validate():
return make_response(import_corpus_form.errors, 400) return make_response(form.errors, 400)
corpus = Corpus(creator=current_user, corpus = Corpus(creator=current_user,
description=import_corpus_form.description.data, description=form.description.data,
status='unprepared', title=form.title.data)
title=import_corpus_form.title.data)
db.session.add(corpus) db.session.add(corpus)
db.session.commit() db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try: try:
os.makedirs(dir) os.makedirs(corpus.path)
except OSError: except OSError:
flash('[ERROR]: Could not import corpus!', 'corpus') logging.error('Make dir {} led to an OSError!'.format(corpus.path))
corpus.delete() db.session.delete(corpus)
else: db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.import_corpus')}, 500)
# Upload zip # Upload zip
archive_file = os.path.join(current_app.config['DATA_DIR'], dir, archive_file = os.path.join(corpus.path, form.file.data.filename)
import_corpus_form.file.data.filename) form.file.data.save(archive_file)
corpus_dir = os.path.dirname(archive_file)
import_corpus_form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus # Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip: with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist() contents = zip.namelist()
if set(check_zip_contents).issubset(contents): if set(check_zip_contents).issubset(contents):
# Unzip # Unzip
shutil.unpack_archive(archive_file, corpus_dir) shutil.unpack_archive(archive_file, corpus.path)
# Register vrt files to corpus # Register vrt files to corpus
vrts = glob.glob(corpus_dir + '/*.vrt') vrts = glob.glob(corpus.path + '/*.vrt')
for file in vrts: for file in vrts:
element_tree = ET.parse(file) element_tree = ET.parse(file)
text_node = element_tree.find('text') text_node = element_tree.find('text')
@ -89,7 +84,6 @@ def import_corpus():
booktitle=text_node.get('booktitle', 'NULL'), booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'), chapter=text_node.get('chapter', 'NULL'),
corpus=corpus, corpus=corpus,
dir=dir,
editor=text_node.get('editor', 'NULL'), editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file), filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'), institution=text_node.get('institution', 'NULL'),
@ -98,30 +92,23 @@ def import_corpus():
publisher=text_node.get('publisher', 'NULL'), publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''), publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'), school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL')) title=text_node.get('title', 'NULL')
)
db.session.add(corpus_file) db.session.add(corpus_file)
# finish import and got to imported corpus # finish import and redirect to imported corpus
url = url_for('corpora.corpus', corpus_id=corpus.id)
corpus.status = 'prepared' corpus.status = 'prepared'
db.session.commit() db.session.commit()
os.remove(archive_file) os.remove(archive_file)
flash('[<a href="{}">{}</a>] imported'.format(url, flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
corpus.title),
'corpus')
return make_response( return make_response(
{'redirect_url': url_for('corpora.corpus', {'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
corpus_id=corpus.id)},
201)
else: else:
# If imported zip is not valid delete corpus and give feedback # If imported zip is not valid delete corpus and give feedback
corpus.delete() flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
db.session.commit() tasks.delete_corpus(corpus.id)
flash('Imported corpus is not valid.', 'error')
return make_response( return make_response(
{'redirect_url': url_for('corpora.import_corpus')}, {'redirect_url': url_for('.import_corpus')}, 201)
201) return render_template('corpora/import_corpus.html.j2', form=form,
return render_template('corpora/import_corpus.html.j2',
import_corpus_form=import_corpus_form,
title='Import Corpus') title='Import Corpus')
@ -131,17 +118,9 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
corpus_files = [dict(filename=corpus_file.filename, corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
author=corpus_file.author, return render_template('corpora/corpus.html.j2', corpus=corpus,
title=corpus_file.title, corpus_files=corpus_files, title='Corpus')
publishing_year=corpus_file.publishing_year,
corpus_id=corpus.id,
id=corpus_file.id)
for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus')
@corpora.route('/<int:corpus_id>/export') @corpora.route('/<int:corpus_id>/export')
@ -150,12 +129,11 @@ def export_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
# TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file) dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file) filename = os.path.basename(corpus.archive_file)
return send_from_directory(directory=dir, return send_from_directory(as_attachment=True, directory=dir,
filename=filename, filename=filename, mimetype='zip')
mimetype='zip',
as_attachment=True)
@corpora.route('/<int:corpus_id>/analyse') @corpora.route('/<int:corpus_id>/analyse')
@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm( display_options_form = DisplayOptionsForm(
prefix='display-options-form', prefix='display-options-form',
result_context=request.args.get('context', 20), result_context=request.args.get('context', 20),
results_per_page=request.args.get('results_per_page', 30)) results_per_page=request.args.get('results_per_page', 30)
)
query_form = QueryForm(prefix='query-form', query_form = QueryForm(prefix='query-form',
query=request.args.get('query')) query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form') query_download_form = QueryDownloadForm(prefix='query-download-form')
@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template( return render_template(
'corpora/analyse_corpus.html.j2', 'corpora/analyse_corpus.html.j2',
corpus=corpus, corpus=corpus,
corpus_id=corpus_id,
display_options_form=display_options_form, display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_form=query_form, query_form=query_form,
query_download_form=query_download_form, query_download_form=query_download_form,
inspect_display_options_form=inspect_display_options_form, title='Corpus analysis'
title='Corpus analysis') )
@corpora.route('/<int:corpus_id>/delete') @corpora.route('/<int:corpus_id>/delete')
@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id) tasks.delete_corpus(corpus_id)
flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
add_corpus_file_form = AddCorpusFileForm(corpus, form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
prefix='add-corpus-file-form') if form.is_submitted():
if add_corpus_file_form.is_submitted(): if not form.validate():
if not add_corpus_file_form.validate(): return make_response(form.errors, 400)
return make_response(add_corpus_file_form.errors, 400)
# Save the file # Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id)) form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
add_corpus_file_form.file.data.save( corpus_file = CorpusFile(address=form.address.data,
os.path.join(current_app.config['DATA_DIR'], dir, author=form.author.data,
add_corpus_file_form.file.data.filename)) booktitle=form.booktitle.data,
corpus_file = CorpusFile( chapter=form.chapter.data,
address=add_corpus_file_form.address.data,
author=add_corpus_file_form.author.data,
booktitle=add_corpus_file_form.booktitle.data,
chapter=add_corpus_file_form.chapter.data,
corpus=corpus, corpus=corpus,
dir=dir, editor=form.editor.data,
editor=add_corpus_file_form.editor.data, filename=form.file.data.filename,
filename=add_corpus_file_form.file.data.filename, institution=form.institution.data,
institution=add_corpus_file_form.institution.data, journal=form.journal.data,
journal=add_corpus_file_form.journal.data, pages=form.pages.data,
pages=add_corpus_file_form.pages.data, publisher=form.publisher.data,
publisher=add_corpus_file_form.publisher.data, publishing_year=form.publishing_year.data,
publishing_year=add_corpus_file_form.publishing_year.data, school=form.school.data,
school=add_corpus_file_form.school.data, title=form.title.data)
title=add_corpus_file_form.title.data)
db.session.add(corpus_file) db.session.add(corpus_file)
corpus.status = 'unprepared' corpus.status = 'unprepared'
db.session.commit() db.session.commit()
flash('Corpus file added!', 'corpus') flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
return make_response( return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
{'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)}, return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
201) form=form, title='Add corpus file')
return render_template('corpora/add_corpus_file.html.j2',
corpus=corpus,
add_corpus_file_form=add_corpus_file_form,
title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete') @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id) tasks.delete_corpus_file(corpus_file_id)
flash('Corpus file deleted!', 'corpus') return redirect(url_for('.corpus', corpus_id=corpus_id))
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download') @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
corpus_file.dir) directory=corpus_file.corpus.path,
return send_from_directory(as_attachment=True, directory=dir,
filename=corpus_file.filename) filename=corpus_file.filename)
@ -274,48 +242,45 @@ def download_corpus_file(corpus_id, corpus_file_id):
methods=['GET', 'POST']) methods=['GET', 'POST'])
@login_required @login_required
def corpus_file(corpus_id, corpus_file_id): def corpus_file(corpus_id, corpus_file_id):
corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id) corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if not corpus_file.corpus_id == corpus_id: if corpus_file.corpus_id != corpus_id:
abort(404) abort(404)
if not (corpus_file.corpus.creator == current_user if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form') form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if edit_corpus_file_form.validate_on_submit(): if form.validate_on_submit():
corpus_file.address = edit_corpus_file_form.address.data corpus_file.address = form.address.data
corpus_file.author = edit_corpus_file_form.author.data corpus_file.author = form.author.data
corpus_file.booktitle = edit_corpus_file_form.booktitle.data corpus_file.booktitle = form.booktitle.data
corpus_file.chapter = edit_corpus_file_form.chapter.data corpus_file.chapter = form.chapter.data
corpus_file.editor = edit_corpus_file_form.editor.data corpus_file.editor = form.editor.data
corpus_file.institution = edit_corpus_file_form.institution.data corpus_file.institution = form.institution.data
corpus_file.journal = edit_corpus_file_form.journal.data corpus_file.journal = form.journal.data
corpus_file.pages = edit_corpus_file_form.pages.data corpus_file.pages = form.pages.data
corpus_file.publisher = edit_corpus_file_form.publisher.data corpus_file.publisher = form.publisher.data
corpus_file.publishing_year = \ corpus_file.publishing_year = form.publishing_year.data
edit_corpus_file_form.publishing_year.data corpus_file.school = form.school.data
corpus_file.school = edit_corpus_file_form.school.data corpus_file.title = form.title.data
corpus_file.title = edit_corpus_file_form.title.data
corpus.status = 'unprepared' corpus.status = 'unprepared'
db.session.commit() db.session.commit()
flash('Corpus file edited!', 'corpus') flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values # If no form is submitted or valid, fill out fields with current values
edit_corpus_file_form.address.data = corpus_file.address form.address.data = corpus_file.address
edit_corpus_file_form.author.data = corpus_file.author form.author.data = corpus_file.author
edit_corpus_file_form.booktitle.data = corpus_file.booktitle form.booktitle.data = corpus_file.booktitle
edit_corpus_file_form.chapter.data = corpus_file.chapter form.chapter.data = corpus_file.chapter
edit_corpus_file_form.editor.data = corpus_file.editor form.editor.data = corpus_file.editor
edit_corpus_file_form.institution.data = corpus_file.institution form.institution.data = corpus_file.institution
edit_corpus_file_form.journal.data = corpus_file.journal form.journal.data = corpus_file.journal
edit_corpus_file_form.pages.data = corpus_file.pages form.pages.data = corpus_file.pages
edit_corpus_file_form.publisher.data = corpus_file.publisher form.publisher.data = corpus_file.publisher
edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year form.publishing_year.data = corpus_file.publishing_year
edit_corpus_file_form.school.data = corpus_file.school form.school.data = corpus_file.school
edit_corpus_file_form.title.data = corpus_file.title form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2', return render_template('corpora/corpus_file.html.j2', corpus=corpus,
corpus_file=corpus_file, corpus=corpus, corpus_file=corpus_file, form=form,
edit_corpus_file_form=edit_corpus_file_form,
title='Edit corpus file') title='Edit corpus file')
@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403) abort(403)
if corpus.files.all(): if corpus.files.all():
tasks.build_corpus(corpus_id) tasks.build_corpus(corpus_id)
flash('Building Corpus...', 'corpus') flash('Corpus "{}" has been marked to get build!', 'corpus')
else: else:
flash('Can not build corpus, please add corpus file(s).', 'corpus') flash('Can not build corpus "{}": No corpus file(s)!', 'error')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results. # Following are view functions to add, view etc. exported results.
@ -340,35 +305,29 @@ def add_query_result():
''' '''
View to import a result as a json file. View to import a result as a json file.
''' '''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form') form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted(): if form.is_submitted():
if not add_query_result_form.validate(): if not form.validate():
return make_response(add_query_result_form.errors, 400) return make_response(form.errors, 400)
query_result = QueryResult( query_result = QueryResult(creator=current_user,
creator=current_user, description=form.description.data,
description=add_query_result_form.description.data, filename=form.file.data.filename,
filename=add_query_result_form.file.data.filename, title=form.title.data)
title=add_query_result_form.title.data
)
db.session.add(query_result) db.session.add(query_result)
db.session.commit() db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try: try:
os.makedirs(query_result_dir) os.makedirs(query_result.path)
except Exception: except OSError:
logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result) db.session.delete(query_result)
db.session.commit() db.session.commit()
flash('Internal Server Error', 'error') flash('Internal Server Error', 'error')
redirect_url = url_for('corpora.add_query_result') return make_response(
return make_response({'redirect_url': redirect_url}, 500) {'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file # save the uploaded file
query_result_file_path = os.path.join(query_result_dir, query_result_file_path = os.path.join(query_result.path,
query_result.filename) query_result.filename)
add_query_result_form.file.data.save(query_result_file_path) form.file.data.save(query_result_file_path)
# parse json from file # parse json from file
with open(query_result_file_path, 'r') as file: with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file) query_result_file_content = json.load(file)
@ -381,19 +340,16 @@ def add_query_result():
except Exception: except Exception:
tasks.delete_query_result(query_result.id) tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result') flash('Uploaded file is invalid', 'result')
redirect_url = url_for('corpora.add_query_result') return make_response(
return make_response({'redirect_url': redirect_url}, 201) {'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches') query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup') query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content query_result.query_metadata = query_result_file_content
db.session.commit() db.session.commit()
flash('Query result added!', 'result') flash('Query result added!', 'result')
redirect_url = url_for('corpora.query_result', return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2', return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form, form=form, title='Add query result')
title='Add query result')
@corpora.route('/result/<int:query_result_id>') @corpora.route('/result/<int:query_result_id>')
@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
return render_template('corpora/query_results/query_result.html.j2', return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result, query_result=query_result, title='Query result')
title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect') @corpora.route('/result/<int:query_result_id>/inspect')
@ -427,13 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm( inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form' prefix='inspect-display-options-form'
) )
query_result_file_path = os.path.join( query_result_file_path = os.path.join(query_result.path, query_result.filename) # noqa
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file: with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file) query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2', return render_template('corpora/query_results/inspect.html.j2',
@ -452,8 +401,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user if not (query_result.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id) tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis")) return redirect(url_for('services.service', service="corpus_analysis"))
@ -464,10 +413,5 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user if not (query_result.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True, directory=query_result.path,
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename) filename=query_result.filename)

4
web/app/email.py
View File

@ -1,11 +1,11 @@
from flask import render_template from flask import current_app, render_template
from flask_mail import Message from flask_mail import Message
from . import mail from . import mail
from .decorators import background from .decorators import background
def create_message(recipient, subject, template, **kwargs): def create_message(recipient, subject, template, **kwargs):
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient]) msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs) msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs) msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg return msg

2
web/app/jobs/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__) jobs = Blueprint('jobs', __name__)
from . import views # noqa from . import views

30
web/app/jobs/views.py
View File

@ -1,11 +1,10 @@
from flask import (abort, current_app, flash, redirect, render_template, from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for) send_from_directory, url_for)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from . import jobs from . import jobs
from . import tasks from . import tasks
from ..decorators import admin_required from ..decorators import admin_required
from ..models import Job, JobInput, JobResult from ..models import Job, JobInput, JobResult
import os
@jobs.route('/<int:job_id>') @jobs.route('/<int:job_id>')
@ -14,13 +13,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id) job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()): if not (job.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
job_inputs = [dict(filename=input.filename, job_inputs = [job_input.to_dict() for job_input in job.inputs]
id=input.id, return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
job_id=job.id)
for input in job.inputs]
return render_template('jobs/job.html.j2',
job=job,
job_inputs=job_inputs,
title='Job') title='Job')
@ -31,7 +25,7 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()): if not (job.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
tasks.delete_job(job_id) tasks.delete_job(job_id)
flash('Job has been deleted!', 'job') flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
@ -44,9 +38,8 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user if not (job_input.job.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
job_input.dir) directory=job_input.job.path,
return send_from_directory(as_attachment=True, directory=dir,
filename=job_input.filename) filename=job_input.filename)
@ -56,11 +49,11 @@ def download_job_input(job_id, job_input_id):
def restart(job_id): def restart(job_id):
job = Job.query.get_or_404(job_id) job = Job.query.get_or_404(job_id)
if job.status != 'failed': if job.status != 'failed':
flash('Could not restart job: status is not "failed"', 'error') flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else: else:
tasks.restart_job(job_id) tasks.restart_job(job_id)
flash('Job has been restarted!', 'job') flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
return redirect(url_for('jobs.job', job_id=job_id)) return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download') @jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@ -72,7 +65,6 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user if not (job_result.job.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
job_result.dir) directory=job_result.job.path,
return send_from_directory(as_attachment=True, directory=dir,
filename=job_result.filename) filename=job_result.filename)

2
web/app/main/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__) main = Blueprint('main', __name__)
from . import views # noqa from . import views

25
web/app/main/views.py
View File

@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST']) @main.route('/', methods=['GET', 'POST'])
def index(): def index():
login_form = LoginForm(prefix='login-form') form = LoginForm(prefix='login-form')
if login_form.validate_on_submit(): if form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first() user = User.query.filter_by(username=form.user.data).first()
if user is None: if user is None:
user = User.query.filter_by(email=login_form.user.data).first() user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(login_form.password.data): if user is not None and user.verify_password(form.password.data):
login_user(user, login_form.remember_me.data) login_user(user, form.remember_me.data)
return redirect(url_for('main.dashboard')) return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.') flash('Invalid email/username or password.')
return render_template('main/index.html.j2', login_form=login_form, return render_template('main/index.html.j2', form=form, title='nopaque')
title='nopaque')
@main.route('/about_and_faq') @main.route('/about_and_faq')
@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard') return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/news') @main.route('/news')
def news(): def news():
return render_template('main/news.html.j2', title='News') return render_template('main/news.html.j2', title='News')
@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy') @main.route('/privacy_policy')
def privacy_policy(): def privacy_policy():
return render_template('main/privacy_policy.html.j2', return render_template('main/privacy_policy.html.j2',
title=('Information on the processing of personal' title='Privacy statement (GDPR)')
' data for the nopaque platform (GDPR)'))
@main.route('/terms_of_use') @main.route('/terms_of_use')
def terms_of_use(): def terms_of_use():
return render_template('main/terms_of_use.html.j2', return render_template('main/terms_of_use.html.j2', title='Terms of Use')
title='General Terms of Use of the platform '
'nopaque')

181
web/app/models.py
View File

@ -7,6 +7,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from . import db, login_manager from . import db, login_manager
import logging
import os import os
import shutil import shutil
@ -54,7 +55,7 @@ class Role(db.Model):
''' '''
String representation of the Role. For human readability. String representation of the Role. For human readability.
''' '''
return '<Role {role_name}>'.format(role_name=self.name) return '<Role {}>'.format(self.name)
def add_permission(self, perm): def add_permission(self, perm):
''' '''
@ -138,6 +139,18 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete', cascade='save-update, merge, delete',
lazy='dynamic') lazy='dynamic')
@property
def path(self):
return os.path.join(current_app.config['NOPAQUE_DATA_DIR'], str(self.id))
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'id': self.id,
'role_id': self.role_id, 'role_id': self.role_id,
@ -162,7 +175,7 @@ class User(UserMixin, db.Model):
''' '''
String representation of the User. For human readability. String representation of the User. For human readability.
''' '''
return '<User {username}>'.format(username=self.username) return '<User {}>'.format(self.username)
def __init__(self, **kwargs): def __init__(self, **kwargs):
super(User, self).__init__(**kwargs) super(User, self).__init__(**kwargs)
@ -220,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user) db.session.add(user)
return True return True
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password): def verify_password(self, password):
return check_password_hash(self.password_hash, password) return check_password_hash(self.password_hash, password)
@ -244,17 +249,11 @@ class User(UserMixin, db.Model):
''' '''
return self.can(Permission.ADMIN) return self.can(Permission.ADMIN)
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def delete(self): def delete(self):
''' '''
Delete the user and its corpora and jobs from database and filesystem. Delete the user and its corpora and jobs from database and filesystem.
''' '''
user_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
@ -280,14 +279,17 @@ class JobInput(db.Model):
# Foreign keys # Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id')) job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields # Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self): def __repr__(self):
''' '''
String representation of the JobInput. For human readability. String representation of the JobInput. For human readability.
''' '''
return '<JobInput {filename}>'.format(filename=self.filename) return '<JobInput {}>'.format(self.filename)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'id': self.id,
@ -305,14 +307,17 @@ class JobResult(db.Model):
# Foreign keys # Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id')) job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields # Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self): def __repr__(self):
''' '''
String representation of the JobResult. For human readability. String representation of the JobResult. For human readability.
''' '''
return '<JobResult {filename}>'.format(filename=self.filename) return '<JobResult {}>'.format(self.filename)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'id': self.id,
@ -351,19 +356,16 @@ class Job(db.Model):
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic', results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
notification_data = db.relationship('NotificationData',
cascade='save-update, merge, delete', @property
uselist=False, def path(self):
back_populates='job') # One-to-One relationship return os.path.join(self.creator.path, 'jobs', str(self.id))
notification_email_data = db.relationship('NotificationEmailData',
cascade='save-update, merge, delete',
back_populates='job')
def __repr__(self): def __repr__(self):
''' '''
String representation of the Job. For human readability. String representation of the Job. For human readability.
''' '''
return '<Job {job_title}>'.format(job_title=self.title) return '<Job {}>'.format(self.title)
def create_secure_filename(self): def create_secure_filename(self):
''' '''
@ -385,11 +387,7 @@ class Job(db.Model):
db.session.commit() db.session.commit()
sleep(1) sleep(1)
db.session.refresh(self) db.session.refresh(self)
job_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(job_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def restart(self): def restart(self):
@ -399,12 +397,8 @@ class Job(db.Model):
if self.status != 'failed': if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"') raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
str(self.user_id), shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
'jobs',
str(self.id))
shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
self.end_date = None self.end_date = None
self.status = 'submitted' self.status = 'submitted'
@ -425,63 +419,6 @@ class Job(db.Model):
for result in self.results}} for result in self.results}}
class NotificationData(db.Model):
'''
Class to define notification data used for sending a notification mail with
nopaque_notify.
'''
__tablename__ = 'notification_data'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_data')
# Fields
notified_on = db.Column(db.String(16), default=None)
def __repr__(self):
'''
String representation of the NotificationData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notified': self.notified}
class NotificationEmailData(db.Model):
'''
Class to define data that will be used to send a corresponding Notification
via email.
'''
__tablename__ = 'notification_email_data'
# Primary Key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_email_data')
notify_status = db.Column(db.String(16), default=None)
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
def __repr__(self):
'''
String representation of the NotificationEmailData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notify_status': self.notify_status,
'creation_date': self.creation_date}
class CorpusFile(db.Model): class CorpusFile(db.Model):
''' '''
Class to define Files. Class to define Files.
@ -496,7 +433,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255)) author = db.Column(db.String(255))
booktitle = db.Column(db.String(255)) booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255)) chapter = db.Column(db.String(255))
dir = db.Column(db.String(255))
editor = db.Column(db.String(255)) editor = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
institution = db.Column(db.String(255)) institution = db.Column(db.String(255))
@ -507,15 +443,15 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255)) school = db.Column(db.String(255))
title = db.Column(db.String(255)) title = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.corpus.path, self.filename)
def delete(self): def delete(self):
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
self.filename)
try: try:
os.remove(corpus_file_path) os.remove(self.path)
except OSError: except OSError:
logging.error('Removing {} led to an OSError!'.format(self.path))
pass pass
db.session.delete(self) db.session.delete(self)
self.corpus.status = 'unprepared' self.corpus.status = 'unprepared'
@ -553,13 +489,17 @@ class Corpus(db.Model):
description = db.Column(db.String(255)) description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow) last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647) max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
status = db.Column(db.String(16)) status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32)) title = db.Column(db.String(32))
archive_file = db.Column(db.String(255)) archive_file = db.Column(db.String(255))
# Relationships # Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic', files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
@property
def path(self):
return os.path.join(self.creator.path, 'corpora', str(self.id))
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'id': self.id,
'user_id': self.user_id, 'user_id': self.user_id,
@ -571,19 +511,14 @@ class Corpus(db.Model):
'files': {file.id: file.to_dict() for file in self.files}} 'files': {file.id: file.to_dict() for file in self.files}}
def build(self): def build(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], output_dir = os.path.join(self.path, 'merged')
str(self.user_id),
'corpora',
str(self.id))
output_dir = os.path.join(corpus_dir, 'merged')
shutil.rmtree(output_dir, ignore_errors=True) shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir) os.mkdir(output_dir)
master_element_tree = ET.ElementTree( master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>') ET.fromstring('<corpus>\n</corpus>')
) )
for corpus_file in self.files: for corpus_file in self.files:
corpus_file_path = os.path.join(corpus_dir, corpus_file.filename) element_tree = ET.parse(corpus_file.path)
element_tree = ET.parse(corpus_file_path)
text_node = element_tree.find('text') text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL") text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author) text_node.set('author', corpus_file.author)
@ -597,7 +532,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year)) text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL") text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title) text_node.set('title', corpus_file.title)
element_tree.write(corpus_file_path) element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node) master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt') output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file, master_element_tree.write(output_file,
@ -607,18 +542,14 @@ class Corpus(db.Model):
self.status = 'submitted' self.status = 'submitted'
def delete(self): def delete(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'corpora',
str(self.id))
shutil.rmtree(corpus_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def __repr__(self): def __repr__(self):
''' '''
String representation of the corpus. For human readability. String representation of the corpus. For human readability.
''' '''
return '<Corpus {corpus_title}>'.format(corpus_title=self.title) return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model): class QueryResult(db.Model):
@ -636,12 +567,12 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON()) query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32)) title = db.Column(db.String(32))
@property
def path(self):
return os.path.join(self.creator.path, 'query_results', str(self.id))
def delete(self): def delete(self):
query_result_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'query_results',
str(self.id))
shutil.rmtree(query_result_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def to_dict(self): def to_dict(self):
@ -654,7 +585,7 @@ class QueryResult(db.Model):
def __repr__(self): def __repr__(self):
''' '''
String representation of the CorpusAnalysisResult. For human readability. String representation of the QueryResult. For human readability.
''' '''
return '<QueryResult {}>'.format(self.title) return '<QueryResult {}>'.format(self.title)

150
web/app/query_results/views.py
View File

@ -1,150 +0,0 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__) services = Blueprint('services', __name__)
from . import views # noqa from . import views

61
web/app/services/views.py
View File

@ -1,5 +1,4 @@
from flask import (abort, current_app, flash, make_response, render_template, from flask import abort, flash, make_response, render_template, url_for
url_for)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from . import services from . import services
@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput from ..models import Job, JobInput
import json import json
import logging
import os import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'}, SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup', 'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4}, 'resources': {'mem_mb': 4096, 'n_cores': 4},
'add_job_form': AddFileSetupJobForm}, 'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing', 'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2}, 'resources': {'mem_mb': 4096, 'n_cores': 2},
'add_job_form': AddNLPJobForm}, 'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition', 'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4}, 'resources': {'mem_mb': 8192, 'n_cores': 4},
'add_job_form': AddOCRJobForm}} 'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST']) @services.route('/<service>', methods=['GET', 'POST'])
@ -30,54 +30,49 @@ def service(service):
if service == 'corpus_analysis': if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service), return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name']) title=SERVICES[service]['name'])
add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form') form = SERVICES[service]['form'](prefix='add-job-form')
if add_job_form.is_submitted(): if form.is_submitted():
if not add_job_form.validate(): if not form.validate():
return make_response(add_job_form.errors, 400) return make_response(form.errors, 400)
service_args = [] service_args = []
if service == 'nlp': if service == 'nlp':
service_args.append('-l {}'.format(add_job_form.language.data)) service_args.append('-l {}'.format(form.language.data))
if add_job_form.check_encoding.data: if form.check_encoding.data:
service_args.append('--check-encoding') service_args.append('--check-encoding')
if service == 'ocr': if service == 'ocr':
service_args.append('-l {}'.format(add_job_form.language.data)) service_args.append('-l {}'.format(form.language.data))
if add_job_form.binarization.data: if form.binarization.data:
service_args.append('--binarize') service_args.append('--binarize')
job = Job(creator=current_user, job = Job(creator=current_user,
description=add_job_form.description.data, description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'], mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'], n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args), service=service, service_args=json.dumps(service_args),
service_version=add_job_form.version.data, service_version=form.version.data,
status='preparing', title=add_job_form.title.data) status='preparing', title=form.title.data)
if job.service != 'corpus_analysis': if job.service != 'corpus_analysis':
job.create_secure_filename() job.create_secure_filename()
db.session.add(job) db.session.add(job)
db.session.commit() db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try: try:
os.makedirs(absolut_dir) os.makedirs(job.path)
except OSError: except OSError:
job.delete() logging.error('Make dir {} led to an OSError!'.format(job.path))
flash('Internal Server Error', 'job') db.session.delete(job)
return make_response({'redirect_url': url_for('services.service', db.session.commit()
service=service)}, flash('Internal Server Error', 'error')
500) return make_response(
{'redirect_url': url_for('.service', service=service)}, 500)
else: else:
for file in add_job_form.files.data: for file in form.files.data:
filename = secure_filename(file.filename) filename = secure_filename(file.filename)
file.save(os.path.join(absolut_dir, filename)) job_input = JobInput(dir=job.path, filename=filename, job=job)
job_input = JobInput(dir=relative_dir, filename=filename, file.save(job_input.path)
job=job)
db.session.add(job_input) db.session.add(job_input)
job.status = 'submitted' job.status = 'submitted'
db.session.commit() db.session.commit()
url = url_for('jobs.job', job_id=job.id) flash('Job "{}" added'.format(job.title), 'job')
flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
return make_response( return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) {'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service), return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'], form=form, title=SERVICES[service]['name'])
add_job_form=add_job_form)

2
web/app/settings/forms.py
View File

@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername', 'Benutzername',
validators=[DataRequired(), validators=[DataRequired(),
Length(1, 64), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'], Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,' message='Usernames must have only letters, numbers,'
' dots or underscores')] ' dots or underscores')]
) )

19
web/app/settings/views.py
View File

@ -1,13 +1,9 @@
from flask import current_app, flash, redirect, render_template, url_for from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user from flask_login import current_user, login_required, logout_user
from . import settings, tasks from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm, from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm) EditNotificationSettingsForm)
from .. import db from .. import db
from ..decorators import admin_required
from ..models import Role, User
import os
import uuid
@settings.route('/') @settings.route('/')
@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.') flash('Your password has been updated.')
return redirect(url_for('.change_password')) return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2', return render_template('settings/change_password.html.j2',
form=form, form=form, title='Change password')
title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST']) @settings.route('/edit_general_settings', methods=['GET', 'POST'])
@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data current_user.username = form.username.data
db.session.commit() db.session.commit()
flash('Your changes have been saved.') flash('Your changes have been saved.')
return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email form.email.data = current_user.email
form.username.data = current_user.username form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2', return render_template('settings/edit_general_settings.html.j2',
form=form, form=form, title='General settings')
title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST']) @settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data form.job_status_site_notifications.data
db.session.commit() db.session.commit()
flash('Your changes have been saved.') flash('Your changes have been saved.')
return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \ form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \ form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2', return render_template('settings/edit_notification_settings.html.j2',
form=form, form=form, title='Notification settings')
title='Notification settings')
@settings.route('/delete') @settings.route('/delete')
@ -76,5 +71,5 @@ def delete():
""" """
tasks.delete_user(current_user.id) tasks.delete_user(current_user.id)
logout_user() logout_user()
flash('Your account has been deleted!') flash('Your account has been marked for deletion!')
return redirect(url_for('main.index')) return redirect(url_for('main.index'))

14
web/app/tasks/__init__.py
View File

@ -11,15 +11,11 @@ def check_corpora():
corpora = Corpus.query.all() corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora): for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus) corpus_utils.create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: (corpus.status == 'queued' for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
or corpus.status == 'running'),
corpora):
corpus_utils.checkout_build_corpus_service(corpus) corpus_utils.checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis', for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpora):
corpus_utils.create_cqpserver_container(corpus) corpus_utils.create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis', for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpora):
corpus_utils.remove_cqpserver_container(corpus) corpus_utils.remove_cqpserver_container(corpus)
db.session.commit() db.session.commit()
@ -28,8 +24,6 @@ def check_jobs():
jobs = Job.query.all() jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs): for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job) job_utils.create_job_service(job)
for job in filter(lambda job: job.status == 'queued', jobs): for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
for job in filter(lambda job: job.status == 'running', jobs):
job_utils.checkout_job_service(job) job_utils.checkout_job_service(job)
db.session.commit() db.session.commit()

125
web/app/tasks/corpus_utils.py
View File

@ -1,4 +1,3 @@
from flask import current_app
from . import docker_client from . import docker_client
import docker import docker
import logging import logging
@ -7,20 +6,14 @@ import shutil
def create_build_corpus_service(corpus): def create_build_corpus_service(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], corpus_data_dir = os.path.join(corpus.path, 'data')
str(corpus.user_id), shutil.rmtree(corpus_data_dir, ignore_errors=True)
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
if os.path.exists(corpus_data_dir):
shutil.rmtree(corpus_data_dir)
if os.path.exists(corpus_registry_dir):
shutil.rmtree(corpus_registry_dir)
os.mkdir(corpus_data_dir) os.mkdir(corpus_data_dir)
corpus_registry_dir = os.path.join(corpus.path, 'registry')
shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir) os.mkdir(corpus_registry_dir)
service_args = { corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus', 'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'], 'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque', 'labels': {'origin': 'nopaque',
@ -32,30 +25,34 @@ def create_build_corpus_service(corpus):
'name': 'build-corpus_{}'.format(corpus.id), 'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy() 'restart_policy': docker.types.RestartPolicy()
} }
service_image = \ service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try: try:
docker_client.services.create(service_image, **service_args) docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('create_build_corpus_service({}): '.format(corpus.id) logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '{} (status: {} -> failed)'.format(e, corpus.status)) + '[docker-APIError] The server returned an error. '
corpus.status = 'failed' + 'Details: {}'.format(e))
else: else:
corpus.status = 'queued' corpus.status = 'queued'
finally:
# TODO: send email
pass
def checkout_build_corpus_service(corpus): def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id) service_name = 'build-corpus_{}'.format(corpus.id)
try: try:
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound as e: except docker.errors.NotFound:
logging.error('checkout_build_corpus_service({}):'.format(corpus.id) logging.error('Get "{}" service raised '.format(service_name)
+ ' {} (stauts: {} -> failed)'.format(e, corpus.status)) + '[docker-NotFound] The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status))
corpus.status = 'failed' corpus.status = 'failed'
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else: else:
service_tasks = service.tasks() service_tasks = service.tasks()
if not service_tasks: if not service_tasks:
@ -63,25 +60,23 @@ def checkout_build_corpus_service(corpus):
task_state = service_tasks[0].get('Status').get('State') task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending': if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running' corpus.status = 'running'
elif corpus.status == 'running' and task_state == 'complete': elif corpus.status == 'running' and task_state in ['complete', 'failed']: # noqa
try:
service.remove() service.remove()
corpus.status = 'prepared' except docker.errors.APIError as e:
elif corpus.status == 'running' and task_state == 'failed': logging.error('Remove "{}" service raised '.format(service_name) # noqa
service.remove() + '[docker-APIError] The server returned an error. ' # noqa
corpus.status = task_state + 'Details: {}'.format(e))
finally: return
# TODO: send email else:
pass corpus.status = 'prepared' if task_state == 'complete' \
else 'failed'
def create_cqpserver_container(corpus): def create_cqpserver_container(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], corpus_data_dir = os.path.join(corpus.path, 'data')
str(corpus.user_id), corpus_registry_dir = os.path.join(corpus.path, 'registry')
'corpora', container_kwargs = {
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
container_args = {
'command': 'cqpserver', 'command': 'cqpserver',
'detach': True, 'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw', 'volumes': [corpus_data_dir + ':/corpora/data:rw',
@ -89,20 +84,43 @@ def create_cqpserver_container(corpus):
'name': 'cqpserver_{}'.format(corpus.id), 'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default' 'network': 'nopaque_default'
} }
container_image = \ container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one
try: try:
container = docker_client.containers.get(container_args['name']) container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.DockerException: except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return return
else: else:
container.remove(force=True)
try: try:
docker_client.containers.run(container_image, **container_args) container.remove(force=True)
except docker.errors.DockerException: except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return return
try:
docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError:
# This case should not occur, because detach is True.
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ContainerError] The container exits with a '
+ 'non-zero exit code and detach is False.')
corpus.status = 'failed'
except docker.errors.ImageNotFound:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ImageNotFound] The specified image does not '
+ 'exist.')
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else: else:
corpus.status = 'analysing' corpus.status = 'analysing'
@ -113,8 +131,17 @@ def remove_cqpserver_container(corpus):
container = docker_client.containers.get(container_name) container = docker_client.containers.get(container_name)
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.DockerException: except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return return
else: else:
try:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
corpus.status = 'prepared' corpus.status = 'prepared'

74
web/app/tasks/job_utils.py
View File

@ -1,7 +1,7 @@
from datetime import datetime from datetime import datetime
from flask import current_app
from . import docker_client from . import docker_client
from .. import db from .. import db
from ..email import create_message, send
from ..models import JobResult from ..models import JobResult
import docker import docker
import logging import logging
@ -10,51 +10,60 @@ import os
def create_job_service(job): def create_job_service(job):
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(job.user_id),
'jobs',
str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service) cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup': if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename) cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files' cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename) cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args)) cmd += ' ' + ' '.join(json.loads(job.service_args))
service_args = {'command': cmd, service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'], 'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque', 'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service), 'type': 'service.{}'.format(job.service),
'job_id': str(job.id)}, 'job_id': str(job.id)},
'mounts': [job_dir + ':/files:rw'], 'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id), 'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources( 'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9), cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)), mem_reservation=job.mem_mb * (10 ** 6)
),
'restart_policy': docker.types.RestartPolicy()} 'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/' service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version) + job.service + ':' + job.service_version)
try: try:
docker_client.services.create(service_image, **service_args) docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('create_job_service({}): {} '.format(job.id, e) logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '(status: {} -> failed)'.format(job.status)) + '[docker-APIError] The server returned an error. '
job.status = 'failed' + 'Details: {}'.format(e))
else: else:
job.status = 'queued' job.status = 'queued'
finally: msg = create_message(
# TODO: send email job.creator.email,
pass 'Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
def checkout_job_service(job): def checkout_job_service(job):
service_name = 'job_{}'.format(job.id) service_name = 'job_{}'.format(job.id)
try: try:
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound as e: except docker.errors.NotFound:
logging.error('checkout_job_service({}): {} '.format(job.id, e) logging.error('Get "{}" service raised '.format(service_name)
+ '(status: {} -> submitted)'.format(job.status)) + '[docker-NotFound] The service does not exist. '
job.status = 'submitted' + '(job.status: {} -> failed)'.format(job.status))
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion job.status = 'failed'
except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else: else:
service_tasks = service.tasks() service_tasks = service.tasks()
if not service_tasks: if not service_tasks:
@ -62,22 +71,16 @@ def checkout_job_service(job):
task_state = service_tasks[0].get('Status').get('State') task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending': if job.status == 'queued' and task_state != 'pending':
job.status = 'running' job.status = 'running'
elif job.status == 'queued' and task_state == 'complete': elif job.status == 'running' and task_state == 'complete':
service.remove() service.remove()
job.end_date = datetime.utcnow() job.end_date = datetime.utcnow()
job.status = task_state job.status = task_state
if task_state == 'complete': if task_state == 'complete':
results_dir = os.path.join(current_app.config['DATA_DIR'], job_results_dir = os.path.join(job.path, 'output')
str(job.user_id), job_results = filter(lambda x: x.endswith('.zip'),
'jobs', os.listdir(job_results_dir))
str(job.id), for job_result in job_results:
'output') job_result = JobResult(filename=job_result, job=job)
results = filter(lambda x: x.endswith('.zip'),
os.listdir(results_dir))
for result in results:
job_result = JobResult(dir=results_dir,
filename=result,
job_id=job.id)
db.session.add(job_result) db.session.add(job_result)
elif job.status == 'running' and task_state == 'failed': elif job.status == 'running' and task_state == 'failed':
service.remove() service.remove()
@ -85,6 +88,13 @@ def checkout_job_service(job):
job.status = task_state job.status = task_state
finally: finally:
# TODO: send email # TODO: send email
msg = create_message(
job.creator.email,
'[nopaque] Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
pass pass

10
web/app/templates/auth/login.html.j2
View File

@ -35,20 +35,20 @@
<div class="card medium"> <div class="card medium">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ login_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }} {{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }} {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;"> <div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align"> <div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a> <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div> </div>
<div class="col s6 right-align"> <div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }} {{ wtf.render_field(form.remember_me) }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

12
web/app/templates/auth/register.html.j2
View File

@ -34,14 +34,14 @@
<div class="card medium"> <div class="card medium">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ registration_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }} {{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }} {{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }} {{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }} {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(registration_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

8
web/app/templates/auth/reset_password.html.j2
View File

@ -20,12 +20,12 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ reset_password_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(reset_password_form.password, data_length='128') }} {{ wtf.render_field(form.password, data_length='128') }}
{{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }} {{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(reset_password_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

6
web/app/templates/auth/reset_password_request.html.j2
View File

@ -20,11 +20,11 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ reset_password_request_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }} {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

8
web/app/templates/corpora/add_corpus.html.j2
View File

@ -27,18 +27,18 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ add_corpus_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_corpus_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

14
web/app/templates/corpora/add_corpus_file.html.j2
View File

@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ add_corpus_file_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }} {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }} {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }} {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }} {{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
<br> <br>
@ -52,7 +52,7 @@
<li> <li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div> <div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body"> <div class="collapsible-body">
{% for field in add_corpus_file_form {% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %} if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }} {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %} {% endfor %}

2
web/app/templates/corpora/analyse_corpus.html.j2
View File

@ -155,7 +155,7 @@ import {
*/ */
document.addEventListener("DOMContentLoaded", () => { document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode // Initialize the client for server client communication in dynamic mode
let corpusId = {{ corpus_id }} let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId, const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket, 'socket': nopaque.socket,
'logging': true, 'logging': true,

12
web/app/templates/corpora/corpus_file.html.j2
View File

@ -20,23 +20,23 @@
<div class="col s12"> <div class="col s12">
<form method="POST"> <form method="POST">
{{ edit_corpus_file_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }} {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }} {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }} {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
<br> <br>
@ -44,7 +44,7 @@
<li> <li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div> <div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body"> <div class="collapsible-body">
{% for field in edit_corpus_file_form {% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %} if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }} {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %} {% endfor %}

12
web/app/templates/corpora/import_corpus.html.j2
View File

@ -1,4 +1,4 @@
{% extends "nopaque.html.j2" %} {% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %} {% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %} {% import 'materialize/wtf.html.j2' as wtf %}
@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ import_corpus_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
</div> </div>
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }} {{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(import_corpus_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

10
web/app/templates/corpora/query_results/add_query_result.html.j2
View File

@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ add_query_result_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }} {{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_query_result_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
</form> </form>

10
web/app/templates/main/index.html.j2
View File

@ -159,20 +159,20 @@
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
<span class="card-title">Log in</span> <span class="card-title">Log in</span>
{{ login_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }} {{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }} {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;"> <div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align"> <div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a> <a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div> </div>
<div class="col s6 right-align"> <div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }} {{ wtf.render_field(form.remember_me) }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

6
web/app/templates/nopaque.html.j2
View File

@ -231,9 +231,9 @@
</div> </div>
<div class="col s12 m9 right-align"> <div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a> <a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
{% if config.CONTACT_EMAIL_ADRESS %} {% if config.NOPAQUE_CONTACT %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a> <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a> <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %} {% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a> <a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div> </div>

12
web/app/templates/services/file-setup.html.j2
View File

@ -48,24 +48,24 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }} {{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div> </div>
<div class="col s12 hide"> <div class="col s12 hide">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

18
web/app/templates/services/nlp.html.j2
View File

@ -66,34 +66,34 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12 l5"> <div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }} {{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div> </div>
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }} {{ wtf.render_field(form.language, material_icon='language') }}
</div> </div>
<div class="col s12 l3"> <div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
<div class="col s12"> <div class="col s12">
<span class="card-title">Preprocessing</span> <span class="card-title">Preprocessing</span>
</div> </div>
<div class="col s9"> <div class="col s9">
<p>{{ add_job_form.check_encoding.label.text }}</p> <p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p> <p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div> </div>
<div class="col s3 right-align"> <div class="col s3 right-align">
<div class="switch"> <div class="switch">
<label> <label>
{{ add_job_form.check_encoding() }} {{ form.check_encoding() }}
<span class="lever"></span> <span class="lever"></span>
</label> </label>
</div> </div>
@ -107,7 +107,7 @@
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

18
web/app/templates/services/ocr.html.j2
View File

@ -48,34 +48,34 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12 l5"> <div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }} {{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div> </div>
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }} {{ wtf.render_field(form.language, material_icon='language') }}
</div> </div>
<div class="col s12 l3"> <div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
<div class="col s12"> <div class="col s12">
<span class="card-title">Preprocessing</span> <span class="card-title">Preprocessing</span>
</div> </div>
<div class="col s9"> <div class="col s9">
<p>{{ add_job_form.binarization.label.text }}</p> <p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p> <p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div> </div>
<div class="col s3 right-align"> <div class="col s3 right-align">
<div class="switch"> <div class="switch">
<label> <label>
{{ add_job_form.binarization() }} {{ form.binarization() }}
<span class="lever"></span> <span class="lever"></span>
</label> </label>
</div> </div>
@ -134,7 +134,7 @@
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }} {{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

7
web/app/templates/tasks/email/notification.html.j2
View File

@ -1,9 +1,8 @@
<p>Dear <b>{{ user.username }}</b>,</p> <p>Dear <b>{{ job.creator.username }}</b>,</p>
<p>The status of your Job/Corpus({{ job.id }}) with the title <b>"{{ job.title }}"</b> has changed!</p> <p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p> <p>It is now <b>{{ job.status }}</b>!</p>
<p>Time of this status update was: <b>{time} UTC</b></p>
<p>You can access your Job/Corpus here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p> <p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p> <p>Kind regards!<br>Your nopaque team</p>

7
web/app/templates/tasks/email/notification.txt.j2
View File

@ -1,10 +1,9 @@
Dear {{ user.username }}, Dear {{ job.creator.username }},
The status of your Job/Corpus({{ job.id }}) with the title "{{ job.title }}" has changed! The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}! It is now {{ job.status }}!
Time of this status update was: {time} UTC
You can access your Job/Corpus here: {{ url_for('jobs.job', job_id=job.id) }} You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards! Kind regards!
Your nopaque team Your nopaque team

1
web/boot.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash #!/bin/bash
source venv/bin/activate source venv/bin/activate
while true; do while true; do
flask deploy flask deploy
if [[ "$?" == "0" ]]; then if [[ "$?" == "0" ]]; then

139
web/config.py
View File

@ -7,103 +7,96 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config: class Config:
''' # Cookies # ''' ''' # Flask # '''
REMEMBER_COOKIE_HTTPONLY = True SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
REMEMBER_COOKIE_SECURE = os.environ.get( SESSION_COOKIE_SECURE = \
'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true' os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
SESSION_COOKIE_SECURE = os.environ.get(
'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Database # ''' ''' # Flask-Login # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = \
os.environ.get('REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask-Mail # '''
MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
MAIL_PORT = int(os.environ.get('MAIL_PORT'))
MAIL_SERVER = os.environ.get('MAIL_SERVER')
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'false').lower() == 'true'
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_TRACK_MODIFICATIONS = False
''' # Email # ''' ''' # nopaque # '''
MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER') NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD') NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT')) NOPAQUE_DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER') NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME') NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
MAIL_USE_SSL = os.environ.get( os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true' NOPAQUE_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
MAIL_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' # General # '''
ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
ALLOWED_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
''' # Message queue # '''
SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
''' # Proxy fix # '''
PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
@classmethod @classmethod
def init_app(cls, app): def init_app(cls, app):
# Set up logging according to the corresponding (LOG_*) variables # Set up logging according to the corresponding (NOPAQUE_LOG_*)
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT, # environment variables
filename=cls.LOG_FILE, basic_config_kwargs = {
format=cls.LOG_FORMAT, 'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
level=cls.LOG_LEVEL) '%Y-%m-%d %H:%M:%S'),
'filename': os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log')),
'format': os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
'%(message)s'
),
'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
}
logging.basicConfig(**basic_config_kwargs)
# Set up and apply the ProxyFix middleware according to the # Set up and apply the ProxyFix middleware according to the
# corresponding (PROXY_FIX_*) variables # corresponding (NOPAQUE_PROXY_FIX_*) environment variables
app.wsgi_app = ProxyFix(app.wsgi_app, proxy_fix_kwargs = {
x_for=cls.PROXY_FIX_X_FOR, 'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
x_host=cls.PROXY_FIX_X_HOST, 'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
x_port=cls.PROXY_FIX_X_PORT, 'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
x_prefix=cls.PROXY_FIX_X_PREFIX, 'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
x_proto=cls.PROXY_FIX_X_PROTO) 'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
}
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
class DevelopmentConfig(Config): class DevelopmentConfig(Config):
''' # Database # ''' ''' # Flask # '''
DEBUG = True
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get( SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL', 'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_dev' 'postgresql://nopaque:nopaque@db/nopaque_dev'
) )
''' # General # '''
DEBUG = True
class ProductionConfig(Config): class ProductionConfig(Config):
''' # Database # ''' ''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get( SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque') 'SQLALCHEMY_DATABASE_URI', 'postgresql://nopaque:nopaque@db/nopaque')
class TestingConfig(Config): class TestingConfig(Config):
''' # Database # ''' ''' # Flask # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
''' # General # '''
TESTING = True TESTING = True
WTF_CSRF_ENABLED = False WTF_CSRF_ENABLED = False
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
config = {'development': DevelopmentConfig, config = {'development': DevelopmentConfig,
'production': ProductionConfig, 'production': ProductionConfig,

11
web/nopaque.py
View File

@ -17,8 +17,7 @@ if os.path.exists(DOTENV_FILE):
from app import create_app, db, socketio # noqa from app import create_app, db, socketio # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
NotificationData, NotificationEmailData, QueryResult, QueryResult, Role, User) # noqa
Role, User) # noqa
from flask_migrate import Migrate, upgrade # noqa from flask_migrate import Migrate, upgrade # noqa
@ -34,8 +33,6 @@ def make_shell_context():
'Job': Job, 'Job': Job,
'JobInput': JobInput, 'JobInput': JobInput,
'JobResult': JobResult, 'JobResult': JobResult,
'NotificationData': NotificationData,
'NotificationEmailData': NotificationEmailData,
'QueryResult': QueryResult, 'QueryResult': QueryResult,
'Role': Role, 'Role': Role,
'User': User} 'User': User}
@ -53,9 +50,9 @@ def deploy():
@app.cli.command() @app.cli.command()
def tasks(): def tasks():
from app.tasks import process_corpora, process_jobs from app.tasks import check_corpora, check_jobs
process_corpora() check_corpora()
process_jobs() check_jobs()
@app.cli.command() @app.cli.command()