sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-24 02:24:20 +00:00
Code Issues Projects Releases Wiki Activity

More exception handling. Remove unused database models. New common view structure!

Browse Source
This commit is contained in:
Patrick Jentsch 2020-11-13 10:01:51 +01:00
parent cb9da5c7dd
commit 5a06a6b241
45 changed files with 692 additions and 1005 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
.env.tpl
View File

@ -9,128 +9,116 @@
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# Example: 1000
# HINT: Use this bash command `id -g`
HOST_GID=
# DEFAULT: ./nopaqued.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_DAEMON_LOG_FILE=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_LOG_FILE=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# HOST_LOG_FILE=
################################################################################
# Cookies #
# Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# SECRET_KEY=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE=
# SESSION_COOKIE_SECURE=
################################################################################
# Database #
# DATABASE_URI blueprint: #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
# Flask-Login #
# https://flask-login.readthedocs.io/en/latest/ #
################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
# NOPAQUE_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# REMEMBER_COOKIE_SECURE=
################################################################################
# Email #
# Flask-Mail #
# https://pythonhosted.org/Flask-Mail/ #
################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER=
MAIL_DEFAULT_SENDER=
NOPAQUE_SMTP_PASSWORD=
MAIL_PASSWORD=
# EXAMPLE: smtp.example.com
NOPAQUE_SMTP_SERVER=
MAIL_SERVER=
# EXAMPLE: 587
NOPAQUE_SMTP_PORT=
MAIL_PORT=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_SSL=
# MAIL_USE_SSL=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_TLS=
# MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com
NOPAQUE_SMTP_USERNAME=
MAIL_USERNAME=
################################################################################
# General #
# Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################
# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
# SQLALCHEMY_DATABASE_URI=
################################################################################
# nopaque #
################################################################################
# If an account is registered with this email adress gets automatically
# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS=
NOPAQUE_ADMIN=
# DEFAULT: development
# CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG=
# This email adress is used for the contact button in the nopaque footer. If
# not set, no contact button is displayed.
# DEFAULT: None
# EXAMPLE: contact.nopaque@example.com
# NOPAQUE_CONTACT_EMAIL_ADRESS=
# NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
# NOTE: This must be a network share and it must be available on all Docker
# Swarm nodes
# NOPAQUE_DATA_DIR=
# DEFAULT: localhost
# NOPAQUE_DOMAIN=
# DEFAULT: 0.0.0.0
# NOPAQUE_HOST=
# DEFAULT: 5000
# NOPAQUE_PORT=
# CHOOSE ONE: http, https
# DEFAULT: http
# NOPAQUE_PROTOCOL=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# NOPAQUE_SECRET_KEY=
################################################################################
# Logging #
################################################################################
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# transport://[userid:password]@hostname[:port]/[virtual_host]
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
@ -146,37 +134,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR=
# NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST=
# NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT=
# NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX=
# NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO=
# NOPAQUE_PROXY_FIX_X_PROTO=

1
docker-compose.yml
View File

@ -28,5 +28,6 @@ services:
image: nopaque:development
restart: unless-stopped
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"

5
web/Dockerfile
View File

@ -21,8 +21,9 @@ RUN apt-get update \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque
WORKDIR /home/nopaque

2
web/app/__init__.py
View File

@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context():
from . import events

2
web/app/admin/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__)
from . import views # noqa
from . import views

1
web/app/admin/forms.py
View File

@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()]
self.user = user

14
web/app/admin/views.py
View File

@ -29,12 +29,11 @@ def user(user_id):
@admin_required
def delete_user(user_id):
settings_tasks.delete_user(user_id)
flash('User has been deleted!')
flash('User has been marked for deletion!')
return redirect(url_for('.users'))
@admin.route('/users/<int:user_id>/edit_general_settings',
methods=['GET', 'POST'])
@admin.route('/users/<int:user_id>/edit_general_settings', methods=['GET', 'POST']) # noqa
@login_required
@admin_required
def edit_general_settings(user_id):
@ -46,16 +45,13 @@ def edit_general_settings(user_id):
user.username = form.username.data
user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data)
db.session.add(user)
db.session.commit()
flash('The profile has been updated.')
return redirect(url_for('admin.edit_general_settings', user_id=user.id))
flash('Settings have been updated.')
return redirect(url_for('.edit_general_settings', user_id=user.id))
form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email
form.role.data = user.role_id
form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2',
form=form,
title='General settings',
user=user)
form=form, title='General settings', user=user)

2
web/app/auth/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__)
from . import views # noqa
from . import views

2
web/app/auth/forms.py
View File

@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField(
'Username',
validators=[DataRequired(), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

107
web/app/auth/views.py
View File

@ -1,5 +1,5 @@
from flask import (current_app, flash, redirect, render_template, request,
url_for)
from datetime import datetime
from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user
from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db
from ..email import create_message, send
from ..models import User
import logging
import os
import shutil
@auth.before_app_request
@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
current_user.ping()
if not current_user.confirmed \
and request.endpoint \
and request.blueprint != 'auth' \
and request.endpoint != 'static':
current_user.last_seen = datetime.utcnow()
db.session.commit()
if (not current_user.confirmed
and request.endpoint
and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
@ -30,20 +31,19 @@ def before_request():
def login():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
next = request.args.get('next')
if next is None or not next.startswith('/'):
next = url_for('main.dashboard')
return redirect(next)
flash('Invalid email/username or password.')
return render_template('auth/login.html.j2', login_form=login_form,
title='Log in')
return render_template('auth/login.html.j2', form=form, title='Log in')
@auth.route('/logout')
@ -58,26 +58,28 @@ def logout():
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
registration_form = RegistrationForm(prefix='registration-form')
if registration_form.validate_on_submit():
user = User(email=registration_form.email.data.lower(),
password=registration_form.password.data,
username=registration_form.username.data)
form = RegistrationForm(prefix='registration-form')
if form.validate_on_submit():
user = User(email=form.email.data.lower(),
password=form.password.data,
username=form.username.data)
db.session.add(user)
db.session.commit()
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(user.id))
if os.path.exists(user_dir):
shutil.rmtree(user_dir)
os.mkdir(user_dir)
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('auth.login'))
return render_template('auth/register.html.j2',
registration_form=registration_form,
try:
os.makedirs(user.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(user.path))
db.session.delete(user)
db.session.commit()
abort(500)
else:
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('.login'))
return render_template('auth/register.html.j2', form=form,
title='Register')
@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard'))
else:
flash('The confirmation link is invalid or has expired.')
return redirect(url_for('auth.unconfirmed'))
return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed')
@ -119,39 +121,32 @@ def resend_confirmation():
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_request_form = ResetPasswordRequestForm(
prefix='reset-password-request-form')
if reset_password_request_form.validate_on_submit():
submitted_email = reset_password_request_form.email.data
user = User.query.filter_by(email=submitted_email.lower()).first()
if user:
form = ResetPasswordRequestForm(prefix='reset-password-request-form')
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data.lower()).first()
if user is not None:
token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token,
user=user)
send(msg)
flash('An email with instructions to reset your password has been '
'sent to you.')
return redirect(url_for('auth.login'))
return render_template(
'auth/reset_password_request.html.j2',
reset_password_request_form=reset_password_request_form,
title='Password Reset')
flash('An email with instructions to reset your password has been sent to you.') # noqa
return redirect(url_for('.login'))
return render_template('auth/reset_password_request.html.j2', form=form,
title='Password Reset')
@auth.route('/reset/<token>', methods=['GET', 'POST'])
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_form = ResetPasswordForm(prefix='reset-password-form')
if reset_password_form.validate_on_submit():
if User.reset_password(token, reset_password_form.password.data):
form = ResetPasswordForm(prefix='reset-password-form')
if form.validate_on_submit():
if User.reset_password(token, form.password.data):
db.session.commit()
flash('Your password has been updated.')
return redirect(url_for('auth.login'))
return redirect(url_for('.login'))
else:
return redirect(url_for('main.index'))
return render_template('auth/reset_password.html.j2',
reset_password_form=reset_password_form,
title='Password Reset',
token=token)
return render_template('auth/reset_password.html.j2', form=form,
title='Password Reset', token=token)

390
web/app/corpora/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, make_response, redirect, request,
from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory)
from flask_login import current_user, login_required
from . import corpora
@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db
from ..models import Corpus, CorpusFile, QueryResult
import json
import logging
import os
import shutil
import glob
@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST'])
@login_required
def add_corpus():
add_corpus_form = AddCorpusForm()
if add_corpus_form.validate_on_submit():
form = AddCorpusForm()
if form.validate_on_submit():
corpus = Corpus(creator=current_user,
description=add_corpus_form.description.data,
status='unprepared', title=add_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not add corpus!', 'corpus')
corpus.delete()
else:
url = url_for('corpora.corpus', corpus_id=corpus.id)
flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2',
add_corpus_form=add_corpus_form,
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
abort(500)
flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
return redirect(url_for('.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2', form=form,
title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST'])
@login_required
def import_corpus():
import_corpus_form = ImportCorpusForm()
if import_corpus_form.is_submitted():
if not import_corpus_form.validate():
return make_response(import_corpus_form.errors, 400)
form = ImportCorpusForm()
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
corpus = Corpus(creator=current_user,
description=import_corpus_form.description.data,
status='unprepared',
title=import_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not import corpus!', 'corpus')
corpus.delete()
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.import_corpus')}, 500)
# Upload zip
archive_file = os.path.join(corpus.path, form.file.data.filename)
form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus.path)
# Register vrt files to corpus
vrts = glob.glob(corpus.path + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL')
)
db.session.add(corpus_file)
# finish import and redirect to imported corpus
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
return make_response(
{'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
else:
# Upload zip
archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
import_corpus_form.file.data.filename)
corpus_dir = os.path.dirname(archive_file)
import_corpus_form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus_dir)
# Register vrt files to corpus
vrts = glob.glob(corpus_dir + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
dir=dir,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL'))
db.session.add(corpus_file)
# finish import and got to imported corpus
url = url_for('corpora.corpus', corpus_id=corpus.id)
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('[<a href="{}">{}</a>] imported'.format(url,
corpus.title),
'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus',
corpus_id=corpus.id)},
201)
else:
# If imported zip is not valid delete corpus and give feedback
corpus.delete()
db.session.commit()
flash('Imported corpus is not valid.', 'error')
return make_response(
{'redirect_url': url_for('corpora.import_corpus')},
201)
return render_template('corpora/import_corpus.html.j2',
import_corpus_form=import_corpus_form,
# If imported zip is not valid delete corpus and give feedback
flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
tasks.delete_corpus(corpus.id)
return make_response(
{'redirect_url': url_for('.import_corpus')}, 201)
return render_template('corpora/import_corpus.html.j2', form=form,
title='Import Corpus')
@ -131,17 +118,9 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
corpus_files = [dict(filename=corpus_file.filename,
author=corpus_file.author,
title=corpus_file.title,
publishing_year=corpus_file.publishing_year,
corpus_id=corpus.id,
id=corpus_file.id)
for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus')
corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2', corpus=corpus,
corpus_files=corpus_files, title='Corpus')
@corpora.route('/<int:corpus_id>/export')
@ -150,12 +129,11 @@ def export_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
# TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file)
return send_from_directory(directory=dir,
filename=filename,
mimetype='zip',
as_attachment=True)
return send_from_directory(as_attachment=True, directory=dir,
filename=filename, mimetype='zip')
@corpora.route('/<int:corpus_id>/analyse')
@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
result_context=request.args.get('context', 20),
results_per_page=request.args.get('results_per_page', 30))
results_per_page=request.args.get('results_per_page', 30)
)
query_form = QueryForm(prefix='query-form',
query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form')
@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template(
'corpora/analyse_corpus.html.j2',
corpus=corpus,
corpus_id=corpus_id,
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_form=query_form,
query_download_form=query_download_form,
inspect_display_options_form=inspect_display_options_form,
title='Corpus analysis')
title='Corpus analysis'
)
@corpora.route('/<int:corpus_id>/delete')
@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id)
flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard'))
@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
add_corpus_file_form = AddCorpusFileForm(corpus,
prefix='add-corpus-file-form')
if add_corpus_file_form.is_submitted():
if not add_corpus_file_form.validate():
return make_response(add_corpus_file_form.errors, 400)
form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
# Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
add_corpus_file_form.file.data.save(
os.path.join(current_app.config['DATA_DIR'], dir,
add_corpus_file_form.file.data.filename))
corpus_file = CorpusFile(
address=add_corpus_file_form.address.data,
author=add_corpus_file_form.author.data,
booktitle=add_corpus_file_form.booktitle.data,
chapter=add_corpus_file_form.chapter.data,
corpus=corpus,
dir=dir,
editor=add_corpus_file_form.editor.data,
filename=add_corpus_file_form.file.data.filename,
institution=add_corpus_file_form.institution.data,
journal=add_corpus_file_form.journal.data,
pages=add_corpus_file_form.pages.data,
publisher=add_corpus_file_form.publisher.data,
publishing_year=add_corpus_file_form.publishing_year.data,
school=add_corpus_file_form.school.data,
title=add_corpus_file_form.title.data)
form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
corpus_file = CorpusFile(address=form.address.data,
author=form.author.data,
booktitle=form.booktitle.data,
chapter=form.chapter.data,
corpus=corpus,
editor=form.editor.data,
filename=form.file.data.filename,
institution=form.institution.data,
journal=form.journal.data,
pages=form.pages.data,
publisher=form.publisher.data,
publishing_year=form.publishing_year.data,
school=form.school.data,
title=form.title.data)
db.session.add(corpus_file)
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file added!', 'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
201)
return render_template('corpora/add_corpus_file.html.j2',
corpus=corpus,
add_corpus_file_form=add_corpus_file_form,
title='Add corpus file')
flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
form=form, title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id)
flash('Corpus file deleted!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
return redirect(url_for('.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
corpus_file.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=corpus_file.corpus.path,
filename=corpus_file.filename)
@ -274,48 +242,45 @@ def download_corpus_file(corpus_id, corpus_file_id):
methods=['GET', 'POST'])
@login_required
def corpus_file(corpus_id, corpus_file_id):
corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if not corpus_file.corpus_id == corpus_id:
if corpus_file.corpus_id != corpus_id:
abort(404)
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if edit_corpus_file_form.validate_on_submit():
corpus_file.address = edit_corpus_file_form.address.data
corpus_file.author = edit_corpus_file_form.author.data
corpus_file.booktitle = edit_corpus_file_form.booktitle.data
corpus_file.chapter = edit_corpus_file_form.chapter.data
corpus_file.editor = edit_corpus_file_form.editor.data
corpus_file.institution = edit_corpus_file_form.institution.data
corpus_file.journal = edit_corpus_file_form.journal.data
corpus_file.pages = edit_corpus_file_form.pages.data
corpus_file.publisher = edit_corpus_file_form.publisher.data
corpus_file.publishing_year = \
edit_corpus_file_form.publishing_year.data
corpus_file.school = edit_corpus_file_form.school.data
corpus_file.title = edit_corpus_file_form.title.data
form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if form.validate_on_submit():
corpus_file.address = form.address.data
corpus_file.author = form.author.data
corpus_file.booktitle = form.booktitle.data
corpus_file.chapter = form.chapter.data
corpus_file.editor = form.editor.data
corpus_file.institution = form.institution.data
corpus_file.journal = form.journal.data
corpus_file.pages = form.pages.data
corpus_file.publisher = form.publisher.data
corpus_file.publishing_year = form.publishing_year.data
corpus_file.school = form.school.data
corpus_file.title = form.title.data
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file edited!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values
edit_corpus_file_form.address.data = corpus_file.address
edit_corpus_file_form.author.data = corpus_file.author
edit_corpus_file_form.booktitle.data = corpus_file.booktitle
edit_corpus_file_form.chapter.data = corpus_file.chapter
edit_corpus_file_form.editor.data = corpus_file.editor
edit_corpus_file_form.institution.data = corpus_file.institution
edit_corpus_file_form.journal.data = corpus_file.journal
edit_corpus_file_form.pages.data = corpus_file.pages
edit_corpus_file_form.publisher.data = corpus_file.publisher
edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
edit_corpus_file_form.school.data = corpus_file.school
edit_corpus_file_form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2',
corpus_file=corpus_file, corpus=corpus,
edit_corpus_file_form=edit_corpus_file_form,
form.address.data = corpus_file.address
form.author.data = corpus_file.author
form.booktitle.data = corpus_file.booktitle
form.chapter.data = corpus_file.chapter
form.editor.data = corpus_file.editor
form.institution.data = corpus_file.institution
form.journal.data = corpus_file.journal
form.pages.data = corpus_file.pages
form.publisher.data = corpus_file.publisher
form.publishing_year.data = corpus_file.publishing_year
form.school.data = corpus_file.school
form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2', corpus=corpus,
corpus_file=corpus_file, form=form,
title='Edit corpus file')
@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403)
if corpus.files.all():
tasks.build_corpus(corpus_id)
flash('Building Corpus...', 'corpus')
flash('Corpus "{}" has been marked to get build!', 'corpus')
else:
flash('Can not build corpus, please add corpus file(s).', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Can not build corpus "{}": No corpus file(s)!', 'error')
return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results.
@ -340,35 +305,29 @@ def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
form = AddQueryResultForm(prefix='add-query-result-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
query_result = QueryResult(creator=current_user,
description=form.description.data,
filename=form.file.data.filename,
title=form.title.data)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
os.makedirs(query_result.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result_file_path = os.path.join(query_result.path,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
@ -381,19 +340,16 @@ def add_query_result():
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('corpora.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
form=form, title='Add query result')
@corpora.route('/result/<int:query_result_id>')
@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
query_result=query_result, title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect')
@ -427,13 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
query_result_file_path = os.path.join(query_result.path, query_result.filename) # noqa
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
@ -452,8 +401,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@ -464,10 +413,5 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
return send_from_directory(as_attachment=True, directory=query_result.path,
filename=query_result.filename)

4
web/app/email.py
View File

@ -1,11 +1,11 @@
from flask import render_template
from flask import current_app, render_template
from flask_mail import Message
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg

2
web/app/jobs/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__)
from . import views # noqa
from . import views

30
web/app/jobs/views.py
View File

@ -1,11 +1,10 @@
from flask import (abort, current_app, flash, redirect, render_template,
from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for)
from flask_login import current_user, login_required
from . import jobs
from . import tasks
from ..decorators import admin_required
from ..models import Job, JobInput, JobResult
import os
@jobs.route('/<int:job_id>')
@ -14,13 +13,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
job_inputs = [dict(filename=input.filename,
id=input.id,
job_id=job.id)
for input in job.inputs]
return render_template('jobs/job.html.j2',
job=job,
job_inputs=job_inputs,
job_inputs = [job_input.to_dict() for job_input in job.inputs]
return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
title='Job')
@ -31,7 +25,7 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
tasks.delete_job(job_id)
flash('Job has been deleted!', 'job')
flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard'))
@ -44,9 +38,8 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_input.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=job_input.job.path,
filename=job_input.filename)
@ -56,11 +49,11 @@ def download_job_input(job_id, job_input_id):
def restart(job_id):
job = Job.query.get_or_404(job_id)
if job.status != 'failed':
flash('Could not restart job: status is not "failed"', 'error')
flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else:
tasks.restart_job(job_id)
flash('Job has been restarted!', 'job')
return redirect(url_for('jobs.job', job_id=job_id))
flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@ -72,7 +65,6 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_result.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=job_result.job.path,
filename=job_result.filename)

2
web/app/main/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__)
from . import views # noqa
from . import views

25
web/app/main/views.py
View File

@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST'])
def index():
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
return redirect(url_for('main.dashboard'))
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.')
return render_template('main/index.html.j2', login_form=login_form,
title='nopaque')
return render_template('main/index.html.j2', form=form, title='nopaque')
@main.route('/about_and_faq')
@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/news')
def news():
return render_template('main/news.html.j2', title='News')
@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy')
def privacy_policy():
return render_template('main/privacy_policy.html.j2',
title=('Information on the processing of personal'
' data for the nopaque platform (GDPR)'))
title='Privacy statement (GDPR)')
@main.route('/terms_of_use')
def terms_of_use():
return render_template('main/terms_of_use.html.j2',
title='General Terms of Use of the platform '
'nopaque')
return render_template('main/terms_of_use.html.j2', title='Terms of Use')

181
web/app/models.py
View File

@ -7,6 +7,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET
from . import db, login_manager
import logging
import os
import shutil
@ -54,7 +55,7 @@ class Role(db.Model):
'''
String representation of the Role. For human readability.
'''
return '<Role {role_name}>'.format(role_name=self.name)
return '<Role {}>'.format(self.name)
def add_permission(self, perm):
'''
@ -138,6 +139,18 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete',
lazy='dynamic')
@property
def path(self):
return os.path.join(current_app.config['NOPAQUE_DATA_DIR'], str(self.id))
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def to_dict(self):
return {'id': self.id,
'role_id': self.role_id,
@ -162,7 +175,7 @@ class User(UserMixin, db.Model):
'''
String representation of the User. For human readability.
'''
return '<User {username}>'.format(username=self.username)
return '<User {}>'.format(self.username)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
@ -220,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user)
return True
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
@ -244,17 +249,11 @@ class User(UserMixin, db.Model):
'''
return self.can(Permission.ADMIN)
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def delete(self):
'''
Delete the user and its corpora and jobs from database and filesystem.
'''
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
@ -280,14 +279,17 @@ class JobInput(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self):
'''
String representation of the JobInput. For human readability.
'''
return '<JobInput {filename}>'.format(filename=self.filename)
return '<JobInput {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@ -305,14 +307,17 @@ class JobResult(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self):
'''
String representation of the JobResult. For human readability.
'''
return '<JobResult {filename}>'.format(filename=self.filename)
return '<JobResult {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@ -351,19 +356,16 @@ class Job(db.Model):
cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete')
notification_data = db.relationship('NotificationData',
cascade='save-update, merge, delete',
uselist=False,
back_populates='job') # One-to-One relationship
notification_email_data = db.relationship('NotificationEmailData',
cascade='save-update, merge, delete',
back_populates='job')
@property
def path(self):
return os.path.join(self.creator.path, 'jobs', str(self.id))
def __repr__(self):
'''
String representation of the Job. For human readability.
'''
return '<Job {job_title}>'.format(job_title=self.title)
return '<Job {}>'.format(self.title)
def create_secure_filename(self):
'''
@ -385,11 +387,7 @@ class Job(db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(job_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def restart(self):
@ -399,12 +397,8 @@ class Job(db.Model):
if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
self.end_date = None
self.status = 'submitted'
@ -425,63 +419,6 @@ class Job(db.Model):
for result in self.results}}
class NotificationData(db.Model):
'''
Class to define notification data used for sending a notification mail with
nopaque_notify.
'''
__tablename__ = 'notification_data'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_data')
# Fields
notified_on = db.Column(db.String(16), default=None)
def __repr__(self):
'''
String representation of the NotificationData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notified': self.notified}
class NotificationEmailData(db.Model):
'''
Class to define data that will be used to send a corresponding Notification
via email.
'''
__tablename__ = 'notification_email_data'
# Primary Key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_email_data')
notify_status = db.Column(db.String(16), default=None)
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
def __repr__(self):
'''
String representation of the NotificationEmailData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notify_status': self.notify_status,
'creation_date': self.creation_date}
class CorpusFile(db.Model):
'''
Class to define Files.
@ -496,7 +433,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
dir = db.Column(db.String(255))
editor = db.Column(db.String(255))
filename = db.Column(db.String(255))
institution = db.Column(db.String(255))
@ -507,15 +443,15 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255))
title = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.corpus.path, self.filename)
def delete(self):
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
self.filename)
try:
os.remove(corpus_file_path)
os.remove(self.path)
except OSError:
logging.error('Removing {} led to an OSError!'.format(self.path))
pass
db.session.delete(self)
self.corpus.status = 'unprepared'
@ -553,13 +489,17 @@ class Corpus(db.Model):
description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
status = db.Column(db.String(16))
status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32))
archive_file = db.Column(db.String(255))
# Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete')
@property
def path(self):
return os.path.join(self.creator.path, 'corpora', str(self.id))
def to_dict(self):
return {'id': self.id,
'user_id': self.user_id,
@ -571,19 +511,14 @@ class Corpus(db.Model):
'files': {file.id: file.to_dict() for file in self.files}}
def build(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
output_dir = os.path.join(corpus_dir, 'merged')
output_dir = os.path.join(self.path, 'merged')
shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir)
master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>')
)
for corpus_file in self.files:
corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
element_tree = ET.parse(corpus_file_path)
element_tree = ET.parse(corpus_file.path)
text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author)
@ -597,7 +532,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title)
element_tree.write(corpus_file_path)
element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file,
@ -607,18 +542,14 @@ class Corpus(db.Model):
self.status = 'submitted'
def delete(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
shutil.rmtree(corpus_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def __repr__(self):
'''
String representation of the corpus. For human readability.
'''
return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model):
@ -636,12 +567,12 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32))
@property
def path(self):
return os.path.join(self.creator.path, 'query_results', str(self.id))
def delete(self):
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'query_results',
str(self.id))
shutil.rmtree(query_result_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_dict(self):
@ -654,7 +585,7 @@ class QueryResult(db.Model):
def __repr__(self):
'''
String representation of the CorpusAnalysisResult. For human readability.
String representation of the QueryResult. For human readability.
'''
return '<QueryResult {}>'.format(self.title)

150
web/app/query_results/views.py
View File

@ -1,150 +0,0 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__)
from . import views # noqa
from . import views

61
web/app/services/views.py
View File

@ -1,5 +1,4 @@
from flask import (abort, current_app, flash, make_response, render_template,
url_for)
from flask import abort, flash, make_response, render_template, url_for
from flask_login import current_user, login_required
from werkzeug.utils import secure_filename
from . import services
@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput
import json
import logging
import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4},
'add_job_form': AddFileSetupJobForm},
'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2},
'add_job_form': AddNLPJobForm},
'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4},
'add_job_form': AddOCRJobForm}}
'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST'])
@ -30,54 +30,49 @@ def service(service):
if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'])
add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
if add_job_form.is_submitted():
if not add_job_form.validate():
return make_response(add_job_form.errors, 400)
form = SERVICES[service]['form'](prefix='add-job-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
service_args = []
if service == 'nlp':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.check_encoding.data:
service_args.append('-l {}'.format(form.language.data))
if form.check_encoding.data:
service_args.append('--check-encoding')
if service == 'ocr':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.binarization.data:
service_args.append('-l {}'.format(form.language.data))
if form.binarization.data:
service_args.append('--binarize')
job = Job(creator=current_user,
description=add_job_form.description.data,
description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args),
service_version=add_job_form.version.data,
status='preparing', title=add_job_form.title.data)
service_version=form.version.data,
status='preparing', title=form.title.data)
if job.service != 'corpus_analysis':
job.create_secure_filename()
db.session.add(job)
db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try:
os.makedirs(absolut_dir)
os.makedirs(job.path)
except OSError:
job.delete()
flash('Internal Server Error', 'job')
return make_response({'redirect_url': url_for('services.service',
service=service)},
500)
logging.error('Make dir {} led to an OSError!'.format(job.path))
db.session.delete(job)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.service', service=service)}, 500)
else:
for file in add_job_form.files.data:
for file in form.files.data:
filename = secure_filename(file.filename)
file.save(os.path.join(absolut_dir, filename))
job_input = JobInput(dir=relative_dir, filename=filename,
job=job)
job_input = JobInput(dir=job.path, filename=filename, job=job)
file.save(job_input.path)
db.session.add(job_input)
job.status = 'submitted'
db.session.commit()
url = url_for('jobs.job', job_id=job.id)
flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
flash('Job "{}" added'.format(job.title), 'job')
return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'],
add_job_form=add_job_form)
form=form, title=SERVICES[service]['name'])

2
web/app/settings/forms.py
View File

@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername',
validators=[DataRequired(),
Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

19
web/app/settings/views.py
View File

@ -1,13 +1,9 @@
from flask import current_app, flash, redirect, render_template, url_for
from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user
from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm)
from .. import db
from ..decorators import admin_required
from ..models import Role, User
import os
import uuid
@settings.route('/')
@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.')
return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2',
form=form,
title='Change password')
form=form, title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST'])
@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email
form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2',
form=form,
title='General settings')
form=form, title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2',
form=form,
title='Notification settings')
form=form, title='Notification settings')
@settings.route('/delete')
@ -76,5 +71,5 @@ def delete():
"""
tasks.delete_user(current_user.id)
logout_user()
flash('Your account has been deleted!')
flash('Your account has been marked for deletion!')
return redirect(url_for('main.index'))

14
web/app/tasks/__init__.py
View File

@ -11,15 +11,11 @@ def check_corpora():
corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: (corpus.status == 'queued'
or corpus.status == 'running'),
corpora):
for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
corpus_utils.checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis',
corpora):
for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpus_utils.create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
corpora):
for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpus_utils.remove_cqpserver_container(corpus)
db.session.commit()
@ -28,8 +24,6 @@ def check_jobs():
jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job)
for job in filter(lambda job: job.status == 'queued', jobs):
job_utils.checkout_job_service(job)
for job in filter(lambda job: job.status == 'running', jobs):
for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
db.session.commit()

131
web/app/tasks/corpus_utils.py
View File

@ -1,4 +1,3 @@
from flask import current_app
from . import docker_client
import docker
import logging
@ -7,20 +6,14 @@ import shutil
def create_build_corpus_service(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
if os.path.exists(corpus_data_dir):
shutil.rmtree(corpus_data_dir)
if os.path.exists(corpus_registry_dir):
shutil.rmtree(corpus_registry_dir)
corpus_data_dir = os.path.join(corpus.path, 'data')
shutil.rmtree(corpus_data_dir, ignore_errors=True)
os.mkdir(corpus_data_dir)
corpus_registry_dir = os.path.join(corpus.path, 'registry')
shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir)
service_args = {
corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
@ -32,30 +25,34 @@ def create_build_corpus_service(corpus):
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
try:
docker_client.services.create(service_image, **service_args)
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error('create_build_corpus_service({}): '.format(corpus.id)
+ '{} (status: {} -> failed)'.format(e, corpus.status))
corpus.status = 'failed'
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
corpus.status = 'queued'
finally:
# TODO: send email
pass
def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound as e:
logging.error('checkout_build_corpus_service({}):'.format(corpus.id)
+ ' {} (stauts: {} -> failed)'.format(e, corpus.status))
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-NotFound] The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status))
corpus.status = 'failed'
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@ -63,25 +60,23 @@ def checkout_build_corpus_service(corpus):
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif corpus.status == 'running' and task_state == 'complete':
service.remove()
corpus.status = 'prepared'
elif corpus.status == 'running' and task_state == 'failed':
service.remove()
corpus.status = task_state
finally:
# TODO: send email
pass
elif corpus.status == 'running' and task_state in ['complete', 'failed']: # noqa
try:
service.remove()
except docker.errors.APIError as e:
logging.error('Remove "{}" service raised '.format(service_name) # noqa
+ '[docker-APIError] The server returned an error. ' # noqa
+ 'Details: {}'.format(e))
return
else:
corpus.status = 'prepared' if task_state == 'complete' \
else 'failed'
def create_cqpserver_container(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
container_args = {
corpus_data_dir = os.path.join(corpus.path, 'data')
corpus_registry_dir = os.path.join(corpus.path, 'registry')
container_kwargs = {
'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
@ -89,20 +84,43 @@ def create_cqpserver_container(corpus):
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'
}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
# Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one
try:
container = docker_client.containers.get(container_args['name'])
container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
else:
container.remove(force=True)
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
try:
docker_client.containers.run(container_image, **container_args)
except docker.errors.DockerException:
return
docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError:
# This case should not occur, because detach is True.
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ContainerError] The container exits with a '
+ 'non-zero exit code and detach is False.')
corpus.status = 'failed'
except docker.errors.ImageNotFound:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ImageNotFound] The specified image does not '
+ 'exist.')
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
corpus.status = 'analysing'
@ -113,8 +131,17 @@ def remove_cqpserver_container(corpus):
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
else:
container.remove(force=True)
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
corpus.status = 'prepared'

90
web/app/tasks/job_utils.py
View File

@ -1,7 +1,7 @@
from datetime import datetime
from flask import current_app
from . import docker_client
from .. import db
from ..email import create_message, send
from ..models import JobResult
import docker
import logging
@ -10,51 +10,60 @@ import os
def create_job_service(job):
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(job.user_id),
'jobs',
str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_args = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job_dir + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)),
'restart_policy': docker.types.RestartPolicy()}
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)
),
'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version)
try:
docker_client.services.create(service_image, **service_args)
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error('create_job_service({}): {} '.format(job.id, e)
+ '(status: {} -> failed)'.format(job.status))
job.status = 'failed'
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
job.status = 'queued'
finally:
# TODO: send email
pass
msg = create_message(
job.creator.email,
'Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
def checkout_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound as e:
logging.error('checkout_job_service({}): {} '.format(job.id, e)
+ '(status: {} -> submitted)'.format(job.status))
job.status = 'submitted'
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-NotFound] The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status))
job.status = 'failed'
except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@ -62,22 +71,16 @@ def checkout_job_service(job):
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif job.status == 'queued' and task_state == 'complete':
elif job.status == 'running' and task_state == 'complete':
service.remove()
job.end_date = datetime.utcnow()
job.status = task_state
if task_state == 'complete':
results_dir = os.path.join(current_app.config['DATA_DIR'],
str(job.user_id),
'jobs',
str(job.id),
'output')
results = filter(lambda x: x.endswith('.zip'),
os.listdir(results_dir))
for result in results:
job_result = JobResult(dir=results_dir,
filename=result,
job_id=job.id)
job_results_dir = os.path.join(job.path, 'output')
job_results = filter(lambda x: x.endswith('.zip'),
os.listdir(job_results_dir))
for job_result in job_results:
job_result = JobResult(filename=job_result, job=job)
db.session.add(job_result)
elif job.status == 'running' and task_state == 'failed':
service.remove()
@ -85,6 +88,13 @@ def checkout_job_service(job):
job.status = task_state
finally:
# TODO: send email
msg = create_message(
job.creator.email,
'[nopaque] Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
pass

10
web/app/templates/auth/login.html.j2
View File

@ -35,20 +35,20 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

12
web/app/templates/auth/register.html.j2
View File

@ -34,14 +34,14 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ registration_form.hidden_tag() }}
{{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(registration_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/auth/reset_password.html.j2
View File

@ -20,12 +20,12 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_form.hidden_tag() }}
{{ wtf.render_field(reset_password_form.password, data_length='128') }}
{{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.password, data_length='128') }}
{{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

6
web/app/templates/auth/reset_password_request.html.j2
View File

@ -20,11 +20,11 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_request_form.hidden_tag() }}
{{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/corpora/add_corpus.html.j2
View File

@ -27,18 +27,18 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ add_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

14
web/app/templates/corpora/add_corpus_file.html.j2
View File

@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
{{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@ -52,7 +52,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body">
{% for field in add_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

2
web/app/templates/corpora/analyse_corpus.html.j2
View File

@ -155,7 +155,7 @@ import {
*/
document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode
let corpusId = {{ corpus_id }}
let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket,
'logging': true,

12
web/app/templates/corpora/corpus_file.html.j2
View File

@ -20,23 +20,23 @@
<div class="col s12">
<form method="POST">
{{ edit_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="card">
<div class="card-content">
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@ -44,7 +44,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body">
{% for field in edit_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

12
web/app/templates/corpora/import_corpus.html.j2
View File

@ -1,4 +1,4 @@
{% extends "nopaque.html.j2" %}
{% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %}
@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ import_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
<div class="row">
<div class="col s12">
{{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
{{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

10
web/app/templates/corpora/query_results/add_query_result.html.j2
View File

@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_query_result_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
{{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
</form>

10
web/app/templates/main/index.html.j2
View File

@ -159,20 +159,20 @@
<form method="POST">
<div class="card-content">
<span class="card-title">Log in</span>
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

6
web/app/templates/nopaque.html.j2
View File

@ -231,9 +231,9 @@
</div>
<div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
{% if config.CONTACT_EMAIL_ADRESS %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% if config.NOPAQUE_CONTACT %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div>

12
web/app/templates/services/file-setup.html.j2
View File

@ -48,24 +48,24 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
{{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div>
<div class="col s12 hide">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/nlp.html.j2
View File

@ -66,34 +66,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
{{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.check_encoding.label.text }}</p>
<p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.check_encoding() }}
{{ form.check_encoding() }}
<span class="lever"></span>
</label>
</div>
@ -107,7 +107,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/ocr.html.j2
View File

@ -48,34 +48,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
{{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.binarization.label.text }}</p>
<p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.binarization() }}
{{ form.binarization() }}
<span class="lever"></span>
</label>
</div>
@ -134,7 +134,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
{{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div>
</form>
</div>

7
web/app/templates/tasks/email/notification.html.j2
View File

@ -1,9 +1,8 @@
<p>Dear <b>{{ user.username }}</b>,</p>
<p>Dear <b>{{ job.creator.username }}</b>,</p>
<p>The status of your Job/Corpus({{ job.id }}) with the title <b>"{{ job.title }}"</b> has changed!</p>
<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p>
<p>Time of this status update was: <b>{time} UTC</b></p>
<p>You can access your Job/Corpus here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p>

7
web/app/templates/tasks/email/notification.txt.j2
View File

@ -1,10 +1,9 @@
Dear {{ user.username }},
Dear {{ job.creator.username }},
The status of your Job/Corpus({{ job.id }}) with the title "{{ job.title }}" has changed!
The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}!
Time of this status update was: {time} UTC
You can access your Job/Corpus here: {{ url_for('jobs.job', job_id=job.id) }}
You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards!
Your nopaque team

1
web/boot.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash
source venv/bin/activate
while true; do
flask deploy
if [[ "$?" == "0" ]]; then

139
web/config.py
View File

@ -7,103 +7,96 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' # Cookies # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = os.environ.get(
'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
SESSION_COOKIE_SECURE = os.environ.get(
'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask # '''
SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
SESSION_COOKIE_SECURE = \
os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Database # '''
''' # Flask-Login # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = \
os.environ.get('REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask-Mail # '''
MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
MAIL_PORT = int(os.environ.get('MAIL_PORT'))
MAIL_SERVER = os.environ.get('MAIL_SERVER')
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'false').lower() == 'true'
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
''' # Email # '''
MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
MAIL_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' # General # '''
ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
ALLOWED_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
''' # Message queue # '''
SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
''' # Proxy fix # '''
PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
''' # nopaque # '''
NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
NOPAQUE_DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
NOPAQUE_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
@classmethod
def init_app(cls, app):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
# Set up logging according to the corresponding (NOPAQUE_LOG_*)
# environment variables
basic_config_kwargs = {
'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S'),
'filename': os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log')),
'format': os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
'%(message)s'
),
'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
}
logging.basicConfig(**basic_config_kwargs)
# Set up and apply the ProxyFix middleware according to the
# corresponding (PROXY_FIX_*) variables
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=cls.PROXY_FIX_X_FOR,
x_host=cls.PROXY_FIX_X_HOST,
x_port=cls.PROXY_FIX_X_PORT,
x_prefix=cls.PROXY_FIX_X_PREFIX,
x_proto=cls.PROXY_FIX_X_PROTO)
# corresponding (NOPAQUE_PROXY_FIX_*) environment variables
proxy_fix_kwargs = {
'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
}
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
class DevelopmentConfig(Config):
''' # Database # '''
''' # Flask # '''
DEBUG = True
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_dev'
)
''' # General # '''
DEBUG = True
class ProductionConfig(Config):
''' # Database # '''
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
'SQLALCHEMY_DATABASE_URI', 'postgresql://nopaque:nopaque@db/nopaque')
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
''' # General # '''
''' # Flask # '''
TESTING = True
WTF_CSRF_ENABLED = False
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
config = {'development': DevelopmentConfig,
'production': ProductionConfig,

11
web/nopaque.py
View File

@ -17,8 +17,7 @@ if os.path.exists(DOTENV_FILE):
from app import create_app, db, socketio # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
NotificationData, NotificationEmailData, QueryResult,
Role, User) # noqa
QueryResult, Role, User) # noqa
from flask_migrate import Migrate, upgrade # noqa
@ -34,8 +33,6 @@ def make_shell_context():
'Job': Job,
'JobInput': JobInput,
'JobResult': JobResult,
'NotificationData': NotificationData,
'NotificationEmailData': NotificationEmailData,
'QueryResult': QueryResult,
'Role': Role,
'User': User}
@ -53,9 +50,9 @@ def deploy():
@app.cli.command()
def tasks():
from app.tasks import process_corpora, process_jobs
process_corpora()
process_jobs()
from app.tasks import check_corpora, check_jobs
check_corpora()
check_jobs()
@app.cli.command()