sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-10-30 18:22:45 +00:00
Code Issues Projects Releases Wiki Activity

More exception handling. Remove unused database models. New common view structure!

Browse Source
This commit is contained in:
Patrick Jentsch
2020-11-13 10:01:51 +01:00
parent cb9da5c7dd
commit 5a06a6b241
45 changed files with 692 additions and 1005 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
.env.tpl
View File

@@ -9,128 +9,116 @@
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# Example: 1000
# HINT: Use this bash command `id -g`
HOST_GID=
# DEFAULT: ./nopaqued.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_DAEMON_LOG_FILE=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_LOG_FILE=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# HOST_LOG_FILE=
################################################################################
# Cookies #
# Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# SECRET_KEY=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE=
# SESSION_COOKIE_SECURE=
################################################################################
# Database #
# DATABASE_URI blueprint: #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
# Flask-Login #
# https://flask-login.readthedocs.io/en/latest/ #
################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
# NOPAQUE_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# REMEMBER_COOKIE_SECURE=
################################################################################
# Email #
# Flask-Mail #
# https://pythonhosted.org/Flask-Mail/ #
################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER=
MAIL_DEFAULT_SENDER=
NOPAQUE_SMTP_PASSWORD=
MAIL_PASSWORD=
# EXAMPLE: smtp.example.com
NOPAQUE_SMTP_SERVER=
MAIL_SERVER=
# EXAMPLE: 587
NOPAQUE_SMTP_PORT=
MAIL_PORT=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_SSL=
# MAIL_USE_SSL=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_TLS=
# MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com
NOPAQUE_SMTP_USERNAME=
MAIL_USERNAME=
################################################################################
# General #
# Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################
# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
# SQLALCHEMY_DATABASE_URI=
################################################################################
# nopaque #
################################################################################
# If an account is registered with this email adress gets automatically
# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS=
NOPAQUE_ADMIN=
# DEFAULT: development
# CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG=
# This email adress is used for the contact button in the nopaque footer. If
# not set, no contact button is displayed.
# DEFAULT: None
# EXAMPLE: contact.nopaque@example.com
# NOPAQUE_CONTACT_EMAIL_ADRESS=
# NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
# NOTE: This must be a network share and it must be available on all Docker
# Swarm nodes
# NOPAQUE_DATA_DIR=
# DEFAULT: localhost
# NOPAQUE_DOMAIN=
# DEFAULT: 0.0.0.0
# NOPAQUE_HOST=
# DEFAULT: 5000
# NOPAQUE_PORT=
# CHOOSE ONE: http, https
# DEFAULT: http
# NOPAQUE_PROTOCOL=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# NOPAQUE_SECRET_KEY=
################################################################################
# Logging #
################################################################################
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# transport://[userid:password]@hostname[:port]/[virtual_host]
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
@@ -146,37 +134,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR=
# NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST=
# NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT=
# NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX=
# NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO=
# NOPAQUE_PROXY_FIX_X_PROTO=

1
docker-compose.yml
View File

@@ -28,5 +28,6 @@ services:
image: nopaque:development
restart: unless-stopped
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"

5
web/Dockerfile
View File

@@ -21,8 +21,9 @@ RUN apt-get update \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque
WORKDIR /home/nopaque

2
web/app/__init__.py
View File

@@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context():
from . import events

2
web/app/admin/__init__.py
View File

@@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__)
from . import views # noqa
from . import views

1
web/app/admin/forms.py
View File

@@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()]
self.user = user

14
web/app/admin/views.py
View File

@@ -29,12 +29,11 @@ def user(user_id):
@admin_required
def delete_user(user_id):
settings_tasks.delete_user(user_id)
flash('User has been deleted!')
flash('User has been marked for deletion!')
return redirect(url_for('.users'))
@admin.route('/users/<int:user_id>/edit_general_settings',
methods=['GET', 'POST'])
@admin.route('/users/<int:user_id>/edit_general_settings', methods=['GET', 'POST']) # noqa
@login_required
@admin_required
def edit_general_settings(user_id):
@@ -46,16 +45,13 @@ def edit_general_settings(user_id):
user.username = form.username.data
user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data)
db.session.add(user)
db.session.commit()
flash('The profile has been updated.')
return redirect(url_for('admin.edit_general_settings', user_id=user.id))
flash('Settings have been updated.')
return redirect(url_for('.edit_general_settings', user_id=user.id))
form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email
form.role.data = user.role_id
form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2',
form=form,
title='General settings',
user=user)
form=form, title='General settings', user=user)

2
web/app/auth/__init__.py
View File

@@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__)
from . import views # noqa
from . import views

2
web/app/auth/forms.py
View File

@@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField(
'Username',
validators=[DataRequired(), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

107
web/app/auth/views.py
View File

@@ -1,5 +1,5 @@
from flask import (current_app, flash, redirect, render_template, request,
url_for)
from datetime import datetime
from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user
from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db
from ..email import create_message, send
from ..models import User
import logging
import os
import shutil
@auth.before_app_request
@@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
current_user.ping()
if not current_user.confirmed \
and request.endpoint \
and request.blueprint != 'auth' \
and request.endpoint != 'static':
current_user.last_seen = datetime.utcnow()
db.session.commit()
if (not current_user.confirmed
and request.endpoint
and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
@@ -30,20 +31,19 @@ def before_request():
def login():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
next = request.args.get('next')
if next is None or not next.startswith('/'):
next = url_for('main.dashboard')
return redirect(next)
flash('Invalid email/username or password.')
return render_template('auth/login.html.j2', login_form=login_form,
title='Log in')
return render_template('auth/login.html.j2', form=form, title='Log in')
@auth.route('/logout')
@@ -58,26 +58,28 @@ def logout():
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
registration_form = RegistrationForm(prefix='registration-form')
if registration_form.validate_on_submit():
user = User(email=registration_form.email.data.lower(),
password=registration_form.password.data,
username=registration_form.username.data)
form = RegistrationForm(prefix='registration-form')
if form.validate_on_submit():
user = User(email=form.email.data.lower(),
password=form.password.data,
username=form.username.data)
db.session.add(user)
db.session.commit()
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(user.id))
if os.path.exists(user_dir):
shutil.rmtree(user_dir)
os.mkdir(user_dir)
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('auth.login'))
return render_template('auth/register.html.j2',
registration_form=registration_form,
try:
os.makedirs(user.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(user.path))
db.session.delete(user)
db.session.commit()
abort(500)
else:
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('.login'))
return render_template('auth/register.html.j2', form=form,
title='Register')
@@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard'))
else:
flash('The confirmation link is invalid or has expired.')
return redirect(url_for('auth.unconfirmed'))
return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed')
@@ -119,39 +121,32 @@ def resend_confirmation():
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_request_form = ResetPasswordRequestForm(
prefix='reset-password-request-form')
if reset_password_request_form.validate_on_submit():
submitted_email = reset_password_request_form.email.data
user = User.query.filter_by(email=submitted_email.lower()).first()
if user:
form = ResetPasswordRequestForm(prefix='reset-password-request-form')
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data.lower()).first()
if user is not None:
token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token,
user=user)
send(msg)
flash('An email with instructions to reset your password has been '
'sent to you.')
return redirect(url_for('auth.login'))
return render_template(
'auth/reset_password_request.html.j2',
reset_password_request_form=reset_password_request_form,
title='Password Reset')
flash('An email with instructions to reset your password has been sent to you.') # noqa
return redirect(url_for('.login'))
return render_template('auth/reset_password_request.html.j2', form=form,
title='Password Reset')
@auth.route('/reset/<token>', methods=['GET', 'POST'])
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_form = ResetPasswordForm(prefix='reset-password-form')
if reset_password_form.validate_on_submit():
if User.reset_password(token, reset_password_form.password.data):
form = ResetPasswordForm(prefix='reset-password-form')
if form.validate_on_submit():
if User.reset_password(token, form.password.data):
db.session.commit()
flash('Your password has been updated.')
return redirect(url_for('auth.login'))
return redirect(url_for('.login'))
else:
return redirect(url_for('main.index'))
return render_template('auth/reset_password.html.j2',
reset_password_form=reset_password_form,
title='Password Reset',
token=token)
return render_template('auth/reset_password.html.j2', form=form,
title='Password Reset', token=token)

390
web/app/corpora/views.py
View File

@@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, make_response, redirect, request,
from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory)
from flask_login import current_user, login_required
from . import corpora
@@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db
from ..models import Corpus, CorpusFile, QueryResult
import json
import logging
import os
import shutil
import glob
@@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST'])
@login_required
def add_corpus():
add_corpus_form = AddCorpusForm()
if add_corpus_form.validate_on_submit():
form = AddCorpusForm()
if form.validate_on_submit():
corpus = Corpus(creator=current_user,
description=add_corpus_form.description.data,
status='unprepared', title=add_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not add corpus!', 'corpus')
corpus.delete()
else:
url = url_for('corpora.corpus', corpus_id=corpus.id)
flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2',
add_corpus_form=add_corpus_form,
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
abort(500)
flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
return redirect(url_for('.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2', form=form,
title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST'])
@login_required
def import_corpus():
import_corpus_form = ImportCorpusForm()
if import_corpus_form.is_submitted():
if not import_corpus_form.validate():
return make_response(import_corpus_form.errors, 400)
form = ImportCorpusForm()
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
corpus = Corpus(creator=current_user,
description=import_corpus_form.description.data,
status='unprepared',
title=import_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not import corpus!', 'corpus')
corpus.delete()
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.import_corpus')}, 500)
# Upload zip
archive_file = os.path.join(corpus.path, form.file.data.filename)
form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus.path)
# Register vrt files to corpus
vrts = glob.glob(corpus.path + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL')
)
db.session.add(corpus_file)
# finish import and redirect to imported corpus
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
return make_response(
{'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
else:
# Upload zip
archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
import_corpus_form.file.data.filename)
corpus_dir = os.path.dirname(archive_file)
import_corpus_form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus_dir)
# Register vrt files to corpus
vrts = glob.glob(corpus_dir + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
dir=dir,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL'))
db.session.add(corpus_file)
# finish import and got to imported corpus
url = url_for('corpora.corpus', corpus_id=corpus.id)
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('[<a href="{}">{}</a>] imported'.format(url,
corpus.title),
'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus',
corpus_id=corpus.id)},
201)
else:
# If imported zip is not valid delete corpus and give feedback
corpus.delete()
db.session.commit()
flash('Imported corpus is not valid.', 'error')
return make_response(
{'redirect_url': url_for('corpora.import_corpus')},
201)
return render_template('corpora/import_corpus.html.j2',
import_corpus_form=import_corpus_form,
# If imported zip is not valid delete corpus and give feedback
flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
tasks.delete_corpus(corpus.id)
return make_response(
{'redirect_url': url_for('.import_corpus')}, 201)
return render_template('corpora/import_corpus.html.j2', form=form,
title='Import Corpus')
@@ -131,17 +118,9 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
corpus_files = [dict(filename=corpus_file.filename,
author=corpus_file.author,
title=corpus_file.title,
publishing_year=corpus_file.publishing_year,
corpus_id=corpus.id,
id=corpus_file.id)
for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus')
corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2', corpus=corpus,
corpus_files=corpus_files, title='Corpus')
@corpora.route('/<int:corpus_id>/export')
@@ -150,12 +129,11 @@ def export_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
# TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file)
return send_from_directory(directory=dir,
filename=filename,
mimetype='zip',
as_attachment=True)
return send_from_directory(as_attachment=True, directory=dir,
filename=filename, mimetype='zip')
@corpora.route('/<int:corpus_id>/analyse')
@@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
result_context=request.args.get('context', 20),
results_per_page=request.args.get('results_per_page', 30))
results_per_page=request.args.get('results_per_page', 30)
)
query_form = QueryForm(prefix='query-form',
query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form')
@@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template(
'corpora/analyse_corpus.html.j2',
corpus=corpus,
corpus_id=corpus_id,
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_form=query_form,
query_download_form=query_download_form,
inspect_display_options_form=inspect_display_options_form,
title='Corpus analysis')
title='Corpus analysis'
)
@corpora.route('/<int:corpus_id>/delete')
@@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id)
flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard'))
@@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
add_corpus_file_form = AddCorpusFileForm(corpus,
prefix='add-corpus-file-form')
if add_corpus_file_form.is_submitted():
if not add_corpus_file_form.validate():
return make_response(add_corpus_file_form.errors, 400)
form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
# Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
add_corpus_file_form.file.data.save(
os.path.join(current_app.config['DATA_DIR'], dir,
add_corpus_file_form.file.data.filename))
corpus_file = CorpusFile(
address=add_corpus_file_form.address.data,
author=add_corpus_file_form.author.data,
booktitle=add_corpus_file_form.booktitle.data,
chapter=add_corpus_file_form.chapter.data,
corpus=corpus,
dir=dir,
editor=add_corpus_file_form.editor.data,
filename=add_corpus_file_form.file.data.filename,
institution=add_corpus_file_form.institution.data,
journal=add_corpus_file_form.journal.data,
pages=add_corpus_file_form.pages.data,
publisher=add_corpus_file_form.publisher.data,
publishing_year=add_corpus_file_form.publishing_year.data,
school=add_corpus_file_form.school.data,
title=add_corpus_file_form.title.data)
form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
corpus_file = CorpusFile(address=form.address.data,
author=form.author.data,
booktitle=form.booktitle.data,
chapter=form.chapter.data,
corpus=corpus,
editor=form.editor.data,
filename=form.file.data.filename,
institution=form.institution.data,
journal=form.journal.data,
pages=form.pages.data,
publisher=form.publisher.data,
publishing_year=form.publishing_year.data,
school=form.school.data,
title=form.title.data)
db.session.add(corpus_file)
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file added!', 'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
201)
return render_template('corpora/add_corpus_file.html.j2',
corpus=corpus,
add_corpus_file_form=add_corpus_file_form,
title='Add corpus file')
flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
form=form, title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id)
flash('Corpus file deleted!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
return redirect(url_for('.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
corpus_file.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=corpus_file.corpus.path,
filename=corpus_file.filename)
@@ -274,48 +242,45 @@ def download_corpus_file(corpus_id, corpus_file_id):
methods=['GET', 'POST'])
@login_required
def corpus_file(corpus_id, corpus_file_id):
corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if not corpus_file.corpus_id == corpus_id:
if corpus_file.corpus_id != corpus_id:
abort(404)
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if edit_corpus_file_form.validate_on_submit():
corpus_file.address = edit_corpus_file_form.address.data
corpus_file.author = edit_corpus_file_form.author.data
corpus_file.booktitle = edit_corpus_file_form.booktitle.data
corpus_file.chapter = edit_corpus_file_form.chapter.data
corpus_file.editor = edit_corpus_file_form.editor.data
corpus_file.institution = edit_corpus_file_form.institution.data
corpus_file.journal = edit_corpus_file_form.journal.data
corpus_file.pages = edit_corpus_file_form.pages.data
corpus_file.publisher = edit_corpus_file_form.publisher.data
corpus_file.publishing_year = \
edit_corpus_file_form.publishing_year.data
corpus_file.school = edit_corpus_file_form.school.data
corpus_file.title = edit_corpus_file_form.title.data
form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if form.validate_on_submit():
corpus_file.address = form.address.data
corpus_file.author = form.author.data
corpus_file.booktitle = form.booktitle.data
corpus_file.chapter = form.chapter.data
corpus_file.editor = form.editor.data
corpus_file.institution = form.institution.data
corpus_file.journal = form.journal.data
corpus_file.pages = form.pages.data
corpus_file.publisher = form.publisher.data
corpus_file.publishing_year = form.publishing_year.data
corpus_file.school = form.school.data
corpus_file.title = form.title.data
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file edited!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values
edit_corpus_file_form.address.data = corpus_file.address
edit_corpus_file_form.author.data = corpus_file.author
edit_corpus_file_form.booktitle.data = corpus_file.booktitle
edit_corpus_file_form.chapter.data = corpus_file.chapter
edit_corpus_file_form.editor.data = corpus_file.editor
edit_corpus_file_form.institution.data = corpus_file.institution
edit_corpus_file_form.journal.data = corpus_file.journal
edit_corpus_file_form.pages.data = corpus_file.pages
edit_corpus_file_form.publisher.data = corpus_file.publisher
edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
edit_corpus_file_form.school.data = corpus_file.school
edit_corpus_file_form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2',
corpus_file=corpus_file, corpus=corpus,
edit_corpus_file_form=edit_corpus_file_form,
form.address.data = corpus_file.address
form.author.data = corpus_file.author
form.booktitle.data = corpus_file.booktitle
form.chapter.data = corpus_file.chapter
form.editor.data = corpus_file.editor
form.institution.data = corpus_file.institution
form.journal.data = corpus_file.journal
form.pages.data = corpus_file.pages
form.publisher.data = corpus_file.publisher
form.publishing_year.data = corpus_file.publishing_year
form.school.data = corpus_file.school
form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2', corpus=corpus,
corpus_file=corpus_file, form=form,
title='Edit corpus file')
@@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403)
if corpus.files.all():
tasks.build_corpus(corpus_id)
flash('Building Corpus...', 'corpus')
flash('Corpus "{}" has been marked to get build!', 'corpus')
else:
flash('Can not build corpus, please add corpus file(s).', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Can not build corpus "{}": No corpus file(s)!', 'error')
return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results.
@@ -340,35 +305,29 @@ def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
form = AddQueryResultForm(prefix='add-query-result-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
query_result = QueryResult(creator=current_user,
description=form.description.data,
filename=form.file.data.filename,
title=form.title.data)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
os.makedirs(query_result.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result_file_path = os.path.join(query_result.path,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
@@ -381,19 +340,16 @@ def add_query_result():
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('corpora.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
form=form, title='Add query result')
@corpora.route('/result/<int:query_result_id>')
@@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
query_result=query_result, title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect')
@@ -427,13 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
query_result_file_path = os.path.join(query_result.path, query_result.filename) # noqa
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
@@ -452,8 +401,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@@ -464,10 +413,5 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
return send_from_directory(as_attachment=True, directory=query_result.path,
filename=query_result.filename)

4
web/app/email.py
View File

@@ -1,11 +1,11 @@
from flask import render_template
from flask import current_app, render_template
from flask_mail import Message
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg

2
web/app/jobs/__init__.py
View File

@@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__)
from . import views # noqa
from . import views

30
web/app/jobs/views.py
View File

@@ -1,11 +1,10 @@
from flask import (abort, current_app, flash, redirect, render_template,
from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for)
from flask_login import current_user, login_required
from . import jobs
from . import tasks
from ..decorators import admin_required
from ..models import Job, JobInput, JobResult
import os
@jobs.route('/<int:job_id>')
@@ -14,13 +13,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
job_inputs = [dict(filename=input.filename,
id=input.id,
job_id=job.id)
for input in job.inputs]
return render_template('jobs/job.html.j2',
job=job,
job_inputs=job_inputs,
job_inputs = [job_input.to_dict() for job_input in job.inputs]
return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
title='Job')
@@ -31,7 +25,7 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
tasks.delete_job(job_id)
flash('Job has been deleted!', 'job')
flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard'))
@@ -44,9 +38,8 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_input.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=job_input.job.path,
filename=job_input.filename)
@@ -56,11 +49,11 @@ def download_job_input(job_id, job_input_id):
def restart(job_id):
job = Job.query.get_or_404(job_id)
if job.status != 'failed':
flash('Could not restart job: status is not "failed"', 'error')
flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else:
tasks.restart_job(job_id)
flash('Job has been restarted!', 'job')
return redirect(url_for('jobs.job', job_id=job_id))
flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@@ -72,7 +65,6 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_result.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=job_result.job.path,
filename=job_result.filename)

2
web/app/main/__init__.py
View File

@@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__)
from . import views # noqa
from . import views

25
web/app/main/views.py
View File

@@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST'])
def index():
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
return redirect(url_for('main.dashboard'))
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.')
return render_template('main/index.html.j2', login_form=login_form,
title='nopaque')
return render_template('main/index.html.j2', form=form, title='nopaque')
@main.route('/about_and_faq')
@@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/news')
def news():
return render_template('main/news.html.j2', title='News')
@@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy')
def privacy_policy():
return render_template('main/privacy_policy.html.j2',
title=('Information on the processing of personal'
' data for the nopaque platform (GDPR)'))
title='Privacy statement (GDPR)')
@main.route('/terms_of_use')
def terms_of_use():
return render_template('main/terms_of_use.html.j2',
title='General Terms of Use of the platform '
'nopaque')
return render_template('main/terms_of_use.html.j2', title='Terms of Use')

181
web/app/models.py
View File

@@ -7,6 +7,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET
from . import db, login_manager
import logging
import os
import shutil
@@ -54,7 +55,7 @@ class Role(db.Model):
'''
String representation of the Role. For human readability.
'''
return '<Role {role_name}>'.format(role_name=self.name)
return '<Role {}>'.format(self.name)
def add_permission(self, perm):
'''
@@ -138,6 +139,18 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete',
lazy='dynamic')
@property
def path(self):
return os.path.join(current_app.config['NOPAQUE_DATA_DIR'], str(self.id))
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def to_dict(self):
return {'id': self.id,
'role_id': self.role_id,
@@ -162,7 +175,7 @@ class User(UserMixin, db.Model):
'''
String representation of the User. For human readability.
'''
return '<User {username}>'.format(username=self.username)
return '<User {}>'.format(self.username)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
@@ -220,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user)
return True
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
@@ -244,17 +249,11 @@ class User(UserMixin, db.Model):
'''
return self.can(Permission.ADMIN)
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def delete(self):
'''
Delete the user and its corpora and jobs from database and filesystem.
'''
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
@@ -280,14 +279,17 @@ class JobInput(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self):
'''
String representation of the JobInput. For human readability.
'''
return '<JobInput {filename}>'.format(filename=self.filename)
return '<JobInput {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@@ -305,14 +307,17 @@ class JobResult(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.job.path, self.filename)
def __repr__(self):
'''
String representation of the JobResult. For human readability.
'''
return '<JobResult {filename}>'.format(filename=self.filename)
return '<JobResult {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@@ -351,19 +356,16 @@ class Job(db.Model):
cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete')
notification_data = db.relationship('NotificationData',
cascade='save-update, merge, delete',
uselist=False,
back_populates='job') # One-to-One relationship
notification_email_data = db.relationship('NotificationEmailData',
cascade='save-update, merge, delete',
back_populates='job')
@property
def path(self):
return os.path.join(self.creator.path, 'jobs', str(self.id))
def __repr__(self):
'''
String representation of the Job. For human readability.
'''
return '<Job {job_title}>'.format(job_title=self.title)
return '<Job {}>'.format(self.title)
def create_secure_filename(self):
'''
@@ -385,11 +387,7 @@ class Job(db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(job_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def restart(self):
@@ -399,12 +397,8 @@ class Job(db.Model):
if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
self.end_date = None
self.status = 'submitted'
@@ -425,63 +419,6 @@ class Job(db.Model):
for result in self.results}}
class NotificationData(db.Model):
'''
Class to define notification data used for sending a notification mail with
nopaque_notify.
'''
__tablename__ = 'notification_data'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_data')
# Fields
notified_on = db.Column(db.String(16), default=None)
def __repr__(self):
'''
String representation of the NotificationData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notified': self.notified}
class NotificationEmailData(db.Model):
'''
Class to define data that will be used to send a corresponding Notification
via email.
'''
__tablename__ = 'notification_email_data'
# Primary Key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_email_data')
notify_status = db.Column(db.String(16), default=None)
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
def __repr__(self):
'''
String representation of the NotificationEmailData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notify_status': self.notify_status,
'creation_date': self.creation_date}
class CorpusFile(db.Model):
'''
Class to define Files.
@@ -496,7 +433,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
dir = db.Column(db.String(255))
editor = db.Column(db.String(255))
filename = db.Column(db.String(255))
institution = db.Column(db.String(255))
@@ -507,15 +443,15 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255))
title = db.Column(db.String(255))
@property
def path(self):
return os.path.join(self.corpus.path, self.filename)
def delete(self):
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
self.filename)
try:
os.remove(corpus_file_path)
os.remove(self.path)
except OSError:
logging.error('Removing {} led to an OSError!'.format(self.path))
pass
db.session.delete(self)
self.corpus.status = 'unprepared'
@@ -553,13 +489,17 @@ class Corpus(db.Model):
description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
status = db.Column(db.String(16))
status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32))
archive_file = db.Column(db.String(255))
# Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete')
@property
def path(self):
return os.path.join(self.creator.path, 'corpora', str(self.id))
def to_dict(self):
return {'id': self.id,
'user_id': self.user_id,
@@ -571,19 +511,14 @@ class Corpus(db.Model):
'files': {file.id: file.to_dict() for file in self.files}}
def build(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
output_dir = os.path.join(corpus_dir, 'merged')
output_dir = os.path.join(self.path, 'merged')
shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir)
master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>')
)
for corpus_file in self.files:
corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
element_tree = ET.parse(corpus_file_path)
element_tree = ET.parse(corpus_file.path)
text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author)
@@ -597,7 +532,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title)
element_tree.write(corpus_file_path)
element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file,
@@ -607,18 +542,14 @@ class Corpus(db.Model):
self.status = 'submitted'
def delete(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
shutil.rmtree(corpus_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def __repr__(self):
'''
String representation of the corpus. For human readability.
'''
return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model):
@@ -636,12 +567,12 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32))
@property
def path(self):
return os.path.join(self.creator.path, 'query_results', str(self.id))
def delete(self):
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'query_results',
str(self.id))
shutil.rmtree(query_result_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_dict(self):
@@ -654,7 +585,7 @@ class QueryResult(db.Model):
def __repr__(self):
'''
String representation of the CorpusAnalysisResult. For human readability.
String representation of the QueryResult. For human readability.
'''
return '<QueryResult {}>'.format(self.title)

150
web/app/query_results/views.py
View File

@@ -1,150 +0,0 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/__init__.py
View File

@@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__)
from . import views # noqa
from . import views

61
web/app/services/views.py
View File

@@ -1,5 +1,4 @@
from flask import (abort, current_app, flash, make_response, render_template,
url_for)
from flask import abort, flash, make_response, render_template, url_for
from flask_login import current_user, login_required
from werkzeug.utils import secure_filename
from . import services
@@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput
import json
import logging
import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4},
'add_job_form': AddFileSetupJobForm},
'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2},
'add_job_form': AddNLPJobForm},
'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4},
'add_job_form': AddOCRJobForm}}
'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST'])
@@ -30,54 +30,49 @@ def service(service):
if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'])
add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
if add_job_form.is_submitted():
if not add_job_form.validate():
return make_response(add_job_form.errors, 400)
form = SERVICES[service]['form'](prefix='add-job-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
service_args = []
if service == 'nlp':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.check_encoding.data:
service_args.append('-l {}'.format(form.language.data))
if form.check_encoding.data:
service_args.append('--check-encoding')
if service == 'ocr':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.binarization.data:
service_args.append('-l {}'.format(form.language.data))
if form.binarization.data:
service_args.append('--binarize')
job = Job(creator=current_user,
description=add_job_form.description.data,
description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args),
service_version=add_job_form.version.data,
status='preparing', title=add_job_form.title.data)
service_version=form.version.data,
status='preparing', title=form.title.data)
if job.service != 'corpus_analysis':
job.create_secure_filename()
db.session.add(job)
db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try:
os.makedirs(absolut_dir)
os.makedirs(job.path)
except OSError:
job.delete()
flash('Internal Server Error', 'job')
return make_response({'redirect_url': url_for('services.service',
service=service)},
500)
logging.error('Make dir {} led to an OSError!'.format(job.path))
db.session.delete(job)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.service', service=service)}, 500)
else:
for file in add_job_form.files.data:
for file in form.files.data:
filename = secure_filename(file.filename)
file.save(os.path.join(absolut_dir, filename))
job_input = JobInput(dir=relative_dir, filename=filename,
job=job)
job_input = JobInput(dir=job.path, filename=filename, job=job)
file.save(job_input.path)
db.session.add(job_input)
job.status = 'submitted'
db.session.commit()
url = url_for('jobs.job', job_id=job.id)
flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
flash('Job "{}" added'.format(job.title), 'job')
return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'],
add_job_form=add_job_form)
form=form, title=SERVICES[service]['name'])

2
web/app/settings/forms.py
View File

@@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername',
validators=[DataRequired(),
Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

19
web/app/settings/views.py
View File

@@ -1,13 +1,9 @@
from flask import current_app, flash, redirect, render_template, url_for
from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user
from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm)
from .. import db
from ..decorators import admin_required
from ..models import Role, User
import os
import uuid
@settings.route('/')
@@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.')
return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2',
form=form,
title='Change password')
form=form, title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST'])
@@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email
form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2',
form=form,
title='General settings')
form=form, title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2',
form=form,
title='Notification settings')
form=form, title='Notification settings')
@settings.route('/delete')
@@ -76,5 +71,5 @@ def delete():
"""
tasks.delete_user(current_user.id)
logout_user()
flash('Your account has been deleted!')
flash('Your account has been marked for deletion!')
return redirect(url_for('main.index'))

14
web/app/tasks/__init__.py
View File

@@ -11,15 +11,11 @@ def check_corpora():
corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: (corpus.status == 'queued'
or corpus.status == 'running'),
corpora):
for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
corpus_utils.checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis',
corpora):
for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpus_utils.create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
corpora):
for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpus_utils.remove_cqpserver_container(corpus)
db.session.commit()
@@ -28,8 +24,6 @@ def check_jobs():
jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job)
for job in filter(lambda job: job.status == 'queued', jobs):
job_utils.checkout_job_service(job)
for job in filter(lambda job: job.status == 'running', jobs):
for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
db.session.commit()

131
web/app/tasks/corpus_utils.py
View File

@@ -1,4 +1,3 @@
from flask import current_app
from . import docker_client
import docker
import logging
@@ -7,20 +6,14 @@ import shutil
def create_build_corpus_service(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
if os.path.exists(corpus_data_dir):
shutil.rmtree(corpus_data_dir)
if os.path.exists(corpus_registry_dir):
shutil.rmtree(corpus_registry_dir)
corpus_data_dir = os.path.join(corpus.path, 'data')
shutil.rmtree(corpus_data_dir, ignore_errors=True)
os.mkdir(corpus_data_dir)
corpus_registry_dir = os.path.join(corpus.path, 'registry')
shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir)
service_args = {
corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
@@ -32,30 +25,34 @@ def create_build_corpus_service(corpus):
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
try:
docker_client.services.create(service_image, **service_args)
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error('create_build_corpus_service({}): '.format(corpus.id)
+ '{} (status: {} -> failed)'.format(e, corpus.status))
corpus.status = 'failed'
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
corpus.status = 'queued'
finally:
# TODO: send email
pass
def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound as e:
logging.error('checkout_build_corpus_service({}):'.format(corpus.id)
+ ' {} (stauts: {} -> failed)'.format(e, corpus.status))
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-NotFound] The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status))
corpus.status = 'failed'
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@@ -63,25 +60,23 @@ def checkout_build_corpus_service(corpus):
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif corpus.status == 'running' and task_state == 'complete':
service.remove()
corpus.status = 'prepared'
elif corpus.status == 'running' and task_state == 'failed':
service.remove()
corpus.status = task_state
finally:
# TODO: send email
pass
elif corpus.status == 'running' and task_state in ['complete', 'failed']: # noqa
try:
service.remove()
except docker.errors.APIError as e:
logging.error('Remove "{}" service raised '.format(service_name) # noqa
+ '[docker-APIError] The server returned an error. ' # noqa
+ 'Details: {}'.format(e))
return
else:
corpus.status = 'prepared' if task_state == 'complete' \
else 'failed'
def create_cqpserver_container(corpus):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
container_args = {
corpus_data_dir = os.path.join(corpus.path, 'data')
corpus_registry_dir = os.path.join(corpus.path, 'registry')
container_kwargs = {
'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
@@ -89,20 +84,43 @@ def create_cqpserver_container(corpus):
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'
}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
# Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one
try:
container = docker_client.containers.get(container_args['name'])
container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
else:
container.remove(force=True)
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
try:
docker_client.containers.run(container_image, **container_args)
except docker.errors.DockerException:
return
docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError:
# This case should not occur, because detach is True.
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ContainerError] The container exits with a '
+ 'non-zero exit code and detach is False.')
corpus.status = 'failed'
except docker.errors.ImageNotFound:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-ImageNotFound] The specified image does not '
+ 'exist.')
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
corpus.status = 'analysing'
@@ -113,8 +131,17 @@ def remove_cqpserver_container(corpus):
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
else:
container.remove(force=True)
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
return
corpus.status = 'prepared'

90
web/app/tasks/job_utils.py
View File

@@ -1,7 +1,7 @@
from datetime import datetime
from flask import current_app
from . import docker_client
from .. import db
from ..email import create_message, send
from ..models import JobResult
import docker
import logging
@@ -10,51 +10,60 @@ import os
def create_job_service(job):
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(job.user_id),
'jobs',
str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_args = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job_dir + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)),
'restart_policy': docker.types.RestartPolicy()}
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)
),
'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version)
try:
docker_client.services.create(service_image, **service_args)
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error('create_job_service({}): {} '.format(job.id, e)
+ '(status: {} -> failed)'.format(job.status))
job.status = 'failed'
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
else:
job.status = 'queued'
finally:
# TODO: send email
pass
msg = create_message(
job.creator.email,
'Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
def checkout_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound as e:
logging.error('checkout_job_service({}): {} '.format(job.id, e)
+ '(status: {} -> submitted)'.format(job.status))
job.status = 'submitted'
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-NotFound] The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status))
job.status = 'failed'
except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-APIError] The server returned an error. '
+ 'Details: {}'.format(e))
except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-InvalidVersion] One of the arguments is '
+ 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@@ -62,22 +71,16 @@ def checkout_job_service(job):
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif job.status == 'queued' and task_state == 'complete':
elif job.status == 'running' and task_state == 'complete':
service.remove()
job.end_date = datetime.utcnow()
job.status = task_state
if task_state == 'complete':
results_dir = os.path.join(current_app.config['DATA_DIR'],
str(job.user_id),
'jobs',
str(job.id),
'output')
results = filter(lambda x: x.endswith('.zip'),
os.listdir(results_dir))
for result in results:
job_result = JobResult(dir=results_dir,
filename=result,
job_id=job.id)
job_results_dir = os.path.join(job.path, 'output')
job_results = filter(lambda x: x.endswith('.zip'),
os.listdir(job_results_dir))
for job_result in job_results:
job_result = JobResult(filename=job_result, job=job)
db.session.add(job_result)
elif job.status == 'running' and task_state == 'failed':
service.remove()
@@ -85,6 +88,13 @@ def checkout_job_service(job):
job.status = task_state
finally:
# TODO: send email
msg = create_message(
job.creator.email,
'[nopaque] Status update for your Job "{}"'.format(job.title),
'tasks/email/notification',
job=job
)
send(msg)
pass

10
web/app/templates/auth/login.html.j2
View File

@@ -35,20 +35,20 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

12
web/app/templates/auth/register.html.j2
View File

@@ -34,14 +34,14 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ registration_form.hidden_tag() }}
{{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(registration_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/auth/reset_password.html.j2
View File

@@ -20,12 +20,12 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_form.hidden_tag() }}
{{ wtf.render_field(reset_password_form.password, data_length='128') }}
{{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.password, data_length='128') }}
{{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

6
web/app/templates/auth/reset_password_request.html.j2
View File

@@ -20,11 +20,11 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_request_form.hidden_tag() }}
{{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/corpora/add_corpus.html.j2
View File

@@ -27,18 +27,18 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ add_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

14
web/app/templates/corpora/add_corpus_file.html.j2
View File

@@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
{{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@@ -52,7 +52,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body">
{% for field in add_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

2
web/app/templates/corpora/analyse_corpus.html.j2
View File

@@ -155,7 +155,7 @@ import {
*/
document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode
let corpusId = {{ corpus_id }}
let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket,
'logging': true,

12
web/app/templates/corpora/corpus_file.html.j2
View File

@@ -20,23 +20,23 @@
<div class="col s12">
<form method="POST">
{{ edit_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="card">
<div class="card-content">
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@@ -44,7 +44,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body">
{% for field in edit_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

12
web/app/templates/corpora/import_corpus.html.j2
View File

@@ -1,4 +1,4 @@
{% extends "nopaque.html.j2" %}
{% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %}
@@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ import_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
<div class="row">
<div class="col s12">
{{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
{{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

10
web/app/templates/corpora/query_results/add_query_result.html.j2
View File

@@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_query_result_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
{{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
</form>

10
web/app/templates/main/index.html.j2
View File

@@ -159,20 +159,20 @@
<form method="POST">
<div class="card-content">
<span class="card-title">Log in</span>
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

6
web/app/templates/nopaque.html.j2
View File

@@ -231,9 +231,9 @@
</div>
<div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
{% if config.CONTACT_EMAIL_ADRESS %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% if config.NOPAQUE_CONTACT %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div>

12
web/app/templates/services/file-setup.html.j2
View File

@@ -48,24 +48,24 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
{{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div>
<div class="col s12 hide">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/nlp.html.j2
View File

@@ -66,34 +66,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
{{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.check_encoding.label.text }}</p>
<p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.check_encoding() }}
{{ form.check_encoding() }}
<span class="lever"></span>
</label>
</div>
@@ -107,7 +107,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/ocr.html.j2
View File

@@ -48,34 +48,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
{{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.binarization.label.text }}</p>
<p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.binarization() }}
{{ form.binarization() }}
<span class="lever"></span>
</label>
</div>
@@ -134,7 +134,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
{{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div>
</form>
</div>

7
web/app/templates/tasks/email/notification.html.j2
View File

@@ -1,9 +1,8 @@
<p>Dear <b>{{ user.username }}</b>,</p>
<p>Dear <b>{{ job.creator.username }}</b>,</p>
<p>The status of your Job/Corpus({{ job.id }}) with the title <b>"{{ job.title }}"</b> has changed!</p>
<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p>
<p>Time of this status update was: <b>{time} UTC</b></p>
<p>You can access your Job/Corpus here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p>

7
web/app/templates/tasks/email/notification.txt.j2
View File

@@ -1,10 +1,9 @@
Dear {{ user.username }},
Dear {{ job.creator.username }},
The status of your Job/Corpus({{ job.id }}) with the title "{{ job.title }}" has changed!
The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}!
Time of this status update was: {time} UTC
You can access your Job/Corpus here: {{ url_for('jobs.job', job_id=job.id) }}
You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards!
Your nopaque team

1
web/boot.sh
View File

@@ -1,5 +1,6 @@
#!/bin/bash
source venv/bin/activate
while true; do
flask deploy
if [[ "$?" == "0" ]]; then

139
web/config.py
View File

@@ -7,103 +7,96 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' # Cookies # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = os.environ.get(
'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
SESSION_COOKIE_SECURE = os.environ.get(
'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask # '''
SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
SESSION_COOKIE_SECURE = \
os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Database # '''
''' # Flask-Login # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = \
os.environ.get('REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask-Mail # '''
MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
MAIL_PORT = int(os.environ.get('MAIL_PORT'))
MAIL_SERVER = os.environ.get('MAIL_SERVER')
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'false').lower() == 'true'
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
''' # Email # '''
MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
MAIL_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' # General # '''
ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
ALLOWED_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
''' # Message queue # '''
SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
''' # Proxy fix # '''
PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
''' # nopaque # '''
NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
NOPAQUE_DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
NOPAQUE_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
@classmethod
def init_app(cls, app):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
# Set up logging according to the corresponding (NOPAQUE_LOG_*)
# environment variables
basic_config_kwargs = {
'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S'),
'filename': os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log')),
'format': os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
'%(message)s'
),
'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
}
logging.basicConfig(**basic_config_kwargs)
# Set up and apply the ProxyFix middleware according to the
# corresponding (PROXY_FIX_*) variables
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=cls.PROXY_FIX_X_FOR,
x_host=cls.PROXY_FIX_X_HOST,
x_port=cls.PROXY_FIX_X_PORT,
x_prefix=cls.PROXY_FIX_X_PREFIX,
x_proto=cls.PROXY_FIX_X_PROTO)
# corresponding (NOPAQUE_PROXY_FIX_*) environment variables
proxy_fix_kwargs = {
'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
}
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
class DevelopmentConfig(Config):
''' # Database # '''
''' # Flask # '''
DEBUG = True
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_dev'
)
''' # General # '''
DEBUG = True
class ProductionConfig(Config):
''' # Database # '''
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
'SQLALCHEMY_DATABASE_URI', 'postgresql://nopaque:nopaque@db/nopaque')
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
''' # General # '''
''' # Flask # '''
TESTING = True
WTF_CSRF_ENABLED = False
''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
config = {'development': DevelopmentConfig,
'production': ProductionConfig,

11
web/nopaque.py
View File

@@ -17,8 +17,7 @@ if os.path.exists(DOTENV_FILE):
from app import create_app, db, socketio # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
NotificationData, NotificationEmailData, QueryResult,
Role, User) # noqa
QueryResult, Role, User) # noqa
from flask_migrate import Migrate, upgrade # noqa
@@ -34,8 +33,6 @@ def make_shell_context():
'Job': Job,
'JobInput': JobInput,
'JobResult': JobResult,
'NotificationData': NotificationData,
'NotificationEmailData': NotificationEmailData,
'QueryResult': QueryResult,
'Role': Role,
'User': User}
@@ -53,9 +50,9 @@ def deploy():
@app.cli.command()
def tasks():
from app.tasks import process_corpora, process_jobs
process_corpora()
process_jobs()
from app.tasks import check_corpora, check_jobs
check_corpora()
check_jobs()
@app.cli.command()