From df6ab3991c7dfcc8f0522da358947615236214c2 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Tue, 8 Feb 2022 12:26:20 +0100 Subject: [PATCH] Use enums where appropriate. This commit includes new migrations that are NOT compatible with older nopaque instances --- app/admin/forms.py | 17 +- app/admin/routes.py | 109 ++++++--- app/api/auth.py | 2 +- app/api/tokens.py | 2 +- app/auth/forms.py | 53 +++-- app/auth/routes.py | 122 +++++++---- app/contribute/routes.py | 8 +- app/corpora/cqi_over_socketio/__init__.py | 11 +- app/corpora/forms.py | 9 +- app/corpora/query_results_routes.py | 4 +- app/corpora/query_results_tasks.py | 13 ++ app/corpora/routes.py | 181 +++++++++------ app/corpora/tasks.py | 16 +- app/daemon/corpus_utils.py | 58 ++--- app/daemon/job_utils.py | 30 +-- app/events/socketio.py | 7 +- app/events/sqlalchemy.py | 62 +++--- app/jobs/routes.py | 53 +++-- app/jobs/tasks.py | 6 +- app/main/routes.py | 16 +- app/models.py | 206 +++++++++--------- app/query_results_models.py | 62 ++++++ app/services/forms.py | 16 +- app/services/routes.py | 25 ++- app/settings/forms.py | 107 +++++---- app/settings/routes.py | 120 +++++----- app/settings/tasks.py | 6 +- app/static/css/nopaque.css | 62 +++--- app/static/css/nopaque_icons.css | 10 +- app/static/js/App.js | 2 +- app/static/js/JobStatusNotifier.js | 2 +- .../js/RessourceDisplays/CorpusDisplay.js | 17 +- app/static/js/RessourceDisplays/JobDisplay.js | 14 +- .../js/RessourceLists/CorpusFileList.js | 26 +-- app/static/js/RessourceLists/CorpusList.js | 22 +- app/static/js/RessourceLists/JobInputList.js | 14 +- app/static/js/RessourceLists/JobList.js | 34 +-- app/static/js/RessourceLists/JobResultList.js | 12 +- .../js/RessourceLists/QueryResultList.js | 22 +- app/static/js/RessourceLists/UserList.js | 25 ++- app/templates/_colors.html.j2 | 30 ++- app/templates/_sidenav.html.j2 | 12 +- app/templates/_styles.html.j2 | 44 ++-- app/templates/admin/edit_user.html.j2 | 168 +++++++++----- app/templates/corpora/add_corpus.html.j2 | 2 +- app/templates/corpora/add_corpus_file.html.j2 | 2 +- app/templates/corpora/analyse_corpus.html.j2 | 4 +- app/templates/corpora/corpus.html.j2 | 14 +- app/templates/corpora/corpus_file.html.j2 | 2 +- app/templates/corpora/import_corpus.html.j2 | 2 +- app/templates/jobs/job.html.j2 | 10 +- app/templates/main/dashboard.html.j2 | 22 +- app/templates/main/index.html.j2 | 28 +-- .../services/corpus_analysis.html.j2 | 4 +- app/templates/services/file_setup.html.j2 | 6 +- app/templates/services/spacy_nlp.html.j2 | 6 +- app/templates/services/tesseract_ocr.html.j2 | 6 +- app/templates/settings/_breadcrumbs.html.j2 | 8 +- app/templates/settings/_menu.html.j2 | 5 - .../settings/change_password.html.j2 | 36 --- .../settings/edit_general_settings.html.j2 | 81 ------- .../edit_notification_settings.html.j2 | 35 --- app/templates/settings/index.html.j2 | 123 +++++++++++ .../tasks/email/notification.html.j2 | 2 +- app/templates/tasks/email/notification.txt.j2 | 2 +- migrations/README | 2 +- migrations/alembic.ini | 7 +- migrations/env.py | 15 +- .../{da9fd175af8c_.py => 097aae1f02d7_.py} | 79 +++++-- migrations/versions/099037c4aa06_.py | 30 --- migrations/versions/0aa38a7973c5_.py | 28 --- migrations/versions/0d7aed934679_.py | 28 --- migrations/versions/10a92d8f4616_.py | 30 --- migrations/versions/1210adfe1e34_.py | 28 --- migrations/versions/318074622d14_.py | 28 --- migrations/versions/33ec4d09b4ca_.py | 30 --- migrations/versions/389bcf564726_.py | 28 --- migrations/versions/3d9a20b8b26c_.py | 28 --- migrations/versions/421ba4373e50_.py | 33 --- migrations/versions/4638e6509e13_.py | 38 ---- migrations/versions/471aa04c1a92_.py | 38 ---- migrations/versions/4886241e0f5d_.py | 28 --- migrations/versions/49a42c69e523_.py | 36 --- migrations/versions/4cf5e5606a83_.py | 35 --- migrations/versions/55d2b1a82ba9_.py | 30 --- migrations/versions/5ba6786a847e_.py | 42 ---- migrations/versions/62233e0cb2c7_.py | 34 --- migrations/versions/6227310c2112_.py | 30 --- migrations/versions/66253783654f_.py | 30 --- migrations/versions/68772b6560c3_.py | 28 --- migrations/versions/68ed092ffe5e_.py | 50 ----- migrations/versions/6c2227f1cc77_.py | 59 ----- migrations/versions/7378391345fa_.py | 28 --- migrations/versions/776761fb7466_.py | 32 --- migrations/versions/790ce9512e75_.py | 42 ---- migrations/versions/8b2e0d43384a_.py | 28 --- migrations/versions/9d21b228d353_.py | 50 ----- migrations/versions/a4b3cf4ab098_.py | 30 --- migrations/versions/abf60427ff84_.py | 32 --- migrations/versions/ad0d835fe5b1_.py | 45 ---- migrations/versions/b15366b25bea_.py | 42 ---- migrations/versions/be010d5d708d_.py | 28 --- migrations/versions/befe5326787e_.py | 30 --- migrations/versions/c3827cddea6e_.py | 43 ---- migrations/versions/c384d7b3268a_.py | 32 --- migrations/versions/ded5a37f8a7b_.py | 44 ---- migrations/versions/e256f5cac75d_.py | 30 --- migrations/versions/ecaf75fece7b_.py | 28 --- nopaque.py | 14 +- requirements.txt | 3 +- 110 files changed, 1389 insertions(+), 2301 deletions(-) create mode 100644 app/corpora/query_results_tasks.py create mode 100644 app/query_results_models.py delete mode 100644 app/templates/settings/_menu.html.j2 delete mode 100644 app/templates/settings/change_password.html.j2 delete mode 100644 app/templates/settings/edit_general_settings.html.j2 delete mode 100644 app/templates/settings/edit_notification_settings.html.j2 create mode 100644 app/templates/settings/index.html.j2 rename migrations/versions/{da9fd175af8c_.py => 097aae1f02d7_.py} (53%) delete mode 100644 migrations/versions/099037c4aa06_.py delete mode 100644 migrations/versions/0aa38a7973c5_.py delete mode 100644 migrations/versions/0d7aed934679_.py delete mode 100644 migrations/versions/10a92d8f4616_.py delete mode 100644 migrations/versions/1210adfe1e34_.py delete mode 100644 migrations/versions/318074622d14_.py delete mode 100644 migrations/versions/33ec4d09b4ca_.py delete mode 100644 migrations/versions/389bcf564726_.py delete mode 100644 migrations/versions/3d9a20b8b26c_.py delete mode 100644 migrations/versions/421ba4373e50_.py delete mode 100644 migrations/versions/4638e6509e13_.py delete mode 100644 migrations/versions/471aa04c1a92_.py delete mode 100644 migrations/versions/4886241e0f5d_.py delete mode 100644 migrations/versions/49a42c69e523_.py delete mode 100644 migrations/versions/4cf5e5606a83_.py delete mode 100644 migrations/versions/55d2b1a82ba9_.py delete mode 100644 migrations/versions/5ba6786a847e_.py delete mode 100644 migrations/versions/62233e0cb2c7_.py delete mode 100644 migrations/versions/6227310c2112_.py delete mode 100644 migrations/versions/66253783654f_.py delete mode 100644 migrations/versions/68772b6560c3_.py delete mode 100644 migrations/versions/68ed092ffe5e_.py delete mode 100644 migrations/versions/6c2227f1cc77_.py delete mode 100644 migrations/versions/7378391345fa_.py delete mode 100644 migrations/versions/776761fb7466_.py delete mode 100644 migrations/versions/790ce9512e75_.py delete mode 100644 migrations/versions/8b2e0d43384a_.py delete mode 100644 migrations/versions/9d21b228d353_.py delete mode 100644 migrations/versions/a4b3cf4ab098_.py delete mode 100644 migrations/versions/abf60427ff84_.py delete mode 100644 migrations/versions/ad0d835fe5b1_.py delete mode 100644 migrations/versions/b15366b25bea_.py delete mode 100644 migrations/versions/be010d5d708d_.py delete mode 100644 migrations/versions/befe5326787e_.py delete mode 100644 migrations/versions/c3827cddea6e_.py delete mode 100644 migrations/versions/c384d7b3268a_.py delete mode 100644 migrations/versions/ded5a37f8a7b_.py delete mode 100644 migrations/versions/e256f5cac75d_.py delete mode 100644 migrations/versions/ecaf75fece7b_.py diff --git a/app/admin/forms.py b/app/admin/forms.py index eb0ed231..30d99448 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -1,13 +1,16 @@ -from wtforms import BooleanField, SelectField -from ..models import Role -from ..settings.forms import EditGeneralSettingsForm +from app.models import Role +from flask_wtf import FlaskForm +from wtforms import BooleanField, SelectField, SubmitField -class EditGeneralSettingsAdminForm(EditGeneralSettingsForm): +class AdminEditUserForm(FlaskForm): confirmed = BooleanField('Confirmed') - role = SelectField('Role', coerce=int) + role = SelectField('Role') + submit = SubmitField('Submit') def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.role.choices = [(role.id, role.name) - for role in Role.query.order_by(Role.name).all()] + self.role.choices = [ + (role.hashid, role.name) + for role in Role.query.order_by(Role.name).all() + ] diff --git a/app/admin/routes.py b/app/admin/routes.py index 7979243d..79f05e19 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -1,11 +1,16 @@ +from app import db, hashids +from app.decorators import admin_required +from app.models import JobStatusMailNotificationLevel, Role, User +from app.settings import tasks as settings_tasks +from app.settings.forms import ( + EditGeneralSettingsForm, + EditInterfaceSettingsForm, + EditNotificationSettingsForm +) from flask import flash, redirect, render_template, url_for from flask_login import login_required from . import bp -from .forms import EditGeneralSettingsAdminForm -from .. import db -from ..decorators import admin_required -from ..models import Role, User -from ..settings import tasks as settings_tasks +from .forms import AdminEditUserForm @bp.before_request @@ -26,10 +31,15 @@ def index(): @bp.route('/users') def users(): - dict_users = {user.id: user.to_dict(backrefs=True, relationships=False) - for user in User.query.all()} + dict_users = { + user.id: user.to_dict(backrefs=True, relationships=False) + for user in User.query.all() + } return render_template( - 'admin/users.html.j2', title='Users', dict_users=dict_users) + 'admin/users.html.j2', + dict_users=dict_users, + title='Users' + ) @bp.route('/users/') @@ -41,27 +51,76 @@ def user(user_id): @bp.route('/users//delete') def delete_user(user_id): settings_tasks.delete_user(user_id) - flash('User has been marked for deletion!') + flash('User has been marked for deletion') return redirect(url_for('.users')) -@bp.route('/users//edit', methods=['GET', 'POST']) # noqa +@bp.route('/users//edit', methods=['GET', 'POST']) def edit_user(user_id): user = User.query.get_or_404(user_id) - form = EditGeneralSettingsAdminForm(user) - if form.validate_on_submit(): - user.setting_dark_mode = form.dark_mode.data - user.email = form.email.data - user.username = form.username.data - user.confirmed = form.confirmed.data - user.role = Role.query.get(form.role.data) - db.session.commit() - flash('Settings have been updated.') + admin_edit_user_form = AdminEditUserForm( + prefix='admin_edit_user_form' + ) + edit_general_settings_form = EditGeneralSettingsForm( + user, + prefix='edit_general_settings_form' + ) + edit_interface_settings_form = EditInterfaceSettingsForm( + prefix='edit_interface_settings_form' + ) + edit_notification_settings_form = EditNotificationSettingsForm( + prefix='edit_notification_settings_form' + ) + if ( + admin_edit_user_form.submit.data + and admin_edit_user_form.validate() + ): + user.confirmed = admin_edit_user_form.confirmed.data + role_id = hashids.decode(admin_edit_user_form.role.data) + user.role = Role.query.get(role_id) + flash('Your changes have been saved') return redirect(url_for('.edit_user', user_id=user.id)) - form.confirmed.data = user.confirmed - form.dark_mode.data = user.setting_dark_mode - form.email.data = user.email - form.role.data = user.role_id - form.username.data = user.username + if ( + edit_general_settings_form.submit.data + and edit_general_settings_form.validate() + ): + user.email = edit_general_settings_form.email.data + user.username = edit_general_settings_form.username.data + db.session.commit() + flash('Your changes have been saved') + return redirect(url_for('.edit_user', user_id=user.id)) + if ( + edit_interface_settings_form.submit.data + and edit_interface_settings_form.validate() + ): + user.setting_dark_mode = edit_interface_settings_form.dark_mode.data + db.session.commit() + flash('Your changes have been saved') + return redirect(url_for('.edit_user', user_id=user.id)) + if ( + edit_notification_settings_form.submit.data + and edit_notification_settings_form.validate() + ): + user.setting_job_status_mail_notification_level = \ + JobStatusMailNotificationLevel[ + edit_notification_settings_form.job_status_mail_notification_level.data # noqa + ] + db.session.commit() + flash('Your changes have been saved') + return redirect(url_for('.edit_user', user_id=user.id)) + admin_edit_user_form.confirmed.data = user.confirmed + admin_edit_user_form.role.data = user.role.hashid + edit_general_settings_form.email.data = user.email + edit_general_settings_form.username.data = user.username + edit_interface_settings_form.dark_mode.data = user.setting_dark_mode + edit_notification_settings_form.job_status_mail_notification_level.data = \ + user.setting_job_status_mail_notification_level.name return render_template( - 'admin/edit_user.html.j2', form=form, title='Edit user', user=user) + 'admin/edit_user.html.j2', + admin_edit_user_form=admin_edit_user_form, + edit_general_settings_form=edit_general_settings_form, + edit_interface_settings_form=edit_interface_settings_form, + edit_notification_settings_form=edit_notification_settings_form, + title='Edit user', + user=user + ) diff --git a/app/api/auth.py b/app/api/auth.py index fea4123b..4c6a3dd9 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -1,7 +1,7 @@ +from app.models import User from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth from sqlalchemy import or_ from werkzeug.http import HTTP_STATUS_CODES -from ..models import User basic_auth = HTTPBasicAuth() token_auth = HTTPTokenAuth() diff --git a/app/api/tokens.py b/app/api/tokens.py index 2eebbfb7..e1b55527 100644 --- a/app/api/tokens.py +++ b/app/api/tokens.py @@ -1,6 +1,6 @@ +from app import db from flask_restx import Namespace, Resource from .auth import basic_auth, token_auth -from .. import db ns = Namespace('tokens', description='Token operations') diff --git a/app/auth/forms.py b/app/auth/forms.py index 8509bc44..a418bc51 100644 --- a/app/auth/forms.py +++ b/app/auth/forms.py @@ -1,9 +1,14 @@ -from . import USERNAME_REGEX -from ..models import User +from app.models import User from flask_wtf import FlaskForm -from wtforms import (BooleanField, PasswordField, StringField, SubmitField, - ValidationError) +from wtforms import ( + BooleanField, + PasswordField, + StringField, + SubmitField, + ValidationError +) from wtforms.validators import DataRequired, Email, EqualTo, Length, Regexp +from . import USERNAME_REGEX class LoginForm(FlaskForm): @@ -17,42 +22,54 @@ class RegistrationForm(FlaskForm): email = StringField('Email', validators=[DataRequired(), Email()]) username = StringField( 'Username', - validators=[DataRequired(), Length(1, 64), - Regexp(USERNAME_REGEX, - message='Usernames must have only letters, numbers,' - ' dots or underscores')] + validators=[ + DataRequired(), + Length(1, 64), + Regexp( + USERNAME_REGEX, + message='Usernames must have only letters, numbers, dots or underscores' # noqa + ) + ] ) password = PasswordField( 'Password', - validators=[DataRequired(), EqualTo('password_confirmation', - message='Passwords must match.')] + validators=[ + DataRequired(), + EqualTo('password_confirmation', message='Passwords must match') + ] ) password_confirmation = PasswordField( 'Password confirmation', - validators=[DataRequired(), EqualTo('password', - message='Passwords must match.')] + validators=[ + DataRequired(), + EqualTo('password', message='Passwords must match') + ] ) submit = SubmitField('Register') def validate_email(self, field): if User.query.filter_by(email=field.data.lower()).first(): - raise ValidationError('Email already registered.') + raise ValidationError('Email already registered') def validate_username(self, field): if User.query.filter_by(username=field.data).first(): - raise ValidationError('Username already in use.') + raise ValidationError('Username already in use') class ResetPasswordForm(FlaskForm): password = PasswordField( 'New password', - validators=[DataRequired(), EqualTo('password_confirmation', - message='Passwords must match.')] + validators=[ + DataRequired(), + EqualTo('password_confirmation', message='Passwords must match') + ] ) password_confirmation = PasswordField( 'Password confirmation', - validators=[DataRequired(), - EqualTo('password', message='Passwords must match.')] + validators=[ + DataRequired(), + EqualTo('password', message='Passwords must match') + ] ) submit = SubmitField('Reset Password') diff --git a/app/auth/routes.py b/app/auth/routes.py index 35842251..045d0baf 100644 --- a/app/auth/routes.py +++ b/app/auth/routes.py @@ -1,15 +1,25 @@ +from app import db +from app.email import create_message, send +from app.models import User from datetime import datetime -from flask import (abort, current_app, flash, redirect, render_template, - request, url_for) +from flask import ( + abort, + current_app, + flash, + redirect, + render_template, + request, + url_for +) from flask_login import current_user, login_user, login_required, logout_user from sqlalchemy import or_ from . import bp -from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm, - RegistrationForm) -from .. import db -from ..email import create_message, send -from ..models import User -import os +from .forms import ( + LoginForm, + ResetPasswordForm, + ResetPasswordRequestForm, + RegistrationForm +) @bp.before_app_request @@ -21,10 +31,12 @@ def before_request(): if current_user.is_authenticated: current_user.last_seen = datetime.utcnow() db.session.commit() - if (not current_user.confirmed - and request.endpoint - and request.blueprint != 'auth' - and request.endpoint != 'static'): + if ( + not current_user.confirmed + and request.endpoint + and request.blueprint != 'auth' + and request.endpoint != 'static' + ): return redirect(url_for('auth.unconfirmed')) @@ -34,15 +46,19 @@ def login(): return redirect(url_for('main.dashboard')) form = LoginForm(prefix='login-form') if form.validate_on_submit(): - user = User.query.filter(or_(User.username == form.user.data, - User.email == form.user.data.lower())).first() + user = User.query.filter( + or_( + User.username == form.user.data, + User.email == form.user.data.lower() + ) + ).first() if user and user.verify_password(form.password.data): login_user(user, form.remember_me.data) next = request.args.get('next') if next is None or not next.startswith('/'): next = url_for('main.dashboard') return redirect(next) - flash('Invalid email/username or password.') + flash('Invalid email/username or password', category='error') return render_template('auth/login.html.j2', form=form, title='Log in') @@ -50,7 +66,7 @@ def login(): @login_required def logout(): logout_user() - flash('You have been logged out.') + flash('You have been logged out') return redirect(url_for('main.index')) @@ -73,19 +89,20 @@ def register(): except OSError as e: current_app.logger.error(e) db.session.rollback() + flash('Internal Server Error', category='error') abort(500) - else: - token = user.generate_confirmation_token() - msg = create_message( - user.email, - 'Confirm Your Account', - 'auth/email/confirm', - token=token, - user=user - ) - send(msg) - flash('A confirmation email has been sent to you by email.') - return redirect(url_for('.login')) + token = user.generate_confirmation_token() + msg = create_message( + user.email, + 'Confirm Your Account', + 'auth/email/confirm', + token=token, + user=user + ) + send(msg) + flash('A confirmation email has been sent to you by email') + db.session.commit() + return redirect(url_for('.login')) return render_template( 'auth/register.html.j2', form=form, @@ -100,10 +117,13 @@ def confirm(token): return redirect(url_for('main.dashboard')) if current_user.confirm(token): db.session.commit() - flash('You have confirmed your account. Thanks!') + flash('You have confirmed your account') return redirect(url_for('main.dashboard')) else: - flash('The confirmation link is invalid or has expired.') + flash( + 'The confirmation link is invalid or has expired', + category='error' + ) return redirect(url_for('.unconfirmed')) @@ -120,10 +140,15 @@ def unconfirmed(): @login_required def resend_confirmation(): token = current_user.generate_confirmation_token() - msg = create_message(current_user.email, 'Confirm Your Account', - 'auth/email/confirm', token=token, user=current_user) + msg = create_message( + current_user.email, + 'Confirm Your Account', + 'auth/email/confirm', + token=token, + user=current_user + ) send(msg) - flash('A new confirmation email has been sent to you by email.') + flash('A new confirmation email has been sent to you by email') return redirect(url_for('auth.unconfirmed')) @@ -136,14 +161,23 @@ def reset_password_request(): user = User.query.filter_by(email=form.email.data.lower()).first() if user is not None: token = user.generate_reset_token() - msg = create_message(user.email, 'Reset Your Password', - 'auth/email/reset_password', token=token, - user=user) + msg = create_message( + user.email, + 'Reset Your Password', + 'auth/email/reset_password', + token=token, + user=user + ) send(msg) - flash('An email with instructions to reset your password has been sent to you.') # noqa + flash( + 'An email with instructions to reset your password has been sent to you' # noqa + ) return redirect(url_for('.login')) - return render_template('auth/reset_password_request.html.j2', form=form, - title='Password Reset') + return render_template( + 'auth/reset_password_request.html.j2', + form=form, + title='Password Reset' + ) @bp.route('/reset/', methods=['GET', 'POST']) @@ -154,9 +188,13 @@ def reset_password(token): if form.validate_on_submit(): if User.reset_password(token, form.password.data): db.session.commit() - flash('Your password has been updated.') + flash('Your password has been updated') return redirect(url_for('.login')) else: return redirect(url_for('main.index')) - return render_template('auth/reset_password.html.j2', form=form, - title='Password Reset', token=token) + return render_template( + 'auth/reset_password.html.j2', + form=form, + title='Password Reset', + token=token + ) diff --git a/app/contribute/routes.py b/app/contribute/routes.py index e0b43231..8bfd9bd8 100644 --- a/app/contribute/routes.py +++ b/app/contribute/routes.py @@ -1,10 +1,10 @@ +from app import db +from app.decorators import permission_required +from app.models import Permission, Role, User +from app.settings import tasks as settings_tasks from flask import flash, redirect, render_template, url_for from flask_login import login_required from . import bp -from .. import db -from ..decorators import permission_required -from ..models import Permission, Role, User -from ..settings import tasks as settings_tasks @bp.before_request diff --git a/app/corpora/cqi_over_socketio/__init__.py b/app/corpora/cqi_over_socketio/__init__.py index 3a358758..f1f5f826 100644 --- a/app/corpora/cqi_over_socketio/__init__.py +++ b/app/corpora/cqi_over_socketio/__init__.py @@ -1,6 +1,6 @@ from app import db, hashids, socketio from app.decorators import socketio_login_required -from app.models import Corpus +from app.models import Corpus, CorpusStatus from flask import session from flask_login import current_user from flask_socketio import ConnectionRefusedError @@ -65,7 +65,12 @@ def connect(auth): if not (corpus.user == current_user or current_user.is_administrator()): # return {'code': 403, 'msg': 'Forbidden'} raise ConnectionRefusedError('Forbidden') - if corpus.status not in ['prepared', 'start analysis', 'analysing', 'stop analysis']: + if corpus.status not in [ + CorpusStatus.BUILT, + CorpusStatus.STARTING_ANALYSIS_SESSION, + CorpusStatus.RUNNING_ANALYSIS_SESSION, + CorpusStatus.CANCELING_ANALYSIS_SESSION + ]: # return {'code': 424, 'msg': 'Failed Dependency'} raise ConnectionRefusedError('Failed Dependency') if corpus.num_analysis_sessions is None: @@ -74,7 +79,7 @@ def connect(auth): corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1 db.session.commit() retry_counter = 20 - while corpus.status != 'analysing': + while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION: if retry_counter == 0: corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1 db.session.commit() diff --git a/app/corpora/forms.py b/app/corpora/forms.py index c015d87e..9af4a5ce 100644 --- a/app/corpora/forms.py +++ b/app/corpora/forms.py @@ -1,7 +1,12 @@ from flask_wtf import FlaskForm from werkzeug.utils import secure_filename -from wtforms import (FileField, StringField, SubmitField, - ValidationError, IntegerField) +from wtforms import ( + FileField, + StringField, + SubmitField, + ValidationError, + IntegerField +) from wtforms.validators import DataRequired, Length diff --git a/app/corpora/query_results_routes.py b/app/corpora/query_results_routes.py index 27402dc9..6c22e11e 100644 --- a/app/corpora/query_results_routes.py +++ b/app/corpora/query_results_routes.py @@ -58,7 +58,7 @@ def add_query_result(): query_result_file_content.pop('cpos_lookup') query_result.query_metadata = query_result_file_content db.session.commit() - flash('Query result added!', 'result') + flash('Query result added', 'result') return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa return render_template('corpora/query_results/add_query_result.html.j2', form=form, title='Add query result') @@ -117,7 +117,7 @@ def delete_query_result(query_result_id): if not (query_result.user == current_user or current_user.is_administrator()): abort(403) - flash(f'Query result "{query_result}" marked for deletion!', 'result') + flash(f'Query result "{query_result}" marked for deletion', 'result') tasks.delete_query_result(query_result_id) return redirect(url_for('services.service', service="corpus_analysis")) diff --git a/app/corpora/query_results_tasks.py b/app/corpora/query_results_tasks.py new file mode 100644 index 00000000..653096b3 --- /dev/null +++ b/app/corpora/query_results_tasks.py @@ -0,0 +1,13 @@ +from .. import db +from ..decorators import background +from ..models import QueryResult + + +@background +def delete_query_result(query_result_id, *args, **kwargs): + with kwargs['app'].app_context(): + query_result = QueryResult.query.get(query_result_id) + if query_result is None: + raise Exception(f'QueryResult {query_result_id} not found') + query_result.delete() + db.session.commit() diff --git a/app/corpora/routes.py b/app/corpora/routes.py index f6d95b54..fd0085ee 100644 --- a/app/corpora/routes.py +++ b/app/corpora/routes.py @@ -1,19 +1,31 @@ -from flask import (abort, current_app, flash, make_response, redirect, - render_template, url_for, send_from_directory) +from app import db +from app.models import Corpus, CorpusFile, CorpusStatus +from flask import ( + abort, + current_app, + flash, + make_response, + redirect, + render_template, + url_for, + send_from_directory +) from flask_login import current_user, login_required from werkzeug.utils import secure_filename +from zipfile import ZipFile from . import bp from . import tasks -from .forms import (AddCorpusFileForm, AddCorpusForm, EditCorpusFileForm, - ImportCorpusForm) -from .. import db -from ..models import Corpus, CorpusFile +from .forms import ( + AddCorpusFileForm, + AddCorpusForm, + EditCorpusFileForm, + ImportCorpusForm +) +from .import_corpus import check_zip_contents import os import shutil import glob import xml.etree.ElementTree as ET -from zipfile import ZipFile -from .import_corpus import check_zip_contents @bp.route('/add', methods=['GET', 'POST']) @@ -34,10 +46,10 @@ def add_corpus(): except OSError as e: current_app.logger.error(e) db.session.rollback() - flash('Internal Server Error', 'error') + flash('Internal Server Error', category='error') abort(500) db.session.commit() - flash(f'Corpus "{corpus.title}" added', 'corpus') + flash(f'Corpus "{corpus.title}" added', category='corpus') return redirect(url_for('.corpus', corpus_id=corpus.id)) return render_template( 'corpora/add_corpus.html.j2', @@ -46,6 +58,21 @@ def add_corpus(): ) +@bp.route('//export') +@login_required +def export_corpus(corpus_id): + abort(503) + corpus = Corpus.query.get_or_404(corpus_id) + if not (corpus.user == current_user or current_user.is_administrator()): + abort(403) + return send_from_directory( + as_attachment=True, + directory=os.path.join(corpus.user.path, 'corpora'), + filename=corpus.archive_file, + mimetype='zip' + ) + + @bp.route('/import', methods=['GET', 'POST']) @login_required def import_corpus(): @@ -65,11 +92,10 @@ def import_corpus(): try: os.makedirs(corpus.path) except OSError as e: - current_app.logger.error(f'Could not import corpus: {e}') + current_app.logger.error(e) db.session.rollback() - flash('Internal Server Error', 'error') - return make_response( - {'redirect_url': url_for('.import_corpus')}, 500) + flash('Internal Server Error', category='error') + return make_response({'redirect_url': url_for('.import_corpus')}, 500) # noqa # Upload zip archive_file = os.path.join(corpus.path, form.file.data.filename) form.file.data.save(archive_file) @@ -102,20 +128,25 @@ def import_corpus(): ) db.session.add(corpus_file) # finish import and redirect to imported corpus - corpus.status = 'prepared' + corpus.status = CorpusStatus.BUILT db.session.commit() os.remove(archive_file) - flash(f'Corpus "{corpus.title}" imported!', 'corpus') + flash(f'Corpus "{corpus.title}" imported', 'corpus') return make_response( {'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) else: # If imported zip is not valid delete corpus and give feedback - flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa + flash( + f'Can\'t import corpus "{corpus.title}": Invalid archive file', + category='error' + ) tasks.delete_corpus(corpus.id) - return make_response( - {'redirect_url': url_for('.import_corpus')}, 201) - return render_template('corpora/import_corpus.html.j2', form=form, - title='Import Corpus') + return make_response({'redirect_url': url_for('.import_corpus')}, 201) # noqa + return render_template( + 'corpora/import_corpus.html.j2', + form=form, + title='Import Corpus' + ) @bp.route('/') @@ -124,9 +155,11 @@ def corpus(corpus_id): corpus = Corpus.query.get_or_404(corpus_id) if not (corpus.user == current_user or current_user.is_administrator()): abort(403) - corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files] - return render_template('corpora/corpus.html.j2', corpus=corpus, - corpus_files=corpus_files, title='Corpus') + return render_template( + 'corpora/corpus.html.j2', + corpus=corpus, + title='Corpus' + ) @bp.route('//analyse') @@ -140,28 +173,13 @@ def analyse_corpus(corpus_id): ) -@bp.route('//download') -@login_required -def download_corpus(corpus_id): - abort(503) - corpus = Corpus.query.get_or_404(corpus_id) - if not (corpus.user == current_user or current_user.is_administrator()): - abort(403) - return send_from_directory( - as_attachment=True, - directory=os.path.join(corpus.user.path, 'corpora'), - filename=corpus.archive_file, - mimetype='zip' - ) - - @bp.route('//delete') @login_required def delete_corpus(corpus_id): corpus = Corpus.query.get_or_404(corpus_id) if not (corpus.user == current_user or current_user.is_administrator()): abort(403) - flash(f'Corpus "{corpus.title}" marked for deletion!', 'corpus') + flash(f'Corpus "{corpus.title}" marked for deletion', 'corpus') tasks.delete_corpus(corpus_id) return redirect(url_for('main.dashboard')) @@ -203,11 +221,11 @@ def add_corpus_file(corpus_id): except OSError as e: current_app.logger.error(e) db.session.rollback() - flash('Internal Server Error', 'error') + flash('Internal Server Error', category='error') return make_response({'redirect_url': url_for('.add_corpus_file', corpus_id=corpus.id)}, 500) # noqa - corpus.status = 'unprepared' + corpus.status = CorpusStatus.UNPREPARED db.session.commit() - flash(f'Corpus file "{corpus_file.title}" added!', 'corpus') + flash(f'Corpus file "{corpus_file.filename}" added', category='corpus') return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa return render_template( 'corpora/add_corpus_file.html.j2', @@ -220,14 +238,19 @@ def add_corpus_file(corpus_id): @bp.route('//files//delete') @login_required def delete_corpus_file(corpus_id, corpus_file_id): - corpus_file = CorpusFile.query.get_or_404(corpus_file_id) - if not corpus_file.corpus_id == corpus_id: - abort(404) - if not (corpus_file.corpus.user == current_user - or current_user.is_administrator()): + corpus_file = CorpusFile.query.filter( + CorpusFile.corpus_id == corpus_id, + CorpusFile.id == corpus_file_id + ).first_or_404() + if not ( + corpus_file.corpus.user == current_user + or current_user.is_administrator() + ): abort(403) flash( - f'Corpus file "{corpus_file.filename}" marked for deletion!', 'corpus') + f'Corpus file "{corpus_file.filename}" marked for deletion', + category='corpus' + ) tasks.delete_corpus_file(corpus_file_id) return redirect(url_for('.corpus', corpus_id=corpus_id)) @@ -235,26 +258,34 @@ def delete_corpus_file(corpus_id, corpus_file_id): @bp.route('//files//download') @login_required def download_corpus_file(corpus_id, corpus_file_id): - corpus_file = CorpusFile.query.get_or_404(corpus_file_id) - if not corpus_file.corpus_id == corpus_id: - abort(404) - if not (corpus_file.corpus.user == current_user - or current_user.is_administrator()): + corpus_file = CorpusFile.query.filter( + CorpusFile.corpus_id == corpus_id, + CorpusFile.id == corpus_file_id + ).first_or_404() + if not ( + corpus_file.corpus.user == current_user + or current_user.is_administrator() + ): abort(403) - return send_from_directory(as_attachment=True, - directory=os.path.dirname(corpus_file.path), - filename=corpus_file.filename) + return send_from_directory( + as_attachment=True, + directory=os.path.dirname(corpus_file.path), + filename=corpus_file.filename + ) -@bp.route('//files/', methods=['GET', 'POST']) +@bp.route('//files/', methods=['GET', 'POST']) # noqa @login_required def corpus_file(corpus_id, corpus_file_id): - corpus = Corpus.query.get_or_404(corpus_id) - if not (corpus.user == current_user or current_user.is_administrator()): + corpus_file = CorpusFile.query.filter( + CorpusFile.corpus_id == corpus_id, + CorpusFile.id == corpus_file_id + ).first_or_404() + if not ( + corpus_file.corpus.user == current_user + or current_user.is_administrator() + ): abort(403) - corpus_file = CorpusFile.query.get_or_404(corpus_file_id) - if corpus_file.corpus != corpus: - abort(404) form = EditCorpusFileForm(prefix='edit-corpus-file-form') if form.validate_on_submit(): corpus_file.address = form.address.data @@ -269,9 +300,9 @@ def corpus_file(corpus_id, corpus_file_id): corpus_file.publishing_year = form.publishing_year.data corpus_file.school = form.school.data corpus_file.title = form.title.data - corpus.status = 'unprepared' + corpus_file.corpus.status = CorpusStatus.UNPREPARED db.session.commit() - flash(f'Corpus file "{corpus_file.filename}" edited!', 'corpus') + flash(f'Corpus file "{corpus_file.filename}" edited', category='corpus') # noqa return redirect(url_for('.corpus', corpus_id=corpus_id)) # If no form is submitted or valid, fill out fields with current values form.address.data = corpus_file.address @@ -286,9 +317,13 @@ def corpus_file(corpus_id, corpus_file_id): form.publishing_year.data = corpus_file.publishing_year form.school.data = corpus_file.school form.title.data = corpus_file.title - return render_template('corpora/corpus_file.html.j2', corpus=corpus, - corpus_file=corpus_file, form=form, - title='Edit corpus file') + return render_template( + 'corpora/corpus_file.html.j2', + corpus=corpus, + corpus_file=corpus_file, + form=form, + title='Edit corpus file' + ) @bp.route('//build') @@ -299,7 +334,13 @@ def build_corpus(corpus_id): abort(403) if corpus.files.all(): tasks.build_corpus(corpus_id) - flash(f'Corpus "{corpus.title}" marked for building!', 'corpus') + flash( + f'Corpus "{corpus.title}" marked for building', + category='corpus' + ) else: - flash(f'Can\'t build corpus "{corpus.title}": No corpus file(s)!', 'error') # noqa + flash( + f'Can\'t build corpus "{corpus.title}": No corpus file(s)', + category='error' + ) return redirect(url_for('.corpus', corpus_id=corpus_id)) diff --git a/app/corpora/tasks.py b/app/corpora/tasks.py index c37ebd99..c914a25a 100644 --- a/app/corpora/tasks.py +++ b/app/corpora/tasks.py @@ -1,6 +1,6 @@ -from .. import db -from ..decorators import background -from ..models import Corpus, CorpusFile, QueryResult +from app import db +from app.decorators import background +from app.models import Corpus, CorpusFile @background @@ -32,13 +32,3 @@ def delete_corpus_file(corpus_file_id, *args, **kwargs): raise Exception(f'Corpus file {corpus_file_id} not found') corpus_file.delete() db.session.commit() - - -@background -def delete_query_result(query_result_id, *args, **kwargs): - with kwargs['app'].app_context(): - query_result = QueryResult.query.get(query_result_id) - if query_result is None: - raise Exception(f'QueryResult {query_result_id} not found') - query_result.delete() - db.session.commit() diff --git a/app/daemon/corpus_utils.py b/app/daemon/corpus_utils.py index 3962582e..78e17f29 100644 --- a/app/daemon/corpus_utils.py +++ b/app/daemon/corpus_utils.py @@ -1,5 +1,5 @@ +from app.models import Corpus, CorpusStatus from flask import current_app -from ..models import Corpus import docker import os import shutil @@ -8,19 +8,19 @@ import shutil class CheckCorporaMixin: def check_corpora(self): corpora = Corpus.query.all() - for corpus in (x for x in corpora if x.status == 'submitted'): + for corpus in (x for x in corpora if x.status == CorpusStatus.SUBMITTED): # noqa self.create_build_corpus_service(corpus) - for corpus in (x for x in corpora if x.status == 'queued' or x.status == 'running'): # noqa + for corpus in (x for x in corpora if x.status == CorpusStatus.QUEUED or x.status == CorpusStatus.BUILDING): # noqa self.checkout_build_corpus_service(corpus) - for corpus in (x for x in corpora if x.status == 'prepared' and x.num_analysis_sessions > 0): # noqa - corpus.status = 'start analysis' - for corpus in (x for x in corpora if x.status == 'analysing' and x.num_analysis_sessions == 0): # noqa - corpus.status = 'stop analysis' - for corpus in (x for x in corpora if x.status == 'analysing'): + for corpus in (x for x in corpora if x.status == CorpusStatus.BUILT and x.num_analysis_sessions > 0): # noqa + corpus.status = CorpusStatus.STARTING_ANALYSIS_SESSION + for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0): # noqa + corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION + for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION): # noqa self.checkout_analysing_corpus_container(corpus) - for corpus in (x for x in corpora if x.status == 'start analysis'): + for corpus in (x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION): # noqa self.create_cqpserver_container(corpus) - for corpus in (x for x in corpora if x.status == 'stop analysis'): + for corpus in (x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION): # noqa self.remove_cqpserver_container(corpus) def create_build_corpus_service(self, corpus): @@ -95,7 +95,7 @@ class CheckCorporaMixin: f'due to "docker.errors.APIError": {e}' ) return - corpus.status = 'queued' + corpus.status = CorpusStatus.QUEUED def checkout_build_corpus_service(self, corpus): service_name = f'build-corpus_{corpus.id}' @@ -106,7 +106,7 @@ class CheckCorporaMixin: f'Get service "{service_name}" failed ' f'due to "docker.errors.NotFound": {e}' ) - corpus.status = 'failed' + corpus.status = CorpusStatus.FAILED return except docker.errors.APIError as e: current_app.logger.error( @@ -117,22 +117,22 @@ class CheckCorporaMixin: if not service_tasks: return task_state = service_tasks[0].get('Status').get('State') - if corpus.status == 'queued' and task_state != 'pending': - corpus.status = 'running' + if corpus.status == CorpusStatus.QUEUED and task_state != 'pending': # noqa + corpus.status = CorpusStatus.BUILDING return - elif corpus.status == 'running' and task_state == 'complete': - corpus.status = 'prepared' - elif corpus.status == 'running' and task_state == 'failed': - corpus.status = 'failed' + elif corpus.status == CorpusStatus.BUILDING and task_state == 'complete': # noqa + corpus.status = CorpusStatus.BUILT + elif corpus.status == CorpusStatus.BUILDING and task_state == 'failed': # noqa + corpus.status = CorpusStatus.FAILED else: return - # try: - # service.remove() - # except docker.errors.APIError as e: - # current_app.logger.error( - # f'Remove service "{service_name}" failed ' - # f'due to "docker.errors.APIError": {e}' - # ) + try: + service.remove() + except docker.errors.APIError as e: + current_app.logger.error( + f'Remove service "{service_name}" failed ' + f'due to "docker.errors.APIError": {e}' + ) def create_cqpserver_container(self, corpus): ''' # Docker container settings # ''' @@ -203,7 +203,7 @@ class CheckCorporaMixin: f'Run container "{name}" failed ' f'due to "docker.errors.ImageNotFound" error: {e}' ) - corpus.status = 'failed' + corpus.status = CorpusStatus.FAILED return except docker.errors.APIError as e: current_app.logger.error( @@ -211,7 +211,7 @@ class CheckCorporaMixin: f'due to "docker.errors.APIError" error: {e}' ) return - corpus.status = 'analysing' + corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION def checkout_analysing_corpus_container(self, corpus): container_name = f'cqpserver_{corpus.id}' @@ -223,7 +223,7 @@ class CheckCorporaMixin: f'due to "docker.errors.NotFound": {e}' ) corpus.num_analysis_sessions = 0 - corpus.status = 'prepared' + corpus.status = CorpusStatus.BUILT except docker.errors.APIError as e: current_app.logger.error( f'Get container "{container_name}" failed ' @@ -235,7 +235,7 @@ class CheckCorporaMixin: try: container = self.docker.containers.get(container_name) except docker.errors.NotFound: - corpus.status = 'prepared' + corpus.status = CorpusStatus.BUILT return except docker.errors.APIError as e: current_app.logger.error( diff --git a/app/daemon/job_utils.py b/app/daemon/job_utils.py index c640f35c..e4797e26 100644 --- a/app/daemon/job_utils.py +++ b/app/daemon/job_utils.py @@ -1,8 +1,8 @@ +from app import db +from app.models import Job, JobResult, JobStatus, TesseractOCRModel from datetime import datetime from flask import current_app from werkzeug.utils import secure_filename -from .. import db -from ..models import Job, JobResult, TesseractOCRModel import docker import json import os @@ -12,11 +12,11 @@ import shutil class CheckJobsMixin: def check_jobs(self): jobs = Job.query.all() - for job in (x for x in jobs if x.status == 'submitted'): + for job in (x for x in jobs if x.status == JobStatus.SUBMITTED): self.create_job_service(job) - for job in (x for x in jobs if x.status in ['queued', 'running']): + for job in (x for x in jobs if x.status in [JobStatus.QUEUED, JobStatus.RUNNING]): # noqa self.checkout_job_service(job) - for job in (x for x in jobs if x.status == 'canceling'): + for job in (x for x in jobs if x.status == JobStatus.CANCELING): self.remove_job_service(job) def create_job_service(self, job): @@ -74,7 +74,7 @@ class CheckJobsMixin: service_args = json.loads(job.service_args) model = TesseractOCRModel.query.get(service_args['model']) if model is None: - job.status = 'failed' + job.status = JobStatus.FAILED return models_mount_source = model.path models_mount_target = f'/usr/local/share/tessdata/{model.filename}' @@ -122,7 +122,7 @@ class CheckJobsMixin: f'due to "docker.errors.APIError": {e}' ) return - job.status = 'queued' + job.status = JobStatus.QUEUED def checkout_job_service(self, job): service_name = f'job_{job.id}' @@ -133,7 +133,7 @@ class CheckJobsMixin: f'Get service "{service_name}" failed ' f'due to "docker.errors.NotFound": {e}' ) - job.status = 'failed' + job.status = JobStatus.FAILED return except docker.errors.APIError as e: current_app.logger.error( @@ -145,11 +145,11 @@ class CheckJobsMixin: if not service_tasks: return task_state = service_tasks[0].get('Status').get('State') - if job.status == 'queued' and task_state != 'pending': - job.status = 'running' + if job.status == JobStatus.QUEUED and task_state != 'pending': + job.status = JobStatus.RUNNING return - elif job.status == 'running' and task_state == 'complete': - job.status = 'complete' + elif job.status == JobStatus.RUNNING and task_state == 'complete': # noqa + job.status = JobStatus.COMPLETED results_dir = os.path.join(job.path, 'results') with open(os.path.join(results_dir, 'outputs.json')) as f: outputs = json.load(f) @@ -169,8 +169,8 @@ class CheckJobsMixin: os.path.join(results_dir, output['file']), job_result.path ) - elif job.status == 'running' and task_state == 'failed': - job.status = 'failed' + elif job.status == JobStatus.RUNNING and task_state == 'failed': + job.status = JobStatus.FAILED else: return job.end_date = datetime.utcnow() @@ -187,7 +187,7 @@ class CheckJobsMixin: try: service = self.docker.services.get(service_name) except docker.errors.NotFound: - job.status = 'canceled' + job.status = JobStatus.CANCELED return except docker.errors.APIError as e: current_app.logger.error( diff --git a/app/events/socketio.py b/app/events/socketio.py index 94b4c0c7..e892ea9d 100644 --- a/app/events/socketio.py +++ b/app/events/socketio.py @@ -1,9 +1,8 @@ -from app import hashids +from app import hashids, socketio +from app.decorators import socketio_login_required +from app.models import User from flask_login import current_user from flask_socketio import join_room -from .. import socketio -from ..decorators import socketio_login_required -from ..models import User ############################################################################### diff --git a/app/events/sqlalchemy.py b/app/events/sqlalchemy.py index b3362ca0..dd88a0fe 100644 --- a/app/events/sqlalchemy.py +++ b/app/events/sqlalchemy.py @@ -1,8 +1,15 @@ +from app import db, mail, socketio +from app.email import create_message +from app.models import ( + Corpus, + CorpusFile, + Job, + JobInput, + JobResult, + JobStatus, + JobStatusMailNotificationLevel +) from datetime import datetime -from flask import current_app -from .. import db, mail, socketio -from ..email import create_message -from ..models import Corpus, CorpusFile, Job, JobInput, JobResult, QueryResult ############################################################################### @@ -13,7 +20,6 @@ from ..models import Corpus, CorpusFile, Job, JobInput, JobResult, QueryResult @db.event.listens_for(Job, 'after_delete') @db.event.listens_for(JobInput, 'after_delete') @db.event.listens_for(JobResult, 'after_delete') -@db.event.listens_for(QueryResult, 'after_delete') def ressource_after_delete(mapper, connection, ressource): jsonpatch = [{'op': 'remove', 'path': ressource.jsonpatch_path}] room = f'users.{ressource.user_hashid}' @@ -25,14 +31,10 @@ def ressource_after_delete(mapper, connection, ressource): @db.event.listens_for(Job, 'after_insert') @db.event.listens_for(JobInput, 'after_insert') @db.event.listens_for(JobResult, 'after_insert') -@db.event.listens_for(QueryResult, 'after_insert') def ressource_after_insert_handler(mapper, connection, ressource): value = ressource.to_dict(backrefs=False, relationships=False) - if isinstance(ressource, Job): - value['inputs'] = {} - value['results'] = {} - elif isinstance(ressource, Corpus): - value['files'] = {} + for relationship in mapper.relationships: + value[relationship.key] = {} jsonpatch = [ {'op': 'add', 'path': ressource.jsonpatch_path, 'value': value} ] @@ -45,35 +47,43 @@ def ressource_after_insert_handler(mapper, connection, ressource): @db.event.listens_for(Job, 'after_update') @db.event.listens_for(JobInput, 'after_update') @db.event.listens_for(JobResult, 'after_update') -@db.event.listens_for(QueryResult, 'after_update') def ressource_after_update_handler(mapper, connection, ressource): jsonpatch = [] for attr in db.inspect(ressource).attrs: - # We don't want to handle changes in relationship fields - # TODO: Find a way to handle this without a hardcoded list - if attr.key in ['files', 'inputs', 'results']: + # Don't handle changes in relationship fields + if attr.key in mapper.relationships: continue + # Check if their are changes for the current field history = attr.load_history() if not history.has_changes(): continue - new_value = history.added[0] - # In order to be JSON serializable, DateTime attributes must be - # converted to a string - if isinstance(new_value, datetime): - new_value = new_value.isoformat() + 'Z' + if isinstance(history.added[0], datetime): + # In order to be JSON serializable, DateTime attributes must be + # converted to a string + attr_name = attr.key + value = history.added[0].isoformat() + 'Z' + elif attr.key.endswith('_enum_value'): + # Handling fake enum attributes + attr_name = attr.key[:-11] + value = getattr(ressource, attr_name).name + else: + attr_name = attr.key + value = history.added[0] jsonpatch.append( { 'op': 'replace', - 'path': f'{ressource.jsonpatch_path}/{attr.key}', - 'value': new_value + 'path': f'{ressource.jsonpatch_path}/{attr_name}', + 'value': value } ) # Job status update notification if it changed and wanted by the user - if isinstance(ressource, Job) and attr.key == 'status': - if ressource.user.setting_job_status_mail_notifications == 'none': # noqa + if isinstance(ressource, Job) and attr_name == 'status': + if ressource.user.setting_job_status_mail_notification_level == JobStatusMailNotificationLevel.NONE: # noqa pass - elif (ressource.user.setting_job_status_mail_notifications == 'end' # noqa - and ressource.status not in ['complete', 'failed']): + elif ( + ressource.user.setting_job_status_mail_notification_level == JobStatusMailNotificationLevel.END # noqa + and ressource.status not in [JobStatus.COMPLETED, JobStatus.FAILED] # noqa + ): pass else: msg = create_message( diff --git a/app/jobs/routes.py b/app/jobs/routes.py index 4acd7c47..ac484958 100644 --- a/app/jobs/routes.py +++ b/app/jobs/routes.py @@ -1,10 +1,16 @@ -from flask import (abort, flash, redirect, render_template, - send_from_directory, url_for) +from app.decorators import admin_required +from app.models import Job, JobInput, JobResult, JobStatus +from flask import ( + abort, + flash, + redirect, + render_template, + send_from_directory, + url_for +) from flask_login import current_user, login_required from . import bp from . import tasks -from ..decorators import admin_required -from ..models import Job, JobInput, JobResult import os @@ -14,9 +20,11 @@ def job(job_id): job = Job.query.get_or_404(job_id) if not (job.user == current_user or current_user.is_administrator()): abort(403) - job_inputs = [job_input.to_dict() for job_input in job.inputs] - return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs, - title='Job') + return render_template( + 'jobs/job.html.j2', + job=job, + title='Job' + ) @bp.route('//delete') @@ -26,15 +34,21 @@ def delete_job(job_id): if not (job.user == current_user or current_user.is_administrator()): abort(403) tasks.delete_job(job_id) - flash('Job has been marked for deletion!', 'job') + flash(f'Job "{job.title}" marked for deletion', 'job') return redirect(url_for('main.dashboard')) @bp.route('//inputs//download') @login_required def download_job_input(job_id, job_input_id): - job_input = JobInput.query.filter(JobInput.job_id == job_id, JobInput.id == job_input_id).first_or_404() # noqa - if not (job_input.job.user == current_user or current_user.is_administrator()): # noqa + job_input = JobInput.query.filter( + JobInput.job_id == job_id, + JobInput.id == job_input_id + ).first_or_404() + if not ( + job_input.job.user == current_user + or current_user.is_administrator() + ): abort(403) return send_from_directory( as_attachment=True, @@ -49,19 +63,28 @@ def download_job_input(job_id, job_input_id): @admin_required def restart(job_id): job = Job.query.get_or_404(job_id) - if job.status not in ['complete', 'failed']: - flash(f'Can not restart job "{job.title}": Status is not "complete/failed"', 'error') # noqa + if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: + flash( + f'Can\'t restart job "{job.title}": Status is not "Completed/Failed"', # noqa + category='error' + ) else: tasks.restart_job(job_id) - flash(f'Job "{job.title}" marked to get restarted!', 'job') + flash(f'Job "{job.title}" marked to get restarted', category='job') return redirect(url_for('.job', job_id=job_id)) @bp.route('//results//download') @login_required def download_job_result(job_id, job_result_id): - job_result = JobResult.query.filter(JobResult.job_id == job_id, JobResult.id == job_result_id).first_or_404() # noqa - if not (job_result.job.user == current_user or current_user.is_administrator()): # noqa + job_result = JobResult.query.filter( + JobResult.job_id == job_id, + JobResult.id == job_result_id + ).first_or_404() + if not ( + job_result.job.user == current_user + or current_user.is_administrator() + ): abort(403) return send_from_directory( as_attachment=True, diff --git a/app/jobs/tasks.py b/app/jobs/tasks.py index 9ba974e0..1738b0cd 100644 --- a/app/jobs/tasks.py +++ b/app/jobs/tasks.py @@ -1,6 +1,6 @@ -from .. import db -from ..decorators import background -from ..models import Job +from app import db +from app.decorators import background +from app.models import Job @background diff --git a/app/main/routes.py b/app/main/routes.py index a817362f..79f19318 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -1,8 +1,8 @@ +from app.auth.forms import LoginForm +from app.models import User from flask import flash, redirect, render_template, url_for from flask_login import login_required, login_user from . import bp -from ..auth.forms import LoginForm -from ..models import User @bp.route('/', methods=['GET', 'POST']) @@ -21,8 +21,10 @@ def index(): @bp.route('/faq') def faq(): - return render_template('main/faq.html.j2', - title='Frequently Asked Questions') + return render_template( + 'main/faq.html.j2', + title='Frequently Asked Questions' + ) @bp.route('/dashboard') @@ -38,8 +40,10 @@ def news(): @bp.route('/privacy_policy') def privacy_policy(): - return render_template('main/privacy_policy.html.j2', - title='Privacy statement (GDPR)') + return render_template( + 'main/privacy_policy.html.j2', + title='Privacy statement (GDPR)' + ) @bp.route('/terms_of_use') diff --git a/app/models.py b/app/models.py index d02b511c..034edc91 100644 --- a/app/models.py +++ b/app/models.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from enum import IntEnum from flask import current_app, url_for from flask_hashids import HashidMixin from flask_login import UserMixin @@ -8,7 +9,6 @@ from tqdm import tqdm from werkzeug.security import generate_password_hash, check_password_hash from . import db, login import base64 -import enum import json import os import requests @@ -17,7 +17,36 @@ import xml.etree.ElementTree as ET import yaml -class Permission(enum.IntEnum): +class CorpusStatus(IntEnum): + UNPREPARED = 1 + SUBMITTED = 2 + QUEUED = 3 + BUILDING = 4 + BUILT = 5 + FAILED = 6 + STARTING_ANALYSIS_SESSION = 7 + RUNNING_ANALYSIS_SESSION = 8 + CANCELING_ANALYSIS_SESSION = 9 + + +class JobStatus(IntEnum): + INITIALIZING = 1 + SUBMITTED = 2 + QUEUED = 3 + RUNNING = 4 + CANCELING = 5 + CANCELED = 6 + COMPLETED = 7 + FAILED = 8 + + +class JobStatusMailNotificationLevel(IntEnum): + NONE = 1 + END = 2 + ALL = 3 + + +class Permission(IntEnum): ''' Defines User permissions as integers by the power of 2. User permission can be evaluated using the bitwise operator &. @@ -130,10 +159,11 @@ class User(HashidMixin, UserMixin, db.Model): token_expiration = db.Column(db.DateTime) username = db.Column(db.String(64), unique=True, index=True) setting_dark_mode = db.Column(db.Boolean, default=False) - setting_job_status_mail_notifications = db.Column( - db.String(16), default='end') - setting_job_status_site_notifications = db.Column( - db.String(16), default='all') + setting_job_status_mail_notification_level_enum_value = db.Column( + 'setting_job_status_mail_notification_level', + db.Integer, + default=2 + ) # Backrefs: role: Role # Relationships tesseract_ocr_models = db.relationship( @@ -154,12 +184,6 @@ class User(HashidMixin, UserMixin, db.Model): cascade='all, delete-orphan', lazy='dynamic' ) - query_results = db.relationship( - 'QueryResult', - backref='user', - cascade='all, delete-orphan', - lazy='dynamic' - ) def __init__(self, **kwargs): super().__init__(**kwargs) @@ -188,7 +212,20 @@ class User(HashidMixin, UserMixin, db.Model): @property def path(self): return os.path.join( - current_app.config.get('NOPAQUE_DATA_DIR'), str(self.id)) + current_app.config.get('NOPAQUE_DATA_DIR'), 'users', str(self.id)) + + @property + def setting_job_status_mail_notification_level(self): + return JobStatusMailNotificationLevel( + self.setting_job_status_mail_notification_level_enum_value + ) + + @setting_job_status_mail_notification_level.setter + def setting_job_status_mail_notification_level(self, enum_member): + if not isinstance(enum_member, JobStatusMailNotificationLevel): + return TypeError() + self.setting_job_status_mail_notification_level_enum_value = \ + enum_member.value def can(self, permission): return self.role.has_permission(permission) @@ -251,10 +288,8 @@ class User(HashidMixin, UserMixin, db.Model): 'username': self.username, 'settings': { 'dark_mode': self.setting_dark_mode, - 'job_status_mail_notifications': - self.setting_job_status_mail_notifications, - 'job_status_site_notifications': - self.setting_job_status_site_notifications + 'job_status_mail_notification_level': + self.setting_job_status_mail_notification_level.name } } if backrefs: @@ -269,9 +304,9 @@ class User(HashidMixin, UserMixin, db.Model): x.hashid: x.to_dict(backrefs=False, relationships=True) for x in self.jobs } - dict_user['query_results'] = { + dict_user['tesseract_ocr_models'] = { x.hashid: x.to_dict(backrefs=False, relationships=True) - for x in self.query_results + for x in self.tesseract_ocr_models } return dict_user @@ -338,6 +373,25 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model): str(self.id) ) + def to_dict(self, backrefs=False, relationships=False): + compatible_service_versions = json.loads(self.compatible_service_versions) # noqa + dict_tesseract_ocr_model = { + 'id': self.hashid, + 'user_id': self.user.hashid, + 'compatible_service_versions': compatible_service_versions, + 'description': self.description, + 'publisher': self.publisher, + 'publishing_year': self.publishing_year, + 'title': self.title, + **self.file_mixin_to_dict() + } + if backrefs: + dict_tesseract_ocr_model['user'] = self.user.to_dict( + backrefs=True, relationships=False) + if relationships: + pass + return dict_tesseract_ocr_model + @staticmethod def insert_defaults(): user = User.query.filter_by(username='nopaque').first() @@ -519,7 +573,7 @@ class Job(HashidMixin, db.Model): ''' service_args = db.Column(db.String(255)) service_version = db.Column(db.String(16)) - status = db.Column(db.String(16)) + status_enum_value = db.Column('status', db.Integer, default=1) title = db.Column(db.String(32)) # Backrefs: user: User # Relationships @@ -547,6 +601,16 @@ class Job(HashidMixin, db.Model): def path(self): return os.path.join(self.user.path, 'jobs', str(self.id)) + @property + def status(self): + return JobStatus(self.status_enum_value) + + @status.setter + def status(self, enum_member): + if not isinstance(enum_member, JobStatus): + return TypeError() + self.status_enum_value = enum_member.value + @property def url(self): return url_for('jobs.job', job_id=self.id) @@ -559,13 +623,13 @@ class Job(HashidMixin, db.Model): ''' Delete the job and its inputs and results from the database. ''' - if self.status not in ['complete', 'failed']: - self.status = 'canceling' + if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: # noqa + self.status = JobStatus.CANCELING db.session.commit() - while self.status != 'canceled': + while self.status != JobStatus.CANCELED: # In case the daemon handled a job in any way - if self.status != 'canceling': - self.status = 'canceling' + if self.status != JobStatus.CANCELING: + self.status = JobStatus.CANCELING db.session.commit() sleep(1) db.session.refresh(self) @@ -583,14 +647,14 @@ class Job(HashidMixin, db.Model): Restart a job - only if the status is complete or failed ''' - if self.status not in ['complete', 'failed']: - raise Exception('Could not restart job: status is not "complete/failed"') # noqa + if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: # noqa + raise Exception('Could not restart job: status is not "completed/failed"') # noqa shutil.rmtree(os.path.join(self.path, 'results'), ignore_errors=True) shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa for result in self.results: db.session.delete(result) self.end_date = None - self.status = 'submitted' + self.status = JobStatus.SUBMITTED def to_dict(self, backrefs=False, relationships=False): service_args = json.loads(self.service_args) @@ -606,7 +670,7 @@ class Job(HashidMixin, db.Model): 'service': self.service, 'service_args': service_args, 'service_version': self.service_version, - 'status': self.status, + 'status': self.status.name, 'title': self.title, 'url': self.url } @@ -687,7 +751,7 @@ class CorpusFile(FileMixin, HashidMixin, db.Model): ) pass db.session.delete(self) - self.corpus.status = 'unprepared' + self.corpus.status = CorpusStatus.UNPREPARED def to_dict(self, backrefs=False, relationships=False): dict_corpus_file = { @@ -729,7 +793,7 @@ class Corpus(HashidMixin, db.Model): creation_date = db.Column(db.DateTime(), default=datetime.utcnow) description = db.Column(db.String(255)) last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow) - status = db.Column(db.String(16), default='unprepared') + status_enum_value = db.Column('status', db.Integer, default=1) title = db.Column(db.String(32)) num_analysis_sessions = db.Column(db.Integer, default=0) num_tokens = db.Column(db.Integer, default=0) @@ -742,7 +806,7 @@ class Corpus(HashidMixin, db.Model): lazy='dynamic', cascade='all, delete-orphan' ) - # Python class variables + # "static" attributes max_num_tokens = 2147483647 def __repr__(self): @@ -760,6 +824,16 @@ class Corpus(HashidMixin, db.Model): def path(self): return os.path.join(self.user.path, 'corpora', str(self.id)) + @property + def status(self): + return CorpusStatus(self.status_enum_value) + + @status.setter + def status(self, enum_member): + if not isinstance(enum_member, CorpusStatus): + return TypeError() + self.status_enum_value = enum_member.value + @property def url(self): return url_for('corpora.corpus', corpus_id=self.id) @@ -791,7 +865,7 @@ class Corpus(HashidMixin, db.Model): encoding='utf-8' ) self.last_edited_date = datetime.utcnow() - self.status = 'submitted' + self.status = CorpusStatus.SUBMITTED def delete(self): shutil.rmtree(self.path, ignore_errors=True) @@ -815,7 +889,7 @@ class Corpus(HashidMixin, db.Model): 'max_num_tokens': self.max_num_tokens, 'num_analysis_sessions': self.num_analysis_sessions, 'num_tokens': self.num_tokens, - 'status': self.status, + 'status': self.status.name, 'last_edited_date': self.last_edited_date.isoformat() + 'Z', 'title': self.title } @@ -830,70 +904,6 @@ class Corpus(HashidMixin, db.Model): return dict_corpus -class QueryResult(FileMixin, HashidMixin, db.Model): - __tablename__ = 'query_results' - # Primary key - id = db.Column(db.Integer, primary_key=True) - # Foreign keys - user_id = db.Column(db.Integer, db.ForeignKey('users.id')) - # Fields - description = db.Column(db.String(255)) - query_metadata = db.Column(db.JSON()) - title = db.Column(db.String(32)) - # Backrefs: user: User - - def __repr__(self): - ''' - String representation of the QueryResult. For human readability. - ''' - return f'' - - @property - def download_url(self): - return url_for( - 'corpora.download_query_result', query_result_id=self.id) - - @property - def jsonpatch_path(self): - return f'{self.user.jsonpatch_path}/query_results/{self.hashid}' - - @property - def path(self): - return os.path.join( - self.user.path, 'query_results', str(self.id), self.filename) - - @property - def url(self): - return url_for('corpora.query_result', query_result_id=self.id) - - @property - def user_hashid(self): - return self.user.hashid - - def delete(self): - shutil.rmtree(self.path, ignore_errors=True) - db.session.delete(self) - - def to_dict(self, backrefs=False, relationships=False): - dict_query_result = { - 'id': self.hashid, - 'user_id': self.user.hashid, - 'download_url': self.download_url, - 'url': self.url, - 'corpus_title': self.query_metadata['corpus_name'], - 'description': self.description, - 'filename': self.filename, - 'query': self.query_metadata['query'], - 'query_metadata': self.query_metadata, - 'title': self.title, - **self.file_mixin_to_dict( - backrefs=backrefs, relationships=relationships) - } - if backrefs: - dict_query_result['user'] = self.user.to_dict( - backrefs=True, relationships=False) - - @login.user_loader def load_user(user_id): return User.query.get(int(user_id)) diff --git a/app/query_results_models.py b/app/query_results_models.py new file mode 100644 index 00000000..102d2825 --- /dev/null +++ b/app/query_results_models.py @@ -0,0 +1,62 @@ +class QueryResult(FileMixin, HashidMixin, db.Model): + __tablename__ = 'query_results' + # Primary key + id = db.Column(db.Integer, primary_key=True) + # Foreign keys + user_id = db.Column(db.Integer, db.ForeignKey('users.id')) + # Fields + description = db.Column(db.String(255)) + query_metadata = db.Column(db.JSON()) + title = db.Column(db.String(32)) + # Backrefs: user: User + + def __repr__(self): + ''' + String representation of the QueryResult. For human readability. + ''' + return f'' + + @property + def download_url(self): + return url_for( + 'corpora.download_query_result', query_result_id=self.id) + + @property + def jsonpatch_path(self): + return f'{self.user.jsonpatch_path}/query_results/{self.hashid}' + + @property + def path(self): + return os.path.join( + self.user.path, 'query_results', str(self.id), self.filename) + + @property + def url(self): + return url_for('corpora.query_result', query_result_id=self.id) + + @property + def user_hashid(self): + return self.user.hashid + + def delete(self): + shutil.rmtree(self.path, ignore_errors=True) + db.session.delete(self) + + def to_dict(self, backrefs=False, relationships=False): + dict_query_result = { + 'id': self.hashid, + 'user_id': self.user.hashid, + 'download_url': self.download_url, + 'url': self.url, + 'corpus_title': self.query_metadata['corpus_name'], + 'description': self.description, + 'filename': self.filename, + 'query': self.query_metadata['query'], + 'query_metadata': self.query_metadata, + 'title': self.title, + **self.file_mixin_to_dict( + backrefs=backrefs, relationships=relationships) + } + if backrefs: + dict_query_result['user'] = self.user.to_dict( + backrefs=True, relationships=False) diff --git a/app/services/forms.py b/app/services/forms.py index 0bebfb02..4a16ad4a 100644 --- a/app/services/forms.py +++ b/app/services/forms.py @@ -1,7 +1,13 @@ from app.models import TesseractOCRModel from flask_wtf import FlaskForm -from wtforms import (BooleanField, MultipleFileField, SelectField, StringField, - SubmitField, ValidationError) +from wtforms import ( + BooleanField, + MultipleFileField, + SelectField, + StringField, + SubmitField, + ValidationError +) from wtforms.validators import DataRequired, Length from . import SERVICES @@ -25,7 +31,7 @@ class AddSpacyNLPJobForm(AddJobForm): def validate_encoding_detection(self, field): service_info = SERVICES['spacy-nlp']['versions'][self.version.data] - if field.data and 'encoding_detection' not in service_info: + if field.data and 'encoding_detection' not in service_info['methods']: raise ValidationError('Encoding detection is not available') def validate_files(form, field): @@ -41,7 +47,7 @@ class AddSpacyNLPJobForm(AddJobForm): version = kwargs.pop('version', SERVICES['spacy-nlp']['latest_version']) # noqa super().__init__(*args, **kwargs) service_info = SERVICES['spacy-nlp']['versions'][version] - if 'check_encoding' not in service_info['methods']: + if 'encoding_detection' not in service_info['methods']: self.encoding_detection.render_kw = {'disabled': True} self.model.choices += [(x, y) for x, y in service_info['models'].items()] # noqa self.version.choices = [(x, x) for x in SERVICES['spacy-nlp']['versions']] # noqa @@ -60,7 +66,7 @@ class AddTesseractOCRJobForm(AddJobForm): def validate_binarization(self, field): service_info = SERVICES['tesseract-ocr']['versions'][self.version.data] - if field.data and 'binarization' not in service_info: + if field.data and 'binarization' not in service_info['methods']: raise ValidationError('Binarization is not available') def validate_files(self, field): diff --git a/app/services/routes.py b/app/services/routes.py index d430e61e..7a8e520e 100644 --- a/app/services/routes.py +++ b/app/services/routes.py @@ -1,21 +1,29 @@ -from app import hashids -from flask import (abort, current_app, flash, make_response, render_template, - request, url_for) +from app import db, hashids +from app.models import Job, JobInput, JobStatus +from flask import ( + abort, + current_app, + flash, + make_response, + render_template, + request, + url_for +) from flask_login import current_user, login_required from werkzeug.utils import secure_filename from . import bp from . import SERVICES -from .. import db from .forms import AddJobForms -from ..models import Job, JobInput import json @bp.route('/corpus-analysis') @login_required def corpus_analysis(): - return render_template('services/corpus_analysis.html.j2', - title='Corpus analysis') + return render_template( + 'services/corpus_analysis.html.j2', + title='Corpus analysis' + ) @bp.route('/', methods=['GET', 'POST']) @@ -47,7 +55,6 @@ def service(service): service=service, service_args=json.dumps(service_args), service_version=form.version.data, - status='preparing', title=form.title.data ) db.session.add(job) @@ -77,7 +84,7 @@ def service(service): db.session.rollback() flash('Internal Server Error', 'error') return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa - job.status = 'submitted' + job.status = JobStatus.SUBMITTED db.session.commit() flash(f'Job "{job.title}" added', 'job') return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) # noqa diff --git a/app/settings/forms.py b/app/settings/forms.py index d402fad2..03d7f243 100644 --- a/app/settings/forms.py +++ b/app/settings/forms.py @@ -1,21 +1,37 @@ +from app.auth import USERNAME_REGEX +from app.models import JobStatusMailNotificationLevel, User from flask_wtf import FlaskForm -from wtforms import (BooleanField, PasswordField, SelectField, StringField, - SubmitField, ValidationError) +from wtforms import ( + BooleanField, + PasswordField, + SelectField, + StringField, + SubmitField, + ValidationError +) from wtforms.validators import DataRequired, Email, EqualTo, Length, Regexp -from ..auth import USERNAME_REGEX -from ..models import User class ChangePasswordForm(FlaskForm): password = PasswordField('Old password', validators=[DataRequired()]) new_password = PasswordField( 'New password', - validators=[DataRequired(), EqualTo('password_confirmation', - message='Passwords must match.')] + validators=[ + DataRequired(), + EqualTo( + 'new_password_confirmation', + message='Passwords must match' + ) + ] ) - new_password2 = PasswordField( - 'Confirm new password', validators=[DataRequired()]) - submit = SubmitField('Change password') + new_password_confirmation = PasswordField( + 'Confirm new password', + validators=[ + DataRequired(), + EqualTo('new_password', message='Passwords must match') + ] + ) + submit = SubmitField('Submit') def __init__(self, user, *args, **kwargs): super().__init__(*args, **kwargs) @@ -23,20 +39,24 @@ class ChangePasswordForm(FlaskForm): def validate_current_password(self, field): if not self.user.verify_password(field.data): - raise ValidationError('Invalid password.') + raise ValidationError('Invalid password') class EditGeneralSettingsForm(FlaskForm): - dark_mode = BooleanField('Dark mode') - email = StringField('E-Mail', - validators=[DataRequired(), Length(1, 254), Email()]) + email = StringField( + 'E-Mail', + validators=[DataRequired(), Length(1, 254), Email()] + ) username = StringField( - 'Benutzername', - validators=[DataRequired(), - Length(1, 64), - Regexp(USERNAME_REGEX, - message='Usernames must have only letters, numbers,' - ' dots or underscores')] + 'Username', + validators=[ + DataRequired(), + Length(1, 64), + Regexp( + USERNAME_REGEX, + message='Usernames must have only letters, numbers, dots or underscores' # noqa + ) + ] ) submit = SubmitField('Submit') @@ -45,29 +65,36 @@ class EditGeneralSettingsForm(FlaskForm): self.user = user def validate_email(self, field): - if (field.data != self.user.email - and User.query.filter_by(email=field.data).first()): - raise ValidationError('Email already registered.') + if ( + field.data != self.user.email + and User.query.filter_by(email=field.data).first() + ): + raise ValidationError('Email already registered') def validate_username(self, field): - if (field.data != self.user.username - and User.query.filter_by(username=field.data).first()): - raise ValidationError('Username already in use.') + if ( + field.data != self.user.username + and User.query.filter_by(username=field.data).first() + ): + raise ValidationError('Username already in use') + + +class EditInterfaceSettingsForm(FlaskForm): + dark_mode = BooleanField('Dark mode') + submit = SubmitField('Submit') class EditNotificationSettingsForm(FlaskForm): - job_status_mail_notifications = SelectField( - 'Job status mail notifications', - choices=[('', 'Choose your option'), - ('all', 'Notify on all status changes'), - ('end', 'Notify only when a job ended'), - ('none', 'No status update notifications')], - validators=[DataRequired()]) - job_status_site_notifications = SelectField( - 'Job status site notifications', - choices=[('', 'Choose your option'), - ('all', 'Notify on all status changes'), - ('end', 'Notify only when a job ended'), - ('none', 'No status update notifications')], - validators=[DataRequired()]) - submit = SubmitField('Save settings') + job_status_mail_notification_level = SelectField( + 'Job status mail notification level', + choices=[('', 'Choose your option')], + validators=[DataRequired()] + ) + submit = SubmitField('Submit') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.job_status_mail_notification_level.choices += [ + (enum_member.name, enum_member.name.capitalize()) + for enum_member in JobStatusMailNotificationLevel + ] diff --git a/app/settings/routes.py b/app/settings/routes.py index 9cc9960a..c59a55f2 100644 --- a/app/settings/routes.py +++ b/app/settings/routes.py @@ -1,66 +1,82 @@ from flask import flash, redirect, render_template, url_for from flask_login import current_user, login_required, logout_user from . import bp, tasks -from .forms import (ChangePasswordForm, EditGeneralSettingsForm, - EditNotificationSettingsForm) +from .forms import ( + ChangePasswordForm, + EditGeneralSettingsForm, + EditInterfaceSettingsForm, + EditNotificationSettingsForm +) from .. import db +from ..models import JobStatusMailNotificationLevel -@bp.route('/') +@bp.route('', methods=['GET', 'POST']) @login_required def index(): - return redirect(url_for('.edit_general_settings')) + change_password_form = ChangePasswordForm( + current_user._get_current_object(), + prefix='change_password_form' + ) + edit_general_settings_form = EditGeneralSettingsForm( + current_user._get_current_object(), + prefix='edit_general_settings_form' + ) + edit_interface_settings_form = EditInterfaceSettingsForm( + prefix='edit_interface_settings_form' + ) + edit_notification_settings_form = EditNotificationSettingsForm( + prefix='edit_notification_settings_form' + ) - -@bp.route('/change_password', methods=['GET', 'POST']) -@login_required -def change_password(): - form = ChangePasswordForm(current_user._get_current_object()) - if form.validate_on_submit(): - current_user.password = form.new_password.data + if change_password_form.submit.data and change_password_form.validate(): + current_user.password = change_password_form.new_password.data db.session.commit() - flash('Your password has been updated.') - return redirect(url_for('.change_password')) - return render_template('settings/change_password.html.j2', - form=form, title='Change password') - - -@bp.route('/edit_general_settings', methods=['GET', 'POST']) -@login_required -def edit_general_settings(): - form = EditGeneralSettingsForm(current_user._get_current_object()) - if form.validate_on_submit(): - current_user.email = form.email.data - current_user.setting_dark_mode = form.dark_mode.data - current_user.username = form.username.data + flash('Your changes have been saved') + return redirect(url_for('.index')) + if ( + edit_general_settings_form.submit.data + and edit_general_settings_form.validate() + ): + current_user.email = edit_general_settings_form.email.data + current_user.username = edit_general_settings_form.username.data db.session.commit() - flash('Your changes have been saved.') - return redirect(url_for('.edit_general_settings')) - form.dark_mode.data = current_user.setting_dark_mode - form.email.data = current_user.email - form.username.data = current_user.username - return render_template('settings/edit_general_settings.html.j2', - form=form, title='General settings') - - -@bp.route('/edit_notification_settings', methods=['GET', 'POST']) -@login_required -def edit_notification_settings(): - form = EditNotificationSettingsForm() - if form.validate_on_submit(): - current_user.setting_job_status_mail_notifications = \ - form.job_status_mail_notifications.data - current_user.setting_job_status_site_notifications = \ - form.job_status_site_notifications.data + flash('Your changes have been saved') + return redirect(url_for('.index')) + if ( + edit_interface_settings_form.submit.data + and edit_interface_settings_form.validate() + ): + current_user.setting_dark_mode = \ + edit_interface_settings_form.dark_mode.data db.session.commit() - flash('Your changes have been saved.') - return redirect(url_for('.edit_notification_settings')) - form.job_status_mail_notifications.data = \ - current_user.setting_job_status_mail_notifications - form.job_status_site_notifications.data = \ - current_user.setting_job_status_site_notifications - return render_template('settings/edit_notification_settings.html.j2', - form=form, title='Notification settings') + flash('Your changes have been saved') + return redirect(url_for('.index')) + if ( + edit_notification_settings_form.submit.data + and edit_notification_settings_form.validate() + ): + current_user.setting_job_status_mail_notification_level = \ + JobStatusMailNotificationLevel[ + edit_notification_settings_form.job_status_mail_notification_level.data # noqa + ] + db.session.commit() + flash('Your changes have been saved') + return redirect(url_for('.index')) + edit_general_settings_form.email.data = current_user.email + edit_general_settings_form.username.data = current_user.username + edit_interface_settings_form.dark_mode.data = \ + current_user.setting_dark_mode + edit_notification_settings_form.job_status_mail_notification_level.data = \ + current_user.setting_job_status_mail_notification_level.name + return render_template( + 'settings/index.html.j2', + change_password_form=change_password_form, + edit_general_settings_form=edit_general_settings_form, + edit_interface_settings_form=edit_interface_settings_form, + edit_notification_settings_form=edit_notification_settings_form, + title='Settings' + ) @bp.route('/delete') @@ -71,5 +87,5 @@ def delete(): """ tasks.delete_user(current_user.id) logout_user() - flash('Your account has been marked for deletion!') + flash('Your account has been marked for deletion') return redirect(url_for('main.index')) diff --git a/app/settings/tasks.py b/app/settings/tasks.py index cc5d9e36..2bd82ca9 100644 --- a/app/settings/tasks.py +++ b/app/settings/tasks.py @@ -1,6 +1,6 @@ -from .. import db -from ..decorators import background -from ..models import User +from app import db +from app.decorators import background +from app.models import User @background diff --git a/app/static/css/nopaque.css b/app/static/css/nopaque.css index ee5377e1..ce122980 100644 --- a/app/static/css/nopaque.css +++ b/app/static/css/nopaque.css @@ -1,3 +1,7 @@ +:root { + --main-bg-color: brown; +} + /* Change navbar height bacause an extended and fixed navbar is used */ .navbar-fixed { height: 112px; @@ -12,15 +16,6 @@ opacity: 1; } -/* preloader circle in the size of a button icon */ -.button-icon-spinner { - bottom: -5px !important; - right: 55px !important; - margin-right: 12px !important; - width: 19.5px !important; - height: 19.5px !important; -} - /* * changes preoloader size etc. to fit visually better with the chip status * indicator of jobs @@ -39,36 +34,37 @@ transform: scale(2); } -.btn-scale-x2 .nopaque-icons.service-icon { +.btn-scale-x2 .nopaque-icon.nopaque-service-icon { font-size: 2.5rem; } /* Fix material icon vertical alignment when nested in various elements */ -h1 .nopaque-icons, h2 .nopaque-icons, h3 .nopaque-icons, h4 .nopaque-icons, -.tab .nopaque-icons, .tab .material-icons { +h1 .nopaque-icon, h2 .nopaque-icon, h3 .nopaque-icon, h4 .nopaque-icon, .tab .nopaque-icon, .tab .material-icons { line-height: inherit; } -.nopaque-icons.service-icon[data-service="corpus-analysis"]:empty:before {content: "H";} -.nopaque-icons.service-icon[data-service="file-setup"]:empty:before {content: "E";} -.nopaque-icons.service-icon[data-service="spacy-nlp"]:empty:before {content: "G";} -.nopaque-icons.service-icon[data-service="tesseract-ocr"]:empty:before {content: "F";} -.status-text[data-status]:empty:before {content: attr(data-status);} +.nopaque-icon.nopaque-service-icon[data-service="file-setup"]:empty:before {content: "E";} +.nopaque-icon.nopaque-service-icon[data-service="tesseract-ocr"]:empty:before {content: "F";} +.nopaque-icon.nopaque-service-icon[data-service="spacy-nlp"]:empty:before {content: "G";} +.nopaque-icon.nopaque-service-icon[data-service="corpus-analysis"]:empty:before {content: "H";} + +.nopaque-corpus-status-text[data-corpus-status="UNPREPARED"]:empty:before {content: "Unprepared";} +.nopaque-corpus-status-text[data-corpus-status="SUBMITTED"]:empty:before {content: "Submitted";} +.nopaque-corpus-status-text[data-corpus-status="QUEUED"]:empty:before {content: "Queued";} +.nopaque-corpus-status-text[data-corpus-status="BUILDING"]:empty:before {content: "Building";} +.nopaque-corpus-status-text[data-corpus-status="BUILT"]:empty:before {content: "Built";} +.nopaque-corpus-status-text[data-corpus-status="STARTING_ANALYSIS_SESSION"]:empty:before {content: "Starting analysis session";} +.nopaque-corpus-status-text[data-corpus-status="RUNNING_ANALYSIS_SESSION"]:empty:before {content: "Running analysis session";} +.nopaque-corpus-status-text[data-corpus-status="CANCELING_ANALYSIS_SESSION"]:empty:before {content: "Canceling analysis session";} + +.nopaque-job-status-text[data-job-status="INITIALIZING"]:empty:before {content: "Initializing";} +.nopaque-job-status-text[data-job-status="SUBMITTED"]:empty:before {content: "Submitted";} +.nopaque-job-status-text[data-job-status="QUEUED"]:empty:before {content: "Queued";} +.nopaque-job-status-text[data-job-status="RUNNING"]:empty:before {content: "Running";} +.nopaque-job-status-text[data-job-status="CANCELING"]:empty:before {content: "Canceling";} +.nopaque-job-status-text[data-job-status="CANCELED"]:empty:before {content: "Canceled";} +.nopaque-job-status-text[data-job-status="COMPLETED"]:empty:before {content: "Completed";} +.nopaque-job-status-text[data-job-status="FAILED"]:empty:before {content: "Failed";} .hoverable {cursor: pointer;} -.s-attr.chip .p-attr.chip {background-color: inherit;} - - -.responsive-youtube-video-container { - position: relative; - width: 100%; - height: 0; - padding-bottom: 56.25%; -} -.responsive-youtube-video { - position: absolute; - top: 0; - left: 0; - width: 100%; - height: 100%; -} +.chip.s-attr .chip.p-attr {background-color: inherit;} diff --git a/app/static/css/nopaque_icons.css b/app/static/css/nopaque_icons.css index f3e49752..20d1c68c 100644 --- a/app/static/css/nopaque_icons.css +++ b/app/static/css/nopaque_icons.css @@ -1,14 +1,14 @@ @font-face { - font-family: 'nopaque Icons'; + font-family: 'Nopaque Icons'; font-style: normal; font-weight: 400; src: local('nopaque Icons'), - local('nopaqueIcons-Regular'), - url(../fonts/nopaque_icons/nopaqueIcons-Regular.otf) format('opentype'); + local('NopaqueIcons-Regular'), + url(../fonts/nopaque_icons/NopaqueIcons-Regular.otf) format('opentype'); } -.nopaque-icons { - font-family: 'nopaque Icons'; +.nopaque-icon { + font-family: 'Nopaque Icons'; font-weight: normal; font-style: normal; font-size: 24px; /* Preferred icon size */ diff --git a/app/static/js/App.js b/app/static/js/App.js index 3761be8c..6a188e63 100644 --- a/app/static/js/App.js +++ b/app/static/js/App.js @@ -31,7 +31,7 @@ class App { iconPrefix = 'error'; break; case 'job': - iconPrefix = 'J'; + iconPrefix = 'J'; break; default: iconPrefix = 'notifications'; diff --git a/app/static/js/JobStatusNotifier.js b/app/static/js/JobStatusNotifier.js index 0ae834cd..fb258b04 100644 --- a/app/static/js/JobStatusNotifier.js +++ b/app/static/js/JobStatusNotifier.js @@ -16,7 +16,7 @@ class JobStatusNotifier { .filter(operation => re.test(operation.path)); for (operation of filteredPatch) { [match, jobId] = operation.path.match(re); - app.flash(`[${app.users[this.userId].jobs[jobId].title}] New status: ${operation.value}`, 'job'); + app.flash(`[${app.users[this.userId].jobs[jobId].title}] New status: `, 'job'); } } } diff --git a/app/static/js/RessourceDisplays/CorpusDisplay.js b/app/static/js/RessourceDisplays/CorpusDisplay.js index b807241e..ad3cb34b 100644 --- a/app/static/js/RessourceDisplays/CorpusDisplay.js +++ b/app/static/js/RessourceDisplays/CorpusDisplay.js @@ -67,30 +67,29 @@ class CorpusDisplay extends RessourceDisplay { let element; let elements; - this.setElements(this.displayElement.querySelectorAll('.corpus-status'), status); - elements = this.displayElement.querySelectorAll('.analyse-corpus-trigger') + elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger') for (element of elements) { - if (['analysing', 'prepared', 'start analysis'].includes(status)) { + if (['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) { element.classList.remove('disabled'); } else { element.classList.add('disabled'); } } - elements = this.displayElement.querySelectorAll('.build-corpus-trigger'); + elements = this.displayElement.querySelectorAll('.corpus-build-trigger'); for (element of elements) { - if (status === 'unprepared' && Object.values(app.users[this.userId].corpora[this.corpusId].files).length > 0) { + if (status === 'UNPREPARED' && Object.values(app.users[this.userId].corpora[this.corpusId].files).length > 0) { element.classList.remove('disabled'); } else { element.classList.add('disabled'); } } - elements = this.displayElement.querySelectorAll('.status'); + elements = this.displayElement.querySelectorAll('.corpus-status'); for (element of elements) { - element.dataset.status = status; + element.dataset.corpusStatus = status; } - elements = this.displayElement.querySelectorAll('.status-spinner'); + elements = this.displayElement.querySelectorAll('.corpus-status-spinner'); for (element of elements) { - if (['submitted', 'queued', 'running', 'canceling', 'start analysis', 'stop analysis'].includes(status)) { + if (['SUBMITTED', 'QUEUED', 'BUILDING', 'STARTING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) { element.classList.remove('hide'); } else { element.classList.add('hide'); diff --git a/app/static/js/RessourceDisplays/JobDisplay.js b/app/static/js/RessourceDisplays/JobDisplay.js index 92102908..f00242a5 100644 --- a/app/static/js/RessourceDisplays/JobDisplay.js +++ b/app/static/js/RessourceDisplays/JobDisplay.js @@ -57,23 +57,21 @@ class JobDisplay extends RessourceDisplay { let element; let elements; - this.setElements(this.displayElement.querySelectorAll('.job-status'), status); - - elements = this.displayElement.querySelectorAll('.status'); + elements = this.displayElement.querySelectorAll('.job-status'); for (element of elements) { - element.dataset.status = status; + element.dataset.jobStatus = status; } - elements = this.displayElement.querySelectorAll('.status-spinner'); + elements = this.displayElement.querySelectorAll('.job-status-spinner'); for (element of elements) { - if (['complete', 'failed'].includes(status)) { + if (['COMPLETED', 'FAILED'].includes(status)) { element.classList.add('hide'); } else { element.classList.remove('hide'); } } - elements = this.displayElement.querySelectorAll('.restart-job-trigger'); + elements = this.displayElement.querySelectorAll('.job-restart-trigger'); for (element of elements) { - if (['complete', 'failed'].includes(status)) { + if (['COMPLETED', 'FAILED'].includes(status)) { element.classList.remove('hide'); } else { element.classList.add('hide'); diff --git a/app/static/js/RessourceLists/CorpusFileList.js b/app/static/js/RessourceLists/CorpusFileList.js index 6aa3864b..a4d76dad 100644 --- a/app/static/js/RessourceLists/CorpusFileList.js +++ b/app/static/js/RessourceLists/CorpusFileList.js @@ -5,31 +5,31 @@ class CorpusFileList extends RessourceList { - + delete - file_download - send + file_download + send `.trim(), ressourceMapper: corpusFile => { return { - id: corpusFile.id, - author: corpusFile.author, - creationDate: corpusFile.creation_date, - filename: corpusFile.filename, - publishingYear: corpusFile.publishing_year, - title: corpusFile.title + 'id': corpusFile.id, + 'author': corpusFile.author, + 'creation-date': corpusFile.creation_date, + 'filename': corpusFile.filename, + 'publishing-year': corpusFile.publishing_year, + 'title': corpusFile.title }; }, - sortValueName: 'creationDate', + sortValueName: 'creation-date', valueNames: [ {data: ['id']}, - {data: ['creationDate']}, + {data: ['creation-date']}, 'author', 'filename', - 'publishingYear', + 'publishing-year', 'title' ] }; @@ -125,7 +125,7 @@ class CorpusFileList extends RessourceList { re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)/(author|filename|publishing_year|title)$`); if (re.test(operation.path)) { [match, corpusFileId, valueName] = operation.path.match(re); - this.replace(corpusFileId, valueName, operation.value); + this.replace(corpusFileId, valueName.replace('_', '-'), operation.value); } break; default: diff --git a/app/static/js/RessourceLists/CorpusList.js b/app/static/js/RessourceLists/CorpusList.js index 7d4eb077..d939d17e 100644 --- a/app/static/js/RessourceLists/CorpusList.js +++ b/app/static/js/RessourceLists/CorpusList.js @@ -2,29 +2,29 @@ class CorpusList extends RessourceList { static options = { item: ` - book + book
- + delete - send + send `.trim(), ressourceMapper: corpus => { return { - id: corpus.id, - creationDate: corpus.creation_date, - description: corpus.description, - status: corpus.status, - title: corpus.title + 'id': corpus.id, + 'creation-date': corpus.creation_date, + 'description': corpus.description, + 'status': corpus.status, + 'title': corpus.title }; }, - sortValueName: 'creationDate', + sortValueName: 'creation-date', valueNames: [ {data: ['id']}, - {data: ['creationDate']}, - {name: 'status', attr: 'data-status'}, + {data: ['creation-date']}, + {name: 'status', attr: 'data-corpus-status'}, 'description', 'title' ] diff --git a/app/static/js/RessourceLists/JobInputList.js b/app/static/js/RessourceLists/JobInputList.js index 714f3e59..c1f31312 100644 --- a/app/static/js/RessourceLists/JobInputList.js +++ b/app/static/js/RessourceLists/JobInputList.js @@ -10,13 +10,17 @@ class JobInputList extends RessourceList { `.trim(), ressourceMapper: jobInput => { return { - id: jobInput.id, - creationDate: jobInput.creation_date, - filename: jobInput.filename + 'id': jobInput.id, + 'creation-date': jobInput.creation_date, + 'filename': jobInput.filename }; }, - sortValueName: 'creationDate', - valueNames: [{data: ['id']}, {data: ['creationDate']}, 'filename'] + sortValueName: 'creation-date', + valueNames: [ + {data: ['id']}, + {data: ['creation-date']}, + 'filename' + ] }; diff --git a/app/static/js/RessourceLists/JobList.js b/app/static/js/RessourceLists/JobList.js index ecb84dbd..58a0074a 100644 --- a/app/static/js/RessourceLists/JobList.js +++ b/app/static/js/RessourceLists/JobList.js @@ -1,36 +1,36 @@ class JobList extends RessourceList { static options = { item: ` - - + +
- + delete - send + send `.trim(), ressourceMapper: job => { return { - id: job.id, - creationDate: job.creation_date, - description: job.description, - service: job.service, - serviceDuplicate1: job.service, - serviceDuplicate2: job.service, - status: job.status, - title: job.title + 'id': job.id, + 'creation-date': job.creation_date, + 'description': job.description, + 'service': job.service, + 'service-1': job.service, + 'service-2': job.service, + 'status': job.status, + 'title': job.title }; }, - sortValueName: 'creationDate', + sortValueName: 'creation-date', valueNames: [ {data: ['id']}, - {data: ['creationDate']}, + {data: ['creation-date']}, {data: ['service']}, - {name: 'serviceDuplicate1', attr: 'data-service'}, - {name: 'serviceDuplicate2', attr: 'data-service'}, - {name: 'status', attr: 'data-status'}, + {name: 'service-1', attr: 'data-service'}, + {name: 'service-2', attr: 'data-service'}, + {name: 'status', attr: 'data-job-status'}, 'description', 'title' ] diff --git a/app/static/js/RessourceLists/JobResultList.js b/app/static/js/RessourceLists/JobResultList.js index 708b25f2..5f9da3b5 100644 --- a/app/static/js/RessourceLists/JobResultList.js +++ b/app/static/js/RessourceLists/JobResultList.js @@ -11,16 +11,16 @@ class JobResultList extends RessourceList { `.trim(), ressourceMapper: jobResult => { return { - id: jobResult.id, - creationDate: jobResult.creation_date, - description: jobResult.description, - filename: jobResult.filename + 'id': jobResult.id, + 'creation-date': jobResult.creation_date, + 'description': jobResult.description, + 'filename': jobResult.filename }; }, - sortValueName: 'creationDate', + sortValueName: 'creation-date', valueNames: [ {data: ['id']}, - {data: ['creationDate']}, + {data: ['creation-date']}, 'description', 'filename' ] diff --git a/app/static/js/RessourceLists/QueryResultList.js b/app/static/js/RessourceLists/QueryResultList.js index 429cc70a..c78ea3cb 100644 --- a/app/static/js/RessourceLists/QueryResultList.js +++ b/app/static/js/RessourceLists/QueryResultList.js @@ -3,7 +3,7 @@ class QueryResultList extends RessourceList { item: `

-
+
delete send @@ -12,19 +12,19 @@ class QueryResultList extends RessourceList { `.trim(), ressourceMapper: queryResult => { return { - id: queryResult.id, - corpusTitle: queryResult.corpus_title, - creationDate: queryResult.creation_date, - description: queryResult.description, - query: queryResult.query, - title: queryResult.title + 'id': queryResult.id, + 'corpus-title': queryResult.corpus_title, + 'creation-date': queryResult.creation_date, + 'description': queryResult.description, + 'query': queryResult.query, + 'title': queryResult.title }; }, - sortValueName: 'creationDate', + sortValueName: 'creation-date', valueNames: [ {data: ['id']}, - {data: ['creationDate']}, - 'corpusTitle', + {data: ['creation-date']}, + 'corpus-title', 'description', 'query', 'title' @@ -118,7 +118,7 @@ class QueryResultList extends RessourceList { re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)/(corpus_title|description|query|title)$`); if (re.test(operation.path)) { [match, queryResultId, valueName] = operation.path.match(re); - this.replace(queryResultId, valueName, operation.value); + this.replace(queryResultId, valueName.replace('_', '-'), operation.value); } break; default: diff --git a/app/static/js/RessourceLists/UserList.js b/app/static/js/RessourceLists/UserList.js index e52eac48..5f488f47 100644 --- a/app/static/js/RessourceLists/UserList.js +++ b/app/static/js/RessourceLists/UserList.js @@ -2,10 +2,10 @@ class UserList extends RessourceList { static options = { item: ` - + - + delete @@ -16,21 +16,22 @@ class UserList extends RessourceList { `.trim(), ressourceMapper: user => { return { - id: user.id, - idDuplicate: user.id, - username: user.username, - email: user.email, - last_seen: new Date(user.last_seen).toLocaleString("en-US"), - role: user.role.name + 'id': user.id, + 'id-1': user.id, + 'username': user.username, + 'email': user.email, + 'last-seen': new Date(user.last_seen).toLocaleString("en-US"), + 'member-since': user.member_since, + 'role': user.role.name }; }, - sortValueName: 'memberSince', + sortValueName: 'member-since', valueNames: [ {data: ['id']}, - {data: ['memberSince']}, + {data: ['member-since']}, 'email', - 'idDuplicate', - 'last_seen', + 'id-1', + 'last-seen', 'role', 'username' ] diff --git a/app/templates/_colors.html.j2 b/app/templates/_colors.html.j2 index a6ac0ed8..ee170aac 100644 --- a/app/templates/_colors.html.j2 +++ b/app/templates/_colors.html.j2 @@ -32,14 +32,24 @@ } %} {% set status = { - 'unprepared': '#9e9e9e', - 'submitted': '#9e9e9e', - 'queued': '#2196f3', - 'running': '#ffc107', - 'complete': '#4caf50', - 'failed': '#f44336', - 'prepared': '#4caf50', - 'start analysis': '#2196f3', - 'analysing': '#4caf50', - 'stop analysis': '#ff5722' + 'corpus': { + 'UNPREPARED': '#9e9e9e', + 'QUEUED': '#2196f3', + 'BUILDING': '#ffc107', + 'BUILT': '#4caf50', + 'FAILED': '#f44336', + 'STARTING_ANALYSIS_SESSION': '#2196f3', + 'RUNNING_ANALYSIS_SESSION': '#4caf50', + 'CANCELING_ANALYSIS_SESSION': '#ff5722' + }, + 'job': { + 'INITIALIZING': '#9e9e9e', + 'SUBMITTED': '#9e9e9e', + 'QUEUED': '#2196f3', + 'RUNNING': '#ffc107', + 'CANCELING': '#ff5722', + 'CANCELED': '#ff5722', + 'COMPLETED': '#4caf50', + 'FAILED': '#f44336' + } } %} diff --git a/app/templates/_sidenav.html.j2 b/app/templates/_sidenav.html.j2 index 8729f4f8..10986506 100644 --- a/app/templates/_sidenav.html.j2 +++ b/app/templates/_sidenav.html.j2 @@ -10,14 +10,14 @@
  • emailNews
  • linear_scaleWorkflow
  • dashboardDashboard
    • -
  • IMy Corpora
    • -
  • JMy Jobs
    • +
  • IMy Corpora
    • +
  • JMy Jobs
  • Processes & Services
    • -
  • File setup
    • -
  • OCR
    • -
  • NLP
    • -
  • Corpus analysis
    • +
  • File setup
    • +
  • OCR
    • +
  • NLP
    • +
  • Corpus analysis
  • Account
  • settingsSettings
    • diff --git a/app/templates/_styles.html.j2 b/app/templates/_styles.html.j2 index d94765de..14a614f0 100644 --- a/app/templates/_styles.html.j2 +++ b/app/templates/_styles.html.j2 @@ -45,29 +45,31 @@ main .tabs .indicator {background-color: {{ colors['baseline']['secondary'] }};} {% for service in colors['services'] %} - .service-scheme[data-service="{{ service }}"] {background-color: {{ colors['services'][service]['lighten'] }};} - .service-scheme[data-service="{{ service }}"] .btn, .service-scheme[data-service="{{ service }}"] .btn-small, .service-scheme[data-service="{{ service }}"] .btn-large, .service-scheme[data-service="{{ service }}"] .btn-floating {background-color: {{ colors['services'][service]['darken'] }};} - .service-scheme[data-service="{{ service }}"] .btn:hover, .service-scheme[data-service="{{ service }}"] .btn-large:hover, .service-scheme[data-service="{{ service }}"] .btn-small:hover, .service-scheme[data-service="{{ service }}"] .btn-floating:hover {background-color: {{ colors['services'][service]['base'] }};} - .service-scheme[data-service="{{ service }}"] .pagination li.active {background-color: {{ colors['services'][service]['darken'] }};} - .service-scheme[data-service="{{ service }}"] .table-of-contents a.active {border-color: {{ colors['services'][service]['darken'] }};} - .service-scheme[data-service="{{ service }}"] .tabs .tab a {color: inherit;} - .service-scheme[data-service="{{ service }}"] .tabs .tab.disabled a, .service-scheme[data-service="{{ service }}"] .tabs .tab.disabled a:hover {color: {{ colors['services'][service]['darken'] }}28;} - .service-scheme[data-service="{{ service }}"] .tabs .tab a:hover {color: {{ colors['services'][service]['darken'] }};} - .service-scheme[data-service="{{ service }}"] .tabs .tab a.active, .service-scheme[data-service="{{ service }}"] .tabs .tab a:focus.active {color: {{ colors['services'][service]['darken'] }}; background-color: {{ colors['services'][service]['darken'] }}28;} - .service-scheme[data-service="{{ service }}"] .tabs .indicator {background-color: {{ colors['services'][service]['darken'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] {background-color: {{ colors['services'][service]['lighten'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .btn, .nopaque-service-scheme[data-service="{{ service }}"] .btn-small, .nopaque-service-scheme[data-service="{{ service }}"] .btn-large, .nopaque-service-scheme[data-service="{{ service }}"] .btn-floating {background-color: {{ colors['services'][service]['darken'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .btn:hover, .nopaque-service-scheme[data-service="{{ service }}"] .btn-large:hover, .nopaque-service-scheme[data-service="{{ service }}"] .btn-small:hover, .nopaque-service-scheme[data-service="{{ service }}"] .btn-floating:hover {background-color: {{ colors['services'][service]['base'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .pagination li.active {background-color: {{ colors['services'][service]['darken'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .table-of-contents a.active {border-color: {{ colors['services'][service]['darken'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab a {color: inherit;} + .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab.disabled a, .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab.disabled a:hover {color: {{ colors['services'][service]['darken'] }}28;} + .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab a:hover {color: {{ colors['services'][service]['darken'] }};} + .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab a.active, .nopaque-service-scheme[data-service="{{ service }}"] .tabs .tab a:focus.active {color: {{ colors['services'][service]['darken'] }}; background-color: {{ colors['services'][service]['darken'] }}28;} + .nopaque-service-scheme[data-service="{{ service }}"] .tabs .indicator {background-color: {{ colors['services'][service]['darken'] }};} - .service-color[data-service="{{ service }}"] {background-color: {{ colors['services'][service]['base'] }} !important;} - .service-color-text[data-service="{{ service }}"] {color: {{ colors['services'][service]['base'] }} !important;} - .service-color-border[data-service="{{ service }}"] {border-color: {{ colors['services'][service]['base'] }} !important;} - .service-color[data-service="{{ service }}"].darken {background-color: {{ colors['services'][service]['darken'] }} !important;} - .service-color-text[data-service="{{ service }}"].text-darken {color: {{ colors['services'][service]['darken'] }} !important;} - .service-color-border[data-service="{{ service }}"].border-darken {border-color: {{ colors['services'][service]['darken'] }} !important;} - .service-color[data-service="{{ service }}"].lighten {background-color: {{ colors['services'][service]['lighten'] }} !important;} - .service-color-text[data-service="{{ service }}"].text-lighten {color: {{ colors['services'][service]['lighten'] }} !important;} - .service-color-border[data-service="{{ service }}"].border-lighten {border-color: {{ colors['services'][service]['lighten'] }} !important;} + .nopaque-service-color[data-service="{{ service }}"] {background-color: {{ colors['services'][service]['base'] }} !important;} + .nopaque-service-color-text[data-service="{{ service }}"] {color: {{ colors['services'][service]['base'] }} !important;} + .nopaque-service-color-border[data-service="{{ service }}"] {border-color: {{ colors['services'][service]['base'] }} !important;} + .nopaque-service-color[data-service="{{ service }}"].darken {background-color: {{ colors['services'][service]['darken'] }} !important;} + .nopaque-service-color-text[data-service="{{ service }}"].text-darken {color: {{ colors['services'][service]['darken'] }} !important;} + .nopaque-service-color-border[data-service="{{ service }}"].border-darken {border-color: {{ colors['services'][service]['darken'] }} !important;} + .nopaque-service-color[data-service="{{ service }}"].lighten {background-color: {{ colors['services'][service]['lighten'] }} !important;} + .nopaque-service-color-text[data-service="{{ service }}"].text-lighten {color: {{ colors['services'][service]['lighten'] }} !important;} + .nopaque-service-color-border[data-service="{{ service }}"].border-lighten {border-color: {{ colors['services'][service]['lighten'] }} !important;} {% endfor %} - {% for status in colors['status'] %} - .status-color[data-status="{{ status }}"] {background-color: {{ colors['status'][status] }} !important;} + {% for status_type in colors['status'] %} + {% for status in colors['status'][status_type] %} + .nopaque-{{ status_type }}-status-color[data-{{status_type}}-status="{{ status }}"] {background-color: {{ colors['status'][status_type][status] }} !important;} + {% endfor %} {% endfor %} diff --git a/app/templates/admin/edit_user.html.j2 b/app/templates/admin/edit_user.html.j2 index d824c112..f44ccb7c 100644 --- a/app/templates/admin/edit_user.html.j2 +++ b/app/templates/admin/edit_user.html.j2 @@ -6,66 +6,132 @@
    -

    Edit user

    +

    {{ title }}

    -
    -

    {{ user.username }}

    -

    Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,

    - arrow_backBack to user administration -
    - -
    -
    -
    +
    + + {{ edit_general_settings_form.hidden_tag() }} +
    - {{ form.hidden_tag() }} - {{ wtf.render_field(form.username, data_length='64', material_icon='account_circle') }} - {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }} - {{ wtf.render_field(form.role, material_icon='swap_vert') }} -
    -

     

    -
    -

    brightness_3

    -
    -
    -

    {{ form.dark_mode.label.text }}

    -

    Enable dark mode to ease your eyes.

    -
    -
    -
    - -
    -
    -

     

    -
    -

     

    -
    -

    check

    -
    -
    -

    {{ form.confirmed.label.text }}

    -

    Change confirmation status manually.

    -
    -
    -
    - -
    + General settings + {{ wtf.render_field(edit_general_settings_form.username, data_length='64', material_icon='person') }} + {{ wtf.render_field(edit_general_settings_form.email, data_length='254', material_icon='email') }} +
    +
    +
    + {{ wtf.render_field(edit_general_settings_form.submit, material_icon='send') }} +
    +
    + +
    + + +
    + {{ edit_interface_settings_form.hidden_tag() }} +
    +
    + Interface settings +
    +

     

    +
    +

    brightness_3

    +
    +
    +

    {{ edit_interface_settings_form.dark_mode.label.text }}

    +

    Enable dark mode to ease your eyes.

    +
    +
    +
    +
    -
    - {{ wtf.render_field(form.submit, material_icon='send') }} +
    +
    +
    + {{ wtf.render_field(edit_interface_settings_form.submit, material_icon='send') }}
    - +
    +
    + + +
    + {{ edit_notification_settings_form.hidden_tag() }} +
    +
    + Notification settings + {{ wtf.render_field(edit_notification_settings_form.job_status_mail_notification_level, material_icon='notifications') }} +
    +
    +
    + {{ wtf.render_field(edit_notification_settings_form.submit, material_icon='send') }} +
    +
    +
    +
    + +
    + {{ admin_edit_user_form.hidden_tag() }} +
    +
    + Administrator settings + {{ wtf.render_field(admin_edit_user_form.role, material_icon='swap_vert') }} +
    +

     

    +
    +

    check

    +
    +
    +

    {{ admin_edit_user_form.confirmed.label.text }}

    +

    Change confirmation status manually.

    +
    +
    +
    + +
    +
    +
    +
    +
    + {{ wtf.render_field(admin_edit_user_form.submit, material_icon='send') }} +
    +
    +
    + +
    +
    + Delete account +

    Deleting an account has the following effects:

    +
      +
    • All data associated with your corpora and jobs will be permanently deleted.
      • +
    • All settings will be permanently deleted.
      • +
    +
    +
    + deleteDelete
    {% endblock page_content %} + +{% block modals %} +{{ super() }} + +{% endblock modals %} diff --git a/app/templates/corpora/add_corpus.html.j2 b/app/templates/corpora/add_corpus.html.j2 index 9d7f24e4..820230c8 100644 --- a/app/templates/corpora/add_corpus.html.j2 +++ b/app/templates/corpora/add_corpus.html.j2 @@ -2,7 +2,7 @@ {% from "corpora/_breadcrumbs.html.j2" import breadcrumbs with context %} {% import "materialize/wtf.html.j2" as wtf %} -{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} +{% block main_attribs %} class="nopaque-service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} {% block page_content %}
    diff --git a/app/templates/corpora/add_corpus_file.html.j2 b/app/templates/corpora/add_corpus_file.html.j2 index 8246821a..d1d3c4fd 100644 --- a/app/templates/corpora/add_corpus_file.html.j2 +++ b/app/templates/corpora/add_corpus_file.html.j2 @@ -2,7 +2,7 @@ {% from "corpora/_breadcrumbs.html.j2" import breadcrumbs with context %} {% import "materialize/wtf.html.j2" as wtf %} -{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} +{% block main_attribs %} class="nopaque-service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} {% block page_content %}
    diff --git a/app/templates/corpora/analyse_corpus.html.j2 b/app/templates/corpora/analyse_corpus.html.j2 index 11389faf..d94ed274 100644 --- a/app/templates/corpora/analyse_corpus.html.j2 +++ b/app/templates/corpora/analyse_corpus.html.j2 @@ -1,11 +1,11 @@ {% extends "base.html.j2" %} {% import "materialize/wtf.html.j2" as wtf %} -{% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %} +{% block main_attribs %} class="nopaque-service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %} {% block page_content %} diff --git a/app/templates/corpora/corpus.html.j2 b/app/templates/corpora/corpus.html.j2 index 7ca7b413..07f4373b 100644 --- a/app/templates/corpora/corpus.html.j2 +++ b/app/templates/corpora/corpus.html.j2 @@ -1,7 +1,7 @@ {% extends "base.html.j2" %} {% from "corpora/_breadcrumbs.html.j2" import breadcrumbs with context %} -{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} +{% block main_attribs %} class="nopaque-service-scheme" data-service="corpus-analysis"{% endblock main_attribs %} {% block page_content %}
    @@ -14,8 +14,8 @@

     

     

    - -