Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/opaque into development

app/admin/forms.py

@@ -5,29 +5,22 @@ from wtforms import (BooleanField, SelectField, StringField, SubmitField,
 from wtforms.validators import DataRequired, Email, Length, Regexp
 
 
-class EditProfileAdminForm(FlaskForm):
+class EditUserForm(FlaskForm):
     email = StringField('Email',
                         validators=[DataRequired(), Length(1, 64), Email()])
-    username = StringField(
+    username = StringField('Username',
-        'Username',
+                           validators=[DataRequired(), Length(1, 64),
-        validators=[
+                                       Regexp('^[A-Za-z][A-Za-z0-9_.]*$', 0,
-            DataRequired(),
+                                              'Usernames must have only '
-            Length(1, 64),
+                                              'letters, numbers, dots or '
-            Regexp(
+                                              'underscores')])
-                '^[A-Za-z][A-Za-z0-9_.]*$',
-                0,
-                'Usernames must have only letters, numbers, dots or '
-                'underscores'
-            )
-        ]
-    )
     confirmed = BooleanField('Confirmed')
     role = SelectField('Role', coerce=int)
     name = StringField('Real name', validators=[Length(0, 64)])
     submit = SubmitField('Update Profile')
 
     def __init__(self, user, *args, **kwargs):
-        super(EditProfileAdminForm, self).__init__(*args, **kwargs)
+        super(EditUserForm, self).__init__(*args, **kwargs)
         self.role.choices = [(role.id, role.name)
                              for role in Role.query.order_by(Role.name).all()]
         self.user = user

app/admin/tables.py

@@ -21,7 +21,7 @@ class AdminUserTable(Table):
     id = Col('User Id', column_html_attrs={'class': 'id'},
              th_html_attrs={'class': 'sort',
                             'data-sort': 'id'})
-    url = LinkCol('Profile', 'admin.admin_user_page',
+    url = LinkCol('Profile', 'admin.user',
                   url_kwargs=dict(user_id='id'),
                   anchor_attrs={'class': 'waves-effect waves-light btn-small'})
 

app/admin/views.py

@@ -1,88 +1,66 @@
 from app import db
 from app.decorators import admin_required
 from app.models import Role, User
-from app.tables import AdminUserItem, AdminUserTable
+from app.profile.background_functions import delete_user_
-from app.background_functions import delete_user_
 from flask import current_app, flash, redirect, render_template, url_for
 from flask_login import login_required
+from threading import Thread
 from . import admin
-from .forms import EditProfileAdminForm
+from .forms import EditUserForm
-import threading
+from .tables import AdminUserItem, AdminUserTable
 
 
-@admin.route('/overview', methods=['GET', 'POST'])
+@admin.route('/')
 @login_required
 @admin_required
-def for_admins_only():
+def index():
-    users = User.query.order_by(User.username).all()
+    users = User.query.all()
     items = [AdminUserItem(u.username, u.email, u.role_id, u.confirmed, u.id)
              for u in users]
     # Convert table object to html string
     table = AdminUserTable(items).__html__()
     # Add class "list" to tbody element. Needed for "List.js"
     table = table.replace('tbody', 'tbody class="list"', 1)
-    return render_template('admin/admin.html.j2',
+    return render_template('admin/index.html.j2', table=table,
-                           table=table,
                            title='Administration tools')
 
 
-@admin.route('/overview/admin_user_page/<int:user_id>',
+@admin.route('/user/<int:user_id>')
-             methods=['GET', 'POST'])
 @login_required
 @admin_required
-def admin_user_page(user_id):
+def user(user_id):
-    selected_user = User.query.filter_by(id=user_id).first()
-    title = 'Administration of user {} with ID: {}'.format(
-        selected_user.username,
-        selected_user.id
-    )
-    registration_date = selected_user.registration_date.strftime(
-        '%A, %e %B %H:%M'
-    )
-    return render_template('admin/admin_user_page.html.j2',
-                           registration_date=registration_date,
-                           selected_user=selected_user,
-                           title=title)
-
-
-@admin.route('/overview/admin_user_page/delete/<int:user_id>',
-             methods=['GET', 'POST'])
-@login_required
-@admin_required
-def admin_delete_user(user_id):
-    delete_thread = threading.Thread(target=delete_user_,
-                                     args=(current_app._get_current_object(),
-                                           user_id))
-    delete_thread.start()
-    flash('User {} has been deleted!'.format(user_id))
-    return redirect(url_for('admin.for_admins_only'))
-
-
-@admin.route('/overview/admin_user_page/edit_profile_admin/<int:user_id>',
-             methods=['GET', 'POST'])
-@login_required
-@admin_required
-def edit_profile_admin(user_id):
     user = User.query.get_or_404(user_id)
-    form = EditProfileAdminForm(user=user)
+    return render_template('admin/user.html.j2', title='Administration: User',
-    if form.validate_on_submit():
+                           user=user)
-        user.email = form.email.data
+
-        user.username = form.username.data
+
-        user.confirmed = form.confirmed.data
+@admin.route('/user/<int:user_id>/delete')
-        user.role = Role.query.get(form.role.data)
+@login_required
+@admin_required
+def delete_user(user_id):
+    user = User.query.get_or_404(user_id)
+    thread = Thread(target=delete_user_,
+                    args=(current_app._get_current_object(), user.id))
+    thread.start()
+    flash('User has been deleted!')
+    return redirect(url_for('admin.index'))
+
+
+@admin.route('/user/<int:user_id>/edit', methods=['GET', 'POST'])
+@login_required
+@admin_required
+def edit_user(user_id):
+    user = User.query.get_or_404(user_id)
+    edit_user_form = EditUserForm(user=user)
+    if edit_user_form.validate_on_submit():
+        user.email = edit_user_form.email.data
+        user.username = edit_user_form.username.data
+        user.confirmed = edit_user_form.confirmed.data
+        user.role = Role.query.get(edit_user_form.role.data)
         db.session.add(user)
         db.session.commit()
         flash('The profile has been updated.')
-        return redirect(url_for('admin.edit_profile_admin', user_id=user.id))
+        return redirect(url_for('admin.edit_user', user_id=user.id))
-    form.email.data = user.email
+    return render_template('admin/edit_user.html.j2',
-    form.username.data = user.username
+                           edit_user_form=edit_user_form,
-    form.confirmed.data = user.confirmed
+                           title='Administration: Edit user', user=user)
-    form.role.data = user.role_id
-    title = 'Edit profile of user {} with ID {}'.format(
-        user.username,
-        user.id
-    )
-    return render_template('admin/edit_profile_admin.html.j2',
-                           form=form,
-                           title=title,
-                           user=user)

app/auth/views.py

@@ -41,15 +41,15 @@ def logout():
 def register():
     if not current_user.is_anonymous:
         return redirect(url_for('main.dashboard'))
-    form = RegistrationForm()
+    registration_form = RegistrationForm()
-    if form.validate_on_submit():
+    if registration_form.validate_on_submit():
-        user = User(email=form.email.data.lower(),
+        user = User(email=registration_form.email.data.lower(),
-                    password=form.password.data,
+                    password=registration_form.password.data,
-                    username=form.username.data)
+                    username=registration_form.username.data)
         db.session.add(user)
         db.session.commit()
         dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-                           str(job.user_id), 'jobs', str(job.id))
+                           str(user.id))
         try:
             os.makedirs(dir)
         except OSError:
@@ -57,15 +57,11 @@ def register():
             user.delete()
         else:
             token = user.generate_confirmation_token()
-            send_email(user.email,
+            send_email(user.email, 'Confirm Your Account',
-                       'Confirm Your Account',
+                       'auth/email/confirm', token=token, user=user)
-                       'auth/email/confirm',
-                       token=token,
-                       user=user)
             flash('A confirmation email has been sent to you by email.')
             return redirect(url_for('auth.login'))
-    return render_template('auth/register.html.j2',
+    return render_template('auth/register.html.j2', form=registration_form,
-                           form=form,
                            title='Register')
 
 

app/background_functions.py

@@ -1,25 +0,0 @@
-from .models import Corpus, Job, User
-
-
-def delete_corpus_(app, corpus_id):
-    with app.app_context():
-        corpus = Corpus.query.filter_by(id=corpus_id).first()
-        if corpus is None:
-            raise Exception('Corpus {} not found!'.format(corpus_id))
-        corpus.delete()
-
-
-def delete_job_(app, job_id):
-    with app.app_context():
-        job = Job.query.filter_by(id=job_id).first()
-        if job is None:
-            raise Exception('Job {} not found!'.format(job_id))
-        job.delete()
-
-
-def delete_user_(app, user_id):
-    with app.app_context():
-        user = User.query.filter_by(id=user_id).first()
-        if user is None:
-            raise Exception('User {} not found!'.format(user_id))
-        user.delete()

app/corpora/background_functions.py

@@ -3,7 +3,7 @@ from app.models import Corpus, CorpusFile
 
 def delete_corpus_(app, corpus_id):
     with app.app_context():
-        corpus = Corpus.query.filter_by(id=corpus_id).first()
+        corpus = Corpus.query.get(corpus_id)
         if corpus is None:
             raise Exception('Corpus {} not found!'.format(corpus_id))
         corpus.delete()
@@ -11,7 +11,7 @@ def delete_corpus_(app, corpus_id):
 
 def delete_corpus_file_(app, corpus_file_id):
     with app.app_context():
-        corpus_file = CorpusFile.query.filter_by(id=corpus_file_id).first()
+        corpus_file = CorpusFile.query.get(corpus_file_id)
         if corpus_file is None:
             raise Exception('Corpus file {} not found!'.format(corpus_file_id))
         corpus_file.delete()
@@ -19,7 +19,7 @@ def delete_corpus_file_(app, corpus_file_id):
 
 def edit_corpus_file_(app, corpus_file_id):
     with app.app_context():
-        corpus_file = CorpusFile.query.filter_by(id=corpus_file_id).first()
+        corpus_file = CorpusFile.query.get(corpus_file_id)
         if corpus_file is None:
             raise Exception('Corpus file {} not found!'.format(corpus_file_id))
         corpus_file.insert_metadata()

app/corpora/events.py

@@ -21,68 +21,90 @@ analysis_clients = {}
 @socketio.on('init_corpus_analysis')
 @login_required
 def init_corpus_analysis(corpus_id):
-    corpus = Corpus.query.filter_by(id=corpus_id).first()
+    corpus = Corpus.query.get(corpus_id)
     if corpus is None:
         socketio.emit('init_corpus_analysis', '[ERROR 404]: Not Found',
                       room=request.sid)
-    if not (corpus.creator == current_user or current_user.is_administrator()):
+    elif not (corpus.creator == current_user
+              or current_user.is_administrator()):
         socketio.emit('init_corpus_analysis', '[ERROR 403]: Forbidden',
                       room=request.sid)
-    if str(corpus_id) not in analysis_sessions:
+    else:
-        analysis_sessions[str(corpus_id)] = [request.sid]
+        if corpus_id not in analysis_sessions:
-    socketio.start_background_task(observe_corpus_analysis_connection,
+            analysis_sessions[corpus_id] = [request.sid]
-                                   current_app._get_current_object(),
+        else:
-                                   corpus_id, request.sid)
+            analysis_sessions[corpus_id].append(request.sid)
+        while corpus.status != 'analysing':
+            db.session.refresh(corpus)
+            socketio.sleep(3)
+        analysis_clients[request.sid] = CQiClient(
+            host='{}_analysis_container{}'.format(corpus.creator.username,
+                                                  corpus.id))
+        analysis_clients[request.sid].ctrl_connect('opaque', 'opaque')
+        socketio.emit('init_corpus_analysis', 'Ready', room=request.sid)
+        socketio.start_background_task(observe_corpus_analysis_connection,
+                                       current_app._get_current_object(),
+                                       corpus_id, request.sid)
 
 
-@socketio.on('query_event')
+@socketio.on('query')
 @login_required
 def recv_query(message):
-    logger.warning(message)
+    analysis_client = analysis_clients.get(request.sid)
-    analysis_client = analysis_clients[request.sid]
+    if analysis_client is None:
-    analysis_client.connect()
+        socketio.emit('query', '[ERROR 424]: Failed Dependency',
+                      room=request.sid)
+        return
     """ Prepare and execute a query """
     corpus_name = 'CORPUS'
     query = message['query']
-    result_subcorpus_name = 'Results'
+    query_subcorpus = 'Results'
-    analysis_client.set_corpus_name(corpus_name)
+    analysis_client.cqp_query(corpus, query_subcorpus, query)
-    logger.warning('Corpus name has been set.')
+    """ Evaluate query results """
-    analysis_client.create_attribute_strings()
+    match_corpus = '{}:{}'.format(corpus, query_subcorpus)
-    logger.warning('Attribute Strings have been created.')
+    match_num = min(int(message['hits_per_page']) - 1,
-    analysis_client.query_subcorpus(result_subcorpus_name, query)
+                    analysis_client.cqp_subcorpus_size(match_corpus))
-    logger.warning('Subcorpus from query has been created.')
+    if match_num == 0:
-    subcorpora = analysis_client.show_subcorpora()
+        print('No matches found.')
-    logger.warning('Known subcorpora: {}'.format(subcorpora))
+        exit()
-    matches = analysis_client.show_results(result_start_count=1,
+    if not analysis_client.cqp_subcorpus_has_field(match_corpus, CONST_FIELD_MATCH):
-                                           result_max_count=3)
+        print('Error.')
-    logger.warning('Match data: {}'.format(matches))
+        exit()
-    socketio.emit('query_results', matches, room=request.sid)
+    if not analysis_client.cqp_subcorpus_has_field(match_corpus, CONST_FIELD_MATCHEND):
+        print('Error')
+        exit()
+    match_boundaries = zip(analysis_client.cqp_dump_subcorpus(match_corpus, CONST_FIELD_MATCH, 0, match_num - 1),
+                           analysis_client.cqp_dump_subcorpus(match_corpus, CONST_FIELD_MATCHEND, 0, match_num - 1))
+    matches = []
+    for match_start, match_end in match_boundaries:
+        matches.append({'cpos_list': list(range(match_start, match_end + 1))})
+    cpos_list = []
+    for match in matches:
+        cpos_list = cpos_list + match['cpos_list']
+    cpos_list = list(set(cpos_list))
+    pos_list = analysis_client.cl_cpos2str('{}.pos'.format(corpus), cpos_list)
+    word_list = analysis_client.cl_cpos2str('{}.word'.format(corpus), cpos_list)
+    foo = {}
+    for cpos, pos, word in zip(cpos_list, pos_list, word_list):
+        foo[cpos] = {'pos': pos, 'word': word}
+    for match in matches:
+        match['pos_list'] = [foo[cpos]['pos'] for cpos in match['cpos_list']]
+        match['word_list'] = [foo[cpos]['word'] for cpos in match['cpos_list']]
+        match.pop('cpos_list', None)
+    logger.warning(matches)
+    socketio.emit('query', matches, room=request.sid)
 
 
 def observe_corpus_analysis_connection(app, corpus_id, session_id):
     with app.app_context():
-        corpus = Corpus.query.filter_by(id=corpus_id).first()
-        while corpus.status != 'analysing':
-            db.session.refresh(corpus)
-            socketio.sleep(3)
-        analysis_client = CQiWrapper(host='{}_analysis_container{}'.format(corpus.creator.username, corpus.id), port=4877, password='opaque', username='opaque')
-        analysis_clients[session_id] = analysis_client
-        socketio.emit('init_corpus_analysis', 'Ready', room=session_id)
         while session_id in connected_sessions:
-            '''
-            try:
-                analysis_client.ctrl_ping()
-            except Exception as err:
-                logger.warning('[Exception]: {}'.format(err))
-                break
-            else:
-                socketio.sleep(3)
-            '''
             socketio.sleep(3)
-        analysis_client.disconnect()
+        analysis_client = analysis_clients.pop(session_id, None)
-        analysis_clients.pop(session_id, None)
+        if analysis_client is not None:
-        analysis_sessions[str(corpus_id)].remove(session_id)
+            analysis_client.ctrl_bye()
-        if not analysis_sessions[str(corpus_id)]:
+        analysis_sessions[corpus_id].remove(session_id)
-            analysis_sessions.pop(str(corpus_id), None)
+        if not analysis_sessions[corpus_id]:
+            analysis_sessions.pop(corpus_id, None)
+            corpus = Corpus.query.get(corpus_id)
             corpus.status = 'stop analysis'
             db.session.commit()

app/corpora/views.py

@@ -1,16 +1,16 @@
-from app import db, logger
+from app import db
 from app.models import Corpus, CorpusFile
 from flask import (abort, current_app, flash, redirect, request,
                    render_template, url_for, send_from_directory)
 from flask_login import current_user, login_required
+from threading import Thread
 from werkzeug.utils import secure_filename
 from . import corpora
 from .background_functions import (delete_corpus_, delete_corpus_file_,
-                               edit_corpus_file_)
+                                   edit_corpus_file_)
 from .forms import (AddCorpusFileForm, AddCorpusForm, EditCorpusFileForm,
                     QueryDownloadForm, QueryForm)
 import os
-import threading
 
 
 @corpora.route('/add', methods=['GET', 'POST'])
@@ -48,31 +48,21 @@ def corpus(corpus_id):
                            title='Corpus')
 
 
-@corpora.route('/<int:corpus_id>/analysis', methods=['GET', 'POST'])
+@corpora.route('/<int:corpus_id>/analyse')
 @login_required
-def corpus_analysis(corpus_id):
+def analyse_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if corpus.status == 'prepared':
         corpus.status = 'start analysis'
         db.session.commit()
-    query = request.args.get('query')
+    query_download_form = QueryDownloadForm()
-    logger.warning('Query first: {}'.format(query))
+    query_form = QueryForm(context=request.args.get('context', 10),
-    hits_per_page = request.args.get('hits_per_page', 30)
+                           hits_per_page=request.args.get('hits_per_page', 30),
-    context = request.args.get('context', 10)
+                           query=request.args.get('query'))
-    dl_form = QueryDownloadForm()
+    return render_template('corpora/analyse_corpus.html.j2',
-    form = QueryForm(hits_per_page=hits_per_page, context=context, query=query)
-    if form.validate_on_submit():
-        flash('Query has been sent!')
-        query = form.query.data
-        hits_per_page = form.hits_per_page.data
-        context = form.context.data
-        return redirect(url_for('corpora.corpus_analysis', corpus_id=corpus_id,
-                                query=query, hits_per_page=hits_per_page,
-                                context=context))
-    return render_template('corpora/corpus_analysis.html.j2',
                            corpus_id=corpus_id,
-                           form=form, dl_form=dl_form,
+                           query_download_form=query_download_form,
-                           title='Corpus: {}'.format(corpus.title))
+                           query_form=query_form, title='Analyse Corpus')
 
 
 @corpora.route('/<int:corpus_id>/delete')
@@ -81,9 +71,8 @@ def delete_corpus(corpus_id):
     corpus = Corpus.query.get_or_404(corpus_id)
     if not (corpus.creator == current_user or current_user.is_administrator()):
         abort(403)
-    thread = threading.Thread(target=delete_corpus_,
+    thread = Thread(target=delete_corpus_,
-                              args=(current_app._get_current_object(),
+                    args=(current_app._get_current_object(), corpus.id))
-                                    corpus.id))
     thread.start()
     flash('Corpus deleted!')
     return redirect(url_for('main.dashboard'))
@@ -102,23 +91,21 @@ def add_corpus_file(corpus_id):
         for corpus_file in corpus.files:
             if filename == corpus_file.filename:
                 flash('File already registered to this corpus.')
-                return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+                return redirect(url_for('corpora.add_corpus_file',
+                                        corpus_id=corpus_id))
         # Save the file
         dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
         file.save(os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
                                dir, filename))
-
         corpus_file = CorpusFile(author=add_corpus_file_form.author.data,
-                                 corpus=corpus,
+                                 corpus=corpus, dir=dir, filename=filename,
-                                 dir=dir,
-                                 filename=filename,
                                  publishing_year=add_corpus_file_form.publishing_year.data,
                                  title=add_corpus_file_form.title.data)
         db.session.add(corpus_file)
         db.session.commit()
-        thread = threading.Thread(target=edit_corpus_file_,
+        thread = Thread(target=edit_corpus_file_,
-                                  args=(current_app._get_current_object(),
+                        args=(current_app._get_current_object(),
-                                        corpus_file.id))
+                              corpus_file.id))
         thread.start()
         flash('Corpus file added!')
         return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
@@ -136,9 +123,8 @@ def delete_corpus_file(corpus_id, corpus_file_id):
     if not (corpus_file.corpus.creator == current_user
             or current_user.is_administrator()):
         abort(403)
-    thread = threading.Thread(target=delete_corpus_file_,
+    thread = Thread(target=delete_corpus_file_,
-                              args=(current_app._get_current_object(),
+                    args=(current_app._get_current_object(), corpus_file.id))
-                                    corpus_file.id))
     thread.start()
     flash('Corpus file deleted!')
     return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
@@ -175,9 +161,9 @@ def edit_corpus_file(corpus_id, corpus_file_id):
         corpus_file.publishing_year = edit_corpus_file_form.publishing_year.data
         corpus_file.title = edit_corpus_file_form.title.data
         db.session.commit()
-        thread = threading.Thread(target=edit_corpus_file_,
+        thread = Thread(target=edit_corpus_file_,
-                                  args=(current_app._get_current_object(),
+                        args=(current_app._get_current_object(),
-                                        corpus_file.id))
+                              corpus_file.id))
         thread.start()
         flash('Corpus file edited!')
         return redirect(url_for('corpora.corpus', corpus_id=corpus_id))

app/email.py

@@ -13,7 +13,7 @@ def send_email(to, subject, template, **kwargs):
     msg = Message('[Opaque] {}'.format(subject), recipients=[to])
     msg.body = render_template(template + '.txt.j2', **kwargs)
     msg.html = render_template(template + '.html.j2', **kwargs)
-    thr = Thread(target=send_async_email,
+    thread = Thread(target=send_async_email,
-                 args=[current_app._get_current_object(), msg])
+                    args=(current_app._get_current_object(), msg))
-    thr.start()
+    thread.start()
-    return thr
+    return thread

app/events.py

@@ -72,7 +72,10 @@ def user_ressource_subscription_handler(app, user_id, session_id,
                      else 'update-jobs'}
     with app.app_context():
         # Gather current values from database.
-        user = User.query.filter_by(id=user_id).first()
+        user = User.query.get(user_id)
+        if user is None:
+            ''' TODO: Handle this '''
+            return
         corpora = {corpus.id: corpus.to_dict() for corpus in user.corpora}
         jobs = {job.id: job.to_dict() for job in user.jobs}
         # Send initial values to the user.

app/jobs/background_functions.py

@@ -0,0 +1,9 @@
+from app.models import Job
+
+
+def delete_job_(app, job_id):
+    with app.app_context():
+        job = Job.query.get(job_id)
+        if job is None:
+            raise Exception('Job {} not found!'.format(job_id))
+        job.delete()

app/jobs/views.py

@@ -1,11 +1,11 @@
 from app.models import Job, JobInput, JobResult
-from app.background_functions import delete_job_
 from flask import (abort, current_app, flash, redirect, render_template,
                    send_from_directory, url_for)
 from flask_login import current_user, login_required
+from threading import Thread
 from . import jobs
+from .background_functions import delete_job_
 import os
-import threading
 
 
 @jobs.route('/<int:job_id>')
@@ -23,10 +23,9 @@ def delete_job(job_id):
     job = Job.query.get_or_404(job_id)
     if not (job.creator == current_user or current_user.is_administrator()):
         abort(403)
-    delete_thread = threading.Thread(target=delete_job_,
+    thread = Thread(target=delete_job_,
-                                     args=(current_app._get_current_object(),
+                    args=(current_app._get_current_object(), job_id))
-                                           job_id))
+    thread.start()
-    delete_thread.start()
     flash('Job has been deleted!')
     return redirect(url_for('main.dashboard'))
 

app/profile/background_functions.py

@@ -0,0 +1,9 @@
+from app.models import User
+
+
+def delete_user_(app, user_id):
+    with app.app_context():
+        user = User.query.get(user_id)
+        if user is None:
+            raise Exception('User {} not found!'.format(user_id))
+        user.delete()

app/profile/views.py

@@ -1,10 +1,10 @@
 from app import db, logger
-from app.background_functions import delete_user_
 from flask import abort, current_app, flash, redirect, render_template, url_for
 from flask_login import current_user, login_required, logout_user
+from threading import Thread
 from . import profile
+from .background_functions import delete_user_
 from .forms import ChangePasswordForm, EditProfileForm, EditUserSettingsForm
-import threading
 
 
 @profile.route('/', methods=['GET', 'POST'])
@@ -94,10 +94,9 @@ def delete_self():
     """
     View to delete yourslef and all associated data.
     """
-    delete_thread = threading.Thread(target=delete_user_,
+    thread = Thread(target=delete_user_,
-                                     args=(current_app._get_current_object(),
+                    args=(current_app._get_current_object(), current_user.id))
-                                           current_user.id))
+    thread.start()
-    delete_thread.start()
     logout_user()
     flash('Your account has been deleted!')
     return redirect(url_for('main.index'))

app/services/views.py

@@ -1,7 +1,8 @@
-from app import db
+from app import db, logger
 from app.jobs.forms import AddNLPJobForm, AddOCRJobForm
 from app.models import Job, JobInput
-from flask import abort, current_app, flash, redirect, render_template, url_for
+from flask import (abort, current_app, flash, make_response, render_template,
+                   request, url_for)
 from flask_login import current_user, login_required
 from werkzeug.utils import secure_filename
 from . import services
@@ -23,7 +24,9 @@ def service(service):
     if service not in SERVICES:
         abort(404)
     add_job_form = SERVICES[service]['add_job_form']()
-    if add_job_form.validate_on_submit():
+    if add_job_form.is_submitted():
+        if not add_job_form.validate():
+            return make_response(add_job_form.errors, 400)
         service_args = []
         if service == 'nlp':
             service_args.append('-l {}'.format(add_job_form.language.data))
@@ -46,8 +49,11 @@ def service(service):
         try:
             os.makedirs(absolut_dir)
         except OSError:
-            flash('[ERROR]: Could not add job!')
             job.delete()
+            flash('Internal Server Error')
+            return make_response(
+                {'redirect_url': url_for('services.service', service='ocr')},
+                500)
         else:
             for file in add_job_form.files.data:
                 filename = secure_filename(file.filename)
@@ -58,7 +64,8 @@ def service(service):
             job.status = 'submitted'
             db.session.commit()
             flash('Job added!')
-            return redirect(url_for('jobs.job', job_id=job.id))
+            return make_response(
+                {'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
     return render_template('services/{}.html.j2'.format(service),
                            title=SERVICES[service]['name'],
                            add_job_form=add_job_form)

app/static/js/JobList.js

@@ -24,8 +24,7 @@ class JobList extends List {
       pathArray = operation.path.split("/").slice(1);
       switch(operation.op) {
         case "add":
-          console.log(pathArray);
+          if (pathArray.includes("results")) {break;}
-          if (pathArray[1].includes("results")) {break;}
           this.addJob(operation.value);
           this.update();
           List.updatePagination(this);
@@ -80,12 +79,8 @@ class JobList extends List {
     serviceIcon = JobList.SERVICE_ICONS[job.service]
                   || JobList.SERVICE_ICONS['default'];
     jobServiceElement = document.createElement("i");
-    jobServiceElement.classList.add(
+    jobServiceElement.classList.add("circle", "material-icons", "service-icon",
-      "circle",
+                                    serviceColor);
-      "material-icons",
-      "service-icon",
-      serviceColor
-    );
     jobServiceElement.innerText = serviceIcon;
     statusColor = JobList.STATUS_COLORS[job.status]
                   || JobList.STATUS_COLORS['default'];

app/static/js/add_job.js

@@ -14,9 +14,15 @@ function SubmitAddJobForm(newJobFormElement, progressModalElement, request) {
     progressModalElement.querySelector(".determinate").style.width = progressInPercent;
   });
   request.addEventListener("load", function(event) {
-    console.log(request.response);
+    if (request.status === 201) {
-    newJobFormElement.reset();
+      window.location.href = JSON.parse(this.responseText)['redirect_url'];
-    location.reload();
+    }
+    if (request.status === 400) {
+      console.log(JSON.parse(this.responseText));
+    }
+    if (request.status === 500) {
+      location.reload();
+    }
   });
   request.addEventListener("abort", function(event) {
     progressModalElement.querySelector(".progress-in-percent").innerHTML = "0%";

app/templates/admin/admin.html.j2

@@ -1,28 +0,0 @@
-{% extends "full_width.html.j2" %}
-
-{% block page_content %}
-<div class="col s12">
-  <div class="card">
-    <div class="card-content">
-      <span class="card-title">User list</span>
-      <div id="users">
-        <div class="input-field">
-          <i class="material-icons prefix">search</i>
-          <input id="search-corpus" class="search" type="text"></input>
-          <label for="search-corpus">Search users</label>
-        </div>
-        {{ table }}
-        <ul class="pagination"></ul>
-      </div>
-    </div>
-  </div>
-</div>
-<script type="text/javascript">
-var options = {
-  valueNames: ['username', 'email', 'role', 'confirmed', 'id'],
-  page: 10,
-  pagination: true
-};
-var userList = new List('users', options);
-</script>
-{% endblock %}

app/templates/admin/admin_user_page.html.j2

@@ -1,104 +0,0 @@
-{% extends "limited_width.html.j2" %}
-
-{% block page_content %}
-<div class="col s12 m6">
-  <div class="card large">
-    <div class="card-content">
-      <span class="card-title">User information</span>
-        <ul>
-          <li>Username: {{selected_user.username}}</li>
-          <li>Email: {{selected_user.email}}</li>
-          <li>ID: {{selected_user.id}}</li>
-          <li>Registration date: {{registration_date}}</li>
-          <li>Confirmed status: {{selected_user.confirmed}}</li>
-          <li>Role ID: {{selected_user.role_id}}</li>
-          <li>Permissions as Int: {{selected_user.role.permissions}}</li>
-          <li>Role name: {{selected_user.role.name}}</li>
-        </ul>
-        <div class="card-action">
-          <a href="{{url_for('admin.edit_profile_admin', user_id=selected_user.id)}}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit user</a>
-        <a href="#modal-confirm-delete" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete User</a>
-        <!-- Modal Strucutre -->
-        <div id="modal-confirm-delete" class="modal">
-          <div class="modal-content">
-            <h4>Confirm deletion</h4>
-              <p>Do you really want to delete the current selected user ({{selected_user.username}})?
-              All associated jobs and job files will be permanently deleted.</p>
-          </div>
-          <div class="modal-footer">
-            <a href="{{url_for('admin.admin_delete_user', user_id=selected_user.id)}}" class="modal-close waves-effect waves-green btn red"><i class="material-icons left">delete</i>Delete User</a>
-            <a href="#!" class="modal-close waves-effect waves-green btn cancel">Cancel</a>
-          </div>
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-<script>
-  socket.emit('subscribe_foreign_user_ressources', {{ selected_user.id }});
-</script>
-<div class="col s12 m6">
-  <div id="job-foreign-list">
-    <div class="card">
-      <div class="card-content">
-        <span class="card-title">User Jobs</span>
-        <div class="row">
-          <div class="col s12">
-            <div class="input-field">
-              <i class="material-icons prefix">search</i>
-              <input id="search-job" class="search" type="text"></input>
-              <label for="search-job">Search job</label>
-            </div>
-          </div>
-          <div class="col s12">
-            <ul class="pagination"></ul>
-          </div>
-        </div>
-      </div>
-    </div>
-    <div class="collection list"></div>
-  </div>
-</div>
-<script>
-  var jobList = new JobList("job-foreign-list", foreignJobsSubscribers, {
-    item: '<div><span class="title"></span><span class="description"></span></div>',
-    page: 4,
-    pagination: true,
-    valueNames: ["description", "title", {data: ["id"]}]
-  });
-  jobList.on("filterComplete", List.updatePagination);
-  jobList.on("searchComplete", List.updatePagination);
-</script>
-<div class="col s12 m6">
-  <div id="corpus-foreign-list">
-    <div class="card">
-      <div class="card-content">
-        <span class="card-title">User Corpora</span>
-        <div class="row">
-          <div class="col s12">
-            <div class="input-field">
-              <i class="material-icons prefix">search</i>
-              <input id="search-job" class="search" type="text"></input>
-              <label for="search-job">Search corpus</label>
-            </div>
-          </div>
-          <div class="col s12">
-            <ul class="pagination"></ul>
-          </div>
-        </div>
-      </div>
-    </div>
-    <div class="collection list"></div>
-  </div>
-</div>
-<script>
-  var corpusList = new CorpusList("corpus-foreign-list", foreignCorporaSubscribers, {
-    item: '<div><span class="title"></span><span class="description"></span></div>',
-    page: 4,
-    pagination: true,
-    valueNames: ["description", "title", {data: ["id"]}]
-  });
-  corpusList.on("filterComplete", List.updatePagination);
-  corpusList.on("searchComplete", List.updatePagination);
-</script>
-{% endblock %}

app/templates/admin/edit_profile_admin.html.j2

@@ -1,59 +0,0 @@
-{% extends "limited_width.html.j2" %}
-
-{% block page_content %}
-<div class="col s12 m8">
-  <div class="card">
-    <form method="POST">
-      <div class="card-content">
-        {{ form.hidden_tag() }}
-        <div class="input-field ">
-          <i class="material-icons prefix">account_circle</i>
-          {{ form.username() }}
-          {{ form.username.label }}
-          {% for error in form.username.errors %}
-            <span class="helper-text red-text">{{ error }}</span>
-          {% endfor %}
-        </div>
-        <div class="input-field">
-          <i class="material-icons prefix">mail</i>
-          {{ form.email() }}
-          {{ form.email.label }}
-          {% for error in form.email.errors %}
-            <span class="helper-text red-text">{{ error }}</span>
-          {% endfor %}
-        </div>
-        <div class="input-field">
-          <i class="material-icons prefix">swap_vert</i>
-          {{ form.role() }}
-          {{ form.role.label }}
-          {% for error in form.role.errors %}
-            <span class="helper-text red-text">{{ error }}</span>
-          {% endfor %}
-        </div>
-        <div class="switch">
-          <i class="material-icons prefix">check</i>
-          <label class="active" for="{{form.confirmed.name}}">
-            Confirmed status:
-            Off
-            {% if form.confirmed.data == True %}
-              <input type="checkbox" id="{{form.confirmed.name}}" name="{{form.confirmed.name}}" checked="checked">
-            {% else %}
-              <input type="checkbox" id="{{form.confirmed.name}}" name="{{form.confirmed.name}}">
-            {% endif %}
-            <span class="lever"></span>
-            On
-          </label>
-          {% for error in form.confirmed.errors %}
-            <span class="helper-text red-text">{{ error }}</span>
-          {% endfor %}
-        </div>
-      </div>
-      <div class="card-action right-align">
-        {{ form.submit(class='btn') }}
-      </div>
-    </div>
-    </form>
-  </div>
-</div>
-
-{% endblock %}

app/templates/admin/edit_user.html.j2

@@ -0,0 +1,64 @@
+{% extends "limited_width.html.j2" %}
+
+{% block page_content %}
+<div class="col s12 m4">
+  <h3 id="title">{{ user.username }}</h3>
+  <p id="description">Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p>
+  <a class="waves-effect waves-light btn" href="{{ url_for('admin.user', user_id=user.id) }}"><i class="material-icons left">arrow_back</i>Back to user administration</a>
+</div>
+
+<div class="col s12 m8">
+  <div class="card">
+    <form method="POST">
+      <div class="card-content">
+        {{ edit_user_form.hidden_tag() }}
+        <div class="input-field ">
+          <i class="material-icons prefix">account_circle</i>
+          {{ edit_user_form.username() }}
+          {{ edit_user_form.username.label }}
+          {% for error in edit_user_form.username.errors %}
+          <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <div class="input-field">
+          <i class="material-icons prefix">mail</i>
+          {{ edit_user_form.email() }}
+          {{ edit_user_form.email.label }}
+          {% for error in edit_user_form.email.errors %}
+          <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <div class="input-field">
+          <i class="material-icons prefix">swap_vert</i>
+          {{ edit_user_form.role() }}
+          {{ edit_user_form.role.label }}
+          {% for error in edit_user_form.role.errors %}
+          <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <div class="switch">
+          <i class="material-icons prefix">check</i>
+          <label class="active" for="{{ edit_user_form.confirmed.name }}">
+            Confirmed status:
+            Off
+            {% if edit_user_form.confirmed.data == True %}
+            <input type="checkbox" id="{{ edit_user_form.confirmed.name }}" name="{{ edit_user_form.confirmed.name }}" checked="checked">
+            {% else %}
+            <input type="checkbox" id="{{ edit_user_form.confirmed.name }}" name="{{ edit_user_form.confirmed.name }}">
+            {% endif %}
+            <span class="lever"></span>
+            On
+          </label>
+          {% for error in edit_user_form.confirmed.errors %}
+          <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+      </div>
+      <div class="card-action right-align">
+        <button class="btn waves-effect waves-light" id="submit" name="submit" type="submit">Submit<i class="material-icons right">send</i></button>
+      </div>
+    </form>
+  </div>
+</div>
+
+{% endblock %}

app/templates/admin/index.html.j2

@@ -0,0 +1,26 @@
+{% extends "full_width.html.j2" %}
+
+{% block page_content %}
+<div class="col s12">
+  <div id="user-list">
+    <div class="card">
+      <div class="card-content">
+        <span class="card-title">User list</span>
+        <div class="input-field">
+          <i class="material-icons prefix">search</i>
+          <input id="search-user" class="search" type="text"></input>
+          <label for="search-user">Search user</label>
+        </div>
+        {{ table }}
+        <ul class="pagination"></ul>
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+  var options = {page: 10, pagination: true,
+                 valueNames: ['username', 'email', 'role', 'confirmed', 'id']};
+  var userList = new List('user-list', options);
+</script>
+{% endblock %}

app/templates/admin/user.html.j2

@@ -0,0 +1,115 @@
+{% extends "limited_width.html.j2" %}
+
+{% block page_content %}
+<div class="col s12 m4">
+  <h3 id="title">{{ user.username }}</h3>
+  <p id="description">Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p>
+  <a class="waves-effect waves-light btn" href="{{ url_for('admin.index') }}"><i class="material-icons left">arrow_back</i>Back to admin board</a>
+</div>
+
+<div class="col s12 m8">
+  <div class="card">
+    <div class="card-content">
+      <span class="card-title">User information</span>
+      <ul>
+        <li>Username: {{ user.username }}</li>
+        <li>Email: {{ user.email }}</li>
+        <li>ID: {{ user.id }}</li>
+        <li>Registration date: {{ user.registration_date.strftime('%m/%d/%Y, %H:%M:%S %p') }}</li>
+        <li>Confirmed status: {{ user.confirmed }}</li>
+        <li>Role ID: {{ user.role_id }}</li>
+        <li>Permissions as Int: {{ user.role.permissions }}</li>
+        <li>Role name: {{ user.role.name }}</li>
+      </ul>
+    </div>
+    <div class="card-action right-align">
+      <a href="{{ url_for('admin.edit_user', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit user</a>
+      <a data-target="delete-user-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete user</a>
+    </div>
+  </div>
+</div>
+
+<div class="col s12"></div>
+
+<div class="col s12 m6">
+  <div id="corpus-list">
+    <div class="card">
+      <div class="card-content">
+        <span class="card-title">Corpora</span>
+        <div class="row">
+          <div class="col s12">
+            <div class="input-field">
+              <i class="material-icons prefix">search</i>
+              <input id="search-corpus" class="search" type="text"></input>
+              <label for="search-corpus">Search corpus</label>
+            </div>
+          </div>
+          <div class="col s12">
+            <ul class="pagination"></ul>
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="collection list"></div>
+  </div>
+</div>
+
+<div class="col s12 m6">
+  <div id="job-list">
+    <div class="card">
+      <div class="card-content">
+        <span class="card-title">Jobs</span>
+        <div class="row">
+          <div class="col s12">
+            <div class="input-field">
+              <i class="material-icons prefix">search</i>
+              <input id="search-job" class="search" type="text"></input>
+              <label for="search-job">Search job</label>
+            </div>
+          </div>
+          <div class="col s12">
+            <ul class="pagination"></ul>
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="collection list"></div>
+  </div>
+</div>
+
+
+<!-- Modals -->
+<div id="delete-user-modal" class="modal">
+  <div class="modal-content">
+    <h4>Confirm user deletion</h4>
+    <p>Do you really want to delete the user {{ user.username }}? All associated data will be permanently deleted!</p>
+  </div>
+  <div class="modal-footer">
+    <a href="#!" class="modal-close waves-effect waves-green btn cancel">Cancel</a>
+    <a href="{{ url_for('admin.delete_user', user_id=user.id) }}" class="modal-close waves-effect waves-green btn red">Confirm<i class="material-icons right">send</i></a>
+  </div>
+</div>
+
+
+<script>
+  socket.emit('subscribe_foreign_user_ressources', {{ user.id }});
+
+  var corpusList = new CorpusList("corpus-list", foreignCorporaSubscribers, {
+    item: '<div><span class="title"></span><span class="description"></span></div>',
+    page: 4,
+    pagination: true,
+    valueNames: ["description", "title", {data: ["id"]}]
+  });
+  corpusList.on("filterComplete", List.updatePagination);
+  corpusList.on("searchComplete", List.updatePagination);
+
+  var jobList = new JobList("job-list", foreignJobsSubscribers, {
+    item: '<div><span class="title"></span><span class="description"></span></div>',
+    page: 4,
+    pagination: true,
+    valueNames: ["description", "title", {data: ["id"]}]
+  });
+  jobList.on("filterComplete", List.updatePagination);
+  jobList.on("searchComplete", List.updatePagination);
+</script>
+{% endblock %}

app/templates/base.html.j2

@@ -37,7 +37,6 @@
       var jobsSubscribers = [];
       var socket = io();
 
-      socket.emit('subscribe_user_ressources');
       socket.on('init-corpora', function(msg) {
         corpora = JSON.parse(msg);
         for (let subscriber of corporaSubscribers) {subscriber._init(corpora);}
@@ -141,7 +140,7 @@
         {% if current_user.is_administrator() %}
         <li><div class="divider"></div></li>
         <li><a class="subheader">Administration</a></li>
-        <li><a href="{{ url_for('admin.for_admins_only') }}"><i class="material-icons">build</i>Administration tools</a></li>
+        <li><a href="{{ url_for('admin.index') }}"><i class="material-icons">build</i>Administration tools</a></li>
         {% endif %}
         <div class="hide-on-large-only">
           <li><div class="divider"></div></li>
@@ -170,28 +169,31 @@
       </div>
     </footer>
 
-    <script type="text/javascript" src="{{ url_for('static', filename='js/materialize.min.js') }}"></script>
     <script>
-        M.AutoInit();
+      socket.emit('subscribe_user_ressources');
-        M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="text"]'))
+    </script>
-        M.Dropdown.init(
+    <script src="{{ url_for('static', filename='js/materialize.min.js') }}"></script>
-          document.getElementById("nav-notifications"),
+    <script>
-          {"alignment": "right", "constrainWidth": false, "coverTrigger": false}
+      M.AutoInit();
-        );
+      M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="text"]'))
-        M.Dropdown.init(
+      M.Dropdown.init(
-          document.getElementById("nav-account"),
+        document.getElementById("nav-notifications"),
-          {"alignment": "right", "constrainWidth": false, "coverTrigger": false}
+        {"alignment": "right", "constrainWidth": false, "coverTrigger": false}
-        );
+      );
-        // Highlight current navigation entry
+      M.Dropdown.init(
-        var entry;
+        document.getElementById("nav-account"),
-        for (entry of document.querySelectorAll("#slide-out a:not(.subheader)")) {
+        {"alignment": "right", "constrainWidth": false, "coverTrigger": false}
-          if (entry.href === window.location.href) {
+      );
-            entry.parentNode.classList.add("active");
+      // Highlight current navigation entry
-          }
+      var entry;
+      for (entry of document.querySelectorAll("#slide-out a:not(.subheader)")) {
+        if (entry.href === window.location.href) {
+          entry.parentNode.classList.add("active");
         }
-        {% for message in get_flashed_messages() %}
+      }
-        M.toast({html: '{{ message }}'})
+      {% for message in get_flashed_messages() %}
-        {% endfor %}
+      M.toast({html: '{{ message }}'})
+      {% endfor %}
     </script>
   </body>
 </html>

app/templates/corpora/analyse_corpus.html.j2

@@ -0,0 +1,125 @@
+{% extends "full_width.html.j2" %}
+
+{% block page_content %}
+<div class="col s12 m3 sticky">
+  <div class="card">
+    <div class="card-content">
+      <form id="query-form" method="POST">
+        {{ query_form.hidden_tag() }}
+        <span class="card-title">Query and analysis</span>
+        <div class="input-field">
+          {{ query_form.query(class='materialize-textarea') }}
+          {{ query_form.query.label }}
+          {% for error in query_form.query.errors %}
+            <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <button class="btn waves-effect waves-light" id="query-form-submit" style="width: 100%;" type="submit">Start Query</button>
+        <p>&nbsp;</p>
+        <span class="card-title">Help</span>
+        <p><a href="http://cwb.sourceforge.net/files/CQP_Tutorial/">CQP Query Language Tutorial</a></p>
+        <p>&nbsp;</p>
+        <span class="card-title">Options</span>
+        <div class="input-field">
+          <i class="material-icons prefix">format_list_numbered</i>
+          {{ query_form.hits_per_page() }}
+          {{ query_form.hits_per_page.label }}
+          {% for error in query_form.hits_per_page.errors %}
+            <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <div class="input-field">
+          <i class="material-icons prefix">short_text</i>
+          {{ query_form.context() }}
+          {{ query_form.context.label }}
+          {% for error in query_form.context.errors %}
+            <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+      </form>
+    </div>
+  </div>
+
+  <div class="card">
+    <div class="card-content">
+      <form id="download-form" method="POST">
+        {{ query_download_form.hidden_tag() }}
+        <span class="card-title">Download Results</span>
+        <p>Downlaod all results of the current query as csv, excel or json file.</p>
+        <div class="input-field">
+          <i class="material-icons prefix">insert_drive_file</i>
+          {{ query_download_form.file_type() }}
+          {{ query_download_form.file_type.label }}
+          {% for error in query_download_form.file_type.errors %}
+            <span class="helper-text red-text">{{ error }}</span>
+          {% endfor %}
+        </div>
+        <button class="btn waves-effect waves-light" id="download-form-submit" style="width: 100%;" type="submit">Download</button>
+      </form>
+    </div>
+  </div>
+</div>
+
+<div class="col s12 m9">
+  <div class="card">
+    <div class="card-content">
+      <span class="card-title">Query Results</span>
+      <div id="query-results"></div>
+    </div>
+  </div>
+</div>
+
+<div id="loading-modal" class="modal no-autoinit">
+  <div class="modal-content">
+    <h4>Waiting for analysis software</h4>
+    <div class="preloader-wrapper big active">
+      <div class="spinner-layer spinner-blue-only">
+        <div class="circle-clipper left">
+          <div class="circle"></div>
+        </div>
+        <div class="gap-patch">
+          <div class="circle"></div>
+        </div>
+        <div class="circle-clipper right">
+          <div class="circle"></div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+  var loadingModal;
+
+  document.addEventListener('DOMContentLoaded', function() {
+    loadingModal = M.Modal.init(document.getElementById("loading-modal"),
+                                {"dismissible": false});
+    loadingModal.open();
+  });
+  socket.emit('init_corpus_analysis', {{ corpus_id }});
+
+  var queryFormElement = document.getElementById("query-form");
+  var queryFormSubmitElement = document.getElementById("query-form-submit");
+  var queryResultsElement = document.getElementById("query-results");
+
+  queryFormSubmitElement.addEventListener('click', function(event) {
+    event.preventDefault();
+    let formData = new FormData(queryFormElement);
+    let queryData = {'context': formData.get('context'),
+                     'hits_per_page': formData.get('hits_per_page'),
+                     'query': formData.get('query')};
+    socket.emit('query', queryData);
+    M.toast({html: 'Query has been sent!'});
+  });
+
+  socket.on('init_corpus_analysis', function(msg) {
+    if (msg === 'Ready') {loadingModal.close();}
+  });
+  socket.on('query', function(results) {
+    queryResultsElement.innerHTML = '';
+    for (let result of results) {
+      queryResultsElement.innerHTML += '<p>' + result['word_list'] + '</p>';
+    }
+  });
+</script>
+{% endblock %}

app/templates/corpora/corpus.html.j2

@@ -1,83 +1,6 @@
 {% extends "limited_width.html.j2" %}
 
 {% block page_content %}
-<script>
-  {% if corpus.creator == current_user %}
-  var foreignCorpusFlag = false;
-  {% else %}
-  var foreignCorpusFlag = true;
-  socket.emit('subscribe_foreign_user_ressources', {{ corpus.user_id }});
-  {% endif %}
-
-  class InformationUpdater {
-    constructor(corpusId) {
-      this.corpusId = corpusId;
-      if (foreignCorpusFlag) {
-        foreignCorporaSubscribers.push(this);
-      } else {
-        corporaSubscribers.push(this);
-      }
-    }
-
-    _init() {
-      if (foreignCorpusFlag) {
-        this.corpus = foreignCorpora[this.corpusId];
-      } else {
-        this.corpus = corpora[this.corpusId];
-      }
-
-      // Status
-      this.setStatus(this.corpus.status);
-    }
-
-    _update(patch) {
-      var pathArray;
-
-      for (let operation of patch) {
-        /* "/corpusId/valueName" -> ["corpusId", "valueName"] */
-        pathArray = operation.path.split("/").slice(1);
-        if (pathArray[0] != this.corpusId) {continue;}
-        switch(operation.op) {
-          case "add":
-            location.reload();
-            break;
-          case "delete":
-            location.reload();
-            break;
-          case "replace":
-            if (pathArray[1] === "status") {
-              this.setStatus(operation.value);
-            }
-            break;
-          default:
-            break;
-        }
-      }
-    }
-
-    setStatus(status) {
-      var statusElement;
-      statusElement = document.getElementById("status");
-      statusElement.classList.remove(...Object.values(CorpusList.STATUS_COLORS));
-      statusElement.classList.add(CorpusList.STATUS_COLORS[status] || CorpusList.STATUS_COLORS['default']);
-      statusElement.innerText = status;
-      var analyseBtn = document.getElementById('analyse');
-      if (status === 'prepared' || status === 'analysing') {
-        analyseBtn.classList.remove('hide', 'disabled');
-      } else if (status === 'start analysis' || status === 'stop analysis') {
-        analyseBtn.classList.remove('hide');
-        analyseBtn.classList.add('disabled');
-      }
-      if (status === 'prepared' || status === 'preparable' || status === 'preparing' || status === 'start analysis' || status === 'analysing' || status === 'stop analysis') {
-        var prepareBtn = document.getElementById('prepare');
-        prepareBtn.classList.add('hide');
-      }
-    }
-  }
-
-  var informationUpdater = new InformationUpdater({{ corpus.id }});
-</script>
-
 <div class="col s12 m4">
   <h3 id="title">{{ corpus.title }}</h3>
   <p id="description">{{ corpus.description }}</p>
@@ -98,7 +21,7 @@
       </div>
     </div>
     <div class="card-action right-align">
-      <a href="{{ url_for('corpora.corpus_analysis', corpus_id=corpus.id) }}" class="waves-effect waves-light btn hide" id="analyse"><i class="material-icons left">help</i>Analyse</a>
+      <a href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" class="waves-effect waves-light btn hide" id="analyse"><i class="material-icons left">help</i>Analyse</a>
       {% if corpus.files[0] is defined %}
       <a href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}" class="waves-effect waves-light btn" id="prepare"><i class="material-icons left">whatshot</i>Prepare</a>
       {% endif %}
@@ -146,11 +69,12 @@
   </div>
 </div>
 
+
 <!-- Modals -->
 <div id="delete-corpus-modal" class="modal">
   <div class="modal-content">
     <h4>Confirm corpus deletion</h4>
-    <p>Do you really want to delete the Corpus {{corpus.title}}? All files will be permanently deleted!</p>
+    <p>Do you really want to delete the corpus {{corpus.title}}? All files will be permanently deleted!</p>
   </div>
   <div class="modal-footer">
     <a href="#!" class="modal-close waves-effect waves-green btn cancel">Cancel</a>
@@ -169,5 +93,78 @@
     <a class="modal-close waves-effect waves-green btn red" href="{{ url_for('corpora.delete_corpus_file', corpus_file_id=file.id, corpus_id=corpus.id) }}">Confirm<i class="material-icons right">send</i></a>
   </div>
 </div>
+
+
+<script>
+  class InformationUpdater {
+    constructor(corpusId, foreignCorpusFlag) {
+      this.corpusId = corpusId;
+      this.foreignCorpusFlag = foreignCorpusFlag;
+      if (this.foreignCorpusFlag) {
+        foreignCorporaSubscribers.push(this);
+      } else {
+        corporaSubscribers.push(this);
+      }
+    }
+
+    _init() {
+      var corpus = this.foreignCorpusFlag ? foreignCorpora[this.corpusId] : corpora[this.corpusId];
+
+      // Status
+      this.setStatus(corpus.status);
+    }
+
+    _update(patch) {
+      var pathArray;
+
+      for (let operation of patch) {
+        /* "/corpusId/valueName" -> ["corpusId", "valueName"] */
+        pathArray = operation.path.split("/").slice(1);
+        if (pathArray[0] != this.corpusId) {continue;}
+        switch(operation.op) {
+          case "add":
+            location.reload();
+            break;
+          case "delete":
+            location.reload();
+            break;
+          case "replace":
+            if (pathArray[1] === "status") {
+              this.setStatus(operation.value);
+            }
+            break;
+          default:
+            break;
+        }
+      }
+    }
+
+    setStatus(status) {
+      var statusElement;
+      statusElement = document.getElementById("status");
+      statusElement.classList.remove(...Object.values(CorpusList.STATUS_COLORS));
+      statusElement.classList.add(CorpusList.STATUS_COLORS[status] || CorpusList.STATUS_COLORS['default']);
+      statusElement.innerText = status;
+      var analyseBtn = document.getElementById('analyse');
+      if (status === 'prepared' || status === 'analysing') {
+        analyseBtn.classList.remove('hide', 'disabled');
+      } else if (status === 'start analysis' || status === 'stop analysis') {
+        analyseBtn.classList.remove('hide');
+        analyseBtn.classList.add('disabled');
+      }
+      if (status === 'prepared' || status === 'preparable' || status === 'preparing' || status === 'start analysis' || status === 'analysing' || status === 'stop analysis') {
+        var prepareBtn = document.getElementById('prepare');
+        prepareBtn.classList.add('hide');
+      }
+    }
+  }
+
+  {% if corpus.creator == current_user %}
+  var informationUpdater = new InformationUpdater({{ corpus.id }}, false);
+  {% else %}
+  var informationUpdater = new InformationUpdater({{ corpus.id }}, true);
+  socket.emit('subscribe_foreign_user_ressources', {{ corpus.user_id }});
+  {% endif %}
+</script>
 {% endfor %}
 {% endblock %}

app/templates/jobs/job.html.j2

@@ -1,114 +1,6 @@
 {% extends "limited_width.html.j2" %}
 
 {% block page_content %}
-<script>
-  {% if job.creator == current_user %}
-  var foreignJobFlag = false;
-  {% else %}
-  var foreignJobFlag = true;
-  socket.emit('subscribe_foreign_user_ressources', {{ job.user_id }});
-  {% endif %}
-
-  class InformationUpdater {
-    constructor(jobId) {
-      this.jobId = jobId;
-      if (foreignJobFlag) {
-        foreignJobsSubscribers.push(this);
-      } else {
-        jobsSubscribers.push(this);
-      }
-    }
-
-    _init() {
-      if (foreignJobFlag) {
-        this.job = foreignJobs[this.jobId];
-      } else {
-        this.job = jobs[this.jobId];
-      }
-
-      // End date
-      this.setEndDate(this.job.end_date);
-      // Status
-      this.setStatus(this.job.status);
-      // End date
-      if (this.job.end_date) {this.setEndDate(this.job.end_date);}
-      // Input results
-      for (let input of this.job.inputs) {
-        for (let result of input.results) {
-          this.setResult(result);
-        }
-      }
-    }
-
-    _update(patch) {
-      var pathArray;
-
-      for (let operation of patch) {
-        /* "/jobId/valueName" -> ["jobId", "valueName"] */
-        pathArray = operation.path.split("/").slice(1);
-        if (pathArray[0] != this.jobId) {continue;}
-        switch(operation.op) {
-          case "add":
-            if (pathArray[1] === "inputs" && pathArray[3] === "results") {
-              this.setResult(operation.value);
-            }
-            break;
-          case "delete":
-            location.reload();
-            break;
-          case "replace":
-            if (pathArray[1] === "end_date") {
-              this.setEndDate(operation.value);
-            } else if (pathArray[1] === "status") {
-              this.setStatus(operation.value);
-            }
-            break;
-          default:
-            break;
-        }
-      }
-    }
-
-    setEndDate(timestamp) {
-      var end_date;
-      if (timestamp === null) {
-        end_date = "N.a.";
-      } else {
-        end_date = new Date(timestamp * 1000).toLocaleString("en-US");
-      }
-      document.getElementById("end-date").value = end_date;
-      M.updateTextFields();
-    }
-
-    setResult(result) {
-      var resultsElement, resultDownloadButtonElement,
-          resultDownloadButtonIconElement;
-      resultsElement = document.getElementById(`input-${result.job_input_id}-results`);
-      resultDownloadButtonElement = document.createElement("a");
-      resultDownloadButtonElement.classList.add("waves-effect", "waves-light", "btn-small");
-      resultDownloadButtonElement.href = `/jobs/${this.jobId}/results/${result.id}/download`;
-      resultDownloadButtonElement.innerText = result.filename.split(".").reverse()[0];
-      resultDownloadButtonElement.setAttribute("download", "");
-      resultDownloadButtonIconElement = document.createElement("i");
-      resultDownloadButtonIconElement.classList.add("material-icons", "left");
-      resultDownloadButtonIconElement.innerText = "file_download";
-      resultDownloadButtonElement.prepend(resultDownloadButtonIconElement);
-      resultsElement.append(resultDownloadButtonElement);
-      resultsElement.append(" ");
-    }
-
-    setStatus(status) {
-      var statusElement;
-      statusElement = document.getElementById("status");
-      statusElement.classList.remove(...Object.values(JobList.STATUS_COLORS));
-      statusElement.classList.add(JobList.STATUS_COLORS[status] || JobList.STATUS_COLORS['default']);
-      statusElement.innerText = status;
-    }
-  }
-
-  var informationUpdater = new InformationUpdater({{ job.id }});
-</script>
-
 <div class="col s12 m4">
   <h3 id="title">{{ job.title }}</h3>
   <p id="description">{{ job.description }}</p>
@@ -210,6 +102,7 @@
   </div>
 </div>
 
+
 <!-- Modals -->
 <div id="delete-job-modal" class="modal">
   <div class="modal-content">
@@ -221,4 +114,108 @@
     <a class="modal-close waves-effect waves-green btn red" href="{{ url_for('jobs.delete_job', job_id=job.id) }}">Confirm<i class="material-icons right">send</i></a>
   </div>
 </div>
+
+
+<script>
+  class InformationUpdater {
+    constructor(jobId, foreignJobFlag) {
+      this.jobId = jobId;
+      this.foreignJobFlag = foreignJobFlag;
+      if (this.foreignJobFlag) {
+        foreignJobsSubscribers.push(this);
+      } else {
+        jobsSubscribers.push(this);
+      }
+    }
+
+    _init() {
+      var job = this.foreignJobFlag ? foreignJobs[this.jobId] : jobs[this.jobId];
+
+      // End date
+      this.setEndDate(job.end_date);
+      // Status
+      this.setStatus(job.status);
+      // End date
+      if (job.end_date) {this.setEndDate(job.end_date);}
+      // Input results
+      for (let input of job.inputs) {
+        for (let result of input.results) {
+          this.setResult(result);
+        }
+      }
+    }
+
+    _update(patch) {
+      var pathArray;
+
+      for (let operation of patch) {
+        /* "/jobId/valueName" -> ["jobId", "valueName"] */
+        pathArray = operation.path.split("/").slice(1);
+        if (pathArray[0] != this.jobId) {continue;}
+        switch(operation.op) {
+          case "add":
+            if (pathArray[1] === "inputs" && pathArray[3] === "results") {
+              this.setResult(operation.value);
+            }
+            break;
+          case "delete":
+            location.reload();
+            break;
+          case "replace":
+            if (pathArray[1] === "end_date") {
+              this.setEndDate(operation.value);
+            } else if (pathArray[1] === "status") {
+              this.setStatus(operation.value);
+            }
+            break;
+          default:
+            break;
+        }
+      }
+    }
+
+    setEndDate(timestamp) {
+      var end_date;
+      if (timestamp === null) {
+        end_date = "N.a.";
+      } else {
+        end_date = new Date(timestamp * 1000).toLocaleString("en-US");
+      }
+      document.getElementById("end-date").value = end_date;
+      M.updateTextFields();
+    }
+
+    setResult(result) {
+      var resultsElement, resultDownloadButtonElement,
+          resultDownloadButtonIconElement;
+      resultsElement = document.getElementById(`input-${result.job_input_id}-results`);
+      resultDownloadButtonElement = document.createElement("a");
+      resultDownloadButtonElement.classList.add("waves-effect", "waves-light", "btn-small");
+      resultDownloadButtonElement.href = `/jobs/${this.jobId}/results/${result.id}/download`;
+      resultDownloadButtonElement.innerText = result.filename.split(".").reverse()[0];
+      resultDownloadButtonElement.setAttribute("download", "");
+      resultDownloadButtonIconElement = document.createElement("i");
+      resultDownloadButtonIconElement.classList.add("material-icons", "left");
+      resultDownloadButtonIconElement.innerText = "file_download";
+      resultDownloadButtonElement.prepend(resultDownloadButtonIconElement);
+      resultsElement.append(resultDownloadButtonElement);
+      resultsElement.append(" ");
+    }
+
+    setStatus(status) {
+      var statusElement;
+      statusElement = document.getElementById("status");
+      statusElement.classList.remove(...Object.values(JobList.STATUS_COLORS));
+      statusElement.classList.add(JobList.STATUS_COLORS[status] || JobList.STATUS_COLORS['default']);
+      statusElement.innerText = status;
+    }
+  }
+
+  {% if job.creator == current_user %}
+  var informationUpdater = new InformationUpdater({{ job.id }}, false);
+  {% else %}
+  var informationUpdater = new InformationUpdater({{ job.id }}, true);
+  socket.emit('subscribe_foreign_user_ressources', {{ job.user_id }});
+  {% endif %}
+</script>
 {% endblock %}