From 7dd55c47a068c962511c2abb5ea9e40769a6a5ad Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 14 Nov 2019 13:19:05 +0100 Subject: [PATCH 01/14] Code update to new request system --- app/corpora/events.py | 42 ++++++++++--------- app/templates/corpora/corpus_analysis.html.j2 | 2 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/app/corpora/events.py b/app/corpora/events.py index a597f8e6..2f674028 100644 --- a/app/corpora/events.py +++ b/app/corpora/events.py @@ -29,8 +29,10 @@ def init_corpus_analysis(corpus_id): if not (corpus.creator == current_user or current_user.is_administrator()): socketio.emit('init_corpus_analysis', '[ERROR 403]: Forbidden', room=request.sid) - if str(corpus_id) not in analysis_sessions: - analysis_sessions[str(corpus_id)] = [request.sid] + if corpus_id not in analysis_sessions: + analysis_sessions[corpus_id] = [request.sid] + else: + analysis_sessions[corpus_id].append(request.sid) socketio.start_background_task(observe_corpus_analysis_connection, current_app._get_current_object(), corpus_id, request.sid) @@ -48,8 +50,8 @@ def recv_query(message): analysis_client.cqp_query(corpus, query_subcorpus, query) """ Evaluate query results """ match_corpus = '{}:{}'.format(corpus, query_subcorpus) - match_num = analysis_client.cqp_subcorpus_size(match_corpus) - match_num = min(int(message['hits_per_page']) - 1, match_num) + match_num = min(int(message['hits_per_page']) - 1, + analysis_client.cqp_subcorpus_size(match_corpus)) if match_num == 0: print('No matches found.') exit() @@ -63,9 +65,20 @@ def recv_query(message): analysis_client.cqp_dump_subcorpus(match_corpus, CONST_FIELD_MATCHEND, 0, match_num - 1)) matches = [] for match_start, match_end in match_boundaries: - tokens = analysis_client.cl_cpos2str('{}.word'.format(corpus), range(match_start, match_end + 1)) - pos = analysis_client.cl_cpos2str('{}.pos'.format(corpus), range(match_start, match_end + 1)) - matches.append({'tokens': tokens, 'pos': pos}) + matches.append({'cpos_list': list(range(match_start, match_end + 1))}) + cpos_list = [] + for match in matches: + cpos_list = cpos_list + match['cpos_list'] + cpos_list = list(set(cpos_list)) + pos_list = analysis_client.cl_cpos2str('{}.pos'.format(corpus), cpos_list) + word_list = analysis_client.cl_cpos2str('{}.word'.format(corpus), cpos_list) + foo = {} + for cpos, pos, word in zip(cpos_list, pos_list, word_list): + foo[cpos] = {'pos': pos, 'word': word} + for match in matches: + match['pos_list'] = [foo[cpos]['pos'] for cpos in match['cpos_list']] + match['word_list'] = [foo[cpos]['word'] for cpos in match['cpos_list']] + match.pop('cpos_list', None) logger.warning(matches) socketio.emit('query_results', matches, room=request.sid) @@ -81,20 +94,11 @@ def observe_corpus_analysis_connection(app, corpus_id, session_id): analysis_clients[session_id] = analysis_client socketio.emit('init_corpus_analysis', 'Ready', room=session_id) while session_id in connected_sessions: - ''' - try: - analysis_client.ctrl_ping() - except Exception as err: - logger.warning('[Exception]: {}'.format(err)) - break - else: - socketio.sleep(3) - ''' socketio.sleep(3) analysis_client.ctrl_bye() analysis_clients.pop(session_id, None) - analysis_sessions[str(corpus_id)].remove(session_id) - if not analysis_sessions[str(corpus_id)]: - analysis_sessions.pop(str(corpus_id), None) + analysis_sessions[corpus_id].remove(session_id) + if not analysis_sessions[corpus_id]: + analysis_sessions.pop(corpus_id, None) corpus.status = 'stop analysis' db.session.commit() diff --git a/app/templates/corpora/corpus_analysis.html.j2 b/app/templates/corpora/corpus_analysis.html.j2 index ed35532a..b4837650 100644 --- a/app/templates/corpora/corpus_analysis.html.j2 +++ b/app/templates/corpora/corpus_analysis.html.j2 @@ -78,7 +78,7 @@ socket.on('query_results', function(json_results) { console.log(json_results) var queryResultsElement = document.getElementById("query-results"); for (let json_result of json_results) { - queryResultsElement.innerHTML += json_result['tokens'] + '
'; + queryResultsElement.innerHTML += json_result['word_list'] + '
'; } }); From 1b02d73d66c4b3e65ae25e076790ef50efb43f23 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 14 Nov 2019 13:40:05 +0100 Subject: [PATCH 02/14] Update background functions --- app/admin/views.py | 4 ++-- app/auth/views.py | 22 +++++++++------------- app/background_functions.py | 25 ------------------------- app/jobs/background_functions.py | 9 +++++++++ app/jobs/views.py | 2 +- app/profile/background_functions.py | 9 +++++++++ app/profile/views.py | 2 +- 7 files changed, 31 insertions(+), 42 deletions(-) delete mode 100644 app/background_functions.py create mode 100644 app/jobs/background_functions.py create mode 100644 app/profile/background_functions.py diff --git a/app/admin/views.py b/app/admin/views.py index 566fbac9..0cc915b1 100644 --- a/app/admin/views.py +++ b/app/admin/views.py @@ -2,7 +2,7 @@ from app import db from app.decorators import admin_required from app.models import Role, User from app.tables import AdminUserItem, AdminUserTable -from app.background_functions import delete_user_ +from app.profile.background_functions import delete_user_ from flask import current_app, flash, redirect, render_template, url_for from flask_login import login_required from . import admin @@ -31,7 +31,7 @@ def for_admins_only(): @login_required @admin_required def admin_user_page(user_id): - selected_user = User.query.filter_by(id=user_id).first() + selected_user = User.query.get_or_404(user_id) title = 'Administration of user {} with ID: {}'.format( selected_user.username, selected_user.id diff --git a/app/auth/views.py b/app/auth/views.py index 146294f3..74afc490 100644 --- a/app/auth/views.py +++ b/app/auth/views.py @@ -41,15 +41,15 @@ def logout(): def register(): if not current_user.is_anonymous: return redirect(url_for('main.dashboard')) - form = RegistrationForm() - if form.validate_on_submit(): - user = User(email=form.email.data.lower(), - password=form.password.data, - username=form.username.data) + registration_form = RegistrationForm() + if registration_form.validate_on_submit(): + user = User(email=registration_form.email.data.lower(), + password=registration_form.password.data, + username=registration_form.username.data) db.session.add(user) db.session.commit() dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], - str(job.user_id), 'jobs', str(job.id)) + str(user.id)) try: os.makedirs(dir) except OSError: @@ -57,15 +57,11 @@ def register(): user.delete() else: token = user.generate_confirmation_token() - send_email(user.email, - 'Confirm Your Account', - 'auth/email/confirm', - token=token, - user=user) + send_email(user.email, 'Confirm Your Account', + 'auth/email/confirm', token=token, user=user) flash('A confirmation email has been sent to you by email.') return redirect(url_for('auth.login')) - return render_template('auth/register.html.j2', - form=form, + return render_template('auth/register.html.j2', form=registration_form, title='Register') diff --git a/app/background_functions.py b/app/background_functions.py deleted file mode 100644 index b98b3bd1..00000000 --- a/app/background_functions.py +++ /dev/null @@ -1,25 +0,0 @@ -from .models import Corpus, Job, User - - -def delete_corpus_(app, corpus_id): - with app.app_context(): - corpus = Corpus.query.filter_by(id=corpus_id).first() - if corpus is None: - raise Exception('Corpus {} not found!'.format(corpus_id)) - corpus.delete() - - -def delete_job_(app, job_id): - with app.app_context(): - job = Job.query.filter_by(id=job_id).first() - if job is None: - raise Exception('Job {} not found!'.format(job_id)) - job.delete() - - -def delete_user_(app, user_id): - with app.app_context(): - user = User.query.filter_by(id=user_id).first() - if user is None: - raise Exception('User {} not found!'.format(user_id)) - user.delete() diff --git a/app/jobs/background_functions.py b/app/jobs/background_functions.py new file mode 100644 index 00000000..db47332e --- /dev/null +++ b/app/jobs/background_functions.py @@ -0,0 +1,9 @@ +from app.models import Job + + +def delete_job_(app, job_id): + with app.app_context(): + job = Job.query.filter_by(id=job_id).first() + if job is None: + raise Exception('Job {} not found!'.format(job_id)) + job.delete() diff --git a/app/jobs/views.py b/app/jobs/views.py index 20b86a23..7b1b24fd 100644 --- a/app/jobs/views.py +++ b/app/jobs/views.py @@ -1,9 +1,9 @@ from app.models import Job, JobInput, JobResult -from app.background_functions import delete_job_ from flask import (abort, current_app, flash, redirect, render_template, send_from_directory, url_for) from flask_login import current_user, login_required from . import jobs +from .background_functions import delete_job_ import os import threading diff --git a/app/profile/background_functions.py b/app/profile/background_functions.py new file mode 100644 index 00000000..db136f72 --- /dev/null +++ b/app/profile/background_functions.py @@ -0,0 +1,9 @@ +from app.models import User + + +def delete_user_(app, user_id): + with app.app_context(): + user = User.query.filter_by(id=user_id).first() + if user is None: + raise Exception('User {} not found!'.format(user_id)) + user.delete() diff --git a/app/profile/views.py b/app/profile/views.py index 2eaa1b02..82423059 100644 --- a/app/profile/views.py +++ b/app/profile/views.py @@ -1,8 +1,8 @@ from app import db, logger -from app.background_functions import delete_user_ from flask import abort, current_app, flash, redirect, render_template, url_for from flask_login import current_user, login_required, logout_user from . import profile +from .background_functions import delete_user_ from .forms import ChangePasswordForm, EditProfileForm, EditUserSettingsForm import threading From df98f3a9009e1557dbf6ab19e5e1a6b5047377e3 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 14 Nov 2019 15:34:07 +0100 Subject: [PATCH 03/14] Change HTML for corpus analysis --- app/corpora/events.py | 4 +- app/templates/corpora/corpus_analysis.html.j2 | 174 ++++++++---------- 2 files changed, 78 insertions(+), 100 deletions(-) diff --git a/app/corpora/events.py b/app/corpora/events.py index 2f674028..81c184c9 100644 --- a/app/corpora/events.py +++ b/app/corpora/events.py @@ -38,7 +38,7 @@ def init_corpus_analysis(corpus_id): corpus_id, request.sid) -@socketio.on('query_event') +@socketio.on('query') @login_required def recv_query(message): logger.warning(message) @@ -80,7 +80,7 @@ def recv_query(message): match['word_list'] = [foo[cpos]['word'] for cpos in match['cpos_list']] match.pop('cpos_list', None) logger.warning(matches) - socketio.emit('query_results', matches, room=request.sid) + socketio.emit('query', matches, room=request.sid) def observe_corpus_analysis_connection(app, corpus_id, session_id): diff --git a/app/templates/corpora/corpus_analysis.html.j2 b/app/templates/corpora/corpus_analysis.html.j2 index b4837650..7185f13d 100644 --- a/app/templates/corpora/corpus_analysis.html.j2 +++ b/app/templates/corpora/corpus_analysis.html.j2 @@ -1,37 +1,26 @@ {% extends "full_width.html.j2" %} {% block page_content %} -
+
arrow_backBack to corpus overview
-
- {{ form.hidden_tag() }} - Query and analysis -
-
+ + {{ form.hidden_tag() }} + Query and analysis +
{{ form.query(class='materialize-textarea') }} {{ form.query.label }} - {% for error in form.query.errors %} - {{ error }} - {% endfor %} -
-
- -
-
- Help - -
- Options -
+ {% for error in form.query.errors %} + {{ error }} + {% endfor %} +
+ +

 

+ Help +

CQP Query Language Tutorial

+

 

+ Options
format_list_numbered {{ form.hits_per_page() }} @@ -40,78 +29,46 @@ {{ error }} {% endfor %}
-
-
- short_text - {{ form.context() }} - {{ form.context.label }} - {% for error in form.context.errors %} - {{ error }} - {% endfor %} -
-
- - Download Results -

Downlaod all results of the current query as csv, excel or json file.

-
- {{ form.hidden_tag() }} -
- insert_drive_file - {{ dl_form.file_type() }} - {{ dl_form.file_type.label }} - {% for error in dl_form.file_type.errors %} - {{ error }} - {% endfor %} -
-
- -
-
+
+ short_text + {{ form.context() }} + {{ form.context.label }} + {% for error in form.context.errors %} + {{ error }} + {% endfor %} +
+ +
+
+ +
+
+
+ {{ dl_form.hidden_tag() }} + Download Results +

Downlaod all results of the current query as csv, excel or json file.

+
+ insert_drive_file + {{ dl_form.file_type() }} + {{ dl_form.file_type.label }} + {% for error in dl_form.file_type.errors %} + {{ error }} + {% endfor %} +
+ +
- -
+ +
Query Results -

+
-