diff --git a/app/__init__.py b/app/__init__.py
index 3a03e00b..4a9b0652 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -74,6 +74,8 @@ def create_app(config: Config = Config) -> Flask:
app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
from .corpora import bp as corpora_blueprint
+ from .corpora.cqi_over_sio import CQiNamespace
+ socketio.on_namespace(CQiNamespace('/cqi_over_sio'))
default_breadcrumb_root(corpora_blueprint, '.corpora')
app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
diff --git a/app/corpora/__init__.py b/app/corpora/__init__.py
index 9ad47224..8365ce90 100644
--- a/app/corpora/__init__.py
+++ b/app/corpora/__init__.py
@@ -16,4 +16,4 @@ def before_request():
pass
-from . import cli, cqi_over_sio, files, followers, routes, json_routes
+from . import cli, files, followers, routes, json_routes
diff --git a/app/corpora/cqi_over_sio/__init__.py b/app/corpora/cqi_over_sio/__init__.py
index 8afcf502..b2cf02dc 100644
--- a/app/corpora/cqi_over_sio/__init__.py
+++ b/app/corpora/cqi_over_sio/__init__.py
@@ -1,113 +1,199 @@
from cqi import CQiClient
from cqi.errors import CQiException
+from cqi.status import CQiStatus
from flask import session
from flask_login import current_user
-from flask_socketio import ConnectionRefusedError
+from flask_socketio import Namespace
+from inspect import signature
from threading import Lock
+from typing import Callable, Dict, List
from app import db, hashids, socketio
from app.decorators import socketio_login_required
from app.models import Corpus, CorpusStatus
-import math
+from . import extensions
'''
This package tunnels the Corpus Query interface (CQi) protocol through
-Socket.IO (SIO) by wrapping each CQi function in a seperate SIO event.
-
-This module only handles the SIO connect/disconnect, which handles the setup
-and teardown of necessary ressources for later use. Each CQi function has a
-corresponding SIO event. The event handlers are spread across the different
-modules within this package.
+Socket.IO (SIO) by tunneling CQi API calls through an event called "exec".
Basic concept:
-1. A client connects to the SIO namespace and provides the id of a corpus to be
- analysed.
+1. A client connects to the "/cqi_over_sio" namespace.
+2. The client emits the "init" event and provides a corpus id for the corpus
+ that should be analysed in this session.
1.1 The analysis session counter of the corpus is incremented.
1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
1.3 Wait until the CQP server is running.
1.4 Connect the CQiClient to the server.
- 1.5 Save the CQiClient and the Lock in the session for subsequential use.
-2. A client emits an event and may provide a single json object with necessary
- arguments for the targeted CQi function.
-3. A SIO event handler (decorated with cqi_over_socketio) gets executed.
- - The event handler function defines all arguments. Hence the client
- is sent as a single json object, the decorator decomposes it to fit
- the functions signature. This also includes type checking and proper
- use of the lock (acquire/release) mechanism.
+ 1.5 Save the CQiClient, the Lock and the corpus id in the session for
+ subsequential use.
+2. The client emits the "exec" event provides the name of a CQi API function
+ arguments (optional).
+ - The event "exec" handler will execute the function, make sure that the
+ result is serializable and returns the result back to the client.
4. Wait for more events
-5. The client disconnects from the SIO namespace
+5. The client disconnects from the "/cqi_over_sio" namespace
1.1 The analysis session counter of the corpus is decremented.
1.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
'''
-
-NAMESPACE = '/cqi_over_sio'
+CQI_API_FUNCTION_NAMES: List[str] = [
+ 'ask_feature_cl_2_3',
+ 'ask_feature_cqi_1_0',
+ 'ask_feature_cqp_2_3',
+ 'cl_alg2cpos',
+ 'cl_attribute_size',
+ 'cl_cpos2alg',
+ 'cl_cpos2id',
+ 'cl_cpos2lbound',
+ 'cl_cpos2rbound',
+ 'cl_cpos2str',
+ 'cl_cpos2struc',
+ 'cl_drop_attribute',
+ 'cl_id2cpos',
+ 'cl_id2freq',
+ 'cl_id2str',
+ 'cl_idlist2cpos',
+ 'cl_lexicon_size',
+ 'cl_regex2id',
+ 'cl_str2id',
+ 'cl_struc2cpos',
+ 'cl_struc2str',
+ 'corpus_alignment_attributes',
+ 'corpus_charset',
+ 'corpus_drop_corpus',
+ 'corpus_full_name',
+ 'corpus_info',
+ 'corpus_list_corpora',
+ 'corpus_positional_attributes',
+ 'corpus_properties',
+ 'corpus_structural_attribute_has_values',
+ 'corpus_structural_attributes',
+ 'cqp_drop_subcorpus',
+ 'cqp_dump_subcorpus',
+ 'cqp_fdist_1',
+ 'cqp_fdist_2',
+ 'cqp_list_subcorpora',
+ 'cqp_query',
+ 'cqp_subcorpus_has_field',
+ 'cqp_subcorpus_size',
+ 'ctrl_bye',
+ 'ctrl_connect',
+ 'ctrl_last_general_error',
+ 'ctrl_ping',
+ 'ctrl_user_abort'
+]
-from .cqi import * # noqa
-
-
-@socketio.on('connect', namespace=NAMESPACE)
-@socketio_login_required
-def connect(auth):
- # the auth variable is used in a hacky way. It contains the corpus id for
- # which a corpus analysis session should be started.
- corpus_id = hashids.decode(auth['corpus_id'])
- corpus = Corpus.query.get(corpus_id)
- if corpus is None:
- # return {'code': 404, 'msg': 'Not Found'}
- raise ConnectionRefusedError('Not Found')
- if not (corpus.user == current_user
- or current_user.is_following_corpus(corpus)
- or current_user.is_administrator()):
- # return {'code': 403, 'msg': 'Forbidden'}
- raise ConnectionRefusedError('Forbidden')
- if corpus.status not in [
- CorpusStatus.BUILT,
- CorpusStatus.STARTING_ANALYSIS_SESSION,
- CorpusStatus.RUNNING_ANALYSIS_SESSION,
- CorpusStatus.CANCELING_ANALYSIS_SESSION
- ]:
- # return {'code': 424, 'msg': 'Failed Dependency'}
- raise ConnectionRefusedError('Failed Dependency')
- if corpus.num_analysis_sessions is None:
- corpus.num_analysis_sessions = 0
- db.session.commit()
- corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
- db.session.commit()
- retry_counter = 20
- while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
- if retry_counter == 0:
- corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
- db.session.commit()
- return {'code': 408, 'msg': 'Request Timeout'}
- socketio.sleep(3)
- retry_counter -= 1
- db.session.refresh(corpus)
- cqi_client = CQiClient(f'cqpserver_{corpus_id}', timeout=math.inf)
- session['cqi_over_sio'] = {
- 'corpus_id': corpus_id,
- 'cqi_client': cqi_client,
- 'cqi_client_lock': Lock(),
- }
- # return {'code': 200, 'msg': 'OK'}
-
-
-@socketio.on('disconnect', namespace=NAMESPACE)
-def disconnect():
- try:
- cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
- cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
- except KeyError:
- return
- cqi_client_lock.acquire()
- try:
- cqi_client.api.ctrl_bye()
- except (BrokenPipeError, CQiException):
+class CQiNamespace(Namespace):
+ @socketio_login_required
+ def on_connect(self):
pass
- cqi_client_lock.release()
- corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
- corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
- db.session.commit()
- session.pop('cqi_over_sio')
- # return {'code': 200, 'msg': 'OK'}
+
+ @socketio_login_required
+ def on_init(self, db_corpus_hashid: str):
+ db_corpus_id = hashids.decode(db_corpus_hashid)
+ db_corpus = Corpus.query.get(db_corpus_id)
+ if db_corpus is None:
+ return {'code': 404, 'msg': 'Not Found'}
+ if not (db_corpus.user == current_user
+ or current_user.is_following_corpus(db_corpus)
+ or current_user.is_administrator()):
+ return {'code': 403, 'msg': 'Forbidden'}
+ if db_corpus.status not in [
+ CorpusStatus.BUILT,
+ CorpusStatus.STARTING_ANALYSIS_SESSION,
+ CorpusStatus.RUNNING_ANALYSIS_SESSION,
+ CorpusStatus.CANCELING_ANALYSIS_SESSION
+ ]:
+ return {'code': 424, 'msg': 'Failed Dependency'}
+ if db_corpus.num_analysis_sessions is None:
+ db_corpus.num_analysis_sessions = 0
+ db.session.commit()
+ db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+ db.session.commit()
+ retry_counter = 20
+ while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+ if retry_counter == 0:
+ db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+ db.session.commit()
+ return {'code': 408, 'msg': 'Request Timeout'}
+ socketio.sleep(3)
+ retry_counter -= 1
+ db.session.refresh(db_corpus)
+ cqi_client = CQiClient(f'cqpserver_{db_corpus_id}', timeout=None)
+ session['cqi_over_sio'] = {}
+ session['cqi_over_sio']['cqi_client'] = cqi_client
+ session['cqi_over_sio']['cqi_client_lock'] = Lock()
+ session['cqi_over_sio']['db_corpus_id'] = db_corpus_id
+ return {'code': 200, 'msg': 'OK'}
+
+ @socketio_login_required
+ def on_exec(self, fn_name: str, fn_args: Dict = {}):
+ try:
+ cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+ cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+ except KeyError:
+ return {'code': 424, 'msg': 'Failed Dependency'}
+ if fn_name in CQI_API_FUNCTION_NAMES:
+ fn: Callable = getattr(cqi_client.api, fn_name)
+ elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
+ fn: Callable = getattr(extensions, fn_name)
+ else:
+ return {'code': 400, 'msg': 'Bad Request'}
+ for param in signature(fn).parameters.values():
+ if param.default is param.empty:
+ if param.name not in fn_args:
+ return {'code': 400, 'msg': 'Bad Request'}
+ else:
+ if param.name not in fn_args:
+ continue
+ if type(fn_args[param.name]) is not param.annotation:
+ return {'code': 400, 'msg': 'Bad Request'}
+ cqi_client_lock.acquire()
+ try:
+ fn_return_value = fn(**fn_args)
+ except BrokenPipeError as e:
+ return {'code': 500, 'msg': 'Internal Server Error'}
+ except CQiException as e:
+ return {
+ 'code': 502,
+ 'msg': 'Bad Gateway',
+ 'payload': {
+ 'code': e.code,
+ 'desc': e.description,
+ 'msg': e.__class__.__name__
+ }
+ }
+ finally:
+ cqi_client_lock.release()
+ if isinstance(fn_return_value, CQiStatus):
+ payload = {
+ 'code': fn_return_value.code,
+ 'msg': fn_return_value.__class__.__name__
+ }
+ else:
+ payload = fn_return_value
+ return {'code': 200, 'msg': 'OK', 'payload': payload}
+
+ def on_disconnect(self):
+ try:
+ cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+ cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+ db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+ except KeyError:
+ return
+ cqi_client_lock.acquire()
+ try:
+ session.pop('cqi_over_sio')
+ except KeyError:
+ pass
+ try:
+ cqi_client.api.ctrl_bye()
+ except (BrokenPipeError, CQiException):
+ pass
+ cqi_client_lock.release()
+ db_corpus = Corpus.query.get(db_corpus_id)
+ if db_corpus is not None:
+ db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+ db.session.commit()
diff --git a/app/corpora/cqi_over_sio/cqi.py b/app/corpora/cqi_over_sio/cqi.py
deleted file mode 100644
index 2e483b21..00000000
--- a/app/corpora/cqi_over_sio/cqi.py
+++ /dev/null
@@ -1,113 +0,0 @@
-from cqi import CQiClient
-from cqi.errors import CQiException
-from cqi.status import CQiStatus
-from flask import session
-from inspect import signature
-from threading import Lock
-from typing import Callable, Dict, List
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .extensions import CQI_EXTENSION_FUNCTION_NAMES
-from . import extensions as extensions_module
-
-
-CQI_FUNCTION_NAMES: List[str] = [
- 'ask_feature_cl_2_3',
- 'ask_feature_cqi_1_0',
- 'ask_feature_cqp_2_3',
- 'cl_alg2cpos',
- 'cl_attribute_size',
- 'cl_cpos2alg',
- 'cl_cpos2id',
- 'cl_cpos2lbound',
- 'cl_cpos2rbound',
- 'cl_cpos2str',
- 'cl_cpos2struc',
- 'cl_drop_attribute',
- 'cl_id2cpos',
- 'cl_id2freq',
- 'cl_id2str',
- 'cl_idlist2cpos',
- 'cl_lexicon_size',
- 'cl_regex2id',
- 'cl_str2id',
- 'cl_struc2cpos',
- 'cl_struc2str',
- 'corpus_alignment_attributes',
- 'corpus_charset',
- 'corpus_drop_corpus',
- 'corpus_full_name',
- 'corpus_info',
- 'corpus_list_corpora',
- 'corpus_positional_attributes',
- 'corpus_properties',
- 'corpus_structural_attribute_has_values',
- 'corpus_structural_attributes',
- 'cqp_drop_subcorpus',
- 'cqp_dump_subcorpus',
- 'cqp_fdist_1',
- 'cqp_fdist_2',
- 'cqp_list_subcorpora',
- 'cqp_query',
- 'cqp_subcorpus_has_field',
- 'cqp_subcorpus_size',
- 'ctrl_bye',
- 'ctrl_connect',
- 'ctrl_last_general_error',
- 'ctrl_ping',
- 'ctrl_user_abort'
-]
-
-
-@socketio.on('cqi', namespace=ns)
-@socketio_login_required
-def cqi_over_sio(fn_name: str, fn_args: Dict = {}):
- try:
- cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
- cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
- except KeyError:
- return {'code': 424, 'msg': 'Failed Dependency'}
- if fn_name in CQI_FUNCTION_NAMES:
- fn: Callable = getattr(cqi_client.api, fn_name)
- elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
- fn: Callable = getattr(extensions_module, fn_name)
- else:
- return {'code': 400, 'msg': 'Bad Request'}
- for param in signature(fn).parameters.values():
- if param.default is param.empty:
- if param.name not in fn_args:
- return {'code': 400, 'msg': 'Bad Request'}
- else:
- if param.name not in fn_args:
- continue
- if type(fn_args[param.name]) is not param.annotation:
- return {'code': 400, 'msg': 'Bad Request'}
- cqi_client_lock.acquire()
- try:
- fn_return_value = fn(**fn_args)
- except BrokenPipeError:
- fn_return_value = {
- 'code': 500,
- 'msg': 'Internal Server Error'
- }
- except CQiException as e:
- return {
- 'code': 502,
- 'msg': 'Bad Gateway',
- 'payload': {
- 'code': e.code,
- 'desc': e.description,
- 'msg': e.__class__.__name__
- }
- }
- finally:
- cqi_client_lock.release()
- if isinstance(fn_return_value, CQiStatus):
- payload = {
- 'code': fn_return_value.code,
- 'msg': fn_return_value.__class__.__name__
- }
- else:
- payload = fn_return_value
- return {'code': 200, 'msg': 'OK', 'payload': payload}
diff --git a/app/corpora/cqi_over_sio/extensions.py b/app/corpora/cqi_over_sio/extensions.py
index 903e51c3..70ee5d97 100644
--- a/app/corpora/cqi_over_sio/extensions.py
+++ b/app/corpora/cqi_over_sio/extensions.py
@@ -28,8 +28,9 @@ CQI_EXTENSION_FUNCTION_NAMES: List[str] = [
def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
- db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+ db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+ db_corpus: Corpus = Corpus.query.get(db_corpus_id)
cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
db_corpus.num_tokens = cqi_corpus.size
db.session.commit()
@@ -37,10 +38,11 @@ def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
def ext_corpus_static_data(corpus: str) -> Dict:
- db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
- static_corpus_data_file = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
- if os.path.exists(static_corpus_data_file):
- with open(static_corpus_data_file, 'rb') as f:
+ db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+ db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+ cache_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
+ if os.path.exists(cache_file_path):
+ with open(cache_file_path, 'rb') as f:
return f.read()
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
@@ -189,10 +191,10 @@ def ext_corpus_static_data(corpus: str) -> Dict:
} for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
}
del sub_s_attr_values
- with gzip.open(static_corpus_data_file, 'wt') as f:
+ with gzip.open(cache_file_path, 'wt') as f:
json.dump(static_corpus_data, f)
del static_corpus_data
- with open(static_corpus_data_file, 'rb') as f:
+ with open(cache_file_path, 'rb') as f:
return f.read()
diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js b/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js
index ea7aed3a..2d69fed7 100644
--- a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js
+++ b/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js
@@ -1,13 +1,15 @@
class CorpusAnalysisApp {
constructor(corpusId) {
+ this.corpusId = corpusId;
+
this.data = {};
// HTML elements
this.elements = {
container: document.querySelector('#corpus-analysis-app-container'),
+ extensionCards: document.querySelector('#corpus-analysis-app-extension-cards'),
extensionTabs: document.querySelector('#corpus-analysis-app-extension-tabs'),
- initModal: document.querySelector('#corpus-analysis-app-init-modal'),
- overview: document.querySelector('#corpus-analysis-app-overview')
+ initModal: document.querySelector('#corpus-analysis-app-init-modal')
};
// Materialize elements
this.elements.m = {
@@ -17,80 +19,97 @@ class CorpusAnalysisApp {
this.extensions = {};
- this.settings = {
- corpusId: corpusId
- };
+ this.settings = {};
}
- init() {
+ async init() {
this.disableActionElements();
this.elements.m.initModal.open();
- // Init data
- this.data.cqiClient = new cqi.CQiClient('/cqi_over_sio', this.settings.corpusId);
- this.data.cqiClient.connect('anonymous', '')
- .then((cqiStatus) => {
- return this.data.cqiClient.corpora.list();
- })
- .then((cqiCorpora) => {
- this.data.corpus = {o: cqiCorpora[0]};
- console.log(this.data.corpus.o.staticData);
-
- // TODO: Don't do this hgere
- this.data.corpus.o.updateDb();
- this.enableActionElements();
- for (let extension of Object.values(this.extensions)) {extension.init();}
- this.elements.m.initModal.close();
- },
- (cqiError) => {
- let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
- let errorsElement = this.elements.initModal.querySelector('.errors');
- let progressElement = this.elements.initModal.querySelector('.progress');
- errorsElement.innerText = errorString;
- errorsElement.classList.remove('hide');
- app.flash(errorString, 'error');
- progressElement.classList.add('hide');
- }
- );
-
- // Add event listeners
- for (let extensionSelectorElement of this.elements.overview.querySelectorAll('.extension-selector')) {
+
+ const statusTextElement = this.elements.initModal.querySelector('.status-text');
+
+ // Setup CQi over SocketIO connection and gather data from the CQPServer
+ try {
+ statusTextElement.innerText = 'Creating CQi over SocketIO client...';
+ const cqiClient = new cqi.CQiClient('/cqi_over_sio');
+ statusTextElement.innerText += ' Done';
+ statusTextElement.innerHTML += '
Waiting for the CQP server...';
+ const response = await cqiClient.api.socket.emitWithAck('init', this.corpusId);
+ if (response.code !== 200) {throw new Error();}
+ statusTextElement.innerText += ' Done';
+ statusTextElement.innerHTML += '
Connecting to the CQP server...';
+ await cqiClient.connect('anonymous', '');
+ statusTextElement.innerText += ' Done';
+ statusTextElement.innerHTML += '
Building and receiving corpus data cache from the server (This may take a while)...';
+ const cqiCorpus = await cqiClient.corpora.get(`NOPAQUE-${this.corpusId.toUpperCase()}`);
+ statusTextElement.innerText += ' Done';
+ // TODO: Don't do this hgere
+ await cqiCorpus.updateDb();
+ this.data.cqiClient = cqiClient;
+ this.data.cqiCorpus = cqiCorpus;
+ this.data.corpus = {o: cqiCorpus}; // legacy
+ } catch (error) {
+ let errorString = '';
+ if ('code' in error) {errorString += `[${error.code}] `;}
+ errorString += `${error.constructor.name}`;
+ if ('description' in error) {errorString += `: ${error.description}`;}
+ const errorsElement = this.elements.initModal.querySelector('.errors');
+ const progressElement = this.elements.initModal.querySelector('.progress');
+ errorsElement.innerText = errorString;
+ errorsElement.classList.remove('hide');
+ progressElement.classList.add('hide');
+ return;
+ }
+
+ // Initialize extensions
+ for (const extension of Object.values(this.extensions)) {
+ statusTextElement.innerHTML += `
Initializing ${extension.name} extension...`;
+ await extension.init();
+ statusTextElement.innerText += ' Done'
+ }
+ for (const extensionSelectorElement of this.elements.extensionCards.querySelectorAll('.extension-selector')) {
extensionSelectorElement.addEventListener('click', () => {
this.elements.m.extensionTabs.select(extensionSelectorElement.dataset.target);
});
}
+
+ this.enableActionElements();
+ this.elements.m.initModal.close();
}
registerExtension(extension) {
- if (extension.name in this.extensions) {
- console.error(`Can't register extension ${extension.name}: Already registered`);
- return;
- }
+ if (extension.name in this.extensions) {return;}
this.extensions[extension.name] = extension;
- if ('cQiClient' in this.data && this.data.cQiClient.connected) {extension.init();}
}
disableActionElements() {
- let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
- for (let actionElement of actionElements) {
- if (actionElement.nodeName === 'INPUT') {
- actionElement.disabled = true;
- } else if (actionElement.nodeName === 'SELECT') {
- actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
- } else {
- actionElement.classList.add('disabled');
+ const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
+ for (const actionElement of actionElements) {
+ switch(actionElement.nodeName) {
+ case 'INPUT':
+ actionElement.disabled = true;
+ break;
+ case 'SELECT':
+ actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
+ break;
+ default:
+ actionElement.classList.add('disabled');
}
}
}
enableActionElements() {
- let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
- for (let actionElement of actionElements) {
- if (actionElement.nodeName === 'INPUT') {
- actionElement.disabled = false;
- } else if (actionElement.nodeName === 'SELECT') {
- actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
- } else {
- actionElement.classList.remove('disabled');
+ const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
+ for (const actionElement of actionElements) {
+ switch(actionElement.nodeName) {
+ case 'INPUT':
+ actionElement.disabled = false;
+ break;
+ case 'SELECT':
+ actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
+ break;
+ default:
+ actionElement.classList.remove('disabled');
}
}
}
diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js b/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js
index e6c00e73..11ac9d2f 100644
--- a/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js
+++ b/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js
@@ -30,62 +30,60 @@ class CorpusAnalysisConcordance {
this.app.registerExtension(this);
}
- init() {
+ async submitForm() {
+ this.app.disableActionElements();
+ let query = this.elements.form.query.value.trim();
+ let subcorpusName = this.elements.form['subcorpus-name'].value;
+ this.elements.error.innerText = '';
+ this.elements.error.classList.add('hide');
+ this.elements.progress.classList.remove('hide');
+ try {
+ const subcorpus = {};
+ subcorpus.q = query;
+ subcorpus.selectedItems = new Set();
+ await this.data.corpus.o.query(subcorpusName, query);
+ if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
+ const cqiSubcorpus = await this.data.corpus.o.subcorpora.get(subcorpusName);
+ subcorpus.o = cqiSubcorpus;
+ const paginatedSubcorpus = await cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
+ subcorpus.p = paginatedSubcorpus;
+ this.data.subcorpora[subcorpusName] = subcorpus;
+ this.settings.selectedSubcorpus = subcorpusName;
+ this.renderSubcorpusList();
+ this.renderSubcorpusInfo();
+ this.renderSubcorpusActions();
+ this.renderSubcorpusItems();
+ this.renderSubcorpusPagination();
+ this.elements.progress.classList.add('hide');
+ } catch (error) {
+ let errorString = '';
+ if ('code' in error) {errorString += `[${error.code}] `;}
+ errorString += `${error.constructor.name}`;
+ this.elements.error.innerText = errorString;
+ this.elements.error.classList.remove('hide');
+ app.flash(errorString, 'error');
+ this.elements.progress.classList.add('hide');
+ }
+ this.app.enableActionElements();
+ }
+
+ async init() {
// Init data
this.data.corpus = this.app.data.corpus;
this.data.subcorpora = {};
// Add event listeners
- this.elements.form.addEventListener('submit', event => {
+ this.elements.form.addEventListener('submit', (event) => {
event.preventDefault();
- this.app.disableActionElements();
- let query = this.elements.form.query.value.trim();
- let subcorpusName = this.elements.form['subcorpus-name'].value;
- this.elements.error.innerText = '';
- this.elements.error.classList.add('hide');
- this.elements.progress.classList.remove('hide');
- let subcorpus = {};
- this.data.corpus.o.query(subcorpusName, query)
- .then((cqiStatus) => {
- subcorpus.q = query;
- subcorpus.selectedItems = new Set();
- if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
- return this.data.corpus.o.subcorpora.get(subcorpusName);
- })
- .then((cqiSubcorpus) => {
- subcorpus.o = cqiSubcorpus;
- return cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
- })
- .then(
- (paginatedSubcorpus) => {
- subcorpus.p = paginatedSubcorpus;
- this.data.subcorpora[subcorpusName] = subcorpus;
- this.settings.selectedSubcorpus = subcorpusName;
- this.renderSubcorpusList();
- this.renderSubcorpusInfo();
- this.renderSubcorpusActions();
- this.renderSubcorpusItems();
- this.renderSubcorpusPagination();
- this.elements.progress.classList.add('hide');
- this.app.enableActionElements();
- },
- (cqiError) => {
- let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
- this.elements.error.innerText = errorString;
- this.elements.error.classList.remove('hide');
- app.flash(errorString, 'error');
- this.elements.progress.classList.add('hide');
- this.app.enableActionElements();
- }
- );
+ this.submitForm();
});
- this.elements.form.addEventListener('change', event => {
+ this.elements.form.addEventListener('change', (event) => {
if (event.target === this.elements.form['context']) {
this.settings.context = parseInt(this.elements.form['context'].value);
- this.elements.form.submit.click();
+ this.submitForm();
}
if (event.target === this.elements.form['per-page']) {
this.settings.perPage = parseInt(this.elements.form['per-page'].value);
- this.elements.form.submit.click();
+ this.submitForm();
}
if (event.target === this.elements.form['text-style']) {
this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -161,7 +159,7 @@ class CorpusAnalysisConcordance {
`.trim();
M.Tooltip.init(this.elements.subcorpusActions.querySelectorAll('.tooltipped'));
- this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', event => {
+ this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', (event) => {
event.preventDefault();
let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
let modalElementId = Utils.generateElementId('export-subcorpus-modal-');
@@ -218,7 +216,7 @@ class CorpusAnalysisConcordance {
}
}
);
- exportButton.addEventListener('click', event => {
+ exportButton.addEventListener('click', (event) => {
event.preventDefault();
this.app.disableActionElements();
this.elements.progress.classList.remove('hide');
@@ -240,7 +238,7 @@ class CorpusAnalysisConcordance {
promise = subcorpus.o.export(50);
}
promise.then(
- data => {
+ (data) => {
let blob;
if (exportFormat === 'csv') {
let csvContent = 'sep=,\r\n';
@@ -286,7 +284,7 @@ class CorpusAnalysisConcordance {
});
modal.open();
});
- this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', event => {
+ this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', (event) => {
event.preventDefault();
let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
subcorpus.o.drop().then(
@@ -362,7 +360,7 @@ class CorpusAnalysisConcordance {
this.setTextStyle();
this.setTokenRepresentation();
for (let gotoReaderTriggerElement of this.elements.subcorpusItems.querySelectorAll('.goto-reader-trigger')) {
- gotoReaderTriggerElement.addEventListener('click', event => {
+ gotoReaderTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let corpusAnalysisReader = this.app.extensions.Reader;
let itemId = parseInt(gotoReaderTriggerElement.closest('.item').dataset.id);
@@ -384,7 +382,7 @@ class CorpusAnalysisConcordance {
});
}
for (let selectTriggerElement of this.elements.subcorpusItems.querySelectorAll('.select-trigger')) {
- selectTriggerElement.addEventListener('click', event => {
+ selectTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let itemElement = selectTriggerElement.closest('.item');
let itemId = parseInt(itemElement.dataset.id);
@@ -446,14 +444,14 @@ class CorpusAnalysisConcordance {
`.trim();
for (let paginationTriggerElement of this.elements.subcorpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
- paginationTriggerElement.addEventListener('click', event => {
+ paginationTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
this.app.disableActionElements();
this.elements.progress.classList.remove('hide');
let page = parseInt(paginationTriggerElement.dataset.target);
subcorpus.o.paginate(page, this.settings.perPage, this.settings.context)
.then(
- paginatedSubcorpus => {
+ (paginatedSubcorpus) => {
subcorpus.p = paginatedSubcorpus;
this.renderSubcorpusItems();
this.renderSubcorpusPagination();
diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisReader.js b/app/static/js/CorpusAnalysis/CorpusAnalysisReader.js
index d6c99926..eb63cae9 100644
--- a/app/static/js/CorpusAnalysis/CorpusAnalysisReader.js
+++ b/app/static/js/CorpusAnalysis/CorpusAnalysisReader.js
@@ -19,45 +19,52 @@ class CorpusAnalysisReader {
this.settings = {
perPage: parseInt(this.elements.form['per-page'].value),
textStyle: parseInt(this.elements.form['text-style'].value),
- tokenRepresentation: this.elements.form['token-representation'].value
+ tokenRepresentation: this.elements.form['token-representation'].value,
+ pagination: {
+ innerWindow: 5,
+ outerWindow: 1
+ }
}
this.app.registerExtension(this);
}
- init() {
+ async submitForm() {
+ this.app.disableActionElements();
+ this.elements.error.innerText = '';
+ this.elements.error.classList.add('hide');
+ this.elements.progress.classList.remove('hide');
+ try {
+ const paginatedCorpus = await this.data.corpus.o.paginate(1, this.settings.perPage);
+ this.data.corpus.p = paginatedCorpus;
+ this.renderCorpus();
+ this.renderCorpusPagination();
+ this.elements.progress.classList.add('hide');
+ } catch (error) {
+ let errorString = '';
+ if ('code' in error) {errorString += `[${error.code}] `;}
+ errorString += `${error.constructor.name}`;
+ if ('description' in error) {errorString += `: ${error.description}`;}
+ this.elements.error.innerText = errorString;
+ this.elements.error.classList.remove('hide');
+ app.flash(errorString, 'error');
+ this.elements.progress.classList.add('hide');
+ }
+ this.app.enableActionElements();
+ }
+
+ async init() {
// Init data
this.data.corpus = this.app.data.corpus;
// Add event listeners
this.elements.form.addEventListener('submit', (event) => {
event.preventDefault();
- this.app.disableActionElements();
- this.elements.error.innerText = '';
- this.elements.error.classList.add('hide');
- this.elements.progress.classList.remove('hide');
- this.data.corpus.o.paginate(1, this.settings.perPage)
- .then(
- (paginatedCorpus) => {
- this.data.corpus.p = paginatedCorpus;
- this.renderCorpus();
- this.renderCorpusPagination();
- this.elements.progress.classList.add('hide');
- this.app.enableActionElements();
- },
- (cqiError) => {
- let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
- this.elements.error.innerText = errorString;
- this.elements.error.classList.remove('hide');
- app.flash(errorString, 'error');
- this.elements.progress.classList.add('hide');
- this.app.enableActionElements();
- }
- );
+ this.submitForm();
});
- this.elements.form.addEventListener('change', event => {
+ this.elements.form.addEventListener('change', (event) => {
if (event.target === this.elements.form['per-page']) {
this.settings.perPage = parseInt(this.elements.form['per-page'].value);
- this.elements.form.submit.click();
+ this.submitForm();
}
if (event.target === this.elements.form['text-style']) {
this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -69,7 +76,7 @@ class CorpusAnalysisReader {
}
});
// Load initial data
- this.elements.form.submit.click();
+ await this.submitForm();
}
clearCorpus() {
@@ -142,7 +149,7 @@ class CorpusAnalysisReader {
}
// render page buttons (5 before and 5 after current page)
- for (let i = this.data.corpus.p.page -5; i <= this.data.corpus.p.page; i++) {
+ for (let i = this.data.corpus.p.page - this.settings.pagination.innerWindow; i <= this.data.corpus.p.page; i++) {
if (i <= 0) {continue;}
pageElement = Utils.HTMLToElement(
`
@@ -153,7 +160,7 @@ class CorpusAnalysisReader {
);
this.elements.corpusPagination.appendChild(pageElement);
};
- for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page +5; i++) {
+ for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page + this.settings.pagination.innerWindow; i++) {
if (i > this.data.corpus.p.pages) {break;}
pageElement = Utils.HTMLToElement(
`
@@ -201,7 +208,7 @@ class CorpusAnalysisReader {
this.elements.corpusPagination.appendChild(pageElement);
for (let paginateTriggerElement of this.elements.corpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
- paginateTriggerElement.addEventListener('click', event => {
+ paginateTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let page = parseInt(paginateTriggerElement.dataset.target);
this.page(page);
diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js b/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js
index 2d8f9002..09686387 100644
--- a/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js
+++ b/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js
@@ -13,7 +13,7 @@ class CorpusAnalysisStaticVisualization {
this.app.registerExtension(this);
}
- init() {
+ async init() {
// Init data
this.data.corpus = this.app.data.corpus;
this.renderGeneralCorpusInfo();
diff --git a/app/static/js/cqi/api/client.js b/app/static/js/cqi/api/client.js
index daf2a401..82f8176f 100644
--- a/app/static/js/cqi/api/client.js
+++ b/app/static/js/cqi/api/client.js
@@ -1,18 +1,16 @@
cqi.api.APIClient = class APIClient {
/**
* @param {string} host
- * @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
- constructor(host, corpus_id, timeout = 60, version = '0.1') {
+ constructor(host, timeout = 60, version = '0.1') {
this.host = host;
this.timeout = timeout * 1000; // convert seconds to milliseconds
this.version = version;
this.socket = io(
this.host,
{
- auth: {corpus_id: corpus_id},
transports: ['websocket'],
upgrade: false
}
@@ -24,22 +22,16 @@ cqi.api.APIClient = class APIClient {
* @param {object} [fn_args={}]
* @returns {Promise}
*/
- #request(fn_name, fn_args = {}) {
- return new Promise((resolve, reject) => {
- // this.socket.timeout(this.timeout).emit('cqi', {fn_name: fn_name, fn_args: fn_args}, (timeoutError, response) => {
- // if (timeoutError) {
- // reject(timeoutError);
- // }
- this.socket.emit('cqi', fn_name, fn_args, (response) => {
- if (response.code === 200) {
- resolve(response.payload);
- } else if (response.code === 500) {
- reject(new Error(`[${response.code}] ${response.msg}`));
- } else if (response.code === 502) {
- reject(new cqi.errors.lookup[response.payload.code]());
- }
- });
- });
+ async #request(fn_name, fn_args = {}) {
+ // TODO: implement timeout
+ let response = await this.socket.emitWithAck('exec', fn_name, fn_args);
+ if (response.code === 200) {
+ return response.payload;
+ } else if (response.code === 500) {
+ throw new Error(`[${response.code}] ${response.msg}`);
+ } else if (response.code === 502) {
+ throw new cqi.errors.lookup[response.payload.code]();
+ }
}
/**
@@ -630,7 +622,9 @@ cqi.api.APIClient = class APIClient {
async ext_corpus_static_data(corpus) {
const fn_name = 'ext_corpus_static_data';
const fn_args = {corpus: corpus};
- return await this.#request(fn_name, fn_args);
+ let compressedEncodedData = await this.#request(fn_name, fn_args);
+ let data = pako.inflate(compressedEncodedData, {to: 'string'});
+ return JSON.parse(data);
}
/**
diff --git a/app/static/js/cqi/client.js b/app/static/js/cqi/client.js
index 7a92a904..b1d29447 100644
--- a/app/static/js/cqi/client.js
+++ b/app/static/js/cqi/client.js
@@ -1,13 +1,12 @@
cqi.CQiClient = class CQiClient {
/**
* @param {string} host
- * @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
- constructor(host, corpusId, timeout = 60, version = '0.1') {
+ constructor(host, timeout = 60, version = '0.1') {
/** @type {cqi.api.APIClient} */
- this.api = new cqi.api.APIClient(host, corpusId, timeout, version);
+ this.api = new cqi.api.APIClient(host, timeout, version);
}
/**
diff --git a/app/static/js/cqi/models/corpora.js b/app/static/js/cqi/models/corpora.js
index 2b271811..8128c47f 100644
--- a/app/static/js/cqi/models/corpora.js
+++ b/app/static/js/cqi/models/corpora.js
@@ -138,15 +138,7 @@ cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.
/************************************************************************
* Custom additions for nopaque *
************************************************************************/
- // returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
- let tmp = await this.client.api.ext_corpus_static_data(corpusName);
- console.log(tmp);
- let inflated = pako.inflate(tmp);
- console.log(inflated);
- let decoder = new TextDecoder('utf-8');
- console.log(decoder);
- let decoded = decoder.decode(inflated);
- returnValue.static_data = JSON.parse(decoded);
+ returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
return returnValue;
}
diff --git a/app/templates/_scripts.html.j2 b/app/templates/_scripts.html.j2
index dcf37513..c68b6146 100644
--- a/app/templates/_scripts.html.j2
+++ b/app/templates/_scripts.html.j2
@@ -1,6 +1,6 @@
-
+
diff --git a/app/templates/corpora/analysis.html.j2 b/app/templates/corpora/analysis.html.j2
index ee39be59..ae5b1672 100644
--- a/app/templates/corpora/analysis.html.j2
+++ b/app/templates/corpora/analysis.html.j2
@@ -2,7 +2,7 @@
{% import "materialize/wtf.html.j2" as wtf %}
{% import 'corpora/_analysis/concordance.html.j2' as concordance_extension %}
{% import 'corpora/_analysis/reader.html.j2' as reader_extension %}
-{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension%}
+{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension %}
{% set extensions = [concordance_extension, reader_extension, static_visualization_extension] %}
{% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %}
@@ -11,20 +11,17 @@
{% block page_content %}