Merge branch 'visualizations-update' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into visualizations-update

app/__init__.py

@@ -74,6 +74,8 @@ def create_app(config: Config = Config) -> Flask:
     app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
 
     from .corpora import bp as corpora_blueprint
+    from .corpora.cqi_over_sio import CQiNamespace
+    socketio.on_namespace(CQiNamespace('/cqi_over_sio'))
     default_breadcrumb_root(corpora_blueprint, '.corpora')
     app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
 

app/corpora/__init__.py

@@ -16,4 +16,4 @@ def before_request():
     pass
 
 
-from . import cli, cqi_over_sio, files, followers, routes, json_routes
+from . import cli, files, followers, routes, json_routes

app/corpora/cqi_over_sio/__init__.py

@@ -1,113 +1,199 @@
 from cqi import CQiClient
 from cqi.errors import CQiException
+from cqi.status import CQiStatus
 from flask import session
 from flask_login import current_user
-from flask_socketio import ConnectionRefusedError
+from flask_socketio import Namespace
+from inspect import signature
 from threading import Lock
+from typing import Callable, Dict, List
 from app import db, hashids, socketio
 from app.decorators import socketio_login_required
 from app.models import Corpus, CorpusStatus
-import math
+from . import extensions
 
 
 '''
 This package tunnels the Corpus Query interface (CQi) protocol through
-Socket.IO (SIO) by wrapping each CQi function in a seperate SIO event.
+Socket.IO (SIO) by tunneling CQi API calls through an event called "exec".
-
-This module only handles the SIO connect/disconnect, which handles the setup
-and teardown of necessary ressources for later use. Each CQi function has a
-corresponding SIO event. The event handlers are spread across the different
-modules within this package.
 
 Basic concept:
-1. A client connects to the SIO namespace and provides the id of a corpus to be
+1. A client connects to the "/cqi_over_sio" namespace.
-   analysed.
+2. The client emits the "init" event and provides a corpus id for the corpus
+   that should be analysed in this session.
      1.1 The analysis session counter of the corpus is incremented.
      1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
      1.3 Wait until the CQP server is running.
      1.4 Connect the CQiClient to the server.
-     1.5 Save the CQiClient and the Lock in the session for subsequential use.
+     1.5 Save the CQiClient, the Lock and the corpus id in the session for
-2. A client emits an event and may provide a single json object with necessary
+         subsequential use.
-   arguments for the targeted CQi function.
+2. The client emits the "exec" event provides the name of a CQi API function
-3. A SIO event handler (decorated with cqi_over_socketio) gets executed.
+   arguments (optional).
-     - The event handler function defines all arguments. Hence the client
+     - The event "exec" handler will execute the function, make sure that the
-       is sent as a single json object, the decorator decomposes it to fit
+       result is serializable and returns the result back to the client.
-       the functions signature. This also includes type checking and proper
-       use of the lock (acquire/release) mechanism.
 4. Wait for more events
-5. The client disconnects from the SIO namespace
+5. The client disconnects from the "/cqi_over_sio" namespace
      1.1 The analysis session counter of the corpus is decremented.
      1.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
 '''
 
-
+CQI_API_FUNCTION_NAMES: List[str] = [
-NAMESPACE = '/cqi_over_sio'
+    'ask_feature_cl_2_3',
+    'ask_feature_cqi_1_0',
+    'ask_feature_cqp_2_3',
+    'cl_alg2cpos',
+    'cl_attribute_size',
+    'cl_cpos2alg',
+    'cl_cpos2id',
+    'cl_cpos2lbound',
+    'cl_cpos2rbound',
+    'cl_cpos2str',
+    'cl_cpos2struc',
+    'cl_drop_attribute',
+    'cl_id2cpos',
+    'cl_id2freq',
+    'cl_id2str',
+    'cl_idlist2cpos',
+    'cl_lexicon_size',
+    'cl_regex2id',
+    'cl_str2id',
+    'cl_struc2cpos',
+    'cl_struc2str',
+    'corpus_alignment_attributes',
+    'corpus_charset',
+    'corpus_drop_corpus',
+    'corpus_full_name',
+    'corpus_info',
+    'corpus_list_corpora',
+    'corpus_positional_attributes',
+    'corpus_properties',
+    'corpus_structural_attribute_has_values',
+    'corpus_structural_attributes',
+    'cqp_drop_subcorpus',
+    'cqp_dump_subcorpus',
+    'cqp_fdist_1',
+    'cqp_fdist_2',
+    'cqp_list_subcorpora',
+    'cqp_query',
+    'cqp_subcorpus_has_field',
+    'cqp_subcorpus_size',
+    'ctrl_bye',
+    'ctrl_connect',
+    'ctrl_last_general_error',
+    'ctrl_ping',
+    'ctrl_user_abort'
+]
 
 
-from .cqi import *  # noqa
+class CQiNamespace(Namespace):
-
+    @socketio_login_required
-
+    def on_connect(self):
-@socketio.on('connect', namespace=NAMESPACE)
-@socketio_login_required
-def connect(auth):
-    # the auth variable is used in a hacky way. It contains the corpus id for
-    # which a corpus analysis session should be started.
-    corpus_id = hashids.decode(auth['corpus_id'])
-    corpus = Corpus.query.get(corpus_id)
-    if corpus is None:
-        # return {'code': 404, 'msg': 'Not Found'}
-        raise ConnectionRefusedError('Not Found')
-    if not (corpus.user == current_user
-            or current_user.is_following_corpus(corpus)
-            or current_user.is_administrator()):
-        # return {'code': 403, 'msg': 'Forbidden'}
-        raise ConnectionRefusedError('Forbidden')
-    if corpus.status not in [
-        CorpusStatus.BUILT,
-        CorpusStatus.STARTING_ANALYSIS_SESSION,
-        CorpusStatus.RUNNING_ANALYSIS_SESSION,
-        CorpusStatus.CANCELING_ANALYSIS_SESSION
-    ]:
-        # return {'code': 424, 'msg': 'Failed Dependency'}
-        raise ConnectionRefusedError('Failed Dependency')
-    if corpus.num_analysis_sessions is None:
-        corpus.num_analysis_sessions = 0
-        db.session.commit()
-    corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
-    db.session.commit()
-    retry_counter = 20
-    while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
-        if retry_counter == 0:
-            corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
-            db.session.commit()
-            return {'code': 408, 'msg': 'Request Timeout'}
-        socketio.sleep(3)
-        retry_counter -= 1
-        db.session.refresh(corpus)
-    cqi_client = CQiClient(f'cqpserver_{corpus_id}', timeout=math.inf)
-    session['cqi_over_sio'] = {
-        'corpus_id': corpus_id,
-        'cqi_client': cqi_client,
-        'cqi_client_lock': Lock(),
-    }
-    # return {'code': 200, 'msg': 'OK'}
-
-
-@socketio.on('disconnect', namespace=NAMESPACE)
-def disconnect():
-    try:
-        cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-        cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
-    except KeyError:
-        return
-    cqi_client_lock.acquire()
-    try:
-        cqi_client.api.ctrl_bye()
-    except (BrokenPipeError, CQiException):
         pass
-    cqi_client_lock.release()
+
-    corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
+    @socketio_login_required
-    corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+    def on_init(self, db_corpus_hashid: str):
-    db.session.commit()
+        db_corpus_id = hashids.decode(db_corpus_hashid)
-    session.pop('cqi_over_sio')
+        db_corpus = Corpus.query.get(db_corpus_id)
-    # return {'code': 200, 'msg': 'OK'}
+        if db_corpus is None:
+            return {'code': 404, 'msg': 'Not Found'}
+        if not (db_corpus.user == current_user
+                or current_user.is_following_corpus(db_corpus)
+                or current_user.is_administrator()):
+            return {'code': 403, 'msg': 'Forbidden'}
+        if db_corpus.status not in [
+            CorpusStatus.BUILT,
+            CorpusStatus.STARTING_ANALYSIS_SESSION,
+            CorpusStatus.RUNNING_ANALYSIS_SESSION,
+            CorpusStatus.CANCELING_ANALYSIS_SESSION
+        ]:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if db_corpus.num_analysis_sessions is None:
+            db_corpus.num_analysis_sessions = 0
+            db.session.commit()
+        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+        db.session.commit()
+        retry_counter = 20
+        while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+            if retry_counter == 0:
+                db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+                db.session.commit()
+                return {'code': 408, 'msg': 'Request Timeout'}
+            socketio.sleep(3)
+            retry_counter -= 1
+            db.session.refresh(db_corpus)
+        cqi_client = CQiClient(f'cqpserver_{db_corpus_id}', timeout=None)
+        session['cqi_over_sio'] = {}
+        session['cqi_over_sio']['cqi_client'] = cqi_client
+        session['cqi_over_sio']['cqi_client_lock'] = Lock()
+        session['cqi_over_sio']['db_corpus_id'] = db_corpus_id
+        return {'code': 200, 'msg': 'OK'}
+
+    @socketio_login_required
+    def on_exec(self, fn_name: str, fn_args: Dict = {}):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+        except KeyError:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if fn_name in CQI_API_FUNCTION_NAMES:
+            fn: Callable = getattr(cqi_client.api, fn_name)
+        elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
+            fn: Callable = getattr(extensions, fn_name)
+        else:
+            return {'code': 400, 'msg': 'Bad Request'}
+        for param in signature(fn).parameters.values():
+            if param.default is param.empty:
+                if param.name not in fn_args:
+                    return {'code': 400, 'msg': 'Bad Request'}
+            else:
+                if param.name not in fn_args:
+                    continue
+            if type(fn_args[param.name]) is not param.annotation:
+                return {'code': 400, 'msg': 'Bad Request'}
+        cqi_client_lock.acquire()
+        try:
+            fn_return_value = fn(**fn_args)
+        except BrokenPipeError as e:
+            return {'code': 500, 'msg': 'Internal Server Error'}
+        except CQiException as e:
+            return {
+                'code': 502,
+                'msg': 'Bad Gateway',
+                'payload': {
+                    'code': e.code,
+                    'desc': e.description,
+                    'msg': e.__class__.__name__
+                }
+            }
+        finally:
+            cqi_client_lock.release()
+        if isinstance(fn_return_value, CQiStatus):
+            payload = {
+                'code': fn_return_value.code,
+                'msg': fn_return_value.__class__.__name__
+            }
+        else:
+            payload = fn_return_value
+        return {'code': 200, 'msg': 'OK', 'payload': payload}
+
+    def on_disconnect(self):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+            db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+        except KeyError:
+            return
+        cqi_client_lock.acquire()
+        try:
+            session.pop('cqi_over_sio')
+        except KeyError:
+            pass
+        try:
+            cqi_client.api.ctrl_bye()
+        except (BrokenPipeError, CQiException):
+            pass
+        cqi_client_lock.release()
+        db_corpus = Corpus.query.get(db_corpus_id)
+        if db_corpus is not None:
+            db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+            db.session.commit()

app/corpora/cqi_over_sio/cqi.py

@@ -1,113 +0,0 @@
-from cqi import CQiClient
-from cqi.errors import CQiException
-from cqi.status import CQiStatus
-from flask import session
-from inspect import signature
-from threading import Lock
-from typing import Callable, Dict, List
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .extensions import CQI_EXTENSION_FUNCTION_NAMES
-from . import extensions as extensions_module
-
-
-CQI_FUNCTION_NAMES: List[str] = [
-    'ask_feature_cl_2_3',
-    'ask_feature_cqi_1_0',
-    'ask_feature_cqp_2_3',
-    'cl_alg2cpos',
-    'cl_attribute_size',
-    'cl_cpos2alg',
-    'cl_cpos2id',
-    'cl_cpos2lbound',
-    'cl_cpos2rbound',
-    'cl_cpos2str',
-    'cl_cpos2struc',
-    'cl_drop_attribute',
-    'cl_id2cpos',
-    'cl_id2freq',
-    'cl_id2str',
-    'cl_idlist2cpos',
-    'cl_lexicon_size',
-    'cl_regex2id',
-    'cl_str2id',
-    'cl_struc2cpos',
-    'cl_struc2str',
-    'corpus_alignment_attributes',
-    'corpus_charset',
-    'corpus_drop_corpus',
-    'corpus_full_name',
-    'corpus_info',
-    'corpus_list_corpora',
-    'corpus_positional_attributes',
-    'corpus_properties',
-    'corpus_structural_attribute_has_values',
-    'corpus_structural_attributes',
-    'cqp_drop_subcorpus',
-    'cqp_dump_subcorpus',
-    'cqp_fdist_1',
-    'cqp_fdist_2',
-    'cqp_list_subcorpora',
-    'cqp_query',
-    'cqp_subcorpus_has_field',
-    'cqp_subcorpus_size',
-    'ctrl_bye',
-    'ctrl_connect',
-    'ctrl_last_general_error',
-    'ctrl_ping',
-    'ctrl_user_abort'
-]
-
-
-@socketio.on('cqi', namespace=ns)
-@socketio_login_required
-def cqi_over_sio(fn_name: str, fn_args: Dict = {}):
-    try:
-        cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-        cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
-    except KeyError:
-        return {'code': 424, 'msg': 'Failed Dependency'}
-    if fn_name in CQI_FUNCTION_NAMES:
-        fn: Callable = getattr(cqi_client.api, fn_name)
-    elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
-        fn: Callable = getattr(extensions_module, fn_name)
-    else:
-        return {'code': 400, 'msg': 'Bad Request'}
-    for param in signature(fn).parameters.values():
-        if param.default is param.empty:
-            if param.name not in fn_args:
-                return {'code': 400, 'msg': 'Bad Request'}
-        else:
-            if param.name not in fn_args:
-                continue
-        if type(fn_args[param.name]) is not param.annotation:
-            return {'code': 400, 'msg': 'Bad Request'}
-    cqi_client_lock.acquire()
-    try:
-        fn_return_value = fn(**fn_args)
-    except BrokenPipeError:
-        fn_return_value = {
-            'code': 500,
-            'msg': 'Internal Server Error'
-        }
-    except CQiException as e:
-        return {
-            'code': 502,
-            'msg': 'Bad Gateway',
-            'payload': {
-                'code': e.code,
-                'desc': e.description,
-                'msg': e.__class__.__name__
-            }
-        }
-    finally:
-        cqi_client_lock.release()
-    if isinstance(fn_return_value, CQiStatus):
-        payload = {
-            'code': fn_return_value.code,
-            'msg': fn_return_value.__class__.__name__
-        }
-    else:
-        payload = fn_return_value
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_sio/extensions.py

@@ -28,8 +28,9 @@ CQI_EXTENSION_FUNCTION_NAMES: List[str] = [
 
 
 def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
-    db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
     cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
     cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
     db_corpus.num_tokens = cqi_corpus.size
     db.session.commit()
@@ -37,10 +38,11 @@ def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
 
 
 def ext_corpus_static_data(corpus: str) -> Dict:
-    db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
-    static_corpus_data_file = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
-    if os.path.exists(static_corpus_data_file):
+    cache_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
-        with open(static_corpus_data_file, 'rb') as f:
+    if os.path.exists(cache_file_path):
+        with open(cache_file_path, 'rb') as f:
             return f.read()
     cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
     cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
@@ -189,10 +191,10 @@ def ext_corpus_static_data(corpus: str) -> Dict:
             } for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
         }
         del sub_s_attr_values
-    with gzip.open(static_corpus_data_file, 'wt') as f:
+    with gzip.open(cache_file_path, 'wt') as f:
         json.dump(static_corpus_data, f)
     del static_corpus_data
-    with open(static_corpus_data_file, 'rb') as f:
+    with open(cache_file_path, 'rb') as f:
         return f.read()
 
 

app/static/js/CorpusAnalysis/CorpusAnalysisApp.js

@@ -1,13 +1,15 @@
 class CorpusAnalysisApp {
   constructor(corpusId) {
+    this.corpusId = corpusId;
+
     this.data = {};
 
     // HTML elements
     this.elements = {
       container: document.querySelector('#corpus-analysis-app-container'),
+      extensionCards: document.querySelector('#corpus-analysis-app-extension-cards'),
       extensionTabs: document.querySelector('#corpus-analysis-app-extension-tabs'),
-      initModal: document.querySelector('#corpus-analysis-app-init-modal'),
+      initModal: document.querySelector('#corpus-analysis-app-init-modal')
-      overview: document.querySelector('#corpus-analysis-app-overview')
     };
     // Materialize elements
     this.elements.m = {
@@ -17,80 +19,97 @@ class CorpusAnalysisApp {
 
     this.extensions = {};
 
-    this.settings = {
+    this.settings = {};
-      corpusId: corpusId
-    };
   }
 
-  init() {
+  async init() {
     this.disableActionElements();
     this.elements.m.initModal.open();
-    // Init data
-    this.data.cqiClient = new cqi.CQiClient('/cqi_over_sio', this.settings.corpusId);
-    this.data.cqiClient.connect('anonymous', '')
-      .then((cqiStatus) => {
-        return this.data.cqiClient.corpora.list();
-      })
-      .then((cqiCorpora) => {
-          this.data.corpus = {o: cqiCorpora[0]};
-          console.log(this.data.corpus.o.staticData);
 
-          // TODO: Don't do this hgere
+    const statusTextElement = this.elements.initModal.querySelector('.status-text');
-          this.data.corpus.o.updateDb();
-          this.enableActionElements();
-          for (let extension of Object.values(this.extensions)) {extension.init();}
-          this.elements.m.initModal.close();
-        },
-        (cqiError) => {
-          let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
-          let errorsElement = this.elements.initModal.querySelector('.errors');
-          let progressElement = this.elements.initModal.querySelector('.progress');
-          errorsElement.innerText = errorString;
-          errorsElement.classList.remove('hide');
-          app.flash(errorString, 'error');
-          progressElement.classList.add('hide');
-        }
-      );
 
-    // Add event listeners
+    // Setup CQi over SocketIO connection and gather data from the CQPServer
-    for (let extensionSelectorElement of this.elements.overview.querySelectorAll('.extension-selector')) {
+    try {
+      statusTextElement.innerText = 'Creating CQi over SocketIO client...';
+      const cqiClient = new cqi.CQiClient('/cqi_over_sio');
+      statusTextElement.innerText += ' Done';
+      statusTextElement.innerHTML += '<br>Waiting for the CQP server...';
+      const response = await cqiClient.api.socket.emitWithAck('init', this.corpusId);
+      if (response.code !== 200) {throw new Error();}
+      statusTextElement.innerText += ' Done';
+      statusTextElement.innerHTML += '<br>Connecting to the CQP server...';
+      await cqiClient.connect('anonymous', '');
+      statusTextElement.innerText += ' Done';
+      statusTextElement.innerHTML += '<br>Building and receiving corpus data cache from the server (This may take a while)...';
+      const cqiCorpus = await cqiClient.corpora.get(`NOPAQUE-${this.corpusId.toUpperCase()}`);
+      statusTextElement.innerText += ' Done';
+      // TODO: Don't do this hgere
+      await cqiCorpus.updateDb();
+      this.data.cqiClient = cqiClient;
+      this.data.cqiCorpus = cqiCorpus;
+      this.data.corpus = {o: cqiCorpus};  // legacy
+    } catch (error) {
+      let errorString = '';
+      if ('code' in error) {errorString += `[${error.code}] `;}
+      errorString += `${error.constructor.name}`;
+      if ('description' in error) {errorString += `: ${error.description}`;}
+      const errorsElement = this.elements.initModal.querySelector('.errors');
+      const progressElement = this.elements.initModal.querySelector('.progress');
+      errorsElement.innerText = errorString;
+      errorsElement.classList.remove('hide');
+      progressElement.classList.add('hide');
+      return;
+    }
+
+    // Initialize extensions
+    for (const extension of Object.values(this.extensions)) {
+      statusTextElement.innerHTML += `<br>Initializing ${extension.name} extension...`;
+      await extension.init();
+      statusTextElement.innerText += ' Done'
+    }
+    for (const extensionSelectorElement of this.elements.extensionCards.querySelectorAll('.extension-selector')) {
       extensionSelectorElement.addEventListener('click', () => {
         this.elements.m.extensionTabs.select(extensionSelectorElement.dataset.target);
       });
     }
+
+    this.enableActionElements();
+    this.elements.m.initModal.close();
   }
 
   registerExtension(extension) {
-    if (extension.name in this.extensions) {
+    if (extension.name in this.extensions) {return;}
-      console.error(`Can't register extension ${extension.name}: Already registered`);
-      return;
-    }
     this.extensions[extension.name] = extension;
-    if ('cQiClient' in this.data && this.data.cQiClient.connected) {extension.init();}
   }
 
   disableActionElements() {
-    let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
+    const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
-    for (let actionElement of actionElements) {
+    for (const actionElement of actionElements) {
-      if (actionElement.nodeName === 'INPUT') {
+      switch(actionElement.nodeName) {
-        actionElement.disabled = true;
+        case 'INPUT':
-      } else if (actionElement.nodeName === 'SELECT') {
+          actionElement.disabled = true;
-        actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
+          break;
-      } else {
+        case 'SELECT':
-        actionElement.classList.add('disabled');
+          actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
+          break;
+        default:
+          actionElement.classList.add('disabled');
       }
     }
   }
 
   enableActionElements() {
-    let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
+    const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
-    for (let actionElement of actionElements) {
+    for (const actionElement of actionElements) {
-      if (actionElement.nodeName === 'INPUT') {
+      switch(actionElement.nodeName) {
-        actionElement.disabled = false;
+        case 'INPUT':
-      } else if (actionElement.nodeName === 'SELECT') {
+          actionElement.disabled = false;
-        actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
+          break;
-      } else {
+        case 'SELECT':
-        actionElement.classList.remove('disabled');
+          actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
+          break;
+        default:
+          actionElement.classList.remove('disabled');
       }
     }
   }

app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js

@@ -30,62 +30,60 @@ class CorpusAnalysisConcordance {
     this.app.registerExtension(this);
   }
 
-  init() {
+  async submitForm() {
+    this.app.disableActionElements();
+    let query = this.elements.form.query.value.trim();
+    let subcorpusName = this.elements.form['subcorpus-name'].value;
+    this.elements.error.innerText = '';
+    this.elements.error.classList.add('hide');
+    this.elements.progress.classList.remove('hide');
+    try {
+      const subcorpus = {};
+      subcorpus.q = query;
+      subcorpus.selectedItems = new Set();
+      await this.data.corpus.o.query(subcorpusName, query);
+      if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
+      const cqiSubcorpus = await this.data.corpus.o.subcorpora.get(subcorpusName);
+      subcorpus.o = cqiSubcorpus;
+      const paginatedSubcorpus = await cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
+      subcorpus.p = paginatedSubcorpus;
+      this.data.subcorpora[subcorpusName] = subcorpus;
+      this.settings.selectedSubcorpus = subcorpusName;
+      this.renderSubcorpusList();
+      this.renderSubcorpusInfo();
+      this.renderSubcorpusActions();
+      this.renderSubcorpusItems();
+      this.renderSubcorpusPagination();
+      this.elements.progress.classList.add('hide');
+    } catch (error) {
+      let errorString = '';
+      if ('code' in error) {errorString += `[${error.code}] `;}
+      errorString += `${error.constructor.name}`;
+      this.elements.error.innerText = errorString;
+      this.elements.error.classList.remove('hide');
+      app.flash(errorString, 'error');
+      this.elements.progress.classList.add('hide');
+    }
+    this.app.enableActionElements();
+  }
+
+  async init() {
     // Init data
     this.data.corpus = this.app.data.corpus;
     this.data.subcorpora = {};
     // Add event listeners
-    this.elements.form.addEventListener('submit', event => {
+    this.elements.form.addEventListener('submit', (event) => {
       event.preventDefault();
-      this.app.disableActionElements();
+      this.submitForm();
-      let query = this.elements.form.query.value.trim();
-      let subcorpusName = this.elements.form['subcorpus-name'].value;
-      this.elements.error.innerText = '';
-      this.elements.error.classList.add('hide');
-      this.elements.progress.classList.remove('hide');
-      let subcorpus = {};
-      this.data.corpus.o.query(subcorpusName, query)
-        .then((cqiStatus) => {
-          subcorpus.q = query;
-          subcorpus.selectedItems = new Set();
-          if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
-          return this.data.corpus.o.subcorpora.get(subcorpusName);
-        })
-        .then((cqiSubcorpus) => {
-          subcorpus.o = cqiSubcorpus;
-          return cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
-        })
-        .then(
-          (paginatedSubcorpus) => {
-            subcorpus.p = paginatedSubcorpus;
-            this.data.subcorpora[subcorpusName] = subcorpus;
-            this.settings.selectedSubcorpus = subcorpusName;
-            this.renderSubcorpusList();
-            this.renderSubcorpusInfo();
-            this.renderSubcorpusActions();
-            this.renderSubcorpusItems();
-            this.renderSubcorpusPagination();
-            this.elements.progress.classList.add('hide');
-            this.app.enableActionElements();
-          },
-          (cqiError) => {
-            let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
-            this.elements.error.innerText = errorString;
-            this.elements.error.classList.remove('hide');
-            app.flash(errorString, 'error');
-            this.elements.progress.classList.add('hide');
-            this.app.enableActionElements();
-          }
-        );
     });
-    this.elements.form.addEventListener('change', event => {
+    this.elements.form.addEventListener('change', (event) => {
       if (event.target === this.elements.form['context']) {
         this.settings.context = parseInt(this.elements.form['context'].value);
-        this.elements.form.submit.click();
+        this.submitForm();
       }
       if (event.target === this.elements.form['per-page']) {
         this.settings.perPage = parseInt(this.elements.form['per-page'].value);
-        this.elements.form.submit.click();
+        this.submitForm();
       }
       if (event.target === this.elements.form['text-style']) {
         this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -161,7 +159,7 @@ class CorpusAnalysisConcordance {
       </a>
     `.trim();
     M.Tooltip.init(this.elements.subcorpusActions.querySelectorAll('.tooltipped'));
-    this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', event => {
+    this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', (event) => {
       event.preventDefault();
       let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
       let modalElementId = Utils.generateElementId('export-subcorpus-modal-');
@@ -218,7 +216,7 @@ class CorpusAnalysisConcordance {
           }
         }
       );
-      exportButton.addEventListener('click', event => {
+      exportButton.addEventListener('click', (event) => {
         event.preventDefault();
         this.app.disableActionElements();
         this.elements.progress.classList.remove('hide');
@@ -240,7 +238,7 @@ class CorpusAnalysisConcordance {
           promise = subcorpus.o.export(50);
         }
         promise.then(
-          data => {
+          (data) => {
             let blob;
             if (exportFormat === 'csv') {
               let csvContent = 'sep=,\r\n';
@@ -286,7 +284,7 @@ class CorpusAnalysisConcordance {
       });
       modal.open();
     });
-    this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', event => {
+    this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', (event) => {
       event.preventDefault();
       let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
       subcorpus.o.drop().then(
@@ -362,7 +360,7 @@ class CorpusAnalysisConcordance {
     this.setTextStyle();
     this.setTokenRepresentation();
     for (let gotoReaderTriggerElement of this.elements.subcorpusItems.querySelectorAll('.goto-reader-trigger')) {
-      gotoReaderTriggerElement.addEventListener('click', event => {
+      gotoReaderTriggerElement.addEventListener('click', (event) => {
         event.preventDefault();
         let corpusAnalysisReader = this.app.extensions.Reader;
         let itemId = parseInt(gotoReaderTriggerElement.closest('.item').dataset.id);
@@ -384,7 +382,7 @@ class CorpusAnalysisConcordance {
       });
     }
     for (let selectTriggerElement of this.elements.subcorpusItems.querySelectorAll('.select-trigger')) {
-      selectTriggerElement.addEventListener('click', event => {
+      selectTriggerElement.addEventListener('click', (event) => {
         event.preventDefault();
         let itemElement = selectTriggerElement.closest('.item');
         let itemId = parseInt(itemElement.dataset.id);
@@ -446,14 +444,14 @@ class CorpusAnalysisConcordance {
       </li>
     `.trim();
     for (let paginationTriggerElement of this.elements.subcorpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
-      paginationTriggerElement.addEventListener('click', event => {
+      paginationTriggerElement.addEventListener('click', (event) => {
         event.preventDefault();
         this.app.disableActionElements();
         this.elements.progress.classList.remove('hide');
         let page = parseInt(paginationTriggerElement.dataset.target);
         subcorpus.o.paginate(page, this.settings.perPage, this.settings.context)
           .then(
-            paginatedSubcorpus => {
+            (paginatedSubcorpus) => {
               subcorpus.p = paginatedSubcorpus;
               this.renderSubcorpusItems();
               this.renderSubcorpusPagination();

app/static/js/CorpusAnalysis/CorpusAnalysisReader.js

@@ -19,45 +19,52 @@ class CorpusAnalysisReader {
     this.settings = {
       perPage: parseInt(this.elements.form['per-page'].value),
       textStyle: parseInt(this.elements.form['text-style'].value),
-      tokenRepresentation: this.elements.form['token-representation'].value
+      tokenRepresentation: this.elements.form['token-representation'].value,
+      pagination: {
+        innerWindow: 5,
+        outerWindow: 1
+      }
     }
 
     this.app.registerExtension(this);
   }
 
-  init() {
+  async submitForm() {
+    this.app.disableActionElements();
+    this.elements.error.innerText = '';
+    this.elements.error.classList.add('hide');
+    this.elements.progress.classList.remove('hide');
+    try {
+      const paginatedCorpus = await this.data.corpus.o.paginate(1, this.settings.perPage);
+      this.data.corpus.p = paginatedCorpus;
+      this.renderCorpus();
+      this.renderCorpusPagination();
+      this.elements.progress.classList.add('hide');
+    } catch (error) {
+      let errorString = '';
+      if ('code' in error) {errorString += `[${error.code}] `;}
+      errorString += `${error.constructor.name}`;
+      if ('description' in error) {errorString += `: ${error.description}`;}
+      this.elements.error.innerText = errorString;
+      this.elements.error.classList.remove('hide');
+      app.flash(errorString, 'error');
+      this.elements.progress.classList.add('hide');
+    }
+    this.app.enableActionElements();
+  }
+
+  async init() {
     // Init data
     this.data.corpus = this.app.data.corpus;
     // Add event listeners
     this.elements.form.addEventListener('submit', (event) => {
       event.preventDefault();
-      this.app.disableActionElements();
+      this.submitForm();
-      this.elements.error.innerText = '';
-      this.elements.error.classList.add('hide');
-      this.elements.progress.classList.remove('hide');
-      this.data.corpus.o.paginate(1, this.settings.perPage)
-        .then(
-          (paginatedCorpus) => {
-            this.data.corpus.p = paginatedCorpus;
-            this.renderCorpus();
-            this.renderCorpusPagination();
-            this.elements.progress.classList.add('hide');
-            this.app.enableActionElements();
-          },
-          (cqiError) => {
-            let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
-            this.elements.error.innerText = errorString;
-            this.elements.error.classList.remove('hide');
-            app.flash(errorString, 'error');
-            this.elements.progress.classList.add('hide');
-            this.app.enableActionElements();
-          }
-        );
     });
-    this.elements.form.addEventListener('change', event => {
+    this.elements.form.addEventListener('change', (event) => {
       if (event.target === this.elements.form['per-page']) {
         this.settings.perPage = parseInt(this.elements.form['per-page'].value);
-        this.elements.form.submit.click();
+        this.submitForm();
       }
       if (event.target === this.elements.form['text-style']) {
         this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -69,7 +76,7 @@ class CorpusAnalysisReader {
       }
     });
     // Load initial data
-    this.elements.form.submit.click();
+    await this.submitForm();
   }
 
   clearCorpus() {
@@ -142,7 +149,7 @@ class CorpusAnalysisReader {
     }
 
     // render page buttons (5 before and 5 after current page)
-    for (let i = this.data.corpus.p.page -5; i <= this.data.corpus.p.page; i++) {
+    for (let i = this.data.corpus.p.page - this.settings.pagination.innerWindow; i <= this.data.corpus.p.page; i++) {
       if (i <= 0) {continue;}
       pageElement = Utils.HTMLToElement(
         `
@@ -153,7 +160,7 @@ class CorpusAnalysisReader {
       );
       this.elements.corpusPagination.appendChild(pageElement);
     };
-    for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page +5; i++) {
+    for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page + this.settings.pagination.innerWindow; i++) {
       if (i > this.data.corpus.p.pages) {break;}
       pageElement = Utils.HTMLToElement(
         `
@@ -201,7 +208,7 @@ class CorpusAnalysisReader {
     this.elements.corpusPagination.appendChild(pageElement);
     
     for (let paginateTriggerElement of this.elements.corpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
-      paginateTriggerElement.addEventListener('click', event => {
+      paginateTriggerElement.addEventListener('click', (event) => {
         event.preventDefault();
         let page = parseInt(paginateTriggerElement.dataset.target);
         this.page(page);

app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js

@@ -13,7 +13,7 @@ class CorpusAnalysisStaticVisualization {
     this.app.registerExtension(this);
   }
 
-  init() {
+  async init() {
     // Init data
     this.data.corpus = this.app.data.corpus;
     this.renderGeneralCorpusInfo();

app/static/js/cqi/api/client.js

@@ -1,18 +1,16 @@
 cqi.api.APIClient = class APIClient {
   /**
    * @param {string} host
-   * @param {string} corpusId
    * @param {number} [timeout=60] timeout
    * @param {string} [version=0.1] version
    */
-  constructor(host, corpus_id, timeout = 60, version = '0.1') {
+  constructor(host, timeout = 60, version = '0.1') {
     this.host = host;
     this.timeout = timeout * 1000;  // convert seconds to milliseconds
     this.version = version;
     this.socket = io(
       this.host,
       {
-        auth: {corpus_id: corpus_id},
         transports: ['websocket'],
         upgrade: false
       }
@@ -24,22 +22,16 @@ cqi.api.APIClient = class APIClient {
    * @param {object} [fn_args={}]
    * @returns {Promise}
    */
-  #request(fn_name, fn_args = {}) {
+  async #request(fn_name, fn_args = {}) {
-    return new Promise((resolve, reject) => {
+    // TODO: implement timeout
-      // this.socket.timeout(this.timeout).emit('cqi', {fn_name: fn_name, fn_args: fn_args}, (timeoutError, response) => {
+    let response = await this.socket.emitWithAck('exec', fn_name, fn_args);
-      //   if (timeoutError) {
+    if (response.code === 200) {
-      //     reject(timeoutError);
+      return response.payload;
-      //   }
+    } else if (response.code === 500) {
-      this.socket.emit('cqi', fn_name, fn_args, (response) => {
+      throw new Error(`[${response.code}] ${response.msg}`);
-        if (response.code === 200) {
+    } else if (response.code === 502) {
-          resolve(response.payload);
+      throw new cqi.errors.lookup[response.payload.code]();
-        } else if (response.code === 500) {
+    }
-          reject(new Error(`[${response.code}] ${response.msg}`));
-        } else if (response.code === 502) {
-          reject(new cqi.errors.lookup[response.payload.code]());
-        }
-      });
-    });
   }
 
   /**
@@ -630,7 +622,9 @@ cqi.api.APIClient = class APIClient {
   async ext_corpus_static_data(corpus) {
     const fn_name = 'ext_corpus_static_data';
     const fn_args = {corpus: corpus};
-    return await this.#request(fn_name, fn_args);
+    let compressedEncodedData = await this.#request(fn_name, fn_args);
+    let data = pako.inflate(compressedEncodedData, {to: 'string'});
+    return JSON.parse(data);
   }
 
   /**

app/static/js/cqi/client.js

@@ -1,13 +1,12 @@
 cqi.CQiClient = class CQiClient {
   /**
    * @param {string} host
-   * @param {string} corpusId
    * @param {number} [timeout=60] timeout
    * @param {string} [version=0.1] version
    */
-  constructor(host, corpusId, timeout = 60, version = '0.1') {
+  constructor(host, timeout = 60, version = '0.1') {
      /** @type {cqi.api.APIClient} */
-    this.api = new cqi.api.APIClient(host, corpusId, timeout, version);
+    this.api = new cqi.api.APIClient(host, timeout, version);
   }
 
   /**

app/static/js/cqi/models/corpora.js

@@ -138,15 +138,7 @@ cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.
     /************************************************************************
      *                     Custom additions for nopaque                     *
      ************************************************************************/
-    // returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
+    returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
-    let tmp = await this.client.api.ext_corpus_static_data(corpusName);
-    console.log(tmp);
-    let inflated = pako.inflate(tmp);
-    console.log(inflated);
-    let decoder = new TextDecoder('utf-8');
-    console.log(decoder);
-    let decoded = decoder.decode(inflated);
-    returnValue.static_data = JSON.parse(decoded);
     return returnValue;
   }
 

app/templates/_scripts.html.j2

@@ -1,6 +1,6 @@
 <script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.1/fast-json-patch.min.js" integrity="sha512-5uDdefwnzyq4N+SkmMBmekZLZNmc6dLixvVxCdlHBfqpyz0N3bzLdrJ55OLm7QrZmgZuhLGgHLDtJwU6RZoFCA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/2.3.1/list.min.js" integrity="sha512-93wYgwrIFL+b+P3RvYxi/WUFRXXUDSLCT2JQk9zhVGXuS2mHl2axj6d+R6pP+gcU5isMHRj1u0oYE/mWyt/RjA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.4/socket.io.min.js" integrity="sha512-HTENHrkQ/P0NGDFd5nk6ibVtCkcM7jhr2c7GyvXp5O+4X6O5cQO9AhqFzM+MdeBivsX7Hoys2J7pp2wdgMpCvw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.1/socket.io.min.js" integrity="sha512-+NaO7d6gQ1YPxvc/qHIqZEchjGm207SszoNeMgppoqD/67fEqmc1edS8zrbxPD+4RQI3gDgT/83ihpFW61TG/Q==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/2.24.2/plotly.min.js" integrity="sha512-dAXqGCq94D0kgLSPnfvd/pZpCMoJQpGj2S2XQmFQ9Ay1+96kbjss02ISEh+TBNXMggGg/1qoMcOHcxg+Op/Jmw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/pako/2.1.0/pako_inflate.min.js" integrity="sha512-mlnC6JeOvg9V4vBpWMxGKscsCdScB6yvGVCeFF2plnQMRmwH69s9F8SHPbC0oirqfePmRBhqx2s3Bx7WIvHfWg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 

app/templates/corpora/analysis.html.j2

@@ -2,7 +2,7 @@
 {% import "materialize/wtf.html.j2" as wtf %}
 {% import 'corpora/_analysis/concordance.html.j2' as concordance_extension %}
 {% import 'corpora/_analysis/reader.html.j2' as reader_extension %}
-{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension%}
+{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension %}
 
 {% set extensions = [concordance_extension, reader_extension, static_visualization_extension] %}
 {% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %}
@@ -11,20 +11,17 @@
 
 {% block page_content %}
 <ul class="row tabs no-autoinit" id="corpus-analysis-app-extension-tabs">
-  <li class="tab col s3"><a class="active" href="#corpus-analysis-app-overview"><i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus analysis</a></li>
+  <li class="tab col s3"><a class="active" href="#corpus-analysis-app-home-container"><i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus analysis</a></li>
-  {% for extension in extensions %}
+  {% for extension in extensions if extension.name != 'Static Visualization' %}
-  {% if extension.name != 'Static Visualization' %}
   <li class="tab col s3"><a href="#{{ extension.id_prefix }}-container">{{ extension.tab_content }}</a></li>
-  {% endif %}
   {% endfor %}
 </ul>
 
-<div class="row" id="corpus-analysis-app-overview">
+<div id="corpus-analysis-app-home-container">
-  <div class="col s12">
+  <h1>{{ title }}</h1>
-    <h1>{{ title }}</h1>
 
-    {% for extension in extensions %}
+  <div class="row" id="corpus-analysis-app-extension-cards">
-    {% if extension.name != 'Static Visualization' %}
+    {% for extension in extensions if extension.name != 'Static Visualization' %}
     <div class="col s3">
       <div class="card extension-selector hoverable" data-target="{{ extension.id_prefix }}-container">
         <div class="card-content">
@@ -33,17 +30,14 @@
         </div>
       </div>
     </div>
-    {% endif %}
     {% endfor %}
-
-    {{ static_visualization_extension.container_content }}
-
   </div>
+
+  {{ static_visualization_extension.container_content }}
 </div>
 
 
-{% for extension in extensions %}
+{% for extension in extensions if extension.name != 'Static Visualization' %}
-{% if extension.name != 'Static Visualization'%}
 <div id="{{ extension.id_prefix }}-container">
   {{ extension.container_content }}
 </div>
@@ -62,6 +56,7 @@
     <div class="progress">
       <div class="indeterminate"></div>
     </div>
+    <p class="status-text"></p>
     <p class="errors error-color-text hide"></p>
   </div>
 </div>

requirements.txt

@@ -1,5 +1,5 @@
 apifairy
-cqi
+cqi>=0.1.4
 dnspython==2.2.1
 docker
 eventlet