Merge branch 'visualizations-update' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into visualizations-update

app/corpora/__init__.py

@@ -16,4 +16,4 @@ def before_request():
     pass
 
 
-from . import cli, cqi_over_socketio, files, followers, routes, json_routes
+from . import cli, cqi_over_sio, files, followers, routes, json_routes

app/corpora/cqi_over_sio/__init__.py

@@ -1,8 +1,9 @@
+from cqi import CQiClient
+from cqi.errors import CQiException
 from flask import session
 from flask_login import current_user
 from flask_socketio import ConnectionRefusedError
 from threading import Lock
-import cqi
 from app import db, hashids, socketio
 from app.decorators import socketio_login_required
 from app.models import Corpus, CorpusStatus
@@ -39,16 +40,9 @@ Basic concept:
 '''
 
 
-NAMESPACE = '/corpora/corpus/corpus_analysis'
+NAMESPACE = '/cqi_over_sio'
 
 
-# Import all CQi over Socket.IO event handlers
-from .cqi_corpora_corpus_subcorpora import *  # noqa
-from .cqi_corpora_corpus_structural_attributes import *  # noqa
-from .cqi_corpora_corpus_positional_attributes import *  # noqa
-from .cqi_corpora_corpus_alignment_attributes import *  # noqa
-from .cqi_corpora_corpus import *  # noqa
-from .cqi_corpora import *  # noqa
 from .cqi import *  # noqa
 
 
@@ -89,8 +83,8 @@ def connect(auth):
         socketio.sleep(3)
         retry_counter -= 1
         db.session.refresh(corpus)
-    cqi_client = cqi.CQiClient(f'cqpserver_{corpus_id}')
+    cqi_client = CQiClient(f'cqpserver_{corpus_id}')
-    session['d'] = {
+    session['cqi_over_sio'] = {
         'corpus_id': corpus_id,
         'cqi_client': cqi_client,
         'cqi_client_lock': Lock(),
@@ -100,16 +94,19 @@ def connect(auth):
 
 @socketio.on('disconnect', namespace=NAMESPACE)
 def disconnect():
-    if 'd' not in session:
-        return
-    session['d']['cqi_client_lock'].acquire()
     try:
-        session['d']['cqi_client'].disconnect()
+        cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
-    except (BrokenPipeError, cqi.errors.CQiException):
+        cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+    except KeyError:
+        return
+    cqi_client_lock.acquire()
+    try:
+        cqi_client.api.ctrl_bye()
+    except (BrokenPipeError, CQiException):
         pass
-    session['d']['cqi_client_lock'].release()
+    cqi_client_lock.release()
-    corpus = Corpus.query.get(session['d']['corpus_id'])
+    corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
     corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
     db.session.commit()
-    session.pop('d')
+    session.pop('cqi_over_sio')
     # return {'code': 200, 'msg': 'OK'}

app/corpora/cqi_over_sio/cqi.py

@@ -0,0 +1,120 @@
+from cqi import CQiClient
+from cqi.errors import CQiException
+from cqi.status import CQiStatus
+from flask import session
+from inspect import signature
+from threading import Lock
+from typing import Callable, Dict, List
+from app import socketio
+from app.decorators import socketio_login_required
+from . import NAMESPACE as ns
+from .extensions import CQI_EXTENSION_FUNCTION_NAMES
+from . import extensions as extensions_module
+
+
+CQI_FUNCTION_NAMES: List[str] = [
+    'ask_feature_cl_2_3',
+    'ask_feature_cqi_1_0',
+    'ask_feature_cqp_2_3',
+    'cl_alg2cpos',
+    'cl_attribute_size',
+    'cl_cpos2alg',
+    'cl_cpos2id',
+    'cl_cpos2lbound',
+    'cl_cpos2rbound',
+    'cl_cpos2str',
+    'cl_cpos2struc',
+    'cl_drop_attribute',
+    'cl_id2cpos',
+    'cl_id2freq',
+    'cl_id2str',
+    'cl_idlist2cpos',
+    'cl_lexicon_size',
+    'cl_regex2id',
+    'cl_str2id',
+    'cl_struc2cpos',
+    'cl_struc2str',
+    'corpus_alignment_attributes',
+    'corpus_charset',
+    'corpus_drop_corpus',
+    'corpus_full_name',
+    'corpus_info',
+    'corpus_list_corpora',
+    'corpus_positional_attributes',
+    'corpus_properties',
+    'corpus_structural_attribute_has_values',
+    'corpus_structural_attributes',
+    'cqp_drop_subcorpus',
+    'cqp_dump_subcorpus',
+    'cqp_fdist_1',
+    'cqp_fdist_2',
+    'cqp_list_subcorpora',
+    'cqp_query',
+    'cqp_subcorpus_has_field',
+    'cqp_subcorpus_size',
+    'ctrl_bye',
+    'ctrl_connect',
+    'ctrl_last_general_error',
+    'ctrl_ping',
+    'ctrl_user_abort'
+]
+
+
+@socketio.on('cqi', namespace=ns)
+@socketio_login_required
+def cqi_over_sio(fn_data):
+    try:
+        fn_name: str = fn_data['fn_name']
+    except KeyError:
+        return {'code': 400, 'msg': 'Bad Request'}
+    fn_name: str = fn_data['fn_name']
+    fn_args: Dict = fn_data.get('fn_args', {})
+    try:
+        cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+        cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+    except KeyError:
+        return {'code': 424, 'msg': 'Failed Dependency'}
+    if fn_name in CQI_FUNCTION_NAMES:
+        fn: Callable = getattr(cqi_client.api, fn_name)
+    elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
+        fn_args['cqi_client'] = cqi_client
+        fn: Callable = getattr(extensions_module, fn_name)
+    else:
+        return {'code': 400, 'msg': 'Bad Request'}
+    for param in signature(fn).parameters.values():
+        if param.default is param.empty:
+            if param.name not in fn_args:
+                return {'code': 400, 'msg': 'Bad Request'}
+        else:
+            if param.name not in fn_args:
+                continue
+        if type(fn_args[param.name]) is not param.annotation:
+            return {'code': 400, 'msg': 'Bad Request'}
+    cqi_client_lock.acquire()
+    try:
+        return_value = fn(**fn_args)
+    except BrokenPipeError:
+        return_value = {
+            'code': 500,
+            'msg': 'Internal Server Error'
+        }
+    except CQiException as e:
+        return_value = {
+            'code': 502,
+            'msg': 'Bad Gateway',
+            'payload': {
+                'code': e.code,
+                'desc': e.description,
+                'msg': e.__class__.__name__
+            }
+        }
+    finally:
+        cqi_client_lock.release()
+    if isinstance(return_value, CQiStatus):
+        payload = {
+            'code': return_value.code,
+            'msg': return_value.__class__.__name__
+        }
+    else:
+        payload = return_value
+    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_sio/extensions.py

@@ -0,0 +1,254 @@
+from collections import Counter
+from cqi import CQiClient
+from cqi.models.corpora import Corpus
+from cqi.status import StatusOk
+from flask import session
+from typing import Dict, List
+import json
+import math
+import os
+from app import db
+from app.models import Corpus
+from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
+
+
+CQI_EXTENSION_FUNCTION_NAMES: List[str] = [
+    'ext_corpus_update_db',
+    'ext_corpus_static_data',
+    'ext_corpus_paginate_corpus',
+    'ext_cqp_paginate_subcorpus',
+    'ext_cqp_partial_export_subcorpus',
+    'ext_cqp_export_subcorpus',
+]
+
+
+def ext_corpus_update_db(cqi_client: CQiClient, corpus: str):
+    db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    db_corpus.num_tokens = cqi_corpus.size
+    db.session.commit()
+    return StatusOk()
+
+
+def ext_corpus_static_data(cqi_client: CQiClient, corpus: str) -> Dict:
+    db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
+    static_corpus_data_file = os.path.join(db_corpus.path, 'cwb', 'static.json')
+    if os.path.exists(static_corpus_data_file):
+        with open(static_corpus_data_file, 'r') as f:
+            return json.load(f)
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    ##########################################################################
+    # A faster way to get cpos boundaries for smaller s_attrs                #
+    ##########################################################################
+    # cqi_corpus.query('Last', '<s> []* </s>;')
+    # cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
+    # print(cqi_subcorpus.size)
+    # first_match = 0
+    # last_match = cqi_subcorpus.attrs['size'] - 1
+    # match_boundaries = zip(
+    #     list(range(first_match, last_match + 1)),
+    #     cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['match'], first_match, last_match),
+    #     cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['matchend'], first_match, last_match)
+    # )
+    # for x in match_boundaries:
+    #     print(x)
+    cqi_p_attrs = {
+        p_attr.name: p_attr
+        for p_attr in cqi_corpus.positional_attributes.list()
+    }
+    cqi_s_attrs = {
+        s_attr.name: s_attr
+        for s_attr in cqi_corpus.structural_attributes.list()
+    }
+    static_corpus_data = {
+        'corpus': {
+            'bounds': [0, cqi_corpus.size - 1],
+            'counts': {
+                'token': cqi_corpus.size
+            },
+            'freqs': {}
+        },
+        'p_attrs': {},
+        's_attrs': {},
+        'values': {'p_attrs': {}, 's_attrs': {}}
+    }
+    for p_attr in cqi_p_attrs.values():
+        static_corpus_data['corpus']['freqs'][p_attr.name] = dict(
+            zip(
+                range(0, p_attr.lexicon_size),
+                p_attr.freqs_by_ids(list(range(0, p_attr.lexicon_size)))
+            )
+        )
+        static_corpus_data['p_attrs'][p_attr.name] = dict(
+            zip(
+                range(0, cqi_corpus.size),
+                p_attr.ids_by_cpos(list(range(0, cqi_corpus.size)))
+            )
+        )
+        static_corpus_data['values']['p_attrs'][p_attr.name] = dict(
+            zip(
+                range(0, p_attr.lexicon_size),
+                p_attr.values_by_ids(list(range(0, p_attr.lexicon_size)))
+            )
+        )
+    for s_attr in cqi_s_attrs.values():
+        if s_attr.has_values:
+            continue
+        static_corpus_data['corpus']['counts'][s_attr.name] = s_attr.size
+        static_corpus_data['s_attrs'][s_attr.name] = {'lexicon': {}, 'values': None}
+        static_corpus_data['values']['s_attrs'][s_attr.name] = {}
+        for id in range(0, s_attr.size):
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id] = {}
+            lbound, rbound = s_attr.cpos_by_id(id)
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {}
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1
+            if s_attr.name not in ['text', 's']:
+                continue
+            cpos_range = range(lbound, rbound + 1)
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['ent'] = len({x for x in cqi_s_attrs['ent'].ids_by_cpos(list(cpos_range)) if x != -1})
+            if s_attr.name != 'text':
+                continue
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['s'] = len({x for x in cqi_s_attrs['s'].ids_by_cpos(list(cpos_range)) if x != -1})
+            static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
+            for p_attr in cqi_p_attrs.values():
+                static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr.ids_by_cpos(list(cpos_range))))
+        sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
+        s_attr_value_names = [
+            sub_s_attr.name[(len(s_attr.name) + 1):]
+            for sub_s_attr in sub_s_attrs
+        ]
+        sub_s_attr_values = [
+            sub_s_attr.values_by_ids(list(range(0, s_attr.size)))
+            for sub_s_attr in sub_s_attrs
+        ]
+        static_corpus_data['s_attrs'][s_attr.name]['values'] = s_attr_value_names
+        static_corpus_data['values']['s_attrs'][s_attr.name] = {
+            s_attr_id: {
+                s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id_idx]
+                for s_attr_value_name_idx, s_attr_value_name in enumerate(
+                    static_corpus_data['s_attrs'][s_attr.name]['values']
+                )
+            } for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
+        }
+    with open(static_corpus_data_file, 'w') as f:
+        json.dump(static_corpus_data, f)
+    return static_corpus_data
+
+
+def ext_corpus_paginate_corpus(
+    cqi_client: CQiClient,
+    corpus: str,
+    page: int = 1,
+    per_page: int = 20
+) -> Dict:
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_corpus.size > 0
+            and page > math.ceil(cqi_corpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    first_cpos = (page - 1) * per_page
+    last_cpos = min(cqi_corpus.size, first_cpos + per_page)
+    cpos_list = [*range(first_cpos, last_cpos)]
+    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
+    payload = {}
+    # the items for the current page
+    payload['items'] = [cpos_list]
+    # the lookups for the items
+    payload['lookups'] = lookups
+    # the total number of items matching the query
+    payload['total'] = cqi_corpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_paginate_subcorpus(
+    cqi_client: CQiClient,
+    subcorpus: str,
+    context: int = 50,
+    page: int = 1,
+    per_page: int = 20
+) -> Dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_subcorpus.size > 0
+            and page > math.ceil(cqi_subcorpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    offset = (page - 1) * per_page
+    cutoff = per_page
+    cqi_results_export = export_subcorpus(
+        cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
+    payload = {}
+    # the items for the current page
+    payload['items'] = cqi_results_export.pop('matches')
+    # the lookups for the items
+    payload['lookups'] = cqi_results_export
+    # the total number of items matching the query
+    payload['total'] = cqi_subcorpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_partial_export_subcorpus(
+    cqi_client: CQiClient,
+    subcorpus: str,
+    match_id_list: list,
+    context: int = 50
+) -> Dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
+    return cqi_subcorpus_partial_export
+
+
+def ext_cqp_export_subcorpus(
+    cqi_client: CQiClient,
+    subcorpus: str,
+    context: int = 50
+) -> Dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
+    return cqi_subcorpus_export

app/corpora/cqi_over_sio/utils.py

@@ -1,64 +1,10 @@
-from flask import session
+from cqi.models.corpora import Corpus
-from functools import wraps
+from cqi.models.subcorpora import Subcorpus
-from inspect import signature
+from typing import Dict, List
-import cqi
+from app.models import Corpus
 
 
-def cqi_over_socketio(f):
+def lookups_by_cpos(corpus: Corpus, cpos_list: List[int]) -> Dict:
-    @wraps(f)
-    def wrapped(*args):
-        if 'd' not in session:
-            return {'code': 424, 'msg': 'Failed Dependency'}
-        f_args = {}
-        # Check for missing args and if all provided args are of the right type
-        for param in signature(f).parameters.values():
-            if param.name == 'corpus_name':
-                f_args[param.name] = f'NOPAQUE_{session["d"]["corpus_id"]}'
-                continue
-            if param.name == 'cqi_client':
-                f_args[param.name] = session['d']['cqi_client']
-                continue
-            if param.default is param.empty:
-                # args
-                if param.name not in args[0]:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                arg = args[0][param.name]
-                if type(arg) is not param.annotation:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                f_args[param.name] = arg
-            else:
-                # kwargs
-                if param.name not in args[0]:
-                    continue
-                arg = args[0][param.name]
-                if type(arg) is not param.annotation:
-                    return {'code': 400, 'msg': 'Bad Request'}
-                f_args[param.name] = arg
-        session['d']['cqi_client_lock'].acquire()
-        try:
-            return_value = f(**f_args)
-        except BrokenPipeError:
-            return_value = {
-                'code': 500,
-                'msg': 'Internal Server Error'
-            }
-        except cqi.errors.CQiException as e:
-            return_value = {
-                'code': 500,
-                'msg': 'Internal Server Error',
-                'payload': {
-                    'code': e.code,
-                    'desc': e.description,
-                    'msg': e.name
-                }
-            }
-        finally:
-            session['d']['cqi_client_lock'].release()
-        return return_value
-    return wrapped
-
-
-def lookups_by_cpos(corpus, cpos_list):
     lookups = {}
     lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
     for attr in corpus.positional_attributes.list():
@@ -93,18 +39,22 @@ def lookups_by_cpos(corpus, cpos_list):
     return lookups
 
 
-def partial_export_subcorpus(subcorpus, match_id_list, context=25):
+def partial_export_subcorpus(
-    if subcorpus.attrs['size'] == 0:
+    subcorpus: Subcorpus,
+    match_id_list: List[int],
+    context: int = 25
+) -> Dict:
+    if subcorpus.size == 0:
         return {"matches": []}
     match_boundaries = []
     for match_id in match_id_list:
-        if match_id < 0 or match_id >= subcorpus.attrs['size']:
+        if match_id < 0 or match_id >= subcorpus.size:
             continue
         match_boundaries.append(
             (
                 match_id,
-                subcorpus.dump(subcorpus.attrs['fields']['match'], match_id, match_id)[0],
+                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
-                subcorpus.dump(subcorpus.attrs['fields']['matchend'], match_id, match_id)[0]
+                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
             )
         )
     cpos_set = set()
@@ -120,14 +70,14 @@ def partial_export_subcorpus(subcorpus, match_id_list, context=25):
             lc_rbound = match_start - 1
             lc = (lc_lbound, lc_rbound)
             cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
             rc = None
             cpos_list_rbound = match_end
         else:
             rc_lbound = match_end + 1
             rc_rbound = min(
                 (match_end + context),
-                (subcorpus.collection.corpus.attrs['size'] - 1)
+                (subcorpus.collection.corpus.size - 1)
             )
             rc = (rc_lbound, rc_rbound)
             cpos_list_rbound = rc_rbound
@@ -138,15 +88,20 @@ def partial_export_subcorpus(subcorpus, match_id_list, context=25):
     return {'matches': matches, **lookups}
 
 
-def export_subcorpus(subcorpus, context=25, cutoff=float('inf'), offset=0):
+def export_subcorpus(
-    if subcorpus.attrs['size'] == 0:
+    subcorpus: Subcorpus,
+    context: int = 25,
+    cutoff: float = float('inf'),
+    offset: int = 0
+) -> Dict:
+    if subcorpus.size == 0:
         return {"matches": []}
     first_match = max(0, offset)
-    last_match = min((offset + cutoff - 1), (subcorpus.attrs['size'] - 1))
+    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
     match_boundaries = zip(
-        list(range(first_match, last_match + 1)),
+        range(first_match, last_match + 1),
-        subcorpus.dump(subcorpus.attrs['fields']['match'], first_match, last_match),
+        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
-        subcorpus.dump(subcorpus.attrs['fields']['matchend'], first_match, last_match)
+        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
     )
     cpos_set = set()
     matches = []
@@ -160,14 +115,14 @@ def export_subcorpus(subcorpus, context=25, cutoff=float('inf'), offset=0):
             lc_rbound = match_start - 1
             lc = (lc_lbound, lc_rbound)
             cpos_list_lbound = lc_lbound
-        if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
             rc = None
             cpos_list_rbound = match_end
         else:
             rc_lbound = match_end + 1
             rc_rbound = min(
                 (match_end + context),
-                (subcorpus.collection.corpus.attrs['size'] - 1)
+                (subcorpus.collection.corpus.size - 1)
             )
             rc = (rc_lbound, rc_rbound)
             cpos_list_rbound = rc_rbound

app/corpora/cqi_over_socketio/cqi.py

@@ -1,43 +0,0 @@
-from socket import gaierror
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.connect', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_connect(cqi_client: cqi.CQiClient):
-    try:
-        cqi_status = cqi_client.connect()
-    except gaierror as e:
-        return {
-            'code': 500,
-            'msg': 'Internal Server Error',
-            'payload': {'code': e.args[0], 'desc': e.args[1]}
-        }
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.disconnect', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_disconnect(cqi_client: cqi.CQiClient):
-    cqi_status = cqi_client.disconnect()
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.ping', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_ping(cqi_client: cqi.CQiClient):
-    cqi_status = cqi_client.ping()
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora.py

@@ -1,22 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_get(cqi_client: cqi.CQiClient, corpus_name: str):
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = {**cqi_corpus.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_list(cqi_client: cqi.CQiClient):
-    payload = [{**x.attrs} for x in cqi_client.corpora.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus.py

@@ -1,199 +0,0 @@
-from collections import Counter
-from flask import session
-import cqi
-import json
-import math
-import os
-from app import db, socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, lookups_by_cpos
-
-
-@socketio.on('cqi.corpora.corpus.drop', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_drop(cqi_client: cqi.CQiClient, corpus_name: str):
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_status = cqi_corpus.drop()
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.query', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_query(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, query: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_status = cqi_corpus.query(subcorpus_name, query)
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-###############################################################################
-# nopaque specific CQi extensions                                             #
-###############################################################################
-@socketio.on('cqi.corpora.corpus.update_db', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_update_db(cqi_client: cqi.CQiClient, corpus_name: str):
-    corpus = Corpus.query.get(session['d']['corpus_id'])
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    corpus.num_tokens = cqi_corpus.size
-    db.session.commit()
-
-
-@socketio.on('cqi.corpora.corpus.get_visualization_data', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_get_visualization_data(cqi_client: cqi.CQiClient, corpus_name: str):
-    corpus = Corpus.query.get(session['d']['corpus_id'])
-    visualization_data_file_path = os.path.join(corpus.path, 'cwb', 'visualization_data.json')
-    if os.path.exists(visualization_data_file_path):
-        with open(visualization_data_file_path, 'r') as f:
-            payload = json.load(f)
-        return {'code': 200, 'msg': 'OK', 'payload': payload}
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    ##########################################################################
-    # A faster way to get cpos boundaries for smaller s_attrs                #
-    ##########################################################################
-    # cqi_corpus.query('Last', '<s> []* </s>;')
-    # cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
-    # print(cqi_subcorpus.size)
-    # first_match = 0
-    # last_match = cqi_subcorpus.attrs['size'] - 1
-    # match_boundaries = zip(
-    #     list(range(first_match, last_match + 1)),
-    #     cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['match'], first_match, last_match),
-    #     cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['matchend'], first_match, last_match)
-    # )
-    # for x in match_boundaries:
-    #     print(x)
-    cqi_p_attrs = {
-        p_attr.name: p_attr
-        for p_attr in cqi_corpus.positional_attributes.list()
-    }
-    cqi_s_attrs = {
-        s_attr.name: s_attr
-        for s_attr in cqi_corpus.structural_attributes.list()
-    }
-    payload = {
-        'corpus': {
-            'bounds': [0, cqi_corpus.size - 1],
-            'counts': {
-                'token': cqi_corpus.size
-            },
-            'freqs': {}
-        },
-        'p_attrs': {},
-        's_attrs': {},
-        'values': {'p_attrs': {}, 's_attrs': {}}
-    }
-    for p_attr in cqi_p_attrs.values():
-        payload['corpus']['freqs'][p_attr.name] = dict(
-            zip(
-                range(0, p_attr.lexicon_size),
-                p_attr.freqs_by_ids(list(range(0, p_attr.lexicon_size)))
-            )
-        )
-        payload['p_attrs'][p_attr.name] = dict(
-            zip(
-                range(0, cqi_corpus.size),
-                p_attr.ids_by_cpos(list(range(0, cqi_corpus.size)))
-            )
-        )
-        payload['values']['p_attrs'][p_attr.name] = dict(
-            zip(
-                range(0, p_attr.lexicon_size),
-                p_attr.values_by_ids(list(range(0, p_attr.lexicon_size)))
-            )
-        )
-    for s_attr in cqi_s_attrs.values():
-        if s_attr.has_values:
-            continue
-        payload['corpus']['counts'][s_attr.name] = s_attr.size
-        payload['s_attrs'][s_attr.name] = {'lexicon': {}, 'values': None}
-        payload['values']['s_attrs'][s_attr.name] = {}
-        for id in range(0, s_attr.size):
-            payload['s_attrs'][s_attr.name]['lexicon'][id] = {}
-            lbound, rbound = s_attr.cpos_by_id(id)
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {}
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1
-            if s_attr.name not in ['text', 's']:
-                continue
-            cpos_range = range(lbound, rbound + 1)
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['ent'] = len({x for x in cqi_s_attrs['ent'].ids_by_cpos(list(cpos_range)) if x != -1})
-            if s_attr.name != 'text':
-                continue
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['s'] = len({x for x in cqi_s_attrs['s'].ids_by_cpos(list(cpos_range)) if x != -1})
-            payload['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
-            for p_attr in cqi_p_attrs.values():
-                payload['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr.ids_by_cpos(list(cpos_range))))
-        sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
-        s_attr_value_names = [
-            sub_s_attr.name[(len(s_attr.name) + 1):]
-            for sub_s_attr in sub_s_attrs
-        ]
-        sub_s_attr_values = [
-            sub_s_attr.values_by_ids(list(range(0, s_attr.size)))
-            for sub_s_attr in sub_s_attrs
-        ]
-        payload['s_attrs'][s_attr.name]['values'] = s_attr_value_names
-        payload['values']['s_attrs'][s_attr.name] = {
-            s_attr_id: {
-                s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id_idx]
-                for s_attr_value_name_idx, s_attr_value_name in enumerate(
-                    payload['s_attrs'][s_attr.name]['values']
-                )
-            } for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
-        }
-    with open(visualization_data_file_path, 'w') as f:
-        json.dump(payload, f)
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.paginate', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, page: int = 1, per_page: int = 20):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    # Sanity checks
-    if (
-        per_page < 1
-        or page < 1
-        or (
-            cqi_corpus.size > 0
-            and page > math.ceil(cqi_corpus.size / per_page)
-        )
-    ):
-        return {'code': 416, 'msg': 'Range Not Satisfiable'}
-    first_cpos = (page - 1) * per_page
-    last_cpos = min(cqi_corpus.size, first_cpos + per_page)
-    cpos_list = [*range(first_cpos, last_cpos)]
-    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
-    payload = {}
-    # the items for the current page
-    payload['items'] = [cpos_list]
-    # the lookups for the items
-    payload['lookups'] = lookups
-    # the total number of items matching the query
-    payload['total'] = cqi_corpus.size
-    # the number of items to be displayed on a page.
-    payload['per_page'] = per_page
-    # The total number of pages
-    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
-    # the current page number (1 indexed)
-    payload['page'] = page if payload['pages'] > 0 else None
-    # True if a previous page exists
-    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
-    # True if a next page exists.
-    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
-    # Number of the previous page.
-    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
-    # Number of the next page
-    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.alignment_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_alignment_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, alignment_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_alignment_attribute = cqi_corpus.alignment_attributes.get(alignment_attribute_name)  # noqa
-    payload = {**cqi_alignment_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.alignment_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_alignment_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.alignment_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.positional_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_positional_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, positional_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_positional_attribute = cqi_corpus.positional_attributes.get(positional_attribute_name)  # noqa
-    payload = {**cqi_positional_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.positional_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_positional_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.positional_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py

@@ -1,24 +0,0 @@
-import cqi
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio
-
-
-@socketio.on('cqi.corpora.corpus.structural_attributes.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_structural_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, structural_attribute_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_structural_attribute = cqi_corpus.structural_attributes.get(structural_attribute_name)  # noqa
-    payload = {**cqi_structural_attribute.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.structural_attributes.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_structural_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.structural_attributes.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}

app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py

@@ -1,125 +0,0 @@
-import cqi
-import math
-from app import socketio
-from app.decorators import socketio_login_required
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, export_subcorpus, partial_export_subcorpus
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.get', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_get(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    payload = {**cqi_subcorpus.attrs}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.list', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_list(cqi_client: cqi.CQiClient, corpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    payload = [{**x.attrs} for x in cqi_corpus.subcorpora.list()]
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.drop', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_drop(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_status = cqi_subcorpus.drop()
-    payload = {'code': cqi_status.code,
-               'msg': cqi_status.__class__.__name__}
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.dump', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_dump(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_1', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_fdist_1(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_2', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_fdist_2(cqi_client: cqi.CQiClient):
-    return {'code': 501, 'msg': 'Not Implemented'}
-
-
-###############################################################################
-# nopaque specific CQi extensions                                             #
-###############################################################################
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.paginate', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50, page: int = 1, per_page: int = 20):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    # Sanity checks
-    if (
-        per_page < 1
-        or page < 1
-        or (
-            cqi_subcorpus.attrs['size'] > 0
-            and page > math.ceil(cqi_subcorpus.attrs['size'] / per_page)
-        )
-    ):
-        return {'code': 416, 'msg': 'Range Not Satisfiable'}
-    offset = (page - 1) * per_page
-    cutoff = per_page
-    cqi_results_export = export_subcorpus(
-        cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
-    payload = {}
-    # the items for the current page
-    payload['items'] = cqi_results_export.pop('matches')
-    # the lookups for the items
-    payload['lookups'] = cqi_results_export
-    # the total number of items matching the query
-    payload['total'] = cqi_subcorpus.attrs['size']
-    # the number of items to be displayed on a page.
-    payload['per_page'] = per_page
-    # The total number of pages
-    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
-    # the current page number (1 indexed)
-    payload['page'] = page if payload['pages'] > 0 else None
-    # True if a previous page exists
-    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
-    # True if a next page exists.
-    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
-    # Number of the previous page.
-    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
-    # Number of the next page
-    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
-    return {'code': 200, 'msg': 'OK', 'payload': payload}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.partial_export', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_partial_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, match_id_list: list, context: int = 50):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
-    return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_partial_export}
-
-
-@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.export', namespace=ns)
-@socketio_login_required
-@cqi_over_socketio
-def cqi_corpora_corpus_subcorpora_subcorpus_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50):  # noqa
-    cqi_corpus = cqi_client.corpora.get(corpus_name)
-    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
-    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
-    return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_export}

app/static/js/CorpusAnalysis/CorpusAnalysisApp.js

@@ -71,25 +71,19 @@ class CorpusAnalysisApp {
     this.disableActionElements();
     this.elements.m.initModal.open();
     // Init data
-    this.data.cQiClient = new CQiClient(this.settings.corpusId);
+    this.data.cqiClient = new cqi.CQiClient('/cqi_over_sio', this.settings.corpusId);
-    this.data.cQiClient.connect()
+    this.data.cqiClient.connect('anonymous', '')
-      .then(cQiStatus => {
+      .then((cqiStatus) => {
-        return this.data.cQiClient.corpora.get(`NOPAQUE_${this.settings.corpusId}`);
+        return this.data.cqiClient.corpora.list();
       })
-      .then(
+      .then((cqiCorpora) => {
-        cQiCorpus => {
+          this.data.corpus = {o: cqiCorpora[0]};
-          this.data.corpus = {o: cQiCorpus};
+          console.log(this.data.corpus.o.staticData);
-          this.data.corpus.o.getVisualizationData()
+          this.renderGeneralCorpusInfo(this.data.corpus.o.staticData);
-            .then(
+          this.renderTextInfoList(this.data.corpus.o.staticData);
-              (data) => {
+          this.renderTextProportionsGraphic(this.data.corpus.o.staticData);
-                console.log(data);
+          this.renderFrequenciesGraphic(this.data.corpus.o.staticData);
-                this.renderGeneralCorpusInfo(data);
+          this.renderBoundsGraphic(this.data.corpus.o.staticData);
-                this.renderTextInfoList(data);
-                this.renderTextProportionsGraphic(data);
-                this.renderFrequenciesGraphic(data);
-                this.renderBoundsGraphic(data);
-              }
-            );
           // this.data.corpus.o.getCorpusData()
           //   .then(corpusData => {
           //     console.log(corpusData);
@@ -100,19 +94,19 @@ class CorpusAnalysisApp {
           //     this.renderBoundsGraphic(corpusData);
           //   });
           // TODO: Don't do this hgere
-          cQiCorpus.updateDb();
+          this.data.corpus.o.updateDb();
           this.enableActionElements();
           for (let extension of Object.values(this.extensions)) {extension.init();}
           this.elements.m.initModal.close();
         },
-        cQiError => {
+        (cqiError) => {
           let errorsElement = this.elements.initModal.querySelector('.errors');
           let progressElement = this.elements.initModal.querySelector('.progress');
-          errorsElement.innerText = JSON.stringify(cQiError);
+          errorsElement.innerText = JSON.stringify(cqiError);
           errorsElement.classList.remove('hide');
           progressElement.classList.add('hide');
-          if ('payload' in cQiError && 'code' in cQiError.payload && 'msg' in cQiError.payload) {
+          if ('payload' in cqiError && 'code' in cqiError.payload && 'msg' in cqiError.payload) {
-            app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
+            app.flash(`${cqiError.payload.code}: ${cqiError.payload.msg}`, 'error');
           }
         }
       );

app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js

@@ -45,18 +45,18 @@ class CorpusAnalysisConcordance {
       this.elements.progress.classList.remove('hide');
       let subcorpus = {};
       this.data.corpus.o.query(subcorpusName, query)
-        .then(cQiStatus => {
+        .then((cqiStatus) => {
           subcorpus.q = query;
           subcorpus.selectedItems = new Set();
           if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
           return this.data.corpus.o.subcorpora.get(subcorpusName);
         })
-        .then(cQiSubcorpus => {
+        .then((cqiSubcorpus) => {
-          subcorpus.o = cQiSubcorpus;
+          subcorpus.o = cqiSubcorpus;
-          return cQiSubcorpus.paginate(1, this.settings.perPage, this.settings.context);
+          return cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
         })
         .then(
-          paginatedSubcorpus => {
+          (paginatedSubcorpus) => {
             subcorpus.p = paginatedSubcorpus;
             this.data.subcorpora[subcorpusName] = subcorpus;
             this.settings.selectedSubcorpus = subcorpusName;
@@ -68,11 +68,12 @@ class CorpusAnalysisConcordance {
             this.elements.progress.classList.add('hide');
             this.app.enableActionElements();
           },
-          cQiError => {
+          (cqiStatus) => {
-            this.elements.error.innerText = JSON.stringify(cQiError);
+            // TODDO: CHECK THIS!
+            this.elements.error.innerText = JSON.stringify(cqiStatus);
             this.elements.error.classList.remove('hide');
-            if ('payload' in cQiError && 'code' in cQiError.payload && 'msg' in cQiError.payload) {
+            if ('payload' in cqiStatus && 'code' in cqiStatus.payload && 'msg' in cqiStatus.payload) {
-              app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
+              app.flash(`${cqiStatus.payload.code}: ${cqiStatus.payload.msg}`, 'error');
             }
             this.elements.progress.classList.add('hide');
             this.app.enableActionElements();
@@ -236,7 +237,7 @@ class CorpusAnalysisConcordance {
             app.flash('No matches selected', 'error');
             return;
           }
-          promise = subcorpus.o.partial_export([...subcorpus.selectedItems], 50);
+          promise = subcorpus.o.partialExport([...subcorpus.selectedItems], 50);
         } else {
           promise = subcorpus.o.export(50);
         }
@@ -291,7 +292,7 @@ class CorpusAnalysisConcordance {
       event.preventDefault();
       let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
       subcorpus.o.drop().then(
-        cQiStatus => {
+        (cQiStatus) => {
           app.flash(`${subcorpus.o.name} deleted`, 'corpus');
           delete this.data.subcorpora[subcorpus.o.name];
           this.settings.selectedSubcorpus = undefined;
@@ -312,7 +313,7 @@ class CorpusAnalysisConcordance {
             this.clearSubcorpusPagination();
           }
         },
-        cQiError => {
+        (cQiError) => {
           app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
         }
       );

app/static/js/CorpusAnalysis/CorpusAnalysisReader.js

@@ -28,7 +28,6 @@ class CorpusAnalysisReader {
   init() {
     // Init data
     this.data.corpus = this.app.data.corpus;
-    this.data.subcorpora = {};
     // Add event listeners
     this.elements.form.addEventListener('submit', (event) => {
       event.preventDefault();
@@ -38,14 +37,14 @@ class CorpusAnalysisReader {
       this.elements.progress.classList.remove('hide');
       this.data.corpus.o.paginate(1, this.settings.perPage)
         .then(
-          paginatedCorpus => {
+          (paginatedCorpus) => {
             this.data.corpus.p = paginatedCorpus;
             this.renderCorpus();
             this.renderCorpusPagination();
             this.elements.progress.classList.add('hide');
             this.app.enableActionElements();
           },
-          error => {
+          (cqiError) => {
             this.elements.error.innerText = JSON.stringify(error);
             this.elements.error.classList.remove('hide');
             if ('payload' in error && 'code' in error.payload && 'msg' in error.payload) {
@@ -247,7 +246,7 @@ class CorpusAnalysisReader {
     this.elements.progress.classList.remove('hide');
     this.data.corpus.o.paginate(pageNum, this.settings.perPage)
       .then(
-        paginatedCorpus => {
+        (paginatedCorpus) => {
           this.data.corpus.p = paginatedCorpus;
           this.renderCorpus();
           this.renderCorpusPagination();

app/static/js/cqi/api/client.js

@@ -0,0 +1,691 @@
+cqi.api.APIClient = class APIClient {
+  /**
+   * @param {string} host
+   * @param {string} corpusId
+   * @param {number} [timeout=60] timeout
+   * @param {string} [version=0.1] version
+   */
+  constructor(host, corpus_id, timeout = 60, version = '0.1') {
+    this.host = host;
+    this.timeout = timeout * 1000;  // convert seconds to milliseconds
+    this.version = version;
+    this.socket = io(
+      this.host,
+      {
+        auth: {corpus_id: corpus_id},
+        transports: ['websocket'],
+        upgrade: false
+      }
+    );
+  }
+
+  /**
+   * @param {string} fn_name
+   * @param {object} [fn_args={}]
+   * @returns {Promise<cqi.status.StatusConnectOk>}
+   */
+  #request(fn_name, fn_args = {}) {
+    return new Promise((resolve, reject) => {
+      this.socket.timeout(this.timeout).emit('cqi', {fn_name: fn_name, fn_args: fn_args}, (timeoutError, response) => {
+        if (timeoutError) {
+          reject(timeoutError);
+        }
+        if (response.code === 200) {
+          resolve(response.payload);
+        }
+        if (response.code === 500) {
+          reject(new Error(`[${response.code}] ${response.msg}`));
+        }
+        if (response.code === 502) {
+          reject(new cqi.errors.lookup[response.payload.code]());
+        }
+      });
+    });
+  }
+
+  /**
+   * @param {string} username
+   * @param {string} password
+   * @returns {Promise<cqi.status.StatusConnectOk>}
+   */
+  async ctrl_connect(username, password) {
+    const fn_name = 'ctrl_connect';
+    const fn_args = {username: username, password: password};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusByeOk>}
+   */
+  async ctrl_bye() {
+    const fn_name = 'ctrl_bye';
+    let payload = await this.#request(fn_name);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * @returns {Promise<null>}
+   */
+  async ctrl_user_abort() {
+    const fn_name = 'ctrl_user_abort';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusPingOk>}
+   */
+  async ctrl_ping() {
+    const fn_name = 'ctrl_ping';
+    let payload = await this.#request(fn_name);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * Full-text error message for the last general error reported
+   * by the CQi server
+   * 
+   * @returns {Promise<string>}
+   */
+  async ctrl_last_general_error() {
+    const fn_name = 'ctrl_last_general_error';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @returns {Promise<boolean>}
+   */
+  async ask_feature_cqi_1_0() {
+    const fn_name = 'ask_feature_cqi_1_0';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @returns {Promise<boolean>}
+   */
+  async ask_feature_cl_2_3() {
+    const fn_name = 'ask_feature_cl_2_3';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @returns {Promise<boolean>}
+   */
+  async ask_feature_cqp_2_3() {
+    const fn_name = 'ask_feature_cqp_2_3';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @returns {Promise<string[]>}
+   */
+  async corpus_list_corpora() {
+    const fn_name = 'corpus_list_corpora';
+    return await this.#request(fn_name);
+  }
+
+  /**
+   * @param {string} corpus 
+   * @returns {Promise<string>}
+   */
+  async corpus_charset(corpus) {
+    const fn_name = 'corpus_charset';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus 
+   * @returns {Promise<string[]>}
+   */
+  async corpus_properties(corpus) {
+    const fn_name = 'corpus_properties';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<string[]>}
+   */
+  async corpus_positional_attributes(corpus) {
+    const fn_name = 'corpus_positional_attributes';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<string[]>}
+   */
+  async corpus_structural_attributes(corpus) {
+    const fn_name = 'corpus_structural_attributes';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus
+   * @param {string} attribute
+   * @returns {Promise<boolean>}
+   */
+  async corpus_structural_attribute_has_values(corpus, attribute) {
+    const fn_name = 'corpus_structural_attribute_has_values';
+    const fn_args = {corpus: corpus, attribute: attribute};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<string[]>}
+   */
+  async corpus_alignment_attributes(corpus) {
+    const fn_name = 'corpus_alignment_attributes';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * the full name of <corpus> as specified in its registry entry
+   * 
+   * @param {string} corpus
+   * @returns {Promise<string>}
+   */
+  async corpus_full_name(corpus) {
+    const fn_name = 'corpus_full_name';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns the contents of the .info file of <corpus> as a list of lines
+   * 
+   * @param {string} corpus
+   * @returns {Promise<string[]>}
+   */
+  async corpus_info(corpus) {
+    const fn_name = 'corpus_info';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * try to unload a corpus and all its attributes from memory
+   * 
+   * @param {string} corpus
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async corpus_drop_corpus(corpus) {
+    const fn_name = 'corpus_drop_corpus';
+    const fn_args = {corpus: corpus};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * returns the size of <attribute>:
+   * - number of tokens        (positional)
+   * - number of regions       (structural)
+   * - number of alignments    (alignment)
+   * 
+   * @param {string} attribute
+   * @returns {Promise<number>}
+   */
+  async cl_attribute_size(attribute) {
+    const fn_name = 'cl_attribute_size';
+    const fn_args = {attribute: attribute};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns the number of entries in the lexicon of a positional attribute;
+   *
+   * valid lexicon IDs range from 0 .. (lexicon_size - 1)
+   * 
+   * @param {string} attribute
+   * @returns {Promise<number>}
+   */
+  async cl_lexicon_size(attribute) {
+    const fn_name = 'cl_lexicon_size';
+    const fn_args = {attribute: attribute};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * unload attribute from memory
+   * 
+   * @param {string} attribute
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async cl_drop_attribute(attribute) {
+    const fn_name = 'cl_drop_attribute';
+    const fn_args = {attribute: attribute};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * NOTE: simple (scalar) mappings are applied to lists (the returned list
+   *       has exactly the same length as the list passed as an argument)
+   */
+
+  /**
+   * returns -1 for every string in <strings> that is not found in the lexicon
+   * 
+   * @param {string} attribute
+   * @param {strings[]} string
+   * @returns {Promise<number[]>}
+   */
+  async cl_str2id(attribute, strings) {
+    const fn_name = 'cl_str2id';
+    const fn_args = {attribute: attribute, strings: strings};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns "" for every ID in <id> that is out of range
+   * 
+   * @param {string} attribute
+   * @param {number[]} id
+   * @returns {Promise<string[]>}
+   */
+  async cl_id2str(attribute, id) {
+    const fn_name = 'cl_id2str';
+    const fn_args = {attribute: attribute, id: id};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns 0 for every ID in <id> that is out of range
+   * 
+   * @param {string} attribute
+   * @param {number[]} id
+   * @returns {Promise<number[]>}
+   */
+  async cl_id2freq(attribute, id) {
+    const fn_name = 'cl_id2freq';
+    const fn_args = {attribute: attribute, id: id};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns -1 for every corpus position in <cpos> that is out of range
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<number[]>}
+   */
+  async cl_cpos2id(attribute, cpos) {
+    const fn_name = 'cl_cpos2id';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns "" for every corpus position in <cpos> that is out of range
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<string[]>}
+   */
+  async cl_cpos2str(attribute, cpos) {
+    const fn_name = 'cl_cpos2str';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns -1 for every corpus position not inside a structure region
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<number[]>}
+   */
+  async cl_cpos2struc(attribute, cpos) {
+    const fn_name = 'cl_cpos2struc';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * NOTE: temporary addition for the Euralex2000 tutorial, but should
+   * probably be included in CQi specs
+   */
+
+  /**
+   * returns left boundary of s-attribute region enclosing cpos,
+   * -1 if not in region
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<number[]>}
+   */
+  async cl_cpos2lbound(attribute, cpos) {
+    const fn_name = 'cl_cpos2lbound';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns right boundary of s-attribute region enclosing cpos,
+   * -1 if not in region
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<number[]>}
+   */
+  async cl_cpos2rbound(attribute, cpos) {
+    const fn_name = 'cl_cpos2rbound';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns -1 for every corpus position not inside an alignment
+   * 
+   * @param {string} attribute
+   * @param {number[]} cpos
+   * @returns {Promise<number[]>}
+   */
+  async cl_cpos2alg(attribute, cpos) {
+    const fn_name = 'cl_cpos2alg';
+    const fn_args = {attribute: attribute, cpos: cpos};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns annotated string values of structure regions in <strucs>;
+   * "" if out of range
+   *
+   * check corpus_structural_attribute_has_values(<attribute>) first
+   * 
+   * @param {string} attribute
+   * @param {number[]} strucs
+   * @returns {Promise<string[]>}
+   */
+  async cl_struc2str(attribute, strucs) {
+    const fn_name = 'cl_struc2str';
+    const fn_args = {attribute: attribute, strucs: strucs};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * NOTE: the following mappings take a single argument and return multiple
+   * values, including lists of arbitrary size
+   */
+
+  /**
+   * returns all corpus positions where the given token occurs
+   * 
+   * @param {string} attribute
+   * @param {number} id
+   * @returns {Promise<number[]>}
+   */
+  async cl_id2cpos(attribute, id) {
+    const fn_name = 'cl_id2cpos';
+    const fn_args = {attribute: attribute, id: id};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns all corpus positions where one of the tokens in <id_list> occurs;
+   * the returned list is sorted as a whole, not per token id
+   * 
+   * @param {string} attribute
+   * @param {number[]} id_list
+   * @returns {Promise<number[]>}
+   */
+  async cl_idlist2cpos(attribute, id_list) {
+    const fn_name = 'cl_idlist2cpos';
+    const fn_args = {attribute: attribute, id_list: id_list};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns lexicon IDs of all tokens that match <regex>;
+   * the returned list may be empty (size 0);
+   * 
+   * @param {string} attribute
+   * @param {string} regex
+   * @returns {Promise<number[]>}
+   */
+  async cl_regex2id(attribute, regex) {
+    const fn_name = 'cl_regex2id';
+    const fn_args = {attribute: attribute, regex: regex};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns start and end corpus positions of structure region <struc>
+   * 
+   * @param {string} attribute
+   * @param {number} struc
+   * @returns {Promise<[number, number]>}
+   */
+  async cl_struc2cpos(attribute, struc) {
+    const fn_name = 'cl_struc2cpos';
+    const fn_args = {attribute: attribute, struc: struc};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * returns (src_start, src_end, target_start, target_end)
+   * 
+   * @param {string} attribute
+   * @param {number} alg
+   * @returns {Promise<[number, number, number, number]>}
+   */
+  async alg2cpos(attribute, alg) {
+    const fn_name = 'alg2cpos';
+    const fn_args = {attribute: attribute, alg: alg};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * <query> must include the ';' character terminating the query.
+   * 
+   * @param {string} mother_corpus
+   * @param {string} subcorpus_name
+   * @param {string} query
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async cqp_query(mother_corpus, subcorpus_name, query) {
+    const fn_name = 'cqp_query';
+    const fn_args = {mother_corpus: mother_corpus, subcorpus_name: subcorpus_name, query: query};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<string[]>}
+   */
+  async cqp_list_subcorpora(corpus) {
+    const fn_name = 'cqp_list_subcorpora';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} subcorpus
+   * @returns {Promise<number>}
+   */
+  async cqp_subcorpus_size(subcorpus) {
+    const fn_name = 'cqp_subcorpus_size';
+    const fn_args = {subcorpus: subcorpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} subcorpus
+   * @param {number} field
+   * @returns {Promise<boolean>}
+   */
+  async cqp_subcorpus_has_field(subcorpus, field) {
+    const fn_name = 'cqp_subcorpus_has_field';
+    const fn_args = {subcorpus: subcorpus, field: field};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * Dump the values of <field> for match ranges <first> .. <last>
+   * in <subcorpus>. <field> is one of the CQI_CONST_FIELD_* constants.
+   * 
+   * @param {string} subcorpus
+   * @param {number} field
+   * @param {number} first
+   * @param {number} last
+   * @returns {Promise<number[]>}
+   */
+  async cqp_dump_subcorpus(subcorpus, field, first, last) {
+    const fn_name = 'cqp_dump_subcorpus';
+    const fn_args = {subcorpus: subcorpus, field: field, first: first, last: last};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * delete a subcorpus from memory
+   * 
+   * @param {string} subcorpus
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async cqp_drop_subcorpus(subcorpus) {
+    const fn_name = 'cqp_drop_subcorpus';
+    const fn_args = {subcorpus: subcorpus};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * NOTE: The following two functions are temporarily included for the
+   * Euralex 2000 tutorial demo
+   */
+
+  /**
+   * frequency distribution of single tokens
+   *
+   * returns <n> (id, frequency) pairs flattened into a list of size 2*<n>
+   * field is one of
+   * - CQI_CONST_FIELD_MATCH
+   * - CQI_CONST_FIELD_TARGET
+   * - CQI_CONST_FIELD_KEYWORD
+   *
+   * NB: pairs are sorted by frequency desc.
+   * 
+   * @param {string} subcorpus
+   * @param {number} cutoff
+   * @param {number} field
+   * @param {string} attribute
+   * @returns {Promise<number[]>}
+   */
+  async cqp_fdist_1(subcorpus, cutoff, field, attribute) {
+    const fn_name = 'cqp_fdist_1';
+    const fn_args = {subcorpus: subcorpus, cutoff: cutoff, field: field, attribute: attribute};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * frequency distribution of pairs of tokens
+   *
+   * returns <n> (id1, id2, frequency) pairs flattened into a list of
+   * size 3*<n>
+   *
+   * NB: triples are sorted by frequency desc.
+   * 
+   * @param {string} subcorpus
+   * @param {number} cutoff
+   * @param {number} field1
+   * @param {string} attribute1
+   * @param {number} field2
+   * @param {string} attribute2
+   * @returns {Promise<number[]>}
+   */
+  async cqp_fdist_2(subcorpus, cutoff, field1, attribute1, field2, attribute2) {
+    const fn_name = 'cqp_fdist_2';
+    const fn_args = {subcorpus: subcorpus, cutoff: cutoff, field1: field1, attribute1: attribute1, field2: field2, attribute2: attribute2};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**************************************************************************
+   * NOTE: The following is not included in the CQi specification.          *
+   **************************************************************************/
+  /**************************************************************************
+   *                      Custom additions for nopaque                      *
+   **************************************************************************/
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async ext_corpus_update_db(corpus) {
+    const fn_name = 'ext_corpus_update_db';
+    const fn_args = {corpus: corpus};
+    let payload = await this.#request(fn_name, fn_args);
+    return new cqi.status.lookup[payload.code]();
+  }
+
+  /**
+   * @param {string} corpus
+   * @returns {Promise<object>}
+   */
+  async ext_corpus_static_data(corpus) {
+    const fn_name = 'ext_corpus_static_data';
+    const fn_args = {corpus: corpus};
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} corpus
+   * @param {number=} page
+   * @param {number=} per_page
+   * @returns {Promise<object>}
+   */
+  async ext_corpus_paginate_corpus(corpus, page, per_page) {
+    const fn_name = 'ext_corpus_paginate_corpus';
+    const fn_args = {corpus: corpus}
+    if (page !== undefined) {fn_args.page = page;}
+    if (per_page !== undefined) {fn_args.per_page = per_page;}
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} subcorpus
+   * @param {number=} context
+   * @param {number=} page
+   * @param {number=} per_page
+   * @returns {Promise<object>}
+   */
+  async ext_cqp_paginate_subcorpus(subcorpus, context, page, per_page) {
+    const fn_name = 'ext_cqp_paginate_subcorpus';
+    const fn_args = {subcorpus: subcorpus}
+    if (context !== undefined) {fn_args.context = context;}
+    if (page !== undefined) {fn_args.page = page;}
+    if (per_page !== undefined) {fn_args.per_page = per_page;}
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} subcorpus
+   * @param {number[]} match_id_list
+   * @param {number=} context
+   * @returns {Promise<object>}
+   */
+  async ext_cqp_partial_export_subcorpus(subcorpus, match_id_list, context) {
+    const fn_name = 'ext_cqp_partial_export_subcorpus';
+    const fn_args = {subcorpus: subcorpus, match_id_list: match_id_list};
+    if (context !== undefined) {fn_args.context = context;}
+    return await this.#request(fn_name, fn_args);
+  }
+
+  /**
+   * @param {string} subcorpus
+   * @param {number=} context
+   * @returns {Promise<object>}
+   */
+  async ext_cqp_export_subcorpus(subcorpus, context) {
+    const fn_name = 'ext_cqp_export_subcorpus';
+    const fn_args = {subcorpus: subcorpus};
+    if (context !== undefined) {fn_args.context = context;}
+    return await this.#request(fn_name, fn_args);
+  }
+};

app/static/js/cqi/api/package.js

@@ -0,0 +1 @@
+cqi.api = {};

app/static/js/cqi/client.js

@@ -0,0 +1,58 @@
+cqi.CQiClient = class CQiClient {
+  /**
+   * @param {string} host
+   * @param {string} corpusId
+   * @param {number} [timeout=60] timeout
+   * @param {string} [version=0.1] version
+   */
+  constructor(host, corpusId, timeout = 60, version = '0.1') {
+     /** @type {cqi.api.APIClient} */
+    this.api = new cqi.api.APIClient(host, corpusId, timeout, version);
+  }
+
+  /**
+   * @returns {cqi.models.corpora.CorpusCollection}
+   */
+  get corpora() {
+    return new cqi.models.corpora.CorpusCollection(this);
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusByeOk>}
+   */
+  async bye() {
+    return await this.api.ctrl_bye();
+  }
+
+  /**
+   * @param {string} username
+   * @param {string} password
+   * @returns {Promise<cqi.status.StatusConnectOk>}
+   */
+  async connect(username, password) {
+    return await this.api.ctrl_connect(username, password);
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusPingOk>}
+   */
+  async ping() {
+    return await this.api.ctrl_ping();
+  }
+
+  /**
+   * @returns {Promise<null>}
+   */
+  async userAbort() {
+    return await this.api.ctrl_user_abort();
+  }
+
+  /**
+   * Alias for "bye" method
+   * 
+   * @returns {Promise<cqi.status.StatusByeOk>}
+   */
+  async disconnect() {
+    return await this.api.ctrl_bye();
+  }
+};

app/static/js/cqi/errors.js

@@ -0,0 +1,185 @@
+cqi.errors = {};
+
+
+/**
+ * A base class from which all other errors inherit.
+ * If you want to catch all errors that the CQi package might throw,
+ * catch this base error.
+ */
+cqi.errors.CQiError = class CQiError extends Error {
+  constructor(message) {
+    super(message);
+    this.code = undefined;
+    this.description = undefined;
+  }
+};
+
+
+cqi.errors.Error = class Error extends cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 2;
+  }
+};
+
+
+cqi.errors.ErrorGeneralError = class ErrorGeneralError extends cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 513;
+  }
+};
+
+
+cqi.errors.ErrorConnectRefused = class ErrorConnectRefused extends cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 514;
+  }
+};
+
+
+cqi.errors.ErrorUserAbort = class ErrorUserAbort extends cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 515;
+  }
+};
+
+
+cqi.errors.ErrorSyntaxError = class ErrorSyntaxError extends cqi.errors.Error {
+  constructor(message) {
+    super(message);
+    this.code = 516;
+  }
+};
+
+
+cqi.errors.CLError = class Error extends cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 4;
+  }
+};
+
+
+cqi.errors.CLErrorNoSuchAttribute = class CLErrorNoSuchAttribute extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1025;
+    this.description = "CQi server couldn't open attribute";
+  }
+};
+
+
+cqi.errors.CLErrorWrongAttributeType = class CLErrorWrongAttributeType extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1026;
+  }
+};
+
+
+cqi.errors.CLErrorOutOfRange = class CLErrorOutOfRange extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1027;
+  }
+};
+
+
+cqi.errors.CLErrorRegex = class CLErrorRegex extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1028;
+  }
+};
+
+
+cqi.errors.CLErrorCorpusAccess = class CLErrorCorpusAccess extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1029;
+  }
+};
+
+
+cqi.errors.CLErrorOutOfMemory = class CLErrorOutOfMemory extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1030;
+    this.description = 'CQi server has run out of memory; try discarding some other corpora and/or subcorpora';
+  }
+};
+
+
+cqi.errors.CLErrorInternal = class CLErrorInternal extends cqi.errors.CLError {
+  constructor(message) {
+    super(message);
+    this.code = 1031;
+    this.description = "The classical 'please contact technical support' error";
+  }
+};
+
+
+cqi.errors.CQPError = class Error extends cqi.errors.CQiError {
+  constructor(message) {
+    super(message);
+    this.code = 5;
+  }
+};
+
+
+cqi.errors.CQPErrorGeneral = class CQPErrorGeneral extends cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1281;
+  }
+};
+
+
+cqi.errors.CQPErrorNoSuchCorpus = class CQPErrorNoSuchCorpus extends cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1282;
+  }
+};
+
+
+cqi.errors.CQPErrorInvalidField = class CQPErrorInvalidField extends cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1283;
+  }
+};
+
+
+cqi.errors.CQPErrorOutOfRange = class CQPErrorOutOfRange extends cqi.errors.CQPError {
+  constructor(message) {
+    super(message);
+    this.code = 1284;
+    this.description = 'A number is out of range';
+  }
+};
+
+
+cqi.errors.lookup = {
+  2: cqi.errors.Error,
+  513: cqi.errors.ErrorGeneralError,
+  514: cqi.errors.ErrorConnectRefused,
+  515: cqi.errors.ErrorUserAbort,
+  516: cqi.errors.ErrorSyntaxError,
+  4: cqi.errors.CLError,
+  1025: cqi.errors.CLErrorNoSuchAttribute,
+  1026: cqi.errors.CLErrorWrongAttributeType,
+  1027: cqi.errors.CLErrorOutOfRange,
+  1028: cqi.errors.CLErrorRegex,
+  1029: cqi.errors.CLErrorCorpusAccess,
+  1030: cqi.errors.CLErrorOutOfMemory,
+  1031: cqi.errors.CLErrorInternal,
+  5: cqi.errors.CQPError,
+  1281: cqi.errors.CQPErrorGeneral,
+  1282: cqi.errors.CQPErrorNoSuchCorpus,
+  1283: cqi.errors.CQPErrorInvalidField,
+  1284: cqi.errors.CQPErrorOutOfRange
+};

app/static/js/cqi/models/attributes.js

@@ -0,0 +1,289 @@
+cqi.models.attributes = {};
+
+
+cqi.models.attributes.Attribute = class Attribute extends cqi.models.resource.Model {
+  /**
+   * @returns {string}
+   */
+  get apiName() {
+    return this.attrs.api_name;
+  }
+
+  /**
+   * @returns {string}
+   */
+  get name() {
+    return this.attrs.name;
+  }
+
+  /**
+   * @returns {number}
+   */
+  get size() {
+    return this.attrs.size;
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async drop() {
+    return await this.client.api.cl_drop_attribute(this.apiName);
+  }
+};
+
+
+cqi.models.attributes.AttributeCollection = class AttributeCollection extends cqi.models.resource.Collection {
+   /** @type{typeof cqi.models.attributes.Attribute} */
+  static model = cqi.models.attributes.Attribute;
+
+  /**
+   * @param {cqi.CQiClient} client
+   * @param {cqi.models.corpora.Corpus} corpus
+   */
+  constructor(client, corpus) {
+    super(client);
+     /** @type {cqi.models.corpora.Corpus} */
+    this.corpus = corpus;
+  }
+
+  /**
+   * @param {string} attributeName
+   * @returns {Promise<object>}
+   */
+  async _get(attributeName) {
+     /** @type{string} */
+    let apiName = `${this.corpus.apiName}.${attributeName}`;
+    return {
+      api_name: apiName,
+      name: attributeName,
+      size: await this.client.api.cl_attribute_size(apiName)
+    }
+  }
+
+  /**
+   * @param {string} attributeName
+   * @returns {Promise<cqi.models.attributes.Attribute>}
+   */
+  async get(attributeName) {
+    return this.prepareModel(await this._get(attributeName));
+  }
+};
+
+
+cqi.models.attributes.AlignmentAttribute = class AlignmentAttribute extends cqi.models.attributes.Attribute {
+  /**
+   * @param {number} id 
+   * @returns {Promise<[number, number, number, number]>}
+   */
+  async cposById(id) {
+    return await this.client.api.cl_alg2cpos(this.apiName, id);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<number[]>}
+   */
+  async idsByCpos(cposList) {
+    return await this.client.api.cl_cpos2alg(this.apiName, cposList);
+  }
+};
+
+
+cqi.models.attributes.AlignmentAttributeCollection = class AlignmentAttributeCollection extends cqi.models.attributes.AttributeCollection {
+   /** @type{typeof cqi.models.attributes.AlignmentAttribute} */
+  static model = cqi.models.attributes.AlignmentAttribute;
+
+  /**
+   * @returns {Promise<cqi.models.attributes.AlignmentAttribute[]>}
+   */
+  async list() {
+     /** @type {string[]} */
+     let alignmentAttributeNames = await this.client.api.corpus_alignment_attributes(this.corpus.apiName);
+     /** @type {cqi.models.attributes.AlignmentAttribute[]} */
+    let alignmentAttributes = [];
+    for (let alignmentAttributeName of alignmentAttributeNames) {
+      alignmentAttributes.push(await this.get(alignmentAttributeName));
+    }
+    return alignmentAttributes;
+  }
+};
+
+
+cqi.models.attributes.PositionalAttribute = class PositionalAttribute extends cqi.models.attributes.Attribute {
+  /**
+   * @returns {number}
+   */
+  get lexiconSize() {
+    return this.attrs.lexicon_size;
+  }
+
+  /**
+   * @param {number} id
+   * @returns {Promise<number[]>}
+   */
+  async cposById(id) {
+    return await this.client.api.cl_id2cpos(this.apiName, id);
+  }
+
+  /**
+   * @param {number[]} idList
+   * @returns {Promise<number[]>}
+   */
+  async cposByIds(idList) {
+    return await this.client.api.cl_idlist2cpos(this.apiName, idList);
+  }
+
+  /**
+   * @param {number[]} idList
+   * @returns {Promise<number[]>}
+   */
+  async freqsByIds(idList) {
+    return await this.client.api.cl_id2freq(this.apiName, idList);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<number[]>}
+   */
+  async idsByCpos(cposList) {
+    return await this.client.api.cl_cpos2id(this.apiName, cposList);
+  }
+
+  /**
+   * @param {string} regex
+   * @returns {Promise<number[]>}
+   */
+  async idsByRegex(regex) {
+    return await this.client.api.cl_regex2id(this.apiName, regex);
+  }
+
+  /**
+   * @param {string[]} valueList
+   * @returns {Promise<number[]>}
+   */
+  async idsByValues(valueList) {
+    return await this.client.api.cl_str2id(this.apiName, valueList);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<string[]>}
+   */
+  async valuesByCpos(cposList) {
+    return await this.client.api.cl_cpos2str(this.apiName, cposList);
+  }
+
+  /**
+   * @param {number[]} idList
+   * @returns {Promise<string[]>}
+   */
+  async valuesByIds(idList) {
+    return await this.client.api.cl_id2str(this.apiName, idList);
+  }
+};
+
+
+cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeCollection extends cqi.models.attributes.AttributeCollection {
+   /** @type{typeof cqi.models.attributes.PositionalAttribute} */
+  static model = cqi.models.attributes.PositionalAttribute;
+
+  /**
+   * @param {string} positionalAttributeName
+   * @returns {Promise<object>}
+   */
+  async _get(positionalAttributeName) {
+    let positionalAttribute = await super._get(positionalAttributeName);
+    positionalAttribute.lexicon_size = await this.client.api.cl_lexicon_size(positionalAttribute.api_name);
+    return positionalAttribute;
+  }
+
+  /**
+   * @returns {Promise<cqi.models.attributes.PositionalAttribute[]>}
+   */
+  async list() {
+    let positionalAttributeNames = await this.client.api.corpus_positional_attributes(this.corpus.apiName);
+    let positionalAttributes = [];
+    for (let positionalAttributeName of positionalAttributeNames) {
+      positionalAttributes.push(await this.get(positionalAttributeName));
+    }
+    return positionalAttributes;
+  }
+};
+
+
+cqi.models.attributes.StructuralAttribute = class StructuralAttribute extends cqi.models.attributes.Attribute {
+  /**
+   * @returns {boolean}
+   */
+  get hasValues() {
+    return this.attrs.has_values;
+  }
+
+  /**
+   * @param {number} id
+   * @returns {Promise<[number, number]>}
+   */
+  async cposById(id) {
+    return await this.client.api.cl_struc2cpos(this.apiName, id);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<number[]>}
+   */
+  async idsByCpos(cposList) {
+    return await this.client.api.cl_cpos2struc(this.apiName, cposList);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<number[]>}
+   */
+  async lboundByCpos(cposList) {
+    return await this.client.api.cl_cpos2lbound(this.apiName, cposList);
+  }
+
+  /**
+   * @param {number[]} cposList
+   * @returns {Promise<number[]>}
+   */
+  async rboundByCpos(cposList) {
+    return await this.client.api.cl_cpos2rbound(this.apiName, cposList);
+  }
+
+  /**
+   * @param {number[]} idList
+   * @returns {Promise<string[]>}
+   */
+  async valuesByIds(idList) {
+    return await this.client.api.cl_struc2str(this.apiName, idList);
+  }
+};
+
+
+cqi.models.attributes.StructuralAttributeCollection = class StructuralAttributeCollection extends cqi.models.attributes.AttributeCollection {
+   /** @type{typeof cqi.models.attributes.StructuralAttribute} */
+  static model = cqi.models.attributes.StructuralAttribute;
+
+  /**
+   * @param {string} structuralAttributeName
+   * @returns {Promise<object>}
+   */
+  async _get(structuralAttributeName) {
+    let structuralAttribute = await super._get(structuralAttributeName);
+    structuralAttribute.has_values = await this.client.api.cl_has_values(structuralAttribute.api_name);
+    return structuralAttribute;
+  }
+
+  /**
+   * @returns {Promise<cqi.models.attributes.StructuralAttribute[]>}
+   */
+  async list() {
+    let structuralAttributeNames = await this.client.api.corpus_structural_attributes(this.corpus.apiName);
+    let structuralAttributes = [];
+    for (let structuralAttributeName of structuralAttributeNames) {
+      structuralAttributes.push(await this.get(structuralAttributeName));
+    }
+    return structuralAttributes;
+  }
+};

app/static/js/cqi/models/corpora.js

@@ -0,0 +1,166 @@
+cqi.models.corpora = {};
+
+
+cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
+  /**
+   * @returns {string}
+   */
+  get apiName() {
+    return this.attrs.api_name;
+  }
+
+  /**
+   * @returns {string}
+   */
+  get name() {
+    return this.attrs.name;
+  }
+
+  /**
+   * @returns {number}
+   */
+  get size() {
+    return this.attrs.size;
+  }
+
+  /**
+   * @returns {string}
+   */
+  get charset() {
+    return this.attrs.charset;
+  }
+
+  /**
+   * @returns {string[]}
+   */
+  get properties() {
+    return this.attrs?.properties;
+  }
+
+  /**
+   * @returns {cqi.models.attributes.AlignmentAttributeCollection}
+   */
+  get alignmentAttributes() {
+    return new cqi.models.attributes.AlignmentAttributeCollection(this.client, this);
+  }
+
+  /**
+   * @returns {cqi.models.attributes.PositionalAttributeCollection}
+   */
+  get positionalAttributes() {
+    return new cqi.models.attributes.PositionalAttributeCollection(this.client, this);
+  }
+
+  /**
+   * @returns {cqi.models.attributes.StructuralAttributeCollection}
+   */
+  get structuralAttributes() {
+    return new cqi.models.attributes.StructuralAttributeCollection(this.client, this);
+  }
+
+  /**
+   * @returns {cqi.models.subcorpora.SubcorpusCollection}
+   */
+  get subcorpora() {
+    return new cqi.models.subcorpora.SubcorpusCollection(this.client, this);
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async drop() {
+    return await this.client.api.corpus_drop_corpus(this.apiName);
+  }
+
+  /**
+   * @param {string} subcorpusName
+   * @param {string} query
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async query(subcorpusName, query) {
+    return await this.client.api.cqp_query(this.apiName, subcorpusName, query);
+  }
+
+  /**************************************************************************
+   * NOTE: The following is not included in the CQi specification.          *
+   **************************************************************************/
+  /**************************************************************************
+   *                      Custom additions for nopaque                      *
+   **************************************************************************/
+
+  /**
+   * @returns {string}
+   */
+  get staticData() {
+    return this.attrs.static_data;
+  }
+
+  /**
+   * @returns {cqi.status.StatusOk}
+   */
+  async updateDb() {
+    return await this.client.api.ext_corpus_update_db(this.apiName);
+  }
+
+  /**
+   * @param {number=} page
+   * @param {number=} per_page
+   * @returns {Promise<object>}
+   */
+  async paginate(page, per_page) {
+    return await this.client.api.ext_corpus_paginate_corpus(this.apiName, page, per_page);
+  }
+};
+
+
+cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.resource.Collection {
+   /** @type {typeof cqi.models.corpora.Corpus} */
+  static model = cqi.models.corpora.Corpus;
+
+  /**
+   * @param {string} corpusName
+   * @returns {Promise<object>}
+   */
+  async _get(corpusName) {
+    const returnValue = {
+      api_name: corpusName,
+      charset: await this.client.api.corpus_charset(corpusName),
+      // full_name: await this.client.api.corpus_full_name(corpusName),
+      // info: await this.client.api.corpus_info(corpusName),
+      name: corpusName,
+      properties: await this.client.api.corpus_properties(corpusName),
+      size: await this.client.api.cl_attribute_size(`${corpusName}.word`)
+    };
+
+    /************************************************************************
+     * NOTE: The following is not included in the CQi specification.        *
+     ************************************************************************/
+    /************************************************************************
+     *                     Custom additions for nopaque                     *
+     ************************************************************************/
+    returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
+    return returnValue;
+  }
+
+  /**
+   * @param {string} corpusName
+   * @returns {Promise<cqi.models.corpora.Corpus>}
+   */
+  async get(corpusName) {
+    return this.prepareModel(await this._get(corpusName));
+  }
+
+  /**
+   * @returns {Promise<cqi.models.corpora.Corpus[]>}
+   */
+  async list() {
+     /** @type {string[]} */
+    let corpusNames = await this.client.api.corpus_list_corpora();
+     /** @type {cqi.models.corpora.Corpus[]} */
+    let corpora = [];
+    for (let corpusName of corpusNames) {
+      corpora.push(await this.get(corpusName));
+    }
+    return corpora;
+  }
+};

app/static/js/cqi/models/package.js

@@ -0,0 +1 @@
+cqi.models = {};

app/static/js/cqi/models/resource.js

@@ -0,0 +1,90 @@
+cqi.models.resource = {};
+
+
+/**
+ * A base class for representing a single object on the server.
+ */
+cqi.models.resource.Model = class Model {
+  /**
+   * @param {object} attrs
+   * @param {cqi.CQiClient} client
+   * @param {cqi.models.resource.Collection} collection
+   */
+  constructor(attrs, client, collection) {
+     /**
+      * A client pointing at the server that this object is on.
+      *
+      * @type {cqi.CQiClient}
+      */
+    this.client = client;
+     /**
+      * The collection that this model is part of.
+      *
+      * @type {cqi.models.resource.Collection}
+      */
+    this.collection = collection;
+     /**
+      * The raw representation of this object from the API
+      *
+      * @type {object} 
+      */
+    this.attrs = attrs;
+  }
+
+  /**
+   * @returns {string}
+   */
+  get apiName() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {Promise<void>}
+   */
+  async reload() {
+    this.attrs = await this.collection.get(this.apiName).attrs;
+  }
+};
+
+
+/**
+ * A base class for representing all objects of a particular type on the server.
+ */
+cqi.models.resource.Collection = class Collection {
+   /** 
+    * The type of object this collection represents, set by subclasses
+    * 
+    * @type {typeof cqi.models.resource.Model}
+    */
+  static model;
+
+  /**
+   * @param {cqi.CQiClient} client
+   */
+  constructor(client) {
+     /**
+      * A client pointing at the server that this object is on.
+      *
+      * @type {cqi.CQiClient}
+      */
+     this.client = client;
+  }
+
+  async list() {
+    throw new Error('Not implemented');
+  }
+
+  async get() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Create a model from a set of attributes.
+   * 
+   * @param {object} attrs
+   * @returns {cqi.models.resource.Model}
+   */
+  prepareModel(attrs) {
+    return new this.constructor.model(attrs, this.client, this);
+  }
+};

app/static/js/cqi/models/subcorpora.js

@@ -0,0 +1,189 @@
+cqi.models.subcorpora = {};
+
+
+cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Model {
+  /**
+   * @returns {string}
+   */
+  get apiName() {
+    return this.attrs.api_name;
+  }
+
+  /**
+   * @returns {object}
+   */
+  get fields() {
+    return this.attrs.fields;
+  }
+
+  /**
+   * @returns {string}
+   */
+  get name() {
+    return this.attrs.name;
+  }
+
+  /**
+   * @returns {number}
+   */
+  get size() {
+    return this.attrs.size;
+  }
+
+  /**
+   * @returns {Promise<cqi.status.StatusOk>}
+   */
+  async drop() {
+    return await this.client.api.cqp_drop_subcorpus(this.apiName);
+  }
+
+  /**
+   * @param {number} field
+   * @param {number} first
+   * @param {number} last
+   * @returns {Promise<number[]>}
+   */
+  async dump(field, first, last) {
+    return await this.client.api.cqp_dump_subcorpus(
+      this.apiName,
+      field,
+      first,
+      last
+    );
+  }
+
+  /**
+   * @param {number} cutoff
+   * @param {number} field
+   * @param {cqi.models.attributes.PositionalAttribute} attribute
+   * @returns {Promise<number[]>}
+   */
+  async fdist1(cutoff, field, attribute) {
+    return await this.client.api.cqp_fdist_1(
+      this.apiName,
+      cutoff,
+      field,
+      attribute.apiName
+    );
+  }
+
+  /**
+   * @param {number} cutoff
+   * @param {number} field1
+   * @param {cqi.models.attributes.PositionalAttribute} attribute1
+   * @param {number} field2
+   * @param {cqi.models.attributes.PositionalAttribute} attribute2
+   * @returns {Promise<number[]>}
+   */ 
+  async fdist2(cutoff, field1, attribute1, field2, attribute2) {
+    return await this.client.api.cqp_fdist_2(
+      this.apiName,
+      cutoff,
+      field1,
+      attribute1.apiName,
+      field2,
+      attribute2.apiName
+    );
+  }
+
+  /**************************************************************************
+   * NOTE: The following is not included in the CQi specification.          *
+   **************************************************************************/
+  /**************************************************************************
+   *                      Custom additions for nopaque                      *
+   **************************************************************************/
+
+  /**
+   * @param {number=} context
+   * @param {number=} page
+   * @param {number=} perPage
+   * @returns {Promise<object>}
+   */
+  async paginate(context, page, perPage) {
+    return await this.client.api.ext_cqp_paginate_subcorpus(this.apiName, context, page, perPage);
+  }
+
+  /**
+   * @param {number[]} matchIdList
+   * @param {number=} context
+   * @returns {Promise<object>}
+   */
+  async partialExport(matchIdList, context) {
+    return await this.client.api.ext_cqp_partial_export_subcorpus(this.apiName, matchIdList, context);
+  }
+
+  /**
+   * @param {number=} context
+   * @returns {Promise<object>}
+   */
+  async export(context) {
+    return await this.client.api.ext_cqp_export_subcorpus(this.apiName, context);
+  }
+};
+
+
+cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends cqi.models.resource.Collection {
+   /** @type {typeof cqi.models.subcorpora.Subcorpus} */
+  static model = cqi.models.subcorpora.Subcorpus;
+
+  /**
+   * @param {cqi.CQiClient} client
+   * @param {cqi.models.corpora.Corpus} corpus
+   */
+  constructor(client, corpus) {
+    super(client);
+     /** @type {cqi.models.corpora.Corpus} */
+    this.corpus = corpus;
+  }
+
+  /**
+   * @param {string} subcorpusName
+   * @returns {Promise<object>}
+   */
+  async _get(subcorpusName) {
+     /** @type {string} */
+    let apiName = `${this.corpus.apiName}:${subcorpusName}`;
+     /** @type {object} */
+    let fields = {};
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCH)) {
+      fields.match = cqi.CONST_FIELD_MATCH;
+    }
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCHEND)) {
+      fields.matchend = cqi.CONST_FIELD_MATCHEND
+    }
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_TARGET)) {
+      fields.target = cqi.CONST_FIELD_TARGET
+    }
+    if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_KEYWORD)) {
+      fields.keyword = cqi.CONST_FIELD_KEYWORD
+    }
+    return {
+      api_name: apiName,
+      fields: fields,
+      name: subcorpusName,
+      size: await this.client.api.cqp_subcorpus_size(apiName)
+    }
+  }
+
+  /**
+   * @param {string} subcorpusName
+   * @returns {Promise<cqi.models.subcorpora.Subcorpus>}
+   */
+  async get(subcorpusName) {
+    return this.prepareModel(await this._get(subcorpusName));
+  }
+
+  /**
+   * @returns {Promise<cqi.models.subcorpora.Subcorpus[]>}
+   */
+  async list() {
+     /** @type {string[]} */
+    let subcorpusNames = await this.client.api.cqp_list_subcorpora(this.corpus.apiName);
+     /** @type {cqi.models.subcorpora.Subcorpus[]} */
+    let subcorpora = [];
+    for (let subcorpusName of subcorpusNames) {
+      subcorpora.push(await this.get(subcorpusName));
+    }
+    return subcorpora;
+  }
+};

app/static/js/cqi/package.js

@@ -0,0 +1,6 @@
+var cqi = {};
+
+cqi.CONST_FIELD_KEYWORD = 9;
+cqi.CONST_FIELD_MATCH = 16;
+cqi.CONST_FIELD_MATCHEND = 17;
+cqi.CONST_FIELD_TARGET = 0;

app/static/js/cqi/status.js

@@ -0,0 +1,51 @@
+cqi.status = {};
+
+
+/**
+ * A base class from which all other status inherit.
+ */
+cqi.status.CQiStatus = class CQiStatus {
+  constructor() {
+    this.code = undefined;
+  }
+};
+
+
+cqi.status.StatusOk = class StatusOk extends cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 257;
+  }
+};
+
+
+cqi.status.StatusConnectOk = class StatusConnectOk extends cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 258;
+  }
+};
+
+
+cqi.status.StatusByeOk = class StatusByeOk extends cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 259;
+  }
+};
+
+
+cqi.status.StatusPingOk = class StatusPingOk extends cqi.status.CQiStatus {
+  constructor() {
+    super();
+    this.code = 260;
+  }
+};
+
+
+cqi.status.lookup = {
+  257: cqi.status.StatusOk,
+  258: cqi.status.StatusConnectOk,
+  259: cqi.status.StatusByeOk,
+  260: cqi.status.StatusPingOk
+};

app/templates/_scripts.html.j2

@@ -3,12 +3,28 @@
 <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.4/socket.io.min.js" integrity="sha512-HTENHrkQ/P0NGDFd5nk6ibVtCkcM7jhr2c7GyvXp5O+4X6O5cQO9AhqFzM+MdeBivsX7Hoys2J7pp2wdgMpCvw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/2.24.2/plotly.min.js" integrity="sha512-dAXqGCq94D0kgLSPnfvd/pZpCMoJQpGj2S2XQmFQ9Ay1+96kbjss02ISEh+TBNXMggGg/1qoMcOHcxg+Op/Jmw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 
+{%- assets
+  filters='rjsmin',
+  output='gen/cqi.%(version)s.js',
+  'js/cqi/package.js',
+  'js/cqi/errors.js',
+  'js/cqi/status.js',
+  'js/cqi/api/package.js',
+  'js/cqi/api/client.js',
+  'js/cqi/models/package.js',
+  'js/cqi/models/resource.js',
+  'js/cqi/models/attributes.js',
+  'js/cqi/models/subcorpora.js',
+  'js/cqi/models/corpora.js',
+  'js/cqi/client.js'
+%}
+<script src="{{ ASSET_URL }}"></script>
+{%- endassets %}
 {%- assets
   filters='rjsmin',
   output='gen/app.%(version)s.js',
   'js/App.js',
   'js/Utils.js',
-  'js/CorpusAnalysis/CQiClient.js',
   'js/CorpusAnalysis/CorpusAnalysisApp.js',
   'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
   'js/CorpusAnalysis/CorpusAnalysisReader.js',