sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-14 16:55:42 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'visualizations-update' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into visualizations-update

Browse Source
This commit is contained in:
Inga Kirschnick 2023-07-03 11:06:59 +02:00
commit 94dc25750c
28 changed files with 2197 additions and 122941 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/corpora/__init__.py
View File

@ -16,4 +16,4 @@ def before_request():
pass
from . import cli, cqi_over_socketio, files, followers, routes, json_routes
from . import cli, cqi_over_sio, files, followers, routes, json_routes

35
app/corpora/cqi_over_socketio/__init__.py → app/corpora/cqi_over_sio/__init__.py
View File

@ -1,8 +1,9 @@
from cqi import CQiClient
from cqi.errors import CQiException
from flask import session
from flask_login import current_user
from flask_socketio import ConnectionRefusedError
from threading import Lock
import cqi
from app import db, hashids, socketio
from app.decorators import socketio_login_required
from app.models import Corpus, CorpusStatus
@ -39,16 +40,9 @@ Basic concept:
'''
NAMESPACE = '/corpora/corpus/corpus_analysis'
NAMESPACE = '/cqi_over_sio'
# Import all CQi over Socket.IO event handlers
from .cqi_corpora_corpus_subcorpora import * # noqa
from .cqi_corpora_corpus_structural_attributes import * # noqa
from .cqi_corpora_corpus_positional_attributes import * # noqa
from .cqi_corpora_corpus_alignment_attributes import * # noqa
from .cqi_corpora_corpus import * # noqa
from .cqi_corpora import * # noqa
from .cqi import * # noqa
@ -89,8 +83,8 @@ def connect(auth):
socketio.sleep(3)
retry_counter -= 1
db.session.refresh(corpus)
cqi_client = cqi.CQiClient(f'cqpserver_{corpus_id}')
session['d'] = {
cqi_client = CQiClient(f'cqpserver_{corpus_id}')
session['cqi_over_sio'] = {
'corpus_id': corpus_id,
'cqi_client': cqi_client,
'cqi_client_lock': Lock(),
@ -100,16 +94,19 @@ def connect(auth):
@socketio.on('disconnect', namespace=NAMESPACE)
def disconnect():
if 'd' not in session:
return
session['d']['cqi_client_lock'].acquire()
try:
session['d']['cqi_client'].disconnect()
except (BrokenPipeError, cqi.errors.CQiException):
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
except KeyError:
return
cqi_client_lock.acquire()
try:
cqi_client.api.ctrl_bye()
except (BrokenPipeError, CQiException):
pass
session['d']['cqi_client_lock'].release()
corpus = Corpus.query.get(session['d']['corpus_id'])
cqi_client_lock.release()
corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
db.session.commit()
session.pop('d')
session.pop('cqi_over_sio')
# return {'code': 200, 'msg': 'OK'}

120
app/corpora/cqi_over_sio/cqi.py Normal file
View File

@ -0,0 +1,120 @@
from cqi import CQiClient
from cqi.errors import CQiException
from cqi.status import CQiStatus
from flask import session
from inspect import signature
from threading import Lock
from typing import Callable, Dict, List
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .extensions import CQI_EXTENSION_FUNCTION_NAMES
from . import extensions as extensions_module
CQI_FUNCTION_NAMES: List[str] = [
'ask_feature_cl_2_3',
'ask_feature_cqi_1_0',
'ask_feature_cqp_2_3',
'cl_alg2cpos',
'cl_attribute_size',
'cl_cpos2alg',
'cl_cpos2id',
'cl_cpos2lbound',
'cl_cpos2rbound',
'cl_cpos2str',
'cl_cpos2struc',
'cl_drop_attribute',
'cl_id2cpos',
'cl_id2freq',
'cl_id2str',
'cl_idlist2cpos',
'cl_lexicon_size',
'cl_regex2id',
'cl_str2id',
'cl_struc2cpos',
'cl_struc2str',
'corpus_alignment_attributes',
'corpus_charset',
'corpus_drop_corpus',
'corpus_full_name',
'corpus_info',
'corpus_list_corpora',
'corpus_positional_attributes',
'corpus_properties',
'corpus_structural_attribute_has_values',
'corpus_structural_attributes',
'cqp_drop_subcorpus',
'cqp_dump_subcorpus',
'cqp_fdist_1',
'cqp_fdist_2',
'cqp_list_subcorpora',
'cqp_query',
'cqp_subcorpus_has_field',
'cqp_subcorpus_size',
'ctrl_bye',
'ctrl_connect',
'ctrl_last_general_error',
'ctrl_ping',
'ctrl_user_abort'
]
@socketio.on('cqi', namespace=ns)
@socketio_login_required
def cqi_over_sio(fn_data):
try:
fn_name: str = fn_data['fn_name']
except KeyError:
return {'code': 400, 'msg': 'Bad Request'}
fn_name: str = fn_data['fn_name']
fn_args: Dict = fn_data.get('fn_args', {})
try:
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
except KeyError:
return {'code': 424, 'msg': 'Failed Dependency'}
if fn_name in CQI_FUNCTION_NAMES:
fn: Callable = getattr(cqi_client.api, fn_name)
elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
fn_args['cqi_client'] = cqi_client
fn: Callable = getattr(extensions_module, fn_name)
else:
return {'code': 400, 'msg': 'Bad Request'}
for param in signature(fn).parameters.values():
if param.default is param.empty:
if param.name not in fn_args:
return {'code': 400, 'msg': 'Bad Request'}
else:
if param.name not in fn_args:
continue
if type(fn_args[param.name]) is not param.annotation:
return {'code': 400, 'msg': 'Bad Request'}
cqi_client_lock.acquire()
try:
return_value = fn(**fn_args)
except BrokenPipeError:
return_value = {
'code': 500,
'msg': 'Internal Server Error'
}
except CQiException as e:
return_value = {
'code': 502,
'msg': 'Bad Gateway',
'payload': {
'code': e.code,
'desc': e.description,
'msg': e.__class__.__name__
}
}
finally:
cqi_client_lock.release()
if isinstance(return_value, CQiStatus):
payload = {
'code': return_value.code,
'msg': return_value.__class__.__name__
}
else:
payload = return_value
return {'code': 200, 'msg': 'OK', 'payload': payload}

254
app/corpora/cqi_over_sio/extensions.py Normal file
View File

@ -0,0 +1,254 @@
from collections import Counter
from cqi import CQiClient
from cqi.models.corpora import Corpus
from cqi.status import StatusOk
from flask import session
from typing import Dict, List
import json
import math
import os
from app import db
from app.models import Corpus
from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
CQI_EXTENSION_FUNCTION_NAMES: List[str] = [
'ext_corpus_update_db',
'ext_corpus_static_data',
'ext_corpus_paginate_corpus',
'ext_cqp_paginate_subcorpus',
'ext_cqp_partial_export_subcorpus',
'ext_cqp_export_subcorpus',
]
def ext_corpus_update_db(cqi_client: CQiClient, corpus: str):
db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
cqi_corpus = cqi_client.corpora.get(corpus)
db_corpus.num_tokens = cqi_corpus.size
db.session.commit()
return StatusOk()
def ext_corpus_static_data(cqi_client: CQiClient, corpus: str) -> Dict:
db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
static_corpus_data_file = os.path.join(db_corpus.path, 'cwb', 'static.json')
if os.path.exists(static_corpus_data_file):
with open(static_corpus_data_file, 'r') as f:
return json.load(f)
cqi_corpus = cqi_client.corpora.get(corpus)
##########################################################################
# A faster way to get cpos boundaries for smaller s_attrs #
##########################################################################
# cqi_corpus.query('Last', '<s> []* </s>;')
# cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
# print(cqi_subcorpus.size)
# first_match = 0
# last_match = cqi_subcorpus.attrs['size'] - 1
# match_boundaries = zip(
# list(range(first_match, last_match + 1)),
# cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['match'], first_match, last_match),
# cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['matchend'], first_match, last_match)
# )
# for x in match_boundaries:
# print(x)
cqi_p_attrs = {
p_attr.name: p_attr
for p_attr in cqi_corpus.positional_attributes.list()
}
cqi_s_attrs = {
s_attr.name: s_attr
for s_attr in cqi_corpus.structural_attributes.list()
}
static_corpus_data = {
'corpus': {
'bounds': [0, cqi_corpus.size - 1],
'counts': {
'token': cqi_corpus.size
},
'freqs': {}
},
'p_attrs': {},
's_attrs': {},
'values': {'p_attrs': {}, 's_attrs': {}}
}
for p_attr in cqi_p_attrs.values():
static_corpus_data['corpus']['freqs'][p_attr.name] = dict(
zip(
range(0, p_attr.lexicon_size),
p_attr.freqs_by_ids(list(range(0, p_attr.lexicon_size)))
)
)
static_corpus_data['p_attrs'][p_attr.name] = dict(
zip(
range(0, cqi_corpus.size),
p_attr.ids_by_cpos(list(range(0, cqi_corpus.size)))
)
)
static_corpus_data['values']['p_attrs'][p_attr.name] = dict(
zip(
range(0, p_attr.lexicon_size),
p_attr.values_by_ids(list(range(0, p_attr.lexicon_size)))
)
)
for s_attr in cqi_s_attrs.values():
if s_attr.has_values:
continue
static_corpus_data['corpus']['counts'][s_attr.name] = s_attr.size
static_corpus_data['s_attrs'][s_attr.name] = {'lexicon': {}, 'values': None}
static_corpus_data['values']['s_attrs'][s_attr.name] = {}
for id in range(0, s_attr.size):
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id] = {}
lbound, rbound = s_attr.cpos_by_id(id)
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {}
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1
if s_attr.name not in ['text', 's']:
continue
cpos_range = range(lbound, rbound + 1)
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['ent'] = len({x for x in cqi_s_attrs['ent'].ids_by_cpos(list(cpos_range)) if x != -1})
if s_attr.name != 'text':
continue
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['s'] = len({x for x in cqi_s_attrs['s'].ids_by_cpos(list(cpos_range)) if x != -1})
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
for p_attr in cqi_p_attrs.values():
static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr.ids_by_cpos(list(cpos_range))))
sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
s_attr_value_names = [
sub_s_attr.name[(len(s_attr.name) + 1):]
for sub_s_attr in sub_s_attrs
]
sub_s_attr_values = [
sub_s_attr.values_by_ids(list(range(0, s_attr.size)))
for sub_s_attr in sub_s_attrs
]
static_corpus_data['s_attrs'][s_attr.name]['values'] = s_attr_value_names
static_corpus_data['values']['s_attrs'][s_attr.name] = {
s_attr_id: {
s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id_idx]
for s_attr_value_name_idx, s_attr_value_name in enumerate(
static_corpus_data['s_attrs'][s_attr.name]['values']
)
} for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
}
with open(static_corpus_data_file, 'w') as f:
json.dump(static_corpus_data, f)
return static_corpus_data
def ext_corpus_paginate_corpus(
cqi_client: CQiClient,
corpus: str,
page: int = 1,
per_page: int = 20
) -> Dict:
cqi_corpus = cqi_client.corpora.get(corpus)
# Sanity checks
if (
per_page < 1
or page < 1
or (
cqi_corpus.size > 0
and page > math.ceil(cqi_corpus.size / per_page)
)
):
return {'code': 416, 'msg': 'Range Not Satisfiable'}
first_cpos = (page - 1) * per_page
last_cpos = min(cqi_corpus.size, first_cpos + per_page)
cpos_list = [*range(first_cpos, last_cpos)]
lookups = lookups_by_cpos(cqi_corpus, cpos_list)
payload = {}
# the items for the current page
payload['items'] = [cpos_list]
# the lookups for the items
payload['lookups'] = lookups
# the total number of items matching the query
payload['total'] = cqi_corpus.size
# the number of items to be displayed on a page.
payload['per_page'] = per_page
# The total number of pages
payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
# the current page number (1 indexed)
payload['page'] = page if payload['pages'] > 0 else None
# True if a previous page exists
payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
# True if a next page exists.
payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False # noqa
# Number of the previous page.
payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
# Number of the next page
payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
return payload
def ext_cqp_paginate_subcorpus(
cqi_client: CQiClient,
subcorpus: str,
context: int = 50,
page: int = 1,
per_page: int = 20
) -> Dict:
corpus_name, subcorpus_name = subcorpus.split(':', 1)
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
# Sanity checks
if (
per_page < 1
or page < 1
or (
cqi_subcorpus.size > 0
and page > math.ceil(cqi_subcorpus.size / per_page)
)
):
return {'code': 416, 'msg': 'Range Not Satisfiable'}
offset = (page - 1) * per_page
cutoff = per_page
cqi_results_export = export_subcorpus(
cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
payload = {}
# the items for the current page
payload['items'] = cqi_results_export.pop('matches')
# the lookups for the items
payload['lookups'] = cqi_results_export
# the total number of items matching the query
payload['total'] = cqi_subcorpus.size
# the number of items to be displayed on a page.
payload['per_page'] = per_page
# The total number of pages
payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
# the current page number (1 indexed)
payload['page'] = page if payload['pages'] > 0 else None
# True if a previous page exists
payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
# True if a next page exists.
payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False # noqa
# Number of the previous page.
payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
# Number of the next page
payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
return payload
def ext_cqp_partial_export_subcorpus(
cqi_client: CQiClient,
subcorpus: str,
match_id_list: list,
context: int = 50
) -> Dict:
corpus_name, subcorpus_name = subcorpus.split(':', 1)
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
return cqi_subcorpus_partial_export
def ext_cqp_export_subcorpus(
cqi_client: CQiClient,
subcorpus: str,
context: int = 50
) -> Dict:
corpus_name, subcorpus_name = subcorpus.split(':', 1)
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
return cqi_subcorpus_export

103
app/corpora/cqi_over_socketio/utils.py → app/corpora/cqi_over_sio/utils.py
View File

@ -1,64 +1,10 @@
from flask import session
from functools import wraps
from inspect import signature
import cqi
from cqi.models.corpora import Corpus
from cqi.models.subcorpora import Subcorpus
from typing import Dict, List
from app.models import Corpus
def cqi_over_socketio(f):
@wraps(f)
def wrapped(*args):
if 'd' not in session:
return {'code': 424, 'msg': 'Failed Dependency'}
f_args = {}
# Check for missing args and if all provided args are of the right type
for param in signature(f).parameters.values():
if param.name == 'corpus_name':
f_args[param.name] = f'NOPAQUE_{session["d"]["corpus_id"]}'
continue
if param.name == 'cqi_client':
f_args[param.name] = session['d']['cqi_client']
continue
if param.default is param.empty:
# args
if param.name not in args[0]:
return {'code': 400, 'msg': 'Bad Request'}
arg = args[0][param.name]
if type(arg) is not param.annotation:
return {'code': 400, 'msg': 'Bad Request'}
f_args[param.name] = arg
else:
# kwargs
if param.name not in args[0]:
continue
arg = args[0][param.name]
if type(arg) is not param.annotation:
return {'code': 400, 'msg': 'Bad Request'}
f_args[param.name] = arg
session['d']['cqi_client_lock'].acquire()
try:
return_value = f(**f_args)
except BrokenPipeError:
return_value = {
'code': 500,
'msg': 'Internal Server Error'
}
except cqi.errors.CQiException as e:
return_value = {
'code': 500,
'msg': 'Internal Server Error',
'payload': {
'code': e.code,
'desc': e.description,
'msg': e.name
}
}
finally:
session['d']['cqi_client_lock'].release()
return return_value
return wrapped
def lookups_by_cpos(corpus, cpos_list):
def lookups_by_cpos(corpus: Corpus, cpos_list: List[int]) -> Dict:
lookups = {}
lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
for attr in corpus.positional_attributes.list():
@ -93,18 +39,22 @@ def lookups_by_cpos(corpus, cpos_list):
return lookups
def partial_export_subcorpus(subcorpus, match_id_list, context=25):
if subcorpus.attrs['size'] == 0:
def partial_export_subcorpus(
subcorpus: Subcorpus,
match_id_list: List[int],
context: int = 25
) -> Dict:
if subcorpus.size == 0:
return {"matches": []}
match_boundaries = []
for match_id in match_id_list:
if match_id < 0 or match_id >= subcorpus.attrs['size']:
if match_id < 0 or match_id >= subcorpus.size:
continue
match_boundaries.append(
(
match_id,
subcorpus.dump(subcorpus.attrs['fields']['match'], match_id, match_id)[0],
subcorpus.dump(subcorpus.attrs['fields']['matchend'], match_id, match_id)[0]
subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
)
)
cpos_set = set()
@ -120,14 +70,14 @@ def partial_export_subcorpus(subcorpus, match_id_list, context=25):
lc_rbound = match_start - 1
lc = (lc_lbound, lc_rbound)
cpos_list_lbound = lc_lbound
if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
rc = None
cpos_list_rbound = match_end
else:
rc_lbound = match_end + 1
rc_rbound = min(
(match_end + context),
(subcorpus.collection.corpus.attrs['size'] - 1)
(subcorpus.collection.corpus.size - 1)
)
rc = (rc_lbound, rc_rbound)
cpos_list_rbound = rc_rbound
@ -138,15 +88,20 @@ def partial_export_subcorpus(subcorpus, match_id_list, context=25):
return {'matches': matches, **lookups}
def export_subcorpus(subcorpus, context=25, cutoff=float('inf'), offset=0):
if subcorpus.attrs['size'] == 0:
def export_subcorpus(
subcorpus: Subcorpus,
context: int = 25,
cutoff: float = float('inf'),
offset: int = 0
) -> Dict:
if subcorpus.size == 0:
return {"matches": []}
first_match = max(0, offset)
last_match = min((offset + cutoff - 1), (subcorpus.attrs['size'] - 1))
last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
match_boundaries = zip(
list(range(first_match, last_match + 1)),
subcorpus.dump(subcorpus.attrs['fields']['match'], first_match, last_match),
subcorpus.dump(subcorpus.attrs['fields']['matchend'], first_match, last_match)
range(first_match, last_match + 1),
subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
)
cpos_set = set()
matches = []
@ -160,14 +115,14 @@ def export_subcorpus(subcorpus, context=25, cutoff=float('inf'), offset=0):
lc_rbound = match_start - 1
lc = (lc_lbound, lc_rbound)
cpos_list_lbound = lc_lbound
if match_end == (subcorpus.collection.corpus.attrs['size'] - 1) or context == 0:
if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
rc = None
cpos_list_rbound = match_end
else:
rc_lbound = match_end + 1
rc_rbound = min(
(match_end + context),
(subcorpus.collection.corpus.attrs['size'] - 1)
(subcorpus.collection.corpus.size - 1)
)
rc = (rc_lbound, rc_rbound)
cpos_list_rbound = rc_rbound

43
app/corpora/cqi_over_socketio/cqi.py
View File

@ -1,43 +0,0 @@
from socket import gaierror
import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
@socketio.on('cqi.connect', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_connect(cqi_client: cqi.CQiClient):
try:
cqi_status = cqi_client.connect()
except gaierror as e:
return {
'code': 500,
'msg': 'Internal Server Error',
'payload': {'code': e.args[0], 'desc': e.args[1]}
}
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.disconnect', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_disconnect(cqi_client: cqi.CQiClient):
cqi_status = cqi_client.disconnect()
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.ping', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_ping(cqi_client: cqi.CQiClient):
cqi_status = cqi_client.ping()
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}

22
app/corpora/cqi_over_socketio/cqi_corpora.py
View File

@ -1,22 +0,0 @@
import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
@socketio.on('cqi.corpora.get', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_get(cqi_client: cqi.CQiClient, corpus_name: str):
cqi_corpus = cqi_client.corpora.get(corpus_name)
payload = {**cqi_corpus.attrs}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.list', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_list(cqi_client: cqi.CQiClient):
payload = [{**x.attrs} for x in cqi_client.corpora.list()]
return {'code': 200, 'msg': 'OK', 'payload': payload}

199
app/corpora/cqi_over_socketio/cqi_corpora_corpus.py
View File

@ -1,199 +0,0 @@
from collections import Counter
from flask import session
import cqi
import json
import math
import os
from app import db, socketio
from app.decorators import socketio_login_required
from app.models import Corpus
from . import NAMESPACE as ns
from .utils import cqi_over_socketio, lookups_by_cpos
@socketio.on('cqi.corpora.corpus.drop', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_drop(cqi_client: cqi.CQiClient, corpus_name: str):
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_status = cqi_corpus.drop()
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.query', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_query(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, query: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_status = cqi_corpus.query(subcorpus_name, query)
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}
###############################################################################
# nopaque specific CQi extensions #
###############################################################################
@socketio.on('cqi.corpora.corpus.update_db', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_update_db(cqi_client: cqi.CQiClient, corpus_name: str):
corpus = Corpus.query.get(session['d']['corpus_id'])
cqi_corpus = cqi_client.corpora.get(corpus_name)
corpus.num_tokens = cqi_corpus.size
db.session.commit()
@socketio.on('cqi.corpora.corpus.get_visualization_data', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_get_visualization_data(cqi_client: cqi.CQiClient, corpus_name: str):
corpus = Corpus.query.get(session['d']['corpus_id'])
visualization_data_file_path = os.path.join(corpus.path, 'cwb', 'visualization_data.json')
if os.path.exists(visualization_data_file_path):
with open(visualization_data_file_path, 'r') as f:
payload = json.load(f)
return {'code': 200, 'msg': 'OK', 'payload': payload}
cqi_corpus = cqi_client.corpora.get(corpus_name)
##########################################################################
# A faster way to get cpos boundaries for smaller s_attrs #
##########################################################################
# cqi_corpus.query('Last', '<s> []* </s>;')
# cqi_subcorpus = cqi_corpus.subcorpora.get('Last')
# print(cqi_subcorpus.size)
# first_match = 0
# last_match = cqi_subcorpus.attrs['size'] - 1
# match_boundaries = zip(
# list(range(first_match, last_match + 1)),
# cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['match'], first_match, last_match),
# cqi_subcorpus.dump(cqi_subcorpus.attrs['fields']['matchend'], first_match, last_match)
# )
# for x in match_boundaries:
# print(x)
cqi_p_attrs = {
p_attr.name: p_attr
for p_attr in cqi_corpus.positional_attributes.list()
}
cqi_s_attrs = {
s_attr.name: s_attr
for s_attr in cqi_corpus.structural_attributes.list()
}
payload = {
'corpus': {
'bounds': [0, cqi_corpus.size - 1],
'counts': {
'token': cqi_corpus.size
},
'freqs': {}
},
'p_attrs': {},
's_attrs': {},
'values': {'p_attrs': {}, 's_attrs': {}}
}
for p_attr in cqi_p_attrs.values():
payload['corpus']['freqs'][p_attr.name] = dict(
zip(
range(0, p_attr.lexicon_size),
p_attr.freqs_by_ids(list(range(0, p_attr.lexicon_size)))
)
)
payload['p_attrs'][p_attr.name] = dict(
zip(
range(0, cqi_corpus.size),
p_attr.ids_by_cpos(list(range(0, cqi_corpus.size)))
)
)
payload['values']['p_attrs'][p_attr.name] = dict(
zip(
range(0, p_attr.lexicon_size),
p_attr.values_by_ids(list(range(0, p_attr.lexicon_size)))
)
)
for s_attr in cqi_s_attrs.values():
if s_attr.has_values:
continue
payload['corpus']['counts'][s_attr.name] = s_attr.size
payload['s_attrs'][s_attr.name] = {'lexicon': {}, 'values': None}
payload['values']['s_attrs'][s_attr.name] = {}
for id in range(0, s_attr.size):
payload['s_attrs'][s_attr.name]['lexicon'][id] = {}
lbound, rbound = s_attr.cpos_by_id(id)
payload['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
payload['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {}
payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1
if s_attr.name not in ['text', 's']:
continue
cpos_range = range(lbound, rbound + 1)
payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['ent'] = len({x for x in cqi_s_attrs['ent'].ids_by_cpos(list(cpos_range)) if x != -1})
if s_attr.name != 'text':
continue
payload['s_attrs'][s_attr.name]['lexicon'][id]['counts']['s'] = len({x for x in cqi_s_attrs['s'].ids_by_cpos(list(cpos_range)) if x != -1})
payload['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
for p_attr in cqi_p_attrs.values():
payload['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr.ids_by_cpos(list(cpos_range))))
sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
s_attr_value_names = [
sub_s_attr.name[(len(s_attr.name) + 1):]
for sub_s_attr in sub_s_attrs
]
sub_s_attr_values = [
sub_s_attr.values_by_ids(list(range(0, s_attr.size)))
for sub_s_attr in sub_s_attrs
]
payload['s_attrs'][s_attr.name]['values'] = s_attr_value_names
payload['values']['s_attrs'][s_attr.name] = {
s_attr_id: {
s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id_idx]
for s_attr_value_name_idx, s_attr_value_name in enumerate(
payload['s_attrs'][s_attr.name]['values']
)
} for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
}
with open(visualization_data_file_path, 'w') as f:
json.dump(payload, f)
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.paginate', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, page: int = 1, per_page: int = 20): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
# Sanity checks
if (
per_page < 1
or page < 1
or (
cqi_corpus.size > 0
and page > math.ceil(cqi_corpus.size / per_page)
)
):
return {'code': 416, 'msg': 'Range Not Satisfiable'}
first_cpos = (page - 1) * per_page
last_cpos = min(cqi_corpus.size, first_cpos + per_page)
cpos_list = [*range(first_cpos, last_cpos)]
lookups = lookups_by_cpos(cqi_corpus, cpos_list)
payload = {}
# the items for the current page
payload['items'] = [cpos_list]
# the lookups for the items
payload['lookups'] = lookups
# the total number of items matching the query
payload['total'] = cqi_corpus.size
# the number of items to be displayed on a page.
payload['per_page'] = per_page
# The total number of pages
payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
# the current page number (1 indexed)
payload['page'] = page if payload['pages'] > 0 else None
# True if a previous page exists
payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
# True if a next page exists.
payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False # noqa
# Number of the previous page.
payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
# Number of the next page
payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
return {'code': 200, 'msg': 'OK', 'payload': payload}

24
app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py
View File

@ -1,24 +0,0 @@
import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
@socketio.on('cqi.corpora.corpus.alignment_attributes.get', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_alignment_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, alignment_attribute_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_alignment_attribute = cqi_corpus.alignment_attributes.get(alignment_attribute_name) # noqa
payload = {**cqi_alignment_attribute.attrs}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.alignment_attributes.list', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_alignment_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
payload = [{**x.attrs} for x in cqi_corpus.alignment_attributes.list()]
return {'code': 200, 'msg': 'OK', 'payload': payload}

24
app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py
View File

@ -1,24 +0,0 @@
import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
@socketio.on('cqi.corpora.corpus.positional_attributes.get', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_positional_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, positional_attribute_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_positional_attribute = cqi_corpus.positional_attributes.get(positional_attribute_name) # noqa
payload = {**cqi_positional_attribute.attrs}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.positional_attributes.list', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_positional_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
payload = [{**x.attrs} for x in cqi_corpus.positional_attributes.list()]
return {'code': 200, 'msg': 'OK', 'payload': payload}

24
app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py
View File

@ -1,24 +0,0 @@
import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
@socketio.on('cqi.corpora.corpus.structural_attributes.get', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_structural_attributes_get(cqi_client: cqi.CQiClient, corpus_name: str, structural_attribute_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_structural_attribute = cqi_corpus.structural_attributes.get(structural_attribute_name) # noqa
payload = {**cqi_structural_attribute.attrs}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.structural_attributes.list', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_structural_attributes_list(cqi_client: cqi.CQiClient, corpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
payload = [{**x.attrs} for x in cqi_corpus.structural_attributes.list()]
return {'code': 200, 'msg': 'OK', 'payload': payload}

125
app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py
View File

@ -1,125 +0,0 @@
import cqi
import math
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio, export_subcorpus, partial_export_subcorpus
@socketio.on('cqi.corpora.corpus.subcorpora.get', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_get(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
payload = {**cqi_subcorpus.attrs}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.subcorpora.list', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_list(cqi_client: cqi.CQiClient, corpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
payload = [{**x.attrs} for x in cqi_corpus.subcorpora.list()]
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.drop', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_drop(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
cqi_status = cqi_subcorpus.drop()
payload = {'code': cqi_status.code,
'msg': cqi_status.__class__.__name__}
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.dump', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_dump(cqi_client: cqi.CQiClient):
return {'code': 501, 'msg': 'Not Implemented'}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_1', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_fdist_1(cqi_client: cqi.CQiClient):
return {'code': 501, 'msg': 'Not Implemented'}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.fdist_2', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_fdist_2(cqi_client: cqi.CQiClient):
return {'code': 501, 'msg': 'Not Implemented'}
###############################################################################
# nopaque specific CQi extensions #
###############################################################################
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.paginate', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_paginate(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50, page: int = 1, per_page: int = 20): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
# Sanity checks
if (
per_page < 1
or page < 1
or (
cqi_subcorpus.attrs['size'] > 0
and page > math.ceil(cqi_subcorpus.attrs['size'] / per_page)
)
):
return {'code': 416, 'msg': 'Range Not Satisfiable'}
offset = (page - 1) * per_page
cutoff = per_page
cqi_results_export = export_subcorpus(
cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
payload = {}
# the items for the current page
payload['items'] = cqi_results_export.pop('matches')
# the lookups for the items
payload['lookups'] = cqi_results_export
# the total number of items matching the query
payload['total'] = cqi_subcorpus.attrs['size']
# the number of items to be displayed on a page.
payload['per_page'] = per_page
# The total number of pages
payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
# the current page number (1 indexed)
payload['page'] = page if payload['pages'] > 0 else None
# True if a previous page exists
payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
# True if a next page exists.
payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False # noqa
# Number of the previous page.
payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
# Number of the next page
payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
return {'code': 200, 'msg': 'OK', 'payload': payload}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.partial_export', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_partial_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, match_id_list: list, context: int = 50): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_partial_export}
@socketio.on('cqi.corpora.corpus.subcorpora.subcorpus.export', namespace=ns)
@socketio_login_required
@cqi_over_socketio
def cqi_corpora_corpus_subcorpora_subcorpus_export(cqi_client: cqi.CQiClient, corpus_name: str, subcorpus_name: str, context: int = 50): # noqa
cqi_corpus = cqi_client.corpora.get(corpus_name)
cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
return {'code': 200, 'msg': 'OK', 'payload': cqi_subcorpus_export}

122346
app/static/js/CorpusAnalysis/CQiClient.js
View File

File diff suppressed because it is too large Load Diff

40
app/static/js/CorpusAnalysis/CorpusAnalysisApp.js
View File

@ -71,25 +71,19 @@ class CorpusAnalysisApp {
this.disableActionElements();
this.elements.m.initModal.open();
// Init data
this.data.cQiClient = new CQiClient(this.settings.corpusId);
this.data.cQiClient.connect()
.then(cQiStatus => {
return this.data.cQiClient.corpora.get(`NOPAQUE_${this.settings.corpusId}`);
this.data.cqiClient = new cqi.CQiClient('/cqi_over_sio', this.settings.corpusId);
this.data.cqiClient.connect('anonymous', '')
.then((cqiStatus) => {
return this.data.cqiClient.corpora.list();
})
.then(
cQiCorpus => {
this.data.corpus = {o: cQiCorpus};
this.data.corpus.o.getVisualizationData()
.then(
(data) => {
console.log(data);
this.renderGeneralCorpusInfo(data);
this.renderTextInfoList(data);
this.renderTextProportionsGraphic(data);
this.renderFrequenciesGraphic(data);
this.renderBoundsGraphic(data);
}
);
.then((cqiCorpora) => {
this.data.corpus = {o: cqiCorpora[0]};
console.log(this.data.corpus.o.staticData);
this.renderGeneralCorpusInfo(this.data.corpus.o.staticData);
this.renderTextInfoList(this.data.corpus.o.staticData);
this.renderTextProportionsGraphic(this.data.corpus.o.staticData);
this.renderFrequenciesGraphic(this.data.corpus.o.staticData);
this.renderBoundsGraphic(this.data.corpus.o.staticData);
// this.data.corpus.o.getCorpusData()
// .then(corpusData => {
// console.log(corpusData);
@ -100,19 +94,19 @@ class CorpusAnalysisApp {
// this.renderBoundsGraphic(corpusData);
// });
// TODO: Don't do this hgere
cQiCorpus.updateDb();
this.data.corpus.o.updateDb();
this.enableActionElements();
for (let extension of Object.values(this.extensions)) {extension.init();}
this.elements.m.initModal.close();
},
cQiError => {
(cqiError) => {
let errorsElement = this.elements.initModal.querySelector('.errors');
let progressElement = this.elements.initModal.querySelector('.progress');
errorsElement.innerText = JSON.stringify(cQiError);
errorsElement.innerText = JSON.stringify(cqiError);
errorsElement.classList.remove('hide');
progressElement.classList.add('hide');
if ('payload' in cQiError && 'code' in cQiError.payload && 'msg' in cQiError.payload) {
app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
if ('payload' in cqiError && 'code' in cqiError.payload && 'msg' in cqiError.payload) {
app.flash(`${cqiError.payload.code}: ${cqiError.payload.msg}`, 'error');
}
}
);

25
app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js
View File

@ -45,18 +45,18 @@ class CorpusAnalysisConcordance {
this.elements.progress.classList.remove('hide');
let subcorpus = {};
this.data.corpus.o.query(subcorpusName, query)
.then(cQiStatus => {
.then((cqiStatus) => {
subcorpus.q = query;
subcorpus.selectedItems = new Set();
if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
return this.data.corpus.o.subcorpora.get(subcorpusName);
})
.then(cQiSubcorpus => {
subcorpus.o = cQiSubcorpus;
return cQiSubcorpus.paginate(1, this.settings.perPage, this.settings.context);
.then((cqiSubcorpus) => {
subcorpus.o = cqiSubcorpus;
return cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
})
.then(
paginatedSubcorpus => {
(paginatedSubcorpus) => {
subcorpus.p = paginatedSubcorpus;
this.data.subcorpora[subcorpusName] = subcorpus;
this.settings.selectedSubcorpus = subcorpusName;
@ -68,11 +68,12 @@ class CorpusAnalysisConcordance {
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
},
cQiError => {
this.elements.error.innerText = JSON.stringify(cQiError);
(cqiStatus) => {
// TODDO: CHECK THIS!
this.elements.error.innerText = JSON.stringify(cqiStatus);
this.elements.error.classList.remove('hide');
if ('payload' in cQiError && 'code' in cQiError.payload && 'msg' in cQiError.payload) {
app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
if ('payload' in cqiStatus && 'code' in cqiStatus.payload && 'msg' in cqiStatus.payload) {
app.flash(`${cqiStatus.payload.code}: ${cqiStatus.payload.msg}`, 'error');
}
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
@ -236,7 +237,7 @@ class CorpusAnalysisConcordance {
app.flash('No matches selected', 'error');
return;
}
promise = subcorpus.o.partial_export([...subcorpus.selectedItems], 50);
promise = subcorpus.o.partialExport([...subcorpus.selectedItems], 50);
} else {
promise = subcorpus.o.export(50);
}
@ -291,7 +292,7 @@ class CorpusAnalysisConcordance {
event.preventDefault();
let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
subcorpus.o.drop().then(
cQiStatus => {
(cQiStatus) => {
app.flash(`${subcorpus.o.name} deleted`, 'corpus');
delete this.data.subcorpora[subcorpus.o.name];
this.settings.selectedSubcorpus = undefined;
@ -312,7 +313,7 @@ class CorpusAnalysisConcordance {
this.clearSubcorpusPagination();
}
},
cQiError => {
(cQiError) => {
app.flash(`${cQiError.payload.code}: ${cQiError.payload.msg}`, 'error');
}
);

7
app/static/js/CorpusAnalysis/CorpusAnalysisReader.js
View File

@ -28,7 +28,6 @@ class CorpusAnalysisReader {
init() {
// Init data
this.data.corpus = this.app.data.corpus;
this.data.subcorpora = {};
// Add event listeners
this.elements.form.addEventListener('submit', (event) => {
event.preventDefault();
@ -38,14 +37,14 @@ class CorpusAnalysisReader {
this.elements.progress.classList.remove('hide');
this.data.corpus.o.paginate(1, this.settings.perPage)
.then(
paginatedCorpus => {
(paginatedCorpus) => {
this.data.corpus.p = paginatedCorpus;
this.renderCorpus();
this.renderCorpusPagination();
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
},
error => {
(cqiError) => {
this.elements.error.innerText = JSON.stringify(error);
this.elements.error.classList.remove('hide');
if ('payload' in error && 'code' in error.payload && 'msg' in error.payload) {
@ -247,7 +246,7 @@ class CorpusAnalysisReader {
this.elements.progress.classList.remove('hide');
this.data.corpus.o.paginate(pageNum, this.settings.perPage)
.then(
paginatedCorpus => {
(paginatedCorpus) => {
this.data.corpus.p = paginatedCorpus;
this.renderCorpus();
this.renderCorpusPagination();

691
app/static/js/cqi/api/client.js Normal file
View File

@ -0,0 +1,691 @@
cqi.api.APIClient = class APIClient {
/**
* @param {string} host
* @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
constructor(host, corpus_id, timeout = 60, version = '0.1') {
this.host = host;
this.timeout = timeout * 1000; // convert seconds to milliseconds
this.version = version;
this.socket = io(
this.host,
{
auth: {corpus_id: corpus_id},
transports: ['websocket'],
upgrade: false
}
);
}
/**
* @param {string} fn_name
* @param {object} [fn_args={}]
* @returns {Promise<cqi.status.StatusConnectOk>}
*/
#request(fn_name, fn_args = {}) {
return new Promise((resolve, reject) => {
this.socket.timeout(this.timeout).emit('cqi', {fn_name: fn_name, fn_args: fn_args}, (timeoutError, response) => {
if (timeoutError) {
reject(timeoutError);
}
if (response.code === 200) {
resolve(response.payload);
}
if (response.code === 500) {
reject(new Error(`[${response.code}] ${response.msg}`));
}
if (response.code === 502) {
reject(new cqi.errors.lookup[response.payload.code]());
}
});
});
}
/**
* @param {string} username
* @param {string} password
* @returns {Promise<cqi.status.StatusConnectOk>}
*/
async ctrl_connect(username, password) {
const fn_name = 'ctrl_connect';
const fn_args = {username: username, password: password};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* @returns {Promise<cqi.status.StatusByeOk>}
*/
async ctrl_bye() {
const fn_name = 'ctrl_bye';
let payload = await this.#request(fn_name);
return new cqi.status.lookup[payload.code]();
}
/**
* @returns {Promise<null>}
*/
async ctrl_user_abort() {
const fn_name = 'ctrl_user_abort';
return await this.#request(fn_name);
}
/**
* @returns {Promise<cqi.status.StatusPingOk>}
*/
async ctrl_ping() {
const fn_name = 'ctrl_ping';
let payload = await this.#request(fn_name);
return new cqi.status.lookup[payload.code]();
}
/**
* Full-text error message for the last general error reported
* by the CQi server
*
* @returns {Promise<string>}
*/
async ctrl_last_general_error() {
const fn_name = 'ctrl_last_general_error';
return await this.#request(fn_name);
}
/**
* @returns {Promise<boolean>}
*/
async ask_feature_cqi_1_0() {
const fn_name = 'ask_feature_cqi_1_0';
return await this.#request(fn_name);
}
/**
* @returns {Promise<boolean>}
*/
async ask_feature_cl_2_3() {
const fn_name = 'ask_feature_cl_2_3';
return await this.#request(fn_name);
}
/**
* @returns {Promise<boolean>}
*/
async ask_feature_cqp_2_3() {
const fn_name = 'ask_feature_cqp_2_3';
return await this.#request(fn_name);
}
/**
* @returns {Promise<string[]>}
*/
async corpus_list_corpora() {
const fn_name = 'corpus_list_corpora';
return await this.#request(fn_name);
}
/**
* @param {string} corpus
* @returns {Promise<string>}
*/
async corpus_charset(corpus) {
const fn_name = 'corpus_charset';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async corpus_properties(corpus) {
const fn_name = 'corpus_properties';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async corpus_positional_attributes(corpus) {
const fn_name = 'corpus_positional_attributes';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async corpus_structural_attributes(corpus) {
const fn_name = 'corpus_structural_attributes';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @param {string} attribute
* @returns {Promise<boolean>}
*/
async corpus_structural_attribute_has_values(corpus, attribute) {
const fn_name = 'corpus_structural_attribute_has_values';
const fn_args = {corpus: corpus, attribute: attribute};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async corpus_alignment_attributes(corpus) {
const fn_name = 'corpus_alignment_attributes';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* the full name of <corpus> as specified in its registry entry
*
* @param {string} corpus
* @returns {Promise<string>}
*/
async corpus_full_name(corpus) {
const fn_name = 'corpus_full_name';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* returns the contents of the .info file of <corpus> as a list of lines
*
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async corpus_info(corpus) {
const fn_name = 'corpus_info';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* try to unload a corpus and all its attributes from memory
*
* @param {string} corpus
* @returns {Promise<cqi.status.StatusOk>}
*/
async corpus_drop_corpus(corpus) {
const fn_name = 'corpus_drop_corpus';
const fn_args = {corpus: corpus};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* returns the size of <attribute>:
* - number of tokens (positional)
* - number of regions (structural)
* - number of alignments (alignment)
*
* @param {string} attribute
* @returns {Promise<number>}
*/
async cl_attribute_size(attribute) {
const fn_name = 'cl_attribute_size';
const fn_args = {attribute: attribute};
return await this.#request(fn_name, fn_args);
}
/**
* returns the number of entries in the lexicon of a positional attribute;
*
* valid lexicon IDs range from 0 .. (lexicon_size - 1)
*
* @param {string} attribute
* @returns {Promise<number>}
*/
async cl_lexicon_size(attribute) {
const fn_name = 'cl_lexicon_size';
const fn_args = {attribute: attribute};
return await this.#request(fn_name, fn_args);
}
/**
* unload attribute from memory
*
* @param {string} attribute
* @returns {Promise<cqi.status.StatusOk>}
*/
async cl_drop_attribute(attribute) {
const fn_name = 'cl_drop_attribute';
const fn_args = {attribute: attribute};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* NOTE: simple (scalar) mappings are applied to lists (the returned list
* has exactly the same length as the list passed as an argument)
*/
/**
* returns -1 for every string in <strings> that is not found in the lexicon
*
* @param {string} attribute
* @param {strings[]} string
* @returns {Promise<number[]>}
*/
async cl_str2id(attribute, strings) {
const fn_name = 'cl_str2id';
const fn_args = {attribute: attribute, strings: strings};
return await this.#request(fn_name, fn_args);
}
/**
* returns "" for every ID in <id> that is out of range
*
* @param {string} attribute
* @param {number[]} id
* @returns {Promise<string[]>}
*/
async cl_id2str(attribute, id) {
const fn_name = 'cl_id2str';
const fn_args = {attribute: attribute, id: id};
return await this.#request(fn_name, fn_args);
}
/**
* returns 0 for every ID in <id> that is out of range
*
* @param {string} attribute
* @param {number[]} id
* @returns {Promise<number[]>}
*/
async cl_id2freq(attribute, id) {
const fn_name = 'cl_id2freq';
const fn_args = {attribute: attribute, id: id};
return await this.#request(fn_name, fn_args);
}
/**
* returns -1 for every corpus position in <cpos> that is out of range
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<number[]>}
*/
async cl_cpos2id(attribute, cpos) {
const fn_name = 'cl_cpos2id';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* returns "" for every corpus position in <cpos> that is out of range
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<string[]>}
*/
async cl_cpos2str(attribute, cpos) {
const fn_name = 'cl_cpos2str';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* returns -1 for every corpus position not inside a structure region
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<number[]>}
*/
async cl_cpos2struc(attribute, cpos) {
const fn_name = 'cl_cpos2struc';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* NOTE: temporary addition for the Euralex2000 tutorial, but should
* probably be included in CQi specs
*/
/**
* returns left boundary of s-attribute region enclosing cpos,
* -1 if not in region
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<number[]>}
*/
async cl_cpos2lbound(attribute, cpos) {
const fn_name = 'cl_cpos2lbound';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* returns right boundary of s-attribute region enclosing cpos,
* -1 if not in region
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<number[]>}
*/
async cl_cpos2rbound(attribute, cpos) {
const fn_name = 'cl_cpos2rbound';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* returns -1 for every corpus position not inside an alignment
*
* @param {string} attribute
* @param {number[]} cpos
* @returns {Promise<number[]>}
*/
async cl_cpos2alg(attribute, cpos) {
const fn_name = 'cl_cpos2alg';
const fn_args = {attribute: attribute, cpos: cpos};
return await this.#request(fn_name, fn_args);
}
/**
* returns annotated string values of structure regions in <strucs>;
* "" if out of range
*
* check corpus_structural_attribute_has_values(<attribute>) first
*
* @param {string} attribute
* @param {number[]} strucs
* @returns {Promise<string[]>}
*/
async cl_struc2str(attribute, strucs) {
const fn_name = 'cl_struc2str';
const fn_args = {attribute: attribute, strucs: strucs};
return await this.#request(fn_name, fn_args);
}
/**
* NOTE: the following mappings take a single argument and return multiple
* values, including lists of arbitrary size
*/
/**
* returns all corpus positions where the given token occurs
*
* @param {string} attribute
* @param {number} id
* @returns {Promise<number[]>}
*/
async cl_id2cpos(attribute, id) {
const fn_name = 'cl_id2cpos';
const fn_args = {attribute: attribute, id: id};
return await this.#request(fn_name, fn_args);
}
/**
* returns all corpus positions where one of the tokens in <id_list> occurs;
* the returned list is sorted as a whole, not per token id
*
* @param {string} attribute
* @param {number[]} id_list
* @returns {Promise<number[]>}
*/
async cl_idlist2cpos(attribute, id_list) {
const fn_name = 'cl_idlist2cpos';
const fn_args = {attribute: attribute, id_list: id_list};
return await this.#request(fn_name, fn_args);
}
/**
* returns lexicon IDs of all tokens that match <regex>;
* the returned list may be empty (size 0);
*
* @param {string} attribute
* @param {string} regex
* @returns {Promise<number[]>}
*/
async cl_regex2id(attribute, regex) {
const fn_name = 'cl_regex2id';
const fn_args = {attribute: attribute, regex: regex};
return await this.#request(fn_name, fn_args);
}
/**
* returns start and end corpus positions of structure region <struc>
*
* @param {string} attribute
* @param {number} struc
* @returns {Promise<[number, number]>}
*/
async cl_struc2cpos(attribute, struc) {
const fn_name = 'cl_struc2cpos';
const fn_args = {attribute: attribute, struc: struc};
return await this.#request(fn_name, fn_args);
}
/**
* returns (src_start, src_end, target_start, target_end)
*
* @param {string} attribute
* @param {number} alg
* @returns {Promise<[number, number, number, number]>}
*/
async alg2cpos(attribute, alg) {
const fn_name = 'alg2cpos';
const fn_args = {attribute: attribute, alg: alg};
return await this.#request(fn_name, fn_args);
}
/**
* <query> must include the ';' character terminating the query.
*
* @param {string} mother_corpus
* @param {string} subcorpus_name
* @param {string} query
* @returns {Promise<cqi.status.StatusOk>}
*/
async cqp_query(mother_corpus, subcorpus_name, query) {
const fn_name = 'cqp_query';
const fn_args = {mother_corpus: mother_corpus, subcorpus_name: subcorpus_name, query: query};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* @param {string} corpus
* @returns {Promise<string[]>}
*/
async cqp_list_subcorpora(corpus) {
const fn_name = 'cqp_list_subcorpora';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} subcorpus
* @returns {Promise<number>}
*/
async cqp_subcorpus_size(subcorpus) {
const fn_name = 'cqp_subcorpus_size';
const fn_args = {subcorpus: subcorpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} subcorpus
* @param {number} field
* @returns {Promise<boolean>}
*/
async cqp_subcorpus_has_field(subcorpus, field) {
const fn_name = 'cqp_subcorpus_has_field';
const fn_args = {subcorpus: subcorpus, field: field};
return await this.#request(fn_name, fn_args);
}
/**
* Dump the values of <field> for match ranges <first> .. <last>
* in <subcorpus>. <field> is one of the CQI_CONST_FIELD_* constants.
*
* @param {string} subcorpus
* @param {number} field
* @param {number} first
* @param {number} last
* @returns {Promise<number[]>}
*/
async cqp_dump_subcorpus(subcorpus, field, first, last) {
const fn_name = 'cqp_dump_subcorpus';
const fn_args = {subcorpus: subcorpus, field: field, first: first, last: last};
return await this.#request(fn_name, fn_args);
}
/**
* delete a subcorpus from memory
*
* @param {string} subcorpus
* @returns {Promise<cqi.status.StatusOk>}
*/
async cqp_drop_subcorpus(subcorpus) {
const fn_name = 'cqp_drop_subcorpus';
const fn_args = {subcorpus: subcorpus};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* NOTE: The following two functions are temporarily included for the
* Euralex 2000 tutorial demo
*/
/**
* frequency distribution of single tokens
*
* returns <n> (id, frequency) pairs flattened into a list of size 2*<n>
* field is one of
* - CQI_CONST_FIELD_MATCH
* - CQI_CONST_FIELD_TARGET
* - CQI_CONST_FIELD_KEYWORD
*
* NB: pairs are sorted by frequency desc.
*
* @param {string} subcorpus
* @param {number} cutoff
* @param {number} field
* @param {string} attribute
* @returns {Promise<number[]>}
*/
async cqp_fdist_1(subcorpus, cutoff, field, attribute) {
const fn_name = 'cqp_fdist_1';
const fn_args = {subcorpus: subcorpus, cutoff: cutoff, field: field, attribute: attribute};
return await this.#request(fn_name, fn_args);
}
/**
* frequency distribution of pairs of tokens
*
* returns <n> (id1, id2, frequency) pairs flattened into a list of
* size 3*<n>
*
* NB: triples are sorted by frequency desc.
*
* @param {string} subcorpus
* @param {number} cutoff
* @param {number} field1
* @param {string} attribute1
* @param {number} field2
* @param {string} attribute2
* @returns {Promise<number[]>}
*/
async cqp_fdist_2(subcorpus, cutoff, field1, attribute1, field2, attribute2) {
const fn_name = 'cqp_fdist_2';
const fn_args = {subcorpus: subcorpus, cutoff: cutoff, field1: field1, attribute1: attribute1, field2: field2, attribute2: attribute2};
return await this.#request(fn_name, fn_args);
}
/**************************************************************************
* NOTE: The following is not included in the CQi specification. *
**************************************************************************/
/**************************************************************************
* Custom additions for nopaque *
**************************************************************************/
/**
* @param {string} corpus
* @returns {Promise<cqi.status.StatusOk>}
*/
async ext_corpus_update_db(corpus) {
const fn_name = 'ext_corpus_update_db';
const fn_args = {corpus: corpus};
let payload = await this.#request(fn_name, fn_args);
return new cqi.status.lookup[payload.code]();
}
/**
* @param {string} corpus
* @returns {Promise<object>}
*/
async ext_corpus_static_data(corpus) {
const fn_name = 'ext_corpus_static_data';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} corpus
* @param {number=} page
* @param {number=} per_page
* @returns {Promise<object>}
*/
async ext_corpus_paginate_corpus(corpus, page, per_page) {
const fn_name = 'ext_corpus_paginate_corpus';
const fn_args = {corpus: corpus}
if (page !== undefined) {fn_args.page = page;}
if (per_page !== undefined) {fn_args.per_page = per_page;}
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} subcorpus
* @param {number=} context
* @param {number=} page
* @param {number=} per_page
* @returns {Promise<object>}
*/
async ext_cqp_paginate_subcorpus(subcorpus, context, page, per_page) {
const fn_name = 'ext_cqp_paginate_subcorpus';
const fn_args = {subcorpus: subcorpus}
if (context !== undefined) {fn_args.context = context;}
if (page !== undefined) {fn_args.page = page;}
if (per_page !== undefined) {fn_args.per_page = per_page;}
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} subcorpus
* @param {number[]} match_id_list
* @param {number=} context
* @returns {Promise<object>}
*/
async ext_cqp_partial_export_subcorpus(subcorpus, match_id_list, context) {
const fn_name = 'ext_cqp_partial_export_subcorpus';
const fn_args = {subcorpus: subcorpus, match_id_list: match_id_list};
if (context !== undefined) {fn_args.context = context;}
return await this.#request(fn_name, fn_args);
}
/**
* @param {string} subcorpus
* @param {number=} context
* @returns {Promise<object>}
*/
async ext_cqp_export_subcorpus(subcorpus, context) {
const fn_name = 'ext_cqp_export_subcorpus';
const fn_args = {subcorpus: subcorpus};
if (context !== undefined) {fn_args.context = context;}
return await this.#request(fn_name, fn_args);
}
};

1
app/static/js/cqi/api/package.js Normal file
View File

@ -0,0 +1 @@
cqi.api = {};

58
app/static/js/cqi/client.js Normal file
View File

@ -0,0 +1,58 @@
cqi.CQiClient = class CQiClient {
/**
* @param {string} host
* @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
constructor(host, corpusId, timeout = 60, version = '0.1') {
/** @type {cqi.api.APIClient} */
this.api = new cqi.api.APIClient(host, corpusId, timeout, version);
}
/**
* @returns {cqi.models.corpora.CorpusCollection}
*/
get corpora() {
return new cqi.models.corpora.CorpusCollection(this);
}
/**
* @returns {Promise<cqi.status.StatusByeOk>}
*/
async bye() {
return await this.api.ctrl_bye();
}
/**
* @param {string} username
* @param {string} password
* @returns {Promise<cqi.status.StatusConnectOk>}
*/
async connect(username, password) {
return await this.api.ctrl_connect(username, password);
}
/**
* @returns {Promise<cqi.status.StatusPingOk>}
*/
async ping() {
return await this.api.ctrl_ping();
}
/**
* @returns {Promise<null>}
*/
async userAbort() {
return await this.api.ctrl_user_abort();
}
/**
* Alias for "bye" method
*
* @returns {Promise<cqi.status.StatusByeOk>}
*/
async disconnect() {
return await this.api.ctrl_bye();
}
};

185
app/static/js/cqi/errors.js Normal file
View File

@ -0,0 +1,185 @@
cqi.errors = {};
/**
* A base class from which all other errors inherit.
* If you want to catch all errors that the CQi package might throw,
* catch this base error.
*/
cqi.errors.CQiError = class CQiError extends Error {
constructor(message) {
super(message);
this.code = undefined;
this.description = undefined;
}
};
cqi.errors.Error = class Error extends cqi.errors.CQiError {
constructor(message) {
super(message);
this.code = 2;
}
};
cqi.errors.ErrorGeneralError = class ErrorGeneralError extends cqi.errors.Error {
constructor(message) {
super(message);
this.code = 513;
}
};
cqi.errors.ErrorConnectRefused = class ErrorConnectRefused extends cqi.errors.Error {
constructor(message) {
super(message);
this.code = 514;
}
};
cqi.errors.ErrorUserAbort = class ErrorUserAbort extends cqi.errors.Error {
constructor(message) {
super(message);
this.code = 515;
}
};
cqi.errors.ErrorSyntaxError = class ErrorSyntaxError extends cqi.errors.Error {
constructor(message) {
super(message);
this.code = 516;
}
};
cqi.errors.CLError = class Error extends cqi.errors.CQiError {
constructor(message) {
super(message);
this.code = 4;
}
};
cqi.errors.CLErrorNoSuchAttribute = class CLErrorNoSuchAttribute extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1025;
this.description = "CQi server couldn't open attribute";
}
};
cqi.errors.CLErrorWrongAttributeType = class CLErrorWrongAttributeType extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1026;
}
};
cqi.errors.CLErrorOutOfRange = class CLErrorOutOfRange extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1027;
}
};
cqi.errors.CLErrorRegex = class CLErrorRegex extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1028;
}
};
cqi.errors.CLErrorCorpusAccess = class CLErrorCorpusAccess extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1029;
}
};
cqi.errors.CLErrorOutOfMemory = class CLErrorOutOfMemory extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1030;
this.description = 'CQi server has run out of memory; try discarding some other corpora and/or subcorpora';
}
};
cqi.errors.CLErrorInternal = class CLErrorInternal extends cqi.errors.CLError {
constructor(message) {
super(message);
this.code = 1031;
this.description = "The classical 'please contact technical support' error";
}
};
cqi.errors.CQPError = class Error extends cqi.errors.CQiError {
constructor(message) {
super(message);
this.code = 5;
}
};
cqi.errors.CQPErrorGeneral = class CQPErrorGeneral extends cqi.errors.CQPError {
constructor(message) {
super(message);
this.code = 1281;
}
};
cqi.errors.CQPErrorNoSuchCorpus = class CQPErrorNoSuchCorpus extends cqi.errors.CQPError {
constructor(message) {
super(message);
this.code = 1282;
}
};
cqi.errors.CQPErrorInvalidField = class CQPErrorInvalidField extends cqi.errors.CQPError {
constructor(message) {
super(message);
this.code = 1283;
}
};
cqi.errors.CQPErrorOutOfRange = class CQPErrorOutOfRange extends cqi.errors.CQPError {
constructor(message) {
super(message);
this.code = 1284;
this.description = 'A number is out of range';
}
};
cqi.errors.lookup = {
2: cqi.errors.Error,
513: cqi.errors.ErrorGeneralError,
514: cqi.errors.ErrorConnectRefused,
515: cqi.errors.ErrorUserAbort,
516: cqi.errors.ErrorSyntaxError,
4: cqi.errors.CLError,
1025: cqi.errors.CLErrorNoSuchAttribute,
1026: cqi.errors.CLErrorWrongAttributeType,
1027: cqi.errors.CLErrorOutOfRange,
1028: cqi.errors.CLErrorRegex,
1029: cqi.errors.CLErrorCorpusAccess,
1030: cqi.errors.CLErrorOutOfMemory,
1031: cqi.errors.CLErrorInternal,
5: cqi.errors.CQPError,
1281: cqi.errors.CQPErrorGeneral,
1282: cqi.errors.CQPErrorNoSuchCorpus,
1283: cqi.errors.CQPErrorInvalidField,
1284: cqi.errors.CQPErrorOutOfRange
};

289
app/static/js/cqi/models/attributes.js Normal file
View File

@ -0,0 +1,289 @@
cqi.models.attributes = {};
cqi.models.attributes.Attribute = class Attribute extends cqi.models.resource.Model {
/**
* @returns {string}
*/
get apiName() {
return this.attrs.api_name;
}
/**
* @returns {string}
*/
get name() {
return this.attrs.name;
}
/**
* @returns {number}
*/
get size() {
return this.attrs.size;
}
/**
* @returns {Promise<cqi.status.StatusOk>}
*/
async drop() {
return await this.client.api.cl_drop_attribute(this.apiName);
}
};
cqi.models.attributes.AttributeCollection = class AttributeCollection extends cqi.models.resource.Collection {
/** @type{typeof cqi.models.attributes.Attribute} */
static model = cqi.models.attributes.Attribute;
/**
* @param {cqi.CQiClient} client
* @param {cqi.models.corpora.Corpus} corpus
*/
constructor(client, corpus) {
super(client);
/** @type {cqi.models.corpora.Corpus} */
this.corpus = corpus;
}
/**
* @param {string} attributeName
* @returns {Promise<object>}
*/
async _get(attributeName) {
/** @type{string} */
let apiName = `${this.corpus.apiName}.${attributeName}`;
return {
api_name: apiName,
name: attributeName,
size: await this.client.api.cl_attribute_size(apiName)
}
}
/**
* @param {string} attributeName
* @returns {Promise<cqi.models.attributes.Attribute>}
*/
async get(attributeName) {
return this.prepareModel(await this._get(attributeName));
}
};
cqi.models.attributes.AlignmentAttribute = class AlignmentAttribute extends cqi.models.attributes.Attribute {
/**
* @param {number} id
* @returns {Promise<[number, number, number, number]>}
*/
async cposById(id) {
return await this.client.api.cl_alg2cpos(this.apiName, id);
}
/**
* @param {number[]} cposList
* @returns {Promise<number[]>}
*/
async idsByCpos(cposList) {
return await this.client.api.cl_cpos2alg(this.apiName, cposList);
}
};
cqi.models.attributes.AlignmentAttributeCollection = class AlignmentAttributeCollection extends cqi.models.attributes.AttributeCollection {
/** @type{typeof cqi.models.attributes.AlignmentAttribute} */
static model = cqi.models.attributes.AlignmentAttribute;
/**
* @returns {Promise<cqi.models.attributes.AlignmentAttribute[]>}
*/
async list() {
/** @type {string[]} */
let alignmentAttributeNames = await this.client.api.corpus_alignment_attributes(this.corpus.apiName);
/** @type {cqi.models.attributes.AlignmentAttribute[]} */
let alignmentAttributes = [];
for (let alignmentAttributeName of alignmentAttributeNames) {
alignmentAttributes.push(await this.get(alignmentAttributeName));
}
return alignmentAttributes;
}
};
cqi.models.attributes.PositionalAttribute = class PositionalAttribute extends cqi.models.attributes.Attribute {
/**
* @returns {number}
*/
get lexiconSize() {
return this.attrs.lexicon_size;
}
/**
* @param {number} id
* @returns {Promise<number[]>}
*/
async cposById(id) {
return await this.client.api.cl_id2cpos(this.apiName, id);
}
/**
* @param {number[]} idList
* @returns {Promise<number[]>}
*/
async cposByIds(idList) {
return await this.client.api.cl_idlist2cpos(this.apiName, idList);
}
/**
* @param {number[]} idList
* @returns {Promise<number[]>}
*/
async freqsByIds(idList) {
return await this.client.api.cl_id2freq(this.apiName, idList);
}
/**
* @param {number[]} cposList
* @returns {Promise<number[]>}
*/
async idsByCpos(cposList) {
return await this.client.api.cl_cpos2id(this.apiName, cposList);
}
/**
* @param {string} regex
* @returns {Promise<number[]>}
*/
async idsByRegex(regex) {
return await this.client.api.cl_regex2id(this.apiName, regex);
}
/**
* @param {string[]} valueList
* @returns {Promise<number[]>}
*/
async idsByValues(valueList) {
return await this.client.api.cl_str2id(this.apiName, valueList);
}
/**
* @param {number[]} cposList
* @returns {Promise<string[]>}
*/
async valuesByCpos(cposList) {
return await this.client.api.cl_cpos2str(this.apiName, cposList);
}
/**
* @param {number[]} idList
* @returns {Promise<string[]>}
*/
async valuesByIds(idList) {
return await this.client.api.cl_id2str(this.apiName, idList);
}
};
cqi.models.attributes.PositionalAttributeCollection = class PositionalAttributeCollection extends cqi.models.attributes.AttributeCollection {
/** @type{typeof cqi.models.attributes.PositionalAttribute} */
static model = cqi.models.attributes.PositionalAttribute;
/**
* @param {string} positionalAttributeName
* @returns {Promise<object>}
*/
async _get(positionalAttributeName) {
let positionalAttribute = await super._get(positionalAttributeName);
positionalAttribute.lexicon_size = await this.client.api.cl_lexicon_size(positionalAttribute.api_name);
return positionalAttribute;
}
/**
* @returns {Promise<cqi.models.attributes.PositionalAttribute[]>}
*/
async list() {
let positionalAttributeNames = await this.client.api.corpus_positional_attributes(this.corpus.apiName);
let positionalAttributes = [];
for (let positionalAttributeName of positionalAttributeNames) {
positionalAttributes.push(await this.get(positionalAttributeName));
}
return positionalAttributes;
}
};
cqi.models.attributes.StructuralAttribute = class StructuralAttribute extends cqi.models.attributes.Attribute {
/**
* @returns {boolean}
*/
get hasValues() {
return this.attrs.has_values;
}
/**
* @param {number} id
* @returns {Promise<[number, number]>}
*/
async cposById(id) {
return await this.client.api.cl_struc2cpos(this.apiName, id);
}
/**
* @param {number[]} cposList
* @returns {Promise<number[]>}
*/
async idsByCpos(cposList) {
return await this.client.api.cl_cpos2struc(this.apiName, cposList);
}
/**
* @param {number[]} cposList
* @returns {Promise<number[]>}
*/
async lboundByCpos(cposList) {
return await this.client.api.cl_cpos2lbound(this.apiName, cposList);
}
/**
* @param {number[]} cposList
* @returns {Promise<number[]>}
*/
async rboundByCpos(cposList) {
return await this.client.api.cl_cpos2rbound(this.apiName, cposList);
}
/**
* @param {number[]} idList
* @returns {Promise<string[]>}
*/
async valuesByIds(idList) {
return await this.client.api.cl_struc2str(this.apiName, idList);
}
};
cqi.models.attributes.StructuralAttributeCollection = class StructuralAttributeCollection extends cqi.models.attributes.AttributeCollection {
/** @type{typeof cqi.models.attributes.StructuralAttribute} */
static model = cqi.models.attributes.StructuralAttribute;
/**
* @param {string} structuralAttributeName
* @returns {Promise<object>}
*/
async _get(structuralAttributeName) {
let structuralAttribute = await super._get(structuralAttributeName);
structuralAttribute.has_values = await this.client.api.cl_has_values(structuralAttribute.api_name);
return structuralAttribute;
}
/**
* @returns {Promise<cqi.models.attributes.StructuralAttribute[]>}
*/
async list() {
let structuralAttributeNames = await this.client.api.corpus_structural_attributes(this.corpus.apiName);
let structuralAttributes = [];
for (let structuralAttributeName of structuralAttributeNames) {
structuralAttributes.push(await this.get(structuralAttributeName));
}
return structuralAttributes;
}
};

166
app/static/js/cqi/models/corpora.js Normal file
View File

@ -0,0 +1,166 @@
cqi.models.corpora = {};
cqi.models.corpora.Corpus = class Corpus extends cqi.models.resource.Model {
/**
* @returns {string}
*/
get apiName() {
return this.attrs.api_name;
}
/**
* @returns {string}
*/
get name() {
return this.attrs.name;
}
/**
* @returns {number}
*/
get size() {
return this.attrs.size;
}
/**
* @returns {string}
*/
get charset() {
return this.attrs.charset;
}
/**
* @returns {string[]}
*/
get properties() {
return this.attrs?.properties;
}
/**
* @returns {cqi.models.attributes.AlignmentAttributeCollection}
*/
get alignmentAttributes() {
return new cqi.models.attributes.AlignmentAttributeCollection(this.client, this);
}
/**
* @returns {cqi.models.attributes.PositionalAttributeCollection}
*/
get positionalAttributes() {
return new cqi.models.attributes.PositionalAttributeCollection(this.client, this);
}
/**
* @returns {cqi.models.attributes.StructuralAttributeCollection}
*/
get structuralAttributes() {
return new cqi.models.attributes.StructuralAttributeCollection(this.client, this);
}
/**
* @returns {cqi.models.subcorpora.SubcorpusCollection}
*/
get subcorpora() {
return new cqi.models.subcorpora.SubcorpusCollection(this.client, this);
}
/**
* @returns {Promise<cqi.status.StatusOk>}
*/
async drop() {
return await this.client.api.corpus_drop_corpus(this.apiName);
}
/**
* @param {string} subcorpusName
* @param {string} query
* @returns {Promise<cqi.status.StatusOk>}
*/
async query(subcorpusName, query) {
return await this.client.api.cqp_query(this.apiName, subcorpusName, query);
}
/**************************************************************************
* NOTE: The following is not included in the CQi specification. *
**************************************************************************/
/**************************************************************************
* Custom additions for nopaque *
**************************************************************************/
/**
* @returns {string}
*/
get staticData() {
return this.attrs.static_data;
}
/**
* @returns {cqi.status.StatusOk}
*/
async updateDb() {
return await this.client.api.ext_corpus_update_db(this.apiName);
}
/**
* @param {number=} page
* @param {number=} per_page
* @returns {Promise<object>}
*/
async paginate(page, per_page) {
return await this.client.api.ext_corpus_paginate_corpus(this.apiName, page, per_page);
}
};
cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.resource.Collection {
/** @type {typeof cqi.models.corpora.Corpus} */
static model = cqi.models.corpora.Corpus;
/**
* @param {string} corpusName
* @returns {Promise<object>}
*/
async _get(corpusName) {
const returnValue = {
api_name: corpusName,
charset: await this.client.api.corpus_charset(corpusName),
// full_name: await this.client.api.corpus_full_name(corpusName),
// info: await this.client.api.corpus_info(corpusName),
name: corpusName,
properties: await this.client.api.corpus_properties(corpusName),
size: await this.client.api.cl_attribute_size(`${corpusName}.word`)
};
/************************************************************************
* NOTE: The following is not included in the CQi specification. *
************************************************************************/
/************************************************************************
* Custom additions for nopaque *
************************************************************************/
returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
return returnValue;
}
/**
* @param {string} corpusName
* @returns {Promise<cqi.models.corpora.Corpus>}
*/
async get(corpusName) {
return this.prepareModel(await this._get(corpusName));
}
/**
* @returns {Promise<cqi.models.corpora.Corpus[]>}
*/
async list() {
/** @type {string[]} */
let corpusNames = await this.client.api.corpus_list_corpora();
/** @type {cqi.models.corpora.Corpus[]} */
let corpora = [];
for (let corpusName of corpusNames) {
corpora.push(await this.get(corpusName));
}
return corpora;
}
};

1
app/static/js/cqi/models/package.js Normal file
View File

@ -0,0 +1 @@
cqi.models = {};

90
app/static/js/cqi/models/resource.js Normal file
View File

@ -0,0 +1,90 @@
cqi.models.resource = {};
/**
* A base class for representing a single object on the server.
*/
cqi.models.resource.Model = class Model {
/**
* @param {object} attrs
* @param {cqi.CQiClient} client
* @param {cqi.models.resource.Collection} collection
*/
constructor(attrs, client, collection) {
/**
* A client pointing at the server that this object is on.
*
* @type {cqi.CQiClient}
*/
this.client = client;
/**
* The collection that this model is part of.
*
* @type {cqi.models.resource.Collection}
*/
this.collection = collection;
/**
* The raw representation of this object from the API
*
* @type {object}
*/
this.attrs = attrs;
}
/**
* @returns {string}
*/
get apiName() {
throw new Error('Not implemented');
}
/**
* @returns {Promise<void>}
*/
async reload() {
this.attrs = await this.collection.get(this.apiName).attrs;
}
};
/**
* A base class for representing all objects of a particular type on the server.
*/
cqi.models.resource.Collection = class Collection {
/**
* The type of object this collection represents, set by subclasses
*
* @type {typeof cqi.models.resource.Model}
*/
static model;
/**
* @param {cqi.CQiClient} client
*/
constructor(client) {
/**
* A client pointing at the server that this object is on.
*
* @type {cqi.CQiClient}
*/
this.client = client;
}
async list() {
throw new Error('Not implemented');
}
async get() {
throw new Error('Not implemented');
}
/**
* Create a model from a set of attributes.
*
* @param {object} attrs
* @returns {cqi.models.resource.Model}
*/
prepareModel(attrs) {
return new this.constructor.model(attrs, this.client, this);
}
};

189
app/static/js/cqi/models/subcorpora.js Normal file
View File

@ -0,0 +1,189 @@
cqi.models.subcorpora = {};
cqi.models.subcorpora.Subcorpus = class Subcorpus extends cqi.models.resource.Model {
/**
* @returns {string}
*/
get apiName() {
return this.attrs.api_name;
}
/**
* @returns {object}
*/
get fields() {
return this.attrs.fields;
}
/**
* @returns {string}
*/
get name() {
return this.attrs.name;
}
/**
* @returns {number}
*/
get size() {
return this.attrs.size;
}
/**
* @returns {Promise<cqi.status.StatusOk>}
*/
async drop() {
return await this.client.api.cqp_drop_subcorpus(this.apiName);
}
/**
* @param {number} field
* @param {number} first
* @param {number} last
* @returns {Promise<number[]>}
*/
async dump(field, first, last) {
return await this.client.api.cqp_dump_subcorpus(
this.apiName,
field,
first,
last
);
}
/**
* @param {number} cutoff
* @param {number} field
* @param {cqi.models.attributes.PositionalAttribute} attribute
* @returns {Promise<number[]>}
*/
async fdist1(cutoff, field, attribute) {
return await this.client.api.cqp_fdist_1(
this.apiName,
cutoff,
field,
attribute.apiName
);
}
/**
* @param {number} cutoff
* @param {number} field1
* @param {cqi.models.attributes.PositionalAttribute} attribute1
* @param {number} field2
* @param {cqi.models.attributes.PositionalAttribute} attribute2
* @returns {Promise<number[]>}
*/
async fdist2(cutoff, field1, attribute1, field2, attribute2) {
return await this.client.api.cqp_fdist_2(
this.apiName,
cutoff,
field1,
attribute1.apiName,
field2,
attribute2.apiName
);
}
/**************************************************************************
* NOTE: The following is not included in the CQi specification. *
**************************************************************************/
/**************************************************************************
* Custom additions for nopaque *
**************************************************************************/
/**
* @param {number=} context
* @param {number=} page
* @param {number=} perPage
* @returns {Promise<object>}
*/
async paginate(context, page, perPage) {
return await this.client.api.ext_cqp_paginate_subcorpus(this.apiName, context, page, perPage);
}
/**
* @param {number[]} matchIdList
* @param {number=} context
* @returns {Promise<object>}
*/
async partialExport(matchIdList, context) {
return await this.client.api.ext_cqp_partial_export_subcorpus(this.apiName, matchIdList, context);
}
/**
* @param {number=} context
* @returns {Promise<object>}
*/
async export(context) {
return await this.client.api.ext_cqp_export_subcorpus(this.apiName, context);
}
};
cqi.models.subcorpora.SubcorpusCollection = class SubcorpusCollection extends cqi.models.resource.Collection {
/** @type {typeof cqi.models.subcorpora.Subcorpus} */
static model = cqi.models.subcorpora.Subcorpus;
/**
* @param {cqi.CQiClient} client
* @param {cqi.models.corpora.Corpus} corpus
*/
constructor(client, corpus) {
super(client);
/** @type {cqi.models.corpora.Corpus} */
this.corpus = corpus;
}
/**
* @param {string} subcorpusName
* @returns {Promise<object>}
*/
async _get(subcorpusName) {
/** @type {string} */
let apiName = `${this.corpus.apiName}:${subcorpusName}`;
/** @type {object} */
let fields = {};
if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCH)) {
fields.match = cqi.CONST_FIELD_MATCH;
}
if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_MATCHEND)) {
fields.matchend = cqi.CONST_FIELD_MATCHEND
}
if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_TARGET)) {
fields.target = cqi.CONST_FIELD_TARGET
}
if (await this.client.api.cqp_subcorpus_has_field(apiName, cqi.CONST_FIELD_KEYWORD)) {
fields.keyword = cqi.CONST_FIELD_KEYWORD
}
return {
api_name: apiName,
fields: fields,
name: subcorpusName,
size: await this.client.api.cqp_subcorpus_size(apiName)
}
}
/**
* @param {string} subcorpusName
* @returns {Promise<cqi.models.subcorpora.Subcorpus>}
*/
async get(subcorpusName) {
return this.prepareModel(await this._get(subcorpusName));
}
/**
* @returns {Promise<cqi.models.subcorpora.Subcorpus[]>}
*/
async list() {
/** @type {string[]} */
let subcorpusNames = await this.client.api.cqp_list_subcorpora(this.corpus.apiName);
/** @type {cqi.models.subcorpora.Subcorpus[]} */
let subcorpora = [];
for (let subcorpusName of subcorpusNames) {
subcorpora.push(await this.get(subcorpusName));
}
return subcorpora;
}
};

6
app/static/js/cqi/package.js Normal file
View File

@ -0,0 +1,6 @@
var cqi = {};
cqi.CONST_FIELD_KEYWORD = 9;
cqi.CONST_FIELD_MATCH = 16;
cqi.CONST_FIELD_MATCHEND = 17;
cqi.CONST_FIELD_TARGET = 0;

51
app/static/js/cqi/status.js Normal file
View File

@ -0,0 +1,51 @@
cqi.status = {};
/**
* A base class from which all other status inherit.
*/
cqi.status.CQiStatus = class CQiStatus {
constructor() {
this.code = undefined;
}
};
cqi.status.StatusOk = class StatusOk extends cqi.status.CQiStatus {
constructor() {
super();
this.code = 257;
}
};
cqi.status.StatusConnectOk = class StatusConnectOk extends cqi.status.CQiStatus {
constructor() {
super();
this.code = 258;
}
};
cqi.status.StatusByeOk = class StatusByeOk extends cqi.status.CQiStatus {
constructor() {
super();
this.code = 259;
}
};
cqi.status.StatusPingOk = class StatusPingOk extends cqi.status.CQiStatus {
constructor() {
super();
this.code = 260;
}
};
cqi.status.lookup = {
257: cqi.status.StatusOk,
258: cqi.status.StatusConnectOk,
259: cqi.status.StatusByeOk,
260: cqi.status.StatusPingOk
};

18
app/templates/_scripts.html.j2
View File

@ -3,12 +3,28 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.4/socket.io.min.js" integrity="sha512-HTENHrkQ/P0NGDFd5nk6ibVtCkcM7jhr2c7GyvXp5O+4X6O5cQO9AhqFzM+MdeBivsX7Hoys2J7pp2wdgMpCvw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/2.24.2/plotly.min.js" integrity="sha512-dAXqGCq94D0kgLSPnfvd/pZpCMoJQpGj2S2XQmFQ9Ay1+96kbjss02ISEh+TBNXMggGg/1qoMcOHcxg+Op/Jmw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
{%- assets
filters='rjsmin',
output='gen/cqi.%(version)s.js',
'js/cqi/package.js',
'js/cqi/errors.js',
'js/cqi/status.js',
'js/cqi/api/package.js',
'js/cqi/api/client.js',
'js/cqi/models/package.js',
'js/cqi/models/resource.js',
'js/cqi/models/attributes.js',
'js/cqi/models/subcorpora.js',
'js/cqi/models/corpora.js',
'js/cqi/client.js'
%}
<script src="{{ ASSET_URL }}"></script>
{%- endassets %}
{%- assets
filters='rjsmin',
output='gen/app.%(version)s.js',
'js/App.js',
'js/Utils.js',
'js/CorpusAnalysis/CQiClient.js',
'js/CorpusAnalysis/CorpusAnalysisApp.js',
'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
'js/CorpusAnalysis/CorpusAnalysisReader.js',