sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-11-03 20:02:47 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'visualizations-update' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into visualizations-update

Browse Source
This commit is contained in:
Inga Kirschnick
2023-07-18 16:05:09 +02:00
parent 691d4757ff e46f0032bd
commit 7721926d6c
15 changed files with 379 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/__init__.py
View File

@@ -74,6 +74,8 @@ def create_app(config: Config = Config) -> Flask:
app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
from .corpora import bp as corpora_blueprint
from .corpora.cqi_over_sio import CQiNamespace
socketio.on_namespace(CQiNamespace('/cqi_over_sio'))
default_breadcrumb_root(corpora_blueprint, '.corpora')
app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')

2
app/corpora/__init__.py
View File

@@ -16,4 +16,4 @@ def before_request():
pass
from . import cli, cqi_over_sio, files, followers, routes, json_routes
from . import cli, files, followers, routes, json_routes

260
app/corpora/cqi_over_sio/__init__.py
View File

@@ -1,113 +1,199 @@
from cqi import CQiClient
from cqi.errors import CQiException
from cqi.status import CQiStatus
from flask import session
from flask_login import current_user
from flask_socketio import ConnectionRefusedError
from flask_socketio import Namespace
from inspect import signature
from threading import Lock
from typing import Callable, Dict, List
from app import db, hashids, socketio
from app.decorators import socketio_login_required
from app.models import Corpus, CorpusStatus
import math
from . import extensions
'''
This package tunnels the Corpus Query interface (CQi) protocol through
Socket.IO (SIO) by wrapping each CQi function in a seperate SIO event.
This module only handles the SIO connect/disconnect, which handles the setup
and teardown of necessary ressources for later use. Each CQi function has a
corresponding SIO event. The event handlers are spread across the different
modules within this package.
Socket.IO (SIO) by tunneling CQi API calls through an event called "exec".
Basic concept:
1. A client connects to the SIO namespace and provides the id of a corpus to be
analysed.
1. A client connects to the "/cqi_over_sio" namespace.
2. The client emits the "init" event and provides a corpus id for the corpus
that should be analysed in this session.
1.1 The analysis session counter of the corpus is incremented.
1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
1.3 Wait until the CQP server is running.
1.4 Connect the CQiClient to the server.
1.5 Save the CQiClient and the Lock in the session for subsequential use.
2. A client emits an event and may provide a single json object with necessary
arguments for the targeted CQi function.
3. A SIO event handler (decorated with cqi_over_socketio) gets executed.
- The event handler function defines all arguments. Hence the client
is sent as a single json object, the decorator decomposes it to fit
the functions signature. This also includes type checking and proper
use of the lock (acquire/release) mechanism.
1.5 Save the CQiClient, the Lock and the corpus id in the session for
subsequential use.
2. The client emits the "exec" event provides the name of a CQi API function
arguments (optional).
- The event "exec" handler will execute the function, make sure that the
result is serializable and returns the result back to the client.
4. Wait for more events
5. The client disconnects from the SIO namespace
5. The client disconnects from the "/cqi_over_sio" namespace
1.1 The analysis session counter of the corpus is decremented.
1.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
'''
NAMESPACE = '/cqi_over_sio'
CQI_API_FUNCTION_NAMES: List[str] = [
'ask_feature_cl_2_3',
'ask_feature_cqi_1_0',
'ask_feature_cqp_2_3',
'cl_alg2cpos',
'cl_attribute_size',
'cl_cpos2alg',
'cl_cpos2id',
'cl_cpos2lbound',
'cl_cpos2rbound',
'cl_cpos2str',
'cl_cpos2struc',
'cl_drop_attribute',
'cl_id2cpos',
'cl_id2freq',
'cl_id2str',
'cl_idlist2cpos',
'cl_lexicon_size',
'cl_regex2id',
'cl_str2id',
'cl_struc2cpos',
'cl_struc2str',
'corpus_alignment_attributes',
'corpus_charset',
'corpus_drop_corpus',
'corpus_full_name',
'corpus_info',
'corpus_list_corpora',
'corpus_positional_attributes',
'corpus_properties',
'corpus_structural_attribute_has_values',
'corpus_structural_attributes',
'cqp_drop_subcorpus',
'cqp_dump_subcorpus',
'cqp_fdist_1',
'cqp_fdist_2',
'cqp_list_subcorpora',
'cqp_query',
'cqp_subcorpus_has_field',
'cqp_subcorpus_size',
'ctrl_bye',
'ctrl_connect',
'ctrl_last_general_error',
'ctrl_ping',
'ctrl_user_abort'
]
from .cqi import * # noqa
@socketio.on('connect', namespace=NAMESPACE)
@socketio_login_required
def connect(auth):
# the auth variable is used in a hacky way. It contains the corpus id for
# which a corpus analysis session should be started.
corpus_id = hashids.decode(auth['corpus_id'])
corpus = Corpus.query.get(corpus_id)
if corpus is None:
# return {'code': 404, 'msg': 'Not Found'}
raise ConnectionRefusedError('Not Found')
if not (corpus.user == current_user
or current_user.is_following_corpus(corpus)
or current_user.is_administrator()):
# return {'code': 403, 'msg': 'Forbidden'}
raise ConnectionRefusedError('Forbidden')
if corpus.status not in [
CorpusStatus.BUILT,
CorpusStatus.STARTING_ANALYSIS_SESSION,
CorpusStatus.RUNNING_ANALYSIS_SESSION,
CorpusStatus.CANCELING_ANALYSIS_SESSION
]:
# return {'code': 424, 'msg': 'Failed Dependency'}
raise ConnectionRefusedError('Failed Dependency')
if corpus.num_analysis_sessions is None:
corpus.num_analysis_sessions = 0
db.session.commit()
corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
db.session.commit()
retry_counter = 20
while corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
if retry_counter == 0:
corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
db.session.commit()
return {'code': 408, 'msg': 'Request Timeout'}
socketio.sleep(3)
retry_counter -= 1
db.session.refresh(corpus)
cqi_client = CQiClient(f'cqpserver_{corpus_id}', timeout=math.inf)
session['cqi_over_sio'] = {
'corpus_id': corpus_id,
'cqi_client': cqi_client,
'cqi_client_lock': Lock(),
}
# return {'code': 200, 'msg': 'OK'}
@socketio.on('disconnect', namespace=NAMESPACE)
def disconnect():
try:
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
except KeyError:
return
cqi_client_lock.acquire()
try:
cqi_client.api.ctrl_bye()
except (BrokenPipeError, CQiException):
class CQiNamespace(Namespace):
@socketio_login_required
def on_connect(self):
pass
cqi_client_lock.release()
corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
db.session.commit()
session.pop('cqi_over_sio')
# return {'code': 200, 'msg': 'OK'}
@socketio_login_required
def on_init(self, db_corpus_hashid: str):
db_corpus_id = hashids.decode(db_corpus_hashid)
db_corpus = Corpus.query.get(db_corpus_id)
if db_corpus is None:
return {'code': 404, 'msg': 'Not Found'}
if not (db_corpus.user == current_user
or current_user.is_following_corpus(db_corpus)
or current_user.is_administrator()):
return {'code': 403, 'msg': 'Forbidden'}
if db_corpus.status not in [
CorpusStatus.BUILT,
CorpusStatus.STARTING_ANALYSIS_SESSION,
CorpusStatus.RUNNING_ANALYSIS_SESSION,
CorpusStatus.CANCELING_ANALYSIS_SESSION
]:
return {'code': 424, 'msg': 'Failed Dependency'}
if db_corpus.num_analysis_sessions is None:
db_corpus.num_analysis_sessions = 0
db.session.commit()
db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
db.session.commit()
retry_counter = 20
while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
if retry_counter == 0:
db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
db.session.commit()
return {'code': 408, 'msg': 'Request Timeout'}
socketio.sleep(3)
retry_counter -= 1
db.session.refresh(db_corpus)
cqi_client = CQiClient(f'cqpserver_{db_corpus_id}', timeout=None)
session['cqi_over_sio'] = {}
session['cqi_over_sio']['cqi_client'] = cqi_client
session['cqi_over_sio']['cqi_client_lock'] = Lock()
session['cqi_over_sio']['db_corpus_id'] = db_corpus_id
return {'code': 200, 'msg': 'OK'}
@socketio_login_required
def on_exec(self, fn_name: str, fn_args: Dict = {}):
try:
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
except KeyError:
return {'code': 424, 'msg': 'Failed Dependency'}
if fn_name in CQI_API_FUNCTION_NAMES:
fn: Callable = getattr(cqi_client.api, fn_name)
elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
fn: Callable = getattr(extensions, fn_name)
else:
return {'code': 400, 'msg': 'Bad Request'}
for param in signature(fn).parameters.values():
if param.default is param.empty:
if param.name not in fn_args:
return {'code': 400, 'msg': 'Bad Request'}
else:
if param.name not in fn_args:
continue
if type(fn_args[param.name]) is not param.annotation:
return {'code': 400, 'msg': 'Bad Request'}
cqi_client_lock.acquire()
try:
fn_return_value = fn(**fn_args)
except BrokenPipeError as e:
return {'code': 500, 'msg': 'Internal Server Error'}
except CQiException as e:
return {
'code': 502,
'msg': 'Bad Gateway',
'payload': {
'code': e.code,
'desc': e.description,
'msg': e.__class__.__name__
}
}
finally:
cqi_client_lock.release()
if isinstance(fn_return_value, CQiStatus):
payload = {
'code': fn_return_value.code,
'msg': fn_return_value.__class__.__name__
}
else:
payload = fn_return_value
return {'code': 200, 'msg': 'OK', 'payload': payload}
def on_disconnect(self):
try:
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
except KeyError:
return
cqi_client_lock.acquire()
try:
session.pop('cqi_over_sio')
except KeyError:
pass
try:
cqi_client.api.ctrl_bye()
except (BrokenPipeError, CQiException):
pass
cqi_client_lock.release()
db_corpus = Corpus.query.get(db_corpus_id)
if db_corpus is not None:
db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
db.session.commit()

113
app/corpora/cqi_over_sio/cqi.py
View File

@@ -1,113 +0,0 @@
from cqi import CQiClient
from cqi.errors import CQiException
from cqi.status import CQiStatus
from flask import session
from inspect import signature
from threading import Lock
from typing import Callable, Dict, List
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .extensions import CQI_EXTENSION_FUNCTION_NAMES
from . import extensions as extensions_module
CQI_FUNCTION_NAMES: List[str] = [
'ask_feature_cl_2_3',
'ask_feature_cqi_1_0',
'ask_feature_cqp_2_3',
'cl_alg2cpos',
'cl_attribute_size',
'cl_cpos2alg',
'cl_cpos2id',
'cl_cpos2lbound',
'cl_cpos2rbound',
'cl_cpos2str',
'cl_cpos2struc',
'cl_drop_attribute',
'cl_id2cpos',
'cl_id2freq',
'cl_id2str',
'cl_idlist2cpos',
'cl_lexicon_size',
'cl_regex2id',
'cl_str2id',
'cl_struc2cpos',
'cl_struc2str',
'corpus_alignment_attributes',
'corpus_charset',
'corpus_drop_corpus',
'corpus_full_name',
'corpus_info',
'corpus_list_corpora',
'corpus_positional_attributes',
'corpus_properties',
'corpus_structural_attribute_has_values',
'corpus_structural_attributes',
'cqp_drop_subcorpus',
'cqp_dump_subcorpus',
'cqp_fdist_1',
'cqp_fdist_2',
'cqp_list_subcorpora',
'cqp_query',
'cqp_subcorpus_has_field',
'cqp_subcorpus_size',
'ctrl_bye',
'ctrl_connect',
'ctrl_last_general_error',
'ctrl_ping',
'ctrl_user_abort'
]
@socketio.on('cqi', namespace=ns)
@socketio_login_required
def cqi_over_sio(fn_name: str, fn_args: Dict = {}):
try:
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
except KeyError:
return {'code': 424, 'msg': 'Failed Dependency'}
if fn_name in CQI_FUNCTION_NAMES:
fn: Callable = getattr(cqi_client.api, fn_name)
elif fn_name in CQI_EXTENSION_FUNCTION_NAMES:
fn: Callable = getattr(extensions_module, fn_name)
else:
return {'code': 400, 'msg': 'Bad Request'}
for param in signature(fn).parameters.values():
if param.default is param.empty:
if param.name not in fn_args:
return {'code': 400, 'msg': 'Bad Request'}
else:
if param.name not in fn_args:
continue
if type(fn_args[param.name]) is not param.annotation:
return {'code': 400, 'msg': 'Bad Request'}
cqi_client_lock.acquire()
try:
fn_return_value = fn(**fn_args)
except BrokenPipeError:
fn_return_value = {
'code': 500,
'msg': 'Internal Server Error'
}
except CQiException as e:
return {
'code': 502,
'msg': 'Bad Gateway',
'payload': {
'code': e.code,
'desc': e.description,
'msg': e.__class__.__name__
}
}
finally:
cqi_client_lock.release()
if isinstance(fn_return_value, CQiStatus):
payload = {
'code': fn_return_value.code,
'msg': fn_return_value.__class__.__name__
}
else:
payload = fn_return_value
return {'code': 200, 'msg': 'OK', 'payload': payload}

16
app/corpora/cqi_over_sio/extensions.py
View File

@@ -28,8 +28,9 @@ CQI_EXTENSION_FUNCTION_NAMES: List[str] = [
def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
db_corpus: Corpus = Corpus.query.get(db_corpus_id)
cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
db_corpus.num_tokens = cqi_corpus.size
db.session.commit()
@@ -37,10 +38,11 @@ def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
def ext_corpus_static_data(corpus: str) -> Dict:
db_corpus = Corpus.query.get(session['cqi_over_sio']['corpus_id'])
static_corpus_data_file = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
if os.path.exists(static_corpus_data_file):
with open(static_corpus_data_file, 'rb') as f:
db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
db_corpus: Corpus = Corpus.query.get(db_corpus_id)
cache_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz')
if os.path.exists(cache_file_path):
with open(cache_file_path, 'rb') as f:
return f.read()
cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
@@ -189,10 +191,10 @@ def ext_corpus_static_data(corpus: str) -> Dict:
} for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size))
}
del sub_s_attr_values
with gzip.open(static_corpus_data_file, 'wt') as f:
with gzip.open(cache_file_path, 'wt') as f:
json.dump(static_corpus_data, f)
del static_corpus_data
with open(static_corpus_data_file, 'rb') as f:
with open(cache_file_path, 'rb') as f:
return f.read()

127
app/static/js/CorpusAnalysis/CorpusAnalysisApp.js
View File

@@ -1,13 +1,15 @@
class CorpusAnalysisApp {
constructor(corpusId) {
this.corpusId = corpusId;
this.data = {};
// HTML elements
this.elements = {
container: document.querySelector('#corpus-analysis-app-container'),
extensionCards: document.querySelector('#corpus-analysis-app-extension-cards'),
extensionTabs: document.querySelector('#corpus-analysis-app-extension-tabs'),
initModal: document.querySelector('#corpus-analysis-app-init-modal'),
overview: document.querySelector('#corpus-analysis-app-overview')
initModal: document.querySelector('#corpus-analysis-app-init-modal')
};
// Materialize elements
this.elements.m = {
@@ -17,80 +19,97 @@ class CorpusAnalysisApp {
this.extensions = {};
this.settings = {
corpusId: corpusId
};
this.settings = {};
}
init() {
async init() {
this.disableActionElements();
this.elements.m.initModal.open();
// Init data
this.data.cqiClient = new cqi.CQiClient('/cqi_over_sio', this.settings.corpusId);
this.data.cqiClient.connect('anonymous', '')
.then((cqiStatus) => {
return this.data.cqiClient.corpora.list();
})
.then((cqiCorpora) => {
this.data.corpus = {o: cqiCorpora[0]};
console.log(this.data.corpus.o.staticData);
// TODO: Don't do this hgere
this.data.corpus.o.updateDb();
this.enableActionElements();
for (let extension of Object.values(this.extensions)) {extension.init();}
this.elements.m.initModal.close();
},
(cqiError) => {
let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
let errorsElement = this.elements.initModal.querySelector('.errors');
let progressElement = this.elements.initModal.querySelector('.progress');
errorsElement.innerText = errorString;
errorsElement.classList.remove('hide');
app.flash(errorString, 'error');
progressElement.classList.add('hide');
}
);
const statusTextElement = this.elements.initModal.querySelector('.status-text');
// Add event listeners
for (let extensionSelectorElement of this.elements.overview.querySelectorAll('.extension-selector')) {
// Setup CQi over SocketIO connection and gather data from the CQPServer
try {
statusTextElement.innerText = 'Creating CQi over SocketIO client...';
const cqiClient = new cqi.CQiClient('/cqi_over_sio');
statusTextElement.innerText += ' Done';
statusTextElement.innerHTML += '<br>Waiting for the CQP server...';
const response = await cqiClient.api.socket.emitWithAck('init', this.corpusId);
if (response.code !== 200) {throw new Error();}
statusTextElement.innerText += ' Done';
statusTextElement.innerHTML += '<br>Connecting to the CQP server...';
await cqiClient.connect('anonymous', '');
statusTextElement.innerText += ' Done';
statusTextElement.innerHTML += '<br>Building and receiving corpus data cache from the server (This may take a while)...';
const cqiCorpus = await cqiClient.corpora.get(`NOPAQUE-${this.corpusId.toUpperCase()}`);
statusTextElement.innerText += ' Done';
// TODO: Don't do this hgere
await cqiCorpus.updateDb();
this.data.cqiClient = cqiClient;
this.data.cqiCorpus = cqiCorpus;
this.data.corpus = {o: cqiCorpus}; // legacy
} catch (error) {
let errorString = '';
if ('code' in error) {errorString += `[${error.code}] `;}
errorString += `${error.constructor.name}`;
if ('description' in error) {errorString += `: ${error.description}`;}
const errorsElement = this.elements.initModal.querySelector('.errors');
const progressElement = this.elements.initModal.querySelector('.progress');
errorsElement.innerText = errorString;
errorsElement.classList.remove('hide');
progressElement.classList.add('hide');
return;
}
// Initialize extensions
for (const extension of Object.values(this.extensions)) {
statusTextElement.innerHTML += `<br>Initializing ${extension.name} extension...`;
await extension.init();
statusTextElement.innerText += ' Done'
}
for (const extensionSelectorElement of this.elements.extensionCards.querySelectorAll('.extension-selector')) {
extensionSelectorElement.addEventListener('click', () => {
this.elements.m.extensionTabs.select(extensionSelectorElement.dataset.target);
});
}
this.enableActionElements();
this.elements.m.initModal.close();
}
registerExtension(extension) {
if (extension.name in this.extensions) {
console.error(`Can't register extension ${extension.name}: Already registered`);
return;
}
if (extension.name in this.extensions) {return;}
this.extensions[extension.name] = extension;
if ('cQiClient' in this.data && this.data.cQiClient.connected) {extension.init();}
}
disableActionElements() {
let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
for (let actionElement of actionElements) {
if (actionElement.nodeName === 'INPUT') {
actionElement.disabled = true;
} else if (actionElement.nodeName === 'SELECT') {
actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
} else {
actionElement.classList.add('disabled');
const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
for (const actionElement of actionElements) {
switch(actionElement.nodeName) {
case 'INPUT':
actionElement.disabled = true;
break;
case 'SELECT':
actionElement.parentNode.querySelector('input.select-dropdown').disabled = true;
break;
default:
actionElement.classList.add('disabled');
}
}
}
enableActionElements() {
let actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
for (let actionElement of actionElements) {
if (actionElement.nodeName === 'INPUT') {
actionElement.disabled = false;
} else if (actionElement.nodeName === 'SELECT') {
actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
} else {
actionElement.classList.remove('disabled');
const actionElements = this.elements.container.querySelectorAll('.corpus-analysis-action');
for (const actionElement of actionElements) {
switch(actionElement.nodeName) {
case 'INPUT':
actionElement.disabled = false;
break;
case 'SELECT':
actionElement.parentNode.querySelector('input.select-dropdown').disabled = false;
break;
default:
actionElement.classList.remove('disabled');
}
}
}

104
app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js
View File

@@ -30,62 +30,60 @@ class CorpusAnalysisConcordance {
this.app.registerExtension(this);
}
init() {
async submitForm() {
this.app.disableActionElements();
let query = this.elements.form.query.value.trim();
let subcorpusName = this.elements.form['subcorpus-name'].value;
this.elements.error.innerText = '';
this.elements.error.classList.add('hide');
this.elements.progress.classList.remove('hide');
try {
const subcorpus = {};
subcorpus.q = query;
subcorpus.selectedItems = new Set();
await this.data.corpus.o.query(subcorpusName, query);
if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
const cqiSubcorpus = await this.data.corpus.o.subcorpora.get(subcorpusName);
subcorpus.o = cqiSubcorpus;
const paginatedSubcorpus = await cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
subcorpus.p = paginatedSubcorpus;
this.data.subcorpora[subcorpusName] = subcorpus;
this.settings.selectedSubcorpus = subcorpusName;
this.renderSubcorpusList();
this.renderSubcorpusInfo();
this.renderSubcorpusActions();
this.renderSubcorpusItems();
this.renderSubcorpusPagination();
this.elements.progress.classList.add('hide');
} catch (error) {
let errorString = '';
if ('code' in error) {errorString += `[${error.code}] `;}
errorString += `${error.constructor.name}`;
this.elements.error.innerText = errorString;
this.elements.error.classList.remove('hide');
app.flash(errorString, 'error');
this.elements.progress.classList.add('hide');
}
this.app.enableActionElements();
}
async init() {
// Init data
this.data.corpus = this.app.data.corpus;
this.data.subcorpora = {};
// Add event listeners
this.elements.form.addEventListener('submit', event => {
this.elements.form.addEventListener('submit', (event) => {
event.preventDefault();
this.app.disableActionElements();
let query = this.elements.form.query.value.trim();
let subcorpusName = this.elements.form['subcorpus-name'].value;
this.elements.error.innerText = '';
this.elements.error.classList.add('hide');
this.elements.progress.classList.remove('hide');
let subcorpus = {};
this.data.corpus.o.query(subcorpusName, query)
.then((cqiStatus) => {
subcorpus.q = query;
subcorpus.selectedItems = new Set();
if (subcorpusName !== 'Last') {this.data.subcorpora.Last = subcorpus;}
return this.data.corpus.o.subcorpora.get(subcorpusName);
})
.then((cqiSubcorpus) => {
subcorpus.o = cqiSubcorpus;
return cqiSubcorpus.paginate(this.settings.context, 1, this.settings.perPage);
})
.then(
(paginatedSubcorpus) => {
subcorpus.p = paginatedSubcorpus;
this.data.subcorpora[subcorpusName] = subcorpus;
this.settings.selectedSubcorpus = subcorpusName;
this.renderSubcorpusList();
this.renderSubcorpusInfo();
this.renderSubcorpusActions();
this.renderSubcorpusItems();
this.renderSubcorpusPagination();
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
},
(cqiError) => {
let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
this.elements.error.innerText = errorString;
this.elements.error.classList.remove('hide');
app.flash(errorString, 'error');
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
}
);
this.submitForm();
});
this.elements.form.addEventListener('change', event => {
this.elements.form.addEventListener('change', (event) => {
if (event.target === this.elements.form['context']) {
this.settings.context = parseInt(this.elements.form['context'].value);
this.elements.form.submit.click();
this.submitForm();
}
if (event.target === this.elements.form['per-page']) {
this.settings.perPage = parseInt(this.elements.form['per-page'].value);
this.elements.form.submit.click();
this.submitForm();
}
if (event.target === this.elements.form['text-style']) {
this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -161,7 +159,7 @@ class CorpusAnalysisConcordance {
</a>
`.trim();
M.Tooltip.init(this.elements.subcorpusActions.querySelectorAll('.tooltipped'));
this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', event => {
this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', (event) => {
event.preventDefault();
let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
let modalElementId = Utils.generateElementId('export-subcorpus-modal-');
@@ -218,7 +216,7 @@ class CorpusAnalysisConcordance {
}
}
);
exportButton.addEventListener('click', event => {
exportButton.addEventListener('click', (event) => {
event.preventDefault();
this.app.disableActionElements();
this.elements.progress.classList.remove('hide');
@@ -240,7 +238,7 @@ class CorpusAnalysisConcordance {
promise = subcorpus.o.export(50);
}
promise.then(
data => {
(data) => {
let blob;
if (exportFormat === 'csv') {
let csvContent = 'sep=,\r\n';
@@ -286,7 +284,7 @@ class CorpusAnalysisConcordance {
});
modal.open();
});
this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', event => {
this.elements.subcorpusActions.querySelector('.subcorpus-delete-trigger').addEventListener('click', (event) => {
event.preventDefault();
let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus];
subcorpus.o.drop().then(
@@ -362,7 +360,7 @@ class CorpusAnalysisConcordance {
this.setTextStyle();
this.setTokenRepresentation();
for (let gotoReaderTriggerElement of this.elements.subcorpusItems.querySelectorAll('.goto-reader-trigger')) {
gotoReaderTriggerElement.addEventListener('click', event => {
gotoReaderTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let corpusAnalysisReader = this.app.extensions.Reader;
let itemId = parseInt(gotoReaderTriggerElement.closest('.item').dataset.id);
@@ -384,7 +382,7 @@ class CorpusAnalysisConcordance {
});
}
for (let selectTriggerElement of this.elements.subcorpusItems.querySelectorAll('.select-trigger')) {
selectTriggerElement.addEventListener('click', event => {
selectTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let itemElement = selectTriggerElement.closest('.item');
let itemId = parseInt(itemElement.dataset.id);
@@ -446,14 +444,14 @@ class CorpusAnalysisConcordance {
</li>
`.trim();
for (let paginationTriggerElement of this.elements.subcorpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
paginationTriggerElement.addEventListener('click', event => {
paginationTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
this.app.disableActionElements();
this.elements.progress.classList.remove('hide');
let page = parseInt(paginationTriggerElement.dataset.target);
subcorpus.o.paginate(page, this.settings.perPage, this.settings.context)
.then(
paginatedSubcorpus => {
(paginatedSubcorpus) => {
subcorpus.p = paginatedSubcorpus;
this.renderSubcorpusItems();
this.renderSubcorpusPagination();

67
app/static/js/CorpusAnalysis/CorpusAnalysisReader.js
View File

@@ -19,45 +19,52 @@ class CorpusAnalysisReader {
this.settings = {
perPage: parseInt(this.elements.form['per-page'].value),
textStyle: parseInt(this.elements.form['text-style'].value),
tokenRepresentation: this.elements.form['token-representation'].value
tokenRepresentation: this.elements.form['token-representation'].value,
pagination: {
innerWindow: 5,
outerWindow: 1
}
}
this.app.registerExtension(this);
}
init() {
async submitForm() {
this.app.disableActionElements();
this.elements.error.innerText = '';
this.elements.error.classList.add('hide');
this.elements.progress.classList.remove('hide');
try {
const paginatedCorpus = await this.data.corpus.o.paginate(1, this.settings.perPage);
this.data.corpus.p = paginatedCorpus;
this.renderCorpus();
this.renderCorpusPagination();
this.elements.progress.classList.add('hide');
} catch (error) {
let errorString = '';
if ('code' in error) {errorString += `[${error.code}] `;}
errorString += `${error.constructor.name}`;
if ('description' in error) {errorString += `: ${error.description}`;}
this.elements.error.innerText = errorString;
this.elements.error.classList.remove('hide');
app.flash(errorString, 'error');
this.elements.progress.classList.add('hide');
}
this.app.enableActionElements();
}
async init() {
// Init data
this.data.corpus = this.app.data.corpus;
// Add event listeners
this.elements.form.addEventListener('submit', (event) => {
event.preventDefault();
this.app.disableActionElements();
this.elements.error.innerText = '';
this.elements.error.classList.add('hide');
this.elements.progress.classList.remove('hide');
this.data.corpus.o.paginate(1, this.settings.perPage)
.then(
(paginatedCorpus) => {
this.data.corpus.p = paginatedCorpus;
this.renderCorpus();
this.renderCorpusPagination();
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
},
(cqiError) => {
let errorString = `${cqiError.code}: ${cqiError.constructor.name}`;
this.elements.error.innerText = errorString;
this.elements.error.classList.remove('hide');
app.flash(errorString, 'error');
this.elements.progress.classList.add('hide');
this.app.enableActionElements();
}
);
this.submitForm();
});
this.elements.form.addEventListener('change', event => {
this.elements.form.addEventListener('change', (event) => {
if (event.target === this.elements.form['per-page']) {
this.settings.perPage = parseInt(this.elements.form['per-page'].value);
this.elements.form.submit.click();
this.submitForm();
}
if (event.target === this.elements.form['text-style']) {
this.settings.textStyle = parseInt(this.elements.form['text-style'].value);
@@ -69,7 +76,7 @@ class CorpusAnalysisReader {
}
});
// Load initial data
this.elements.form.submit.click();
await this.submitForm();
}
clearCorpus() {
@@ -142,7 +149,7 @@ class CorpusAnalysisReader {
}
// render page buttons (5 before and 5 after current page)
for (let i = this.data.corpus.p.page -5; i <= this.data.corpus.p.page; i++) {
for (let i = this.data.corpus.p.page - this.settings.pagination.innerWindow; i <= this.data.corpus.p.page; i++) {
if (i <= 0) {continue;}
pageElement = Utils.HTMLToElement(
`
@@ -153,7 +160,7 @@ class CorpusAnalysisReader {
);
this.elements.corpusPagination.appendChild(pageElement);
};
for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page +5; i++) {
for (let i = this.data.corpus.p.page +1; i <= this.data.corpus.p.page + this.settings.pagination.innerWindow; i++) {
if (i > this.data.corpus.p.pages) {break;}
pageElement = Utils.HTMLToElement(
`
@@ -201,7 +208,7 @@ class CorpusAnalysisReader {
this.elements.corpusPagination.appendChild(pageElement);
for (let paginateTriggerElement of this.elements.corpusPagination.querySelectorAll('.pagination-trigger[data-target]')) {
paginateTriggerElement.addEventListener('click', event => {
paginateTriggerElement.addEventListener('click', (event) => {
event.preventDefault();
let page = parseInt(paginateTriggerElement.dataset.target);
this.page(page);

2
app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js
View File

@@ -13,7 +13,7 @@ class CorpusAnalysisStaticVisualization {
this.app.registerExtension(this);
}
init() {
async init() {
// Init data
this.data.corpus = this.app.data.corpus;
this.renderGeneralCorpusInfo();

34
app/static/js/cqi/api/client.js
View File

@@ -1,18 +1,16 @@
cqi.api.APIClient = class APIClient {
/**
* @param {string} host
* @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
constructor(host, corpus_id, timeout = 60, version = '0.1') {
constructor(host, timeout = 60, version = '0.1') {
this.host = host;
this.timeout = timeout * 1000; // convert seconds to milliseconds
this.version = version;
this.socket = io(
this.host,
{
auth: {corpus_id: corpus_id},
transports: ['websocket'],
upgrade: false
}
@@ -24,22 +22,16 @@ cqi.api.APIClient = class APIClient {
* @param {object} [fn_args={}]
* @returns {Promise}
*/
#request(fn_name, fn_args = {}) {
return new Promise((resolve, reject) => {
// this.socket.timeout(this.timeout).emit('cqi', {fn_name: fn_name, fn_args: fn_args}, (timeoutError, response) => {
// if (timeoutError) {
// reject(timeoutError);
// }
this.socket.emit('cqi', fn_name, fn_args, (response) => {
if (response.code === 200) {
resolve(response.payload);
} else if (response.code === 500) {
reject(new Error(`[${response.code}] ${response.msg}`));
} else if (response.code === 502) {
reject(new cqi.errors.lookup[response.payload.code]());
}
});
});
async #request(fn_name, fn_args = {}) {
// TODO: implement timeout
let response = await this.socket.emitWithAck('exec', fn_name, fn_args);
if (response.code === 200) {
return response.payload;
} else if (response.code === 500) {
throw new Error(`[${response.code}] ${response.msg}`);
} else if (response.code === 502) {
throw new cqi.errors.lookup[response.payload.code]();
}
}
/**
@@ -630,7 +622,9 @@ cqi.api.APIClient = class APIClient {
async ext_corpus_static_data(corpus) {
const fn_name = 'ext_corpus_static_data';
const fn_args = {corpus: corpus};
return await this.#request(fn_name, fn_args);
let compressedEncodedData = await this.#request(fn_name, fn_args);
let data = pako.inflate(compressedEncodedData, {to: 'string'});
return JSON.parse(data);
}
/**

5
app/static/js/cqi/client.js
View File

@@ -1,13 +1,12 @@
cqi.CQiClient = class CQiClient {
/**
* @param {string} host
* @param {string} corpusId
* @param {number} [timeout=60] timeout
* @param {string} [version=0.1] version
*/
constructor(host, corpusId, timeout = 60, version = '0.1') {
constructor(host, timeout = 60, version = '0.1') {
/** @type {cqi.api.APIClient} */
this.api = new cqi.api.APIClient(host, corpusId, timeout, version);
this.api = new cqi.api.APIClient(host, timeout, version);
}
/**

10
app/static/js/cqi/models/corpora.js
View File

@@ -138,15 +138,7 @@ cqi.models.corpora.CorpusCollection = class CorpusCollection extends cqi.models.
/************************************************************************
* Custom additions for nopaque *
************************************************************************/
// returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
let tmp = await this.client.api.ext_corpus_static_data(corpusName);
console.log(tmp);
let inflated = pako.inflate(tmp);
console.log(inflated);
let decoder = new TextDecoder('utf-8');
console.log(decoder);
let decoded = decoder.decode(inflated);
returnValue.static_data = JSON.parse(decoded);
returnValue.static_data = await this.client.api.ext_corpus_static_data(corpusName);
return returnValue;
}

2
app/templates/_scripts.html.j2
View File

@@ -1,6 +1,6 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.1/fast-json-patch.min.js" integrity="sha512-5uDdefwnzyq4N+SkmMBmekZLZNmc6dLixvVxCdlHBfqpyz0N3bzLdrJ55OLm7QrZmgZuhLGgHLDtJwU6RZoFCA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/2.3.1/list.min.js" integrity="sha512-93wYgwrIFL+b+P3RvYxi/WUFRXXUDSLCT2JQk9zhVGXuS2mHl2axj6d+R6pP+gcU5isMHRj1u0oYE/mWyt/RjA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.4/socket.io.min.js" integrity="sha512-HTENHrkQ/P0NGDFd5nk6ibVtCkcM7jhr2c7GyvXp5O+4X6O5cQO9AhqFzM+MdeBivsX7Hoys2J7pp2wdgMpCvw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.1/socket.io.min.js" integrity="sha512-+NaO7d6gQ1YPxvc/qHIqZEchjGm207SszoNeMgppoqD/67fEqmc1edS8zrbxPD+4RQI3gDgT/83ihpFW61TG/Q==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/2.24.2/plotly.min.js" integrity="sha512-dAXqGCq94D0kgLSPnfvd/pZpCMoJQpGj2S2XQmFQ9Ay1+96kbjss02ISEh+TBNXMggGg/1qoMcOHcxg+Op/Jmw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pako/2.1.0/pako_inflate.min.js" integrity="sha512-mlnC6JeOvg9V4vBpWMxGKscsCdScB6yvGVCeFF2plnQMRmwH69s9F8SHPbC0oirqfePmRBhqx2s3Bx7WIvHfWg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>

27
app/templates/corpora/analysis.html.j2
View File

@@ -2,7 +2,7 @@
{% import "materialize/wtf.html.j2" as wtf %}
{% import 'corpora/_analysis/concordance.html.j2' as concordance_extension %}
{% import 'corpora/_analysis/reader.html.j2' as reader_extension %}
{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension%}
{% import 'corpora/_analysis/static_visualization.html.j2' as static_visualization_extension %}
{% set extensions = [concordance_extension, reader_extension, static_visualization_extension] %}
{% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %}
@@ -11,20 +11,17 @@
{% block page_content %}
<ul class="row tabs no-autoinit" id="corpus-analysis-app-extension-tabs">
<li class="tab col s3"><a class="active" href="#corpus-analysis-app-overview"><i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus analysis</a></li>
{% for extension in extensions %}
{% if extension.name != 'Static Visualization' %}
<li class="tab col s3"><a class="active" href="#corpus-analysis-app-home-container"><i class="nopaque-icons service-icons left" data-service="corpus-analysis"></i>Corpus analysis</a></li>
{% for extension in extensions if extension.name != 'Static Visualization' %}
<li class="tab col s3"><a href="#{{ extension.id_prefix }}-container">{{ extension.tab_content }}</a></li>
{% endif %}
{% endfor %}
</ul>
<div class="row" id="corpus-analysis-app-overview">
<div class="col s12">
<h1>{{ title }}</h1>
<div id="corpus-analysis-app-home-container">
<h1>{{ title }}</h1>
{% for extension in extensions %}
{% if extension.name != 'Static Visualization' %}
<div class="row" id="corpus-analysis-app-extension-cards">
{% for extension in extensions if extension.name != 'Static Visualization' %}
<div class="col s3">
<div class="card extension-selector hoverable" data-target="{{ extension.id_prefix }}-container">
<div class="card-content">
@@ -33,17 +30,14 @@
</div>
</div>
</div>
{% endif %}
{% endfor %}
{{ static_visualization_extension.container_content }}
</div>
{{ static_visualization_extension.container_content }}
</div>
{% for extension in extensions %}
{% if extension.name != 'Static Visualization'%}
{% for extension in extensions if extension.name != 'Static Visualization' %}
<div id="{{ extension.id_prefix }}-container">
{{ extension.container_content }}
</div>
@@ -62,6 +56,7 @@
<div class="progress">
<div class="indeterminate"></div>
</div>
<p class="status-text"></p>
<p class="errors error-color-text hide"></p>
</div>
</div>

2
requirements.txt
View File

@@ -1,5 +1,5 @@
apifairy
cqi
cqi>=0.1.4
dnspython==2.2.1
docker
eventlet