nopaque/app/corpora/events.py

from app import db, logger, socketio
from app.events import connected_sessions
from app.models import Corpus
from flask import current_app, request
from flask_login import current_user, login_required
from .CQiWrapper.CQiWrapper import CQiWrapper
import sys
import gzip
import zlib
import json

'''
' A dictionary containing lists of, with corpus ids associated, Socket.IO
' session ids (sid). {<corpus_id>: [<sid>, ...], ...}
'''
corpus_analysis_sessions = {}
'''
' A dictionary containing Socket.IO session id - CQi client pairs.
' {<sid>: CQiClient, ...}
'''
corpus_analysis_clients = {}


@socketio.on('request_corpus_analysis')
@login_required
def request_corpus_analysis(corpus_id):
    corpus = Corpus.query.get(corpus_id)
    if corpus is None:
        socketio.emit('init_corpus_analysis', '[404]: Not Found',
                      room=request.sid)
    elif not (corpus.creator == current_user
              or current_user.is_administrator()):
        socketio.emit('init_corpus_analysis', '[403]: Forbidden',
                      room=request.sid)
    else:
        socketio.start_background_task(corpus_analysis_session_handler,
                                       current_app._get_current_object(),
                                       corpus_id, request.sid)


@socketio.on('corpus_analysis')
@login_required
def corpus_analysis(message):
    client = corpus_analysis_clients.get(request.sid)
    if client is None:
        socketio.emit('query', '[424]: Failed Dependency',
                      room=request.sid)
        return
    """ Prepare and execute a query """
    corpus = 'CORPUS'
    query = (message['query'])
    query_subcorpus = 'Results'
    client.cqp_query(corpus, query_subcorpus, query)

    data = {'matches': [], 'cpos_lookup': {}, 'text_loopup': {}}

    """ Evaluate query results """
    match_corpus = '{}:{}'.format(corpus, query_subcorpus)
    match_num = min(int(message['hits_per_page']), client.cqp_subcorpus_size(match_corpus))
    match_boundaries = zip(client.cqp_dump_subcorpus(match_corpus,
                                                     0x10,
                                                     0, match_num - 1),
                           client.cqp_dump_subcorpus(match_corpus,
                                                     0x11,
                                                     0, match_num - 1))
    context = 15
    corpus_len = 10000
    for match_start, match_end in match_boundaries:
        data['matches'].append({'lc': list(range(max(0, match_start - int(message['context'])), match_start)),
                                'hit': list(range(match_start, match_end + 1)),
                                'rc': list(range(match_end + 1, min(corpus_len, match_end + 1 + int(message['context']))))})
    cpos_list = []
    for match in data['matches']:
        cpos_list += match['lc'] + match['hit'] + match['rc']
    cpos_list = list(set(cpos_list))
    pos_list = client.cl_cpos2str('{}.pos'.format(corpus), cpos_list)
    simple_pos_list = client.cl_cpos2str('{}.simple_pos'.format(corpus), cpos_list)
    text_id_list = client.cl_cpos2struc('{}.text_title'.format(corpus), cpos_list)
    word_list = client.cl_cpos2str('{}.word'.format(corpus), cpos_list)
    for cpos, pos, simple_pos, text_id, word in zip(cpos_list, pos_list, simple_pos_list, text_id_list, word_list):
        data['cpos_lookup'][cpos] = {'pos': pos, 'simple_pos': simple_pos, 'text_id': text_id, 'word': word}
    text_author_list = client.cl_struc2str('{}.text_author'.format(corpus), text_id_list)
    text_publishing_year_list = client.cl_struc2str('{}.text_publishing_year'.format(corpus), text_id_list)
    text_title_list = client.cl_struc2str('{}.text_title'.format(corpus), text_id_list)
    for text_id, text_author, text_publishing_year, text_title in zip(text_id_list, text_author_list, text_publishing_year_list, text_title_list):
        data['text_loopup'][text_id] = {'author': text_author, 'publishing_year': text_publishing_year, 'title': text_title}
    socketio.emit('corpus_analysis', data, room=request.sid)


def corpus_analysis_session_handler(app, corpus_id, session_id):
    with app.app_context():
        ''' Setup analysis session '''
        corpus = Corpus.query.get(corpus_id)
        while corpus.status != 'analysing':
            db.session.refresh(corpus)
            socketio.sleep(3)
        client = CQiWrapper(host='analyse_corpus_{}'.format(corpus_id))
        client.connect()
        corpus_analysis_clients[session_id] = client
        if corpus_id not in corpus_analysis_sessions:
            corpus_analysis_sessions[corpus_id] = [session_id]
        else:
            corpus_analysis_sessions[corpus_id].append(session_id)
        socketio.emit('request_corpus_analysis', '[201]: Created',
                      room=session_id)
        ''' Observe analysis session '''
        while session_id in connected_sessions:
            socketio.sleep(3)
        ''' Teardown analysis session '''
        client.disconnect()
        corpus_analysis_clients.pop(session_id, None)
        corpus_analysis_sessions[corpus_id].remove(session_id)
        if not corpus_analysis_sessions[corpus_id]:
            corpus_analysis_sessions.pop(corpus_id, None)
            corpus.status = 'stop analysis'
            db.session.commit()