from app import db, logger, socketio from app.events import connected_sessions from app.models import Corpus from flask import current_app, request from flask_login import current_user, login_required from .CQiWrapper.CQiWrapper import CQiWrapper import sys import gzip import zlib import json ''' ' A dictionary containing lists of, with corpus ids associated, Socket.IO ' session ids (sid). {: [, ...], ...} ''' corpus_analysis_sessions = {} ''' ' A dictionary containing Socket.IO session id - CQi client pairs. ' {: CQiClient, ...} ''' corpus_analysis_clients = {} @socketio.on('request_corpus_analysis') @login_required def request_corpus_analysis(corpus_id): corpus = Corpus.query.get(corpus_id) if corpus is None: socketio.emit('init_corpus_analysis', '[404]: Not Found', room=request.sid) elif not (corpus.creator == current_user or current_user.is_administrator()): socketio.emit('init_corpus_analysis', '[403]: Forbidden', room=request.sid) else: socketio.start_background_task(corpus_analysis_session_handler, current_app._get_current_object(), corpus_id, request.sid) @socketio.on('corpus_analysis') @login_required def corpus_analysis(message): client = corpus_analysis_clients.get(request.sid) if client is None: socketio.emit('query', '[424]: Failed Dependency', room=request.sid) return """ Prepare and execute a query """ corpus = 'CORPUS' query = (message['query']) query_subcorpus = 'Results' client.cqp_query(corpus, query_subcorpus, query) data = {'matches': [], 'cpos_lookup': {}, 'text_loopup': {}} """ Evaluate query results """ match_corpus = '{}:{}'.format(corpus, query_subcorpus) match_num = min(int(message['hits_per_page']), client.cqp_subcorpus_size(match_corpus)) match_boundaries = zip(client.cqp_dump_subcorpus(match_corpus, 0x10, 0, match_num - 1), client.cqp_dump_subcorpus(match_corpus, 0x11, 0, match_num - 1)) context = 15 corpus_len = 10000 for match_start, match_end in match_boundaries: data['matches'].append({'lc': list(range(max(0, match_start - int(message['context'])), match_start)), 'hit': list(range(match_start, match_end + 1)), 'rc': list(range(match_end + 1, min(corpus_len, match_end + 1 + int(message['context']))))}) cpos_list = [] for match in data['matches']: cpos_list += match['lc'] + match['hit'] + match['rc'] cpos_list = list(set(cpos_list)) pos_list = client.cl_cpos2str('{}.pos'.format(corpus), cpos_list) simple_pos_list = client.cl_cpos2str('{}.simple_pos'.format(corpus), cpos_list) s_id_list = client.cl_cpos2struc('{}.s'.format(corpus), cpos_list) text_id_list = client.cl_cpos2struc('{}.text'.format(corpus), cpos_list) word_list = client.cl_cpos2str('{}.word'.format(corpus), cpos_list) for cpos, pos, simple_pos, s_id, text_id, word in zip(cpos_list, pos_list, simple_pos_list, s_id_list, text_id_list, word_list): data['cpos_lookup'][cpos] = {'pos': pos, 'simple_pos': simple_pos, 's_id': s_id, 'text_id': text_id, 'word': word} text_author_list = client.cl_struc2str('{}.text_author'.format(corpus), text_id_list) text_publishing_year_list = client.cl_struc2str('{}.text_publishing_year'.format(corpus), text_id_list) text_title_list = client.cl_struc2str('{}.text_title'.format(corpus), text_id_list) for text_id, text_author, text_publishing_year, text_title in zip(text_id_list, text_author_list, text_publishing_year_list, text_title_list): data['text_loopup'][text_id] = {'author': text_author, 'publishing_year': text_publishing_year, 'title': text_title} socketio.emit('corpus_analysis', data, room=request.sid) def corpus_analysis_session_handler(app, corpus_id, session_id): with app.app_context(): ''' Setup analysis session ''' corpus = Corpus.query.get(corpus_id) while corpus.status != 'analysing': db.session.refresh(corpus) socketio.sleep(3) client = CQiWrapper(host='analyse_corpus_{}'.format(corpus_id)) client.connect() corpus_analysis_clients[session_id] = client if corpus_id not in corpus_analysis_sessions: corpus_analysis_sessions[corpus_id] = [session_id] else: corpus_analysis_sessions[corpus_id].append(session_id) socketio.emit('request_corpus_analysis', '[201]: Created', room=session_id) ''' Observe analysis session ''' while session_id in connected_sessions: socketio.sleep(3) ''' Teardown analysis session ''' client.disconnect() corpus_analysis_clients.pop(session_id, None) corpus_analysis_sessions[corpus_id].remove(session_id) if not corpus_analysis_sessions[corpus_id]: corpus_analysis_sessions.pop(corpus_id, None) corpus.status = 'stop analysis' db.session.commit()