sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-15 01:05:42 +00:00
Code Issues Projects Releases Wiki Activity

Add first setup for compression

Browse Source
This commit is contained in:
Stephan Porada 2019-11-19 15:21:42 +01:00
parent 1f74ad8a8e
commit e6b27b0970
4 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/corpora/CQiWrapper/CQiWrapper.py
View File

@ -230,9 +230,11 @@ class CQiWrapper(CQiClient):
for struct_attr_key in self.attr_strings[key].keys():
struct_entry = self.cl_cpos2struc(self.attr_strings[key][struct_attr_key],
all_cpos)
logger.warning("{}: {}".format(self.attr_strings[key][struct_attr_key], struct_entry))
match_strs = self.cl_struc2str(self.attr_strings[key][struct_attr_key], struct_entry)
logger.warning('{}:{}'.format(self.attr_strings[key][struct_attr_key], match_strs))
has_value = self.corpus_structural_attribute_has_values(self.attr_strings[key][struct_attr_key])
if has_value:
match_strs = self.cl_struc2str(self.attr_strings[key][struct_attr_key], struct_entry)
else:
match_strs = [None for i in struct_entry]
cpos_infos[struct_attr_key] = zip(struct_entry, match_strs)
tmp_list = []
attr_key_list = []

17
app/corpora/events.py
View File

@ -4,7 +4,9 @@ from app.models import Corpus
from flask import current_app, request
from flask_login import current_user, login_required
from .CQiWrapper.CQiWrapper import CQiWrapper
import sys
import gzip
import zlib
'''
' A dictionary containing lists of, with corpus ids associated, Socket.IO
@ -55,12 +57,21 @@ def recv_query(message):
room=request.sid)
return
""" Prepare and execute a query """
logger.warning('Payload: {}'.format(message))
corpus_name = 'CORPUS'
query = message['query']
analysis_client.select_corpus(corpus_name)
analysis_client.query_subcorpus(query)
results = analysis_client.show_query_results(result_len=2)
socketio.emit('query', results, room=request.sid)
results = analysis_client.show_query_results(result_len=int(message['hits_per_page']),
context_len=int(message['context']))
size_internal_dict = sys.getsizeof(results) / 1000000
size_dict_to_str = sys.getsizeof(str(results)) / 1000000
compressed_str = gzip.compress(str(results).encode())
size_dict_to_str_compressed = sys.getsizeof(compressed_str) / 1000000
logger.warning('Size of dict for {} hits per page: {} MB'.format(message['hits_per_page'], size_internal_dict))
logger.warning('Size of dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str))
logger.warning('Size of compressed dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str_compressed))
socketio.emit('query', compressed_str, room=request.sid)
def observe_corpus_analysis_connection(app, corpus_id, session_id):

7
app/corpora/forms.py
View File

@ -42,7 +42,12 @@ class QueryForm(FlaskForm):
('20', '20'),
('30', '30'),
('40', '40'),
('50', '50')],
('50', '50'),
('60', '60'),
('70', '70'),
('80', '80'),
('90', '90'),
('100', '100')],
validators=[DataRequired()])
context = SelectField('Context',
choices=[('', 'Words of context around hit'),

2
app/templates/corpora/analyse_corpus.html.j2
View File

@ -118,6 +118,8 @@
socket.on('query', function(results) {
queryResultsElement.innerHTML = '';
console.log(results)
var decoder = new TextDecoder().decode(results)
console.log(results)
for (let key in results) {
console.log(results[key]);
queryResultsElement.innerHTML += '<p>' + results[key]['match_cpos_list'] + '</p>';