sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-06-12 17:10:41 +00:00
Code Issues Projects Releases Wiki Activity

Add compression of result dictionary and size infos for other compression or no compression methods

Browse Source
This commit is contained in:
Stephan Porada
2019-11-21 16:05:00 +01:00
parent e6b27b0970
commit fd2547de0c
3 changed files with 6828 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/corpora/events.py
View File

@ -7,6 +7,7 @@ from .CQiWrapper.CQiWrapper import CQiWrapper
import sys
import gzip
import zlib
import json
'''
' A dictionary containing lists of, with corpus ids associated, Socket.IO
@ -64,14 +65,18 @@ def recv_query(message):
analysis_client.query_subcorpus(query)
results = analysis_client.show_query_results(result_len=int(message['hits_per_page']),
context_len=int(message['context']))
# logger.warning('RESULTS: {}'.format(results))
size_internal_dict = sys.getsizeof(results) / 1000000
size_dict_to_str = sys.getsizeof(str(results)) / 1000000
compressed_str = gzip.compress(str(results).encode())
size_dict_to_str_compressed = sys.getsizeof(compressed_str) / 1000000
logger.warning('Size of dict for {} hits per page: {} MB'.format(message['hits_per_page'], size_internal_dict))
logger.warning('Size of dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str))
logger.warning('Size of compressed dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str_compressed))
socketio.emit('query', compressed_str, room=request.sid)
zlib_compressed = zlib.compress(json.dumps(results).encode('utf-8'))
size_zlib_compressed = sys.getsizeof(zlib_compressed) / 1000000
logger.warning('Internal size of dict for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_internal_dict))
logger.warning('Size of dict as raw string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_dict_to_str))
logger.warning('Size of gzip compressed dict to string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_dict_to_str_compressed))
logger.warning('Size of zlib compressed and utf-8 encoded string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_zlib_compressed))
socketio.emit('query', zlib_compressed, room=request.sid)
def observe_corpus_analysis_connection(app, corpus_id, session_id):