Add new way to get results with full contest to provide moreuser feedback

web/app/corpora/events.py

@@ -9,6 +9,8 @@ import cqi
 import math
 from datetime import datetime
 
+import time
+from app import logger
 
 '''
 ' A dictionary containing lists of, with corpus ids associated, Socket.IO
@@ -141,9 +143,9 @@ def corpus_analysis_query(query):
     client.status = 'ready'
 
 
-@socketio.on('corpus_analysis_inspect_match')
+@socketio.on('corpus_analysis_get_match_with_full_context')
 @socketio_login_required
-def corpus_analysis_inspect_match(payload):
+def corpus_analysis_get_match_with_full_context(payload):
     type = payload['type']
     data_indexes = payload['data_indexes']
     first_cpos = payload['first_cpos']
@@ -155,7 +157,7 @@ def corpus_analysis_inspect_match(payload):
                     'msg': 'Failed Dependency',
                     'type': type,
                     'data_indexes': data_indexes}
-        socketio.emit('corpus_analysis_inspect_match', response,
+        socketio.emit('corpus_analysis_get_match_with_full_context', response,
                       room=request.sid)
         return
     if client.status == 'running':
@@ -170,12 +172,29 @@ def corpus_analysis_inspect_match(payload):
         payload['matches'] = []
         payload['cpos_lookup'] = {}
         payload['text_lookup'] = {}
+        payload['cpos_ranges'] = True
+        payload['progress'] = 0
+        i = 0
+        # Send data one match at a time.
         for index, f_cpos, l_cpos in zip(data_indexes, first_cpos, last_cpos):
+            logger.warning(index)
+            i += 1
             tmp_match = s.export(f_cpos, l_cpos, context=10)
             payload['matches'].append(tmp_match['matches'][0])
             payload['cpos_lookup'].update(tmp_match['cpos_lookup'])
             payload['text_lookup'].update(tmp_match['text_lookup'])
-        payload['cpos_ranges'] = True
+            payload['progress'] = i/len(data_indexes)*100
+            response = {'code': 200,
+                        'desc': None,
+                        'msg': 'OK',
+                        'payload': payload,
+                        'type': type,
+                        'data_indexes': data_indexes}
+            socketio.emit('corpus_analysis_get_match_with_full_context',
+                          response, room=request.sid)
+            payload['matches'] = []
+            payload['cpos_lookup'] = {}
+            payload['text_lookup'] = {}
     except cqi.errors.CQiException as e:
         payload = {'code': e.code, 'desc': e.description, 'msg': e.name}
         response = {'code': 500,
@@ -184,14 +203,7 @@ def corpus_analysis_inspect_match(payload):
                     'payload': payload,
                     'type': type,
                     'data_indexes': data_indexes}
-    else:
+        socketio.emit('corpus_analysis_get_match_with_full_context', response, room=request.sid)
-        response = {'code': 200,
-                    'desc': None,
-                    'msg': 'OK',
-                    'payload': payload,
-                    'type': type,
-                    'data_indexes': data_indexes}
-    socketio.emit('corpus_analysis_inspect_match', response, room=request.sid)
     client.status = 'ready'
 
 

web/app/static/js/modules/corpus_analysis/client/Client.js

@@ -126,7 +126,7 @@ class Client {
       tmp_first_cpos.push(results.data.matches[dataIndex].c[0]);
       tmp_last_cpos.push(results.data.matches[dataIndex].c[1]);
     }
-    nopaque.socket.emit("corpus_analysis_inspect_match",
+    nopaque.socket.emit("corpus_analysis_get_match_with_full_context",
                         {type: resultsType,
                          data_indexes: dataIndexes,
                          first_cpos: tmp_first_cpos,
@@ -135,6 +135,7 @@ class Client {
 
   //
   getResultsDataWithoutContext(resultsType, dataIndexes, results, resultsList) {
+    this.notifyView('results-data-recieving', {fullContext: false});
     let objectKey = '';
     if (resultsType === 'full-results') {
       console.info('Saving full-results data without full context.');
@@ -159,12 +160,10 @@ class Client {
     }
     // Get cpos_lookups from cposes.
     let cpos_lookup = {};
-    let single_lookup = {};
     let textIds = new Set;
     for (let single_cpos of cpos) {
-      single_lookup[single_cpos] = results.data.cpos_lookup[single_cpos];
+      textIds.add(results.data.cpos_lookup[single_cpos].text);
-      textIds.add(single_lookup[single_cpos].text);
+      Object.assign(cpos_lookup, { [single_cpos]: results.data.cpos_lookup[single_cpos]});
-      Object.assign(cpos_lookup, single_lookup);
     }
     let text = {};
     let text_lookup = {};
@@ -189,7 +188,8 @@ class Client {
     console.info('Results data without context has been saved.', results);
     this.isBusy = false;
     this.notifyView('results-data-recieved', {type: resultsType,
-                                                results: results});
+                                              results: results,
+                                              fullContext: false});
   }
 
 }

web/app/static/js/modules/corpus_analysis/client/callbacks.js

@@ -73,11 +73,11 @@ function saveQueryData() {
 function getResultsData() {
   let [resultsType, dataIndexes, resultsList, client, results, rest] = arguments;
   client.isBusy = true;
-  client.notifyView('results-data-recieving');
   if (resultsList.exportFullInspectContext.checked || resultsType === 'inspect-results') {
-    console.log('Get with full context');
+    console.info('Get results with full context');
     client.getResultsData(resultsType, dataIndexes, results);
   } else {
+    console.info('Get results without full context');
     client.getResultsDataWithoutContext(resultsType, dataIndexes, results,
                                         resultsList);
   }
@@ -96,21 +96,26 @@ function saveResultsData() {
     objectKey = 'inspectResultsData'
     console.info('Saving inspect-results data');
   }
-  // Save incoming data
+  // Save incoming data. Data is incoming one match at a time.
-  results[objectKey].init();
   results[objectKey].matches.push(...payload.matches);
   results[objectKey].addData(payload.cpos_lookup, "cpos_lookup");
   results[objectKey].addData(payload.text_lookup, "text_lookup");
   results[objectKey].addData(results.metaData);
   results[objectKey].query = results.data.query;
   results[objectKey].corpus_type = type;
-  results[objectKey].match_count = [...payload.matches].length;
+  results[objectKey].match_count += 1;
   results[objectKey].cpos_ranges = payload.cpos_ranges;
   results[objectKey].fullContext = true;
   console.info('Results data has been saved.', results);
+  // Notify view to update progress bar
+  client.notifyView('results-data-recieving', {type: type,
+    progress: payload.progress})
   client.isBusy = false;
-  client.notifyView('results-data-recieved', {type: type,
+  if (payload.progress === 100) {
-                                              results: results});
+    client.notifyView('results-data-recieved', {type: type,
+                                                results: results,
+                                                fullContext: true});
+  }
 }
 
 // export callbacks

web/app/static/js/modules/corpus_analysis/client/listeners.js

@@ -147,9 +147,9 @@ function recieveResultsData(type, client) {
      */
     if (response.code === 200) {
       console.group('Client recieving results data')
-      console.info('corpus_analysis_inspect_match: Client recieving results data',
+      console.info('corpus_analysis_get_match_with_full_context: Client recieving results data',
       'via socket.on');
-      console.info(`corpus_analysis_inspect_match: ${response.code} - ${response.msg}`);
+      console.info(`corpus_analysis_get_match_with_full_context: ${response.code} - ${response.msg}`);
       console.info(response);
       // executing the registered callbacks
       client.eventListeners[type].executeCallbacks([response.payload,
@@ -158,9 +158,9 @@ function recieveResultsData(type, client) {
     } else {
       let errorText = `Error ${response.payload.code} - ${response.payload.msg}`;
       console.group('Failed to recieve results data.');
-      console.error('corpus_analysis_inspect_match: Client failed to recieve',
+      console.error('corpus_analysis_get_match_with_full_context: Client failed to recieve',
                     'results data via socket.on');
-      console.error(`corpus_analysis_inspect_match: ${errorText}`);
+      console.error(`corpus_analysis_get_match_with_full_context: ${errorText}`);
       client.notifyView('client-failed', { msg: errorText }, 'error');
       console.groupEnd();
     }

web/app/static/js/modules/corpus_analysis/view/ResultsView.js

@@ -219,7 +219,7 @@ class ResultsList extends List {
   // Used in addToSubResults and inspect to toggle the design of the check
   // buttons according to its checked unchecked status.
   helperActivateAddBtn(btn) {
-    btn.classList.remove("grey");
+    btn.classList.remove("corpus-analysis-color.lighten");
     btn.classList.add("green");
     btn.textContent = "check";
   }
@@ -228,7 +228,7 @@ class ResultsList extends List {
   // buttons according to its checked unchecked status.
   helperDeactivateAddBtn(btn) {
     btn.classList.remove("green");
-    btn.classList.add("grey");
+    btn.classList.add("corpus-analysis-color.lighten");
     btn.textContent = "add";
   }
 
@@ -344,7 +344,7 @@ class ResultsList extends List {
     this.contextModal.open();
     // add a button to add this match to sub results with onclick event
     let classes = `btn-floating btn waves-effect` +
-                  ` waves-light grey right`
+                  ` waves-light corpus-analysis-color.lighten right`
     let addToSubResultsIdsBtn = document.createElement("a");
     addToSubResultsIdsBtn.setAttribute("class", classes + ` add`);
     addToSubResultsIdsBtn.innerHTML = '<i class="material-icons">add</i>';
@@ -722,7 +722,7 @@ class ResultsList extends List {
     // # some btn css rules and classes
     let css = `margin-right: 5px; margin-bottom: 5px;`
     let classes = `btn-floating btn waves-effect` +
-                  ` waves-light grey`
+                  ` waves-light corpus-analysis-color.lighten`
     // # add button to trigger more context to every match td
     inspectBtn = document.createElement("a");
     inspectBtn.setAttribute("style", css);

web/app/static/js/modules/corpus_analysis/view/callbacks.js

@@ -151,11 +151,26 @@ function queryDataRecievedCallback(resultsList, detail) {
 }
 
 function resultsDataRecievingCallback(resultsList, detail) {
+  resultsList.getHTMLElements([
+    '#full-results-progress-bar',
+    '#sub-results-progress-bar',
+  ]);
+  // Disable the full context switch when results are being recieved
+  resultsList.exportFullInspectContext.setAttribute('disabled', '');
+  if (detail.type === 'full-results' && detail.progress) {
+    resultsList.fullResultsProgressBar.firstElementChild.style.width = `${detail.progress}%`;
+    resultsList.fullResultsProgressBar.classList.toggle('hide', false);
+  } else if (detail.type === 'sub-results' && detail.progress) {
+    resultsList.subResultsProgressBar.firstElementChild.style.width = `${detail.progress}%`;
+    resultsList.subResultsProgressBar.classList.toggle('hide', false);
+  }
 }
 
 function resultsDataRecievedCallback(resultsList, detail) {
   // create strings for create buttons depending on type
   const handleType = (keyPrefix, text) => {
+    // Enable the full context switch when results have been recieved
+    resultsList.exportFullInspectContext.removeAttribute('disabled', '');
     // hides the create element after results have been recieved and reset it
     resultsList[`${keyPrefix}Create`].classList.toggle('hide');
     resultsList[`${keyPrefix}Create`].textContent = `Create ${text}`;
@@ -171,8 +186,16 @@ function resultsDataRecievedCallback(resultsList, detail) {
   }
   if (detail.type === 'full-results') {
     handleType('fullResults', 'Results');
+    if (detail.fullContext) {
+      resultsList.fullResultsProgressBar.firstElementChild.style.width = `0%`;
+      resultsList.fullResultsProgressBar.classList.toggle('hide', true);
+    }
   } else if (detail.type ==='sub-results') {
     handleType('subResults', 'Sub-Results');
+    if (detail.fullContext) {
+      resultsList.subResultsProgressBar.firstElementChild.style.width = `0%`;
+      resultsList.subResultsProgressBar.classList.toggle('hide', true);
+    }
   } else if (detail.type ==='inspect-results') {
     if (resultsList.addToSubResultsIdsToShow.size === 0) {
       /**

web/app/static/js/nopaque.callbacks.js

@@ -5,7 +5,7 @@ function recvMetaData(payload) {
   console.log("Metada recieved:", results.metaData);
 }
 
-// This callback is called in socket.on "corpus_analysis_inspect_match" but
+// This callback is called in socket.on "corpus_analysis_get_match_with_full_context" but
 // only if the response.type is "sub-results".
 // Saves the incoming inspect match results into results.subResultsData.
 function saveSubResultsChoices(response) {

web/app/templates/corpora/analyse_corpus.html.j2

@@ -3,9 +3,9 @@
 {% set headline = ' ' %}
 
 {% set full_width = True %}
-{% set imported = False %}
 
 {% block page_content %}
+{{ Macros.insert_color_scheme(corpus_analysis_color_darken) }}
 <div class="col s12">
   <div class="card">
     <div class="card-content" style="padding-top: 5px;
@@ -168,9 +168,9 @@ document.addEventListener("DOMContentLoaded", () => {
                                                  saveQueryData,
                                                  [client, results]);
   listenForQueryData.setCallbacks([queryDataCallback]);
-  const listenForResults = new ClientEventListener('corpus_analysis_inspect_match',
+  const listenForResults = new ClientEventListener('corpus_analysis_get_match_with_full_context',
                                                    recieveResultsData);
-  const resultsDataCallback = new ListenerCallback('corpus_analysis_inspect_match',
+  const resultsDataCallback = new ListenerCallback('corpus_analysis_get_match_with_full_context',
                                                    saveResultsData,
                                                    [client, results]);
   listenForResults.setCallbacks([resultsDataCallback]);
@@ -352,6 +352,8 @@ document.addEventListener("DOMContentLoaded", () => {
     resultsList.fullResultsCreate.insertAdjacentHTML('afterbegin',
                                                       loadingSpinnerHTML);
     let dataIndexes = [...Array(results.data.match_count).keys()];
+    // Empty fullResultsData so that no previous data is used.
+    results.fullResultsData.init();
     resultsList.notifyClient('get-results', { resultsType: 'full-results',
                                               dataIndexes: dataIndexes,
                                               resultsList: resultsList,
@@ -367,6 +369,8 @@ document.addEventListener("DOMContentLoaded", () => {
     resultsList.subResultsCreate.innerText = 'Creating...';
     resultsList.subResultsCreate.insertAdjacentHTML('afterbegin',
     loadingSpinnerHTML);
+    // Empty subResultsData so that no previous data is used.
+    results.subResultsData.init();
     resultsList.notifyClient('get-results', { resultsType: 'sub-results',
                                               dataIndexes: dataIndexes,
                                               resultsList: resultsList,

web/app/templates/interactions/export.html.j2

@@ -45,6 +45,11 @@ the selected sub results.-->
         <i class="material-icons left">file_download</i>
       </button>
     </div>
+    <div class="col s12">
+      <div class="progress hide" id="full-results-progress-bar">
+        <div class="determinate"></div>
+      </div>
+    </div>
     <div class="col s12">
       <button class="waves-effect
               waves-light
@@ -65,5 +70,10 @@ the selected sub results.-->
         <i class="material-icons left">file_download</i>
       </button>
     </div>
+    <div class="col s12">
+      <div class="progress hide" id="sub-results-progress-bar">
+        <div class="determinate"></div>
+      </div>
+    </div>
   </div>
 </div>

web/app/templates/interactions/scroll_to_top.html.j2

@@ -1,5 +1,5 @@
 <div id="menu-scroll-to-top-div" class="fixed-action-btn direction-top active hide" style="bottom: 45px; right: 24px;">
-  <a id="menu-scroll-to-top" class="btn btn-floating btn-large cyan">
+  <a id="menu-scroll-to-top" class="btn btn-floating btn-large corpus-analysis-color.lighten">
     <i class="material-icons">arrow_upward</i>
   </a>
 </div>

web/app/templates/query_results/inspect.html.j2

@@ -3,10 +3,9 @@
 {% set headline = ' ' %}
 
 {% set full_width = True %}
-{% set imported = True %}
 
 {% block page_content %}
-
+{{ Macros.insert_color_scheme(corpus_analysis_color_darken) }}
 <div class="col s12">
   <div class="card">
     <div class="card-content" style="padding-top: 5px;