Rework corpus analysis list building part 1

app/corpora/events.py

@@ -45,7 +45,7 @@ def corpus_analysis(message):
     # Prepare and execute a query
     corpus_name = 'CORPUS'
     query = str(message['query'])
-    result_len = 100
+    result_len = 200
     context_len = int(message['context'])
     result_offset = 0
     client.select_corpus(corpus_name)
@@ -60,19 +60,20 @@ def corpus_analysis(message):
                                             context_len=context_len,
                                             result_offset=result_offset)
         result_offset += result_len  # initial offfset is plus result len because client.show_query_results has been already executed once
+        socketio.emit('corpus_analysis', results, room=request.sid)
         while result_offset < client.total_nr_matches:
             logger.warning('====== While loop start for {} ======'.format(query))
             logger.warning('result_offset: {}'.format(result_offset))
-            results_append = client.show_query_results(result_len=result_len,
+            results = client.show_query_results(result_len=result_len,
-                                                       context_len=context_len,
+                                                context_len=context_len,
-                                                       result_offset=result_offset)
+                                                result_offset=result_offset)
-            results['matches'].extend(results_append['matches'])
+            # results['matches'].extend(results_append['matches'])
-            results['cpos_lookup'].update(results_append['cpos_lookup'])
+            # results['cpos_lookup'].update(results_append['cpos_lookup'])
-            results['text_lookup'].update(results_append['text_lookup'])
+            # results['text_lookup'].update(results_append['text_lookup'])
             result_offset += result_len
             result_offset = min(result_offset, client.total_nr_matches)
             logger.warning('result_offset end of while loop: {}'.format(result_offset))
-        socketio.emit('corpus_analysis', results, room=request.sid)
+            socketio.emit('corpus_analysis', results, room=request.sid)
 
 
 @socketio.on('inspect_match')

app/static/js/ResultList.js

@@ -0,0 +1,3 @@
+class JobList extends List {
+  
+}

app/templates/corpora/analyse_corpus.bak.html.j2

@@ -0,0 +1,543 @@
+{% extends "full_width.html.j2" %}
+
+{% block page_content %}
+<style>
+  .token.chip.bold {
+    background-color: #48a54b;
+  }
+  .token.bold {
+    font-weight: bolder;
+  }
+  .token.chip:hover {
+    background-color: #9e9e9e;
+    cursor: pointer;
+  }
+  .material-tooltip {
+      padding: 5px 20px 20px;
+      border-radius: 10px;
+      background: rgba(37, 36, 36);
+      overflow: visible;
+  }
+
+  .material-tooltip .backdrop {
+      display: none !important;
+  }
+  .material-tooltip:after {
+    position: absolute;
+    content: "";
+    top: 100%;
+    left: 50%;
+    margin-left: -8px;
+    width: 0;
+    height: 0;
+    border-style: solid;
+    border-width: 15px 13px 0 13px;
+    border-color: rgba(37, 36, 36) transparent transparent transparent;
+}
+</style>
+
+<!-- Query Form -->
+<form id="query-form">
+  <div class="col s12">
+    <ul class="collapsible expandable">
+      <li class="active">
+        <!-- <div class="collapsible-header">
+          <i class="material-icons">search</i>CQP Query
+        </div> -->
+        <!-- Div element above is part of valid materialize collapsible.
+        Commented out to prevent the user from collapsing it and also to save
+        space -->
+        <div class="collapsible-body" style="padding-top: 10px;
+                                             padding-right: 2rem;
+                                             padding-bottom: 0px;
+                                             padding-left: 2rem;">
+          <div class="row">
+            <div class="col s12 m11">
+              <div class="input-field">
+                <i class="material-icons prefix">search</i>
+                {{ query_form.query() }}
+                {{ query_form.query.label }}
+                <span class="helper-text" data-error="wrong" data-success="right">
+                  <a href="http://cwb.sourceforge.net/files/CQP_Tutorial/">
+                    <i class="material-icons" style="font-size: inherit;">help
+                    </i> CQP query language tutorial
+                  </a>
+                </span>
+                {% for error in query_form.query.errors %}
+                  <span class="helper-text red-text">{{ error }}</span>
+                {% endfor %}
+              </div>
+            </div>
+            <div class="col s12 m1">
+              <p class="hide-on-small-only" style="
+      margin: 0px;">&nbsp;</p>
+              <button class="waves-effect waves-light btn-small right"
+                      type="submit">Send
+                <i class="material-icons right">send</i>
+              </button>
+            </div>
+          </div>
+        </div>
+      </li>
+      <li class="hoverable">
+        <div class="collapsible-header">
+          <i class="material-icons">settings</i>Display Options
+        </div>
+        <div class="collapsible-body">
+          <div class="row">
+            <div class="col s6">
+              <div class="input-field">
+                <i class="material-icons prefix">format_list_numbered</i>
+                {{ query_form.hits_per_page() }}
+                {{ query_form.hits_per_page.label }}
+                {% for error in query_form.hits_per_page.errors %}
+                <span class="helper-text red-text">{{ error }}</span>
+                {% endfor %}
+              </div>
+            </div>
+            <div class="col s6">
+              <div class="input-field">
+                <i class="material-icons prefix">short_text</i>
+                {{ query_form.context() }}
+                {{ query_form.context.label }}
+                {% for error in query_form.context.errors %}
+                <span class="helper-text red-text">{{ error }}</span>
+                {% endfor %}
+              </div>
+            </div>
+          </div>
+          <div class="row">
+            <div class="switch">
+              <label>
+                Expert Mode
+                <input type="checkbox" id="expert-mode-switch">
+                <span class="lever"></span>
+              </label>
+            </div>
+          </div>
+        </div>
+      </li>
+    </ul>
+  </div>
+</form>
+
+<!-- Export query results modal -->
+<div id="export-query-results-modal"
+     class="modal modal-fixed-footer no-autoinit">
+  <div class="modal-content">
+    {{ query_download_form.hidden_tag() }}
+    <h4>Download current query Results</h4>
+    <p>The results of the current query can be downlaoded as several files like csv or json. Those files can be used in other software like excel. Also it is easy to publish your results as raw data like this!</p>
+    <table>
+      <tr>
+        <td>JSON</td>
+        <td>
+          <a class="btn waves-effect waves-light" id="download-results">Download
+            <i class="material-icons right">file_download</i>
+          </a>
+        </td>
+      </tr>
+      <tr>
+        <td>CSV</td>
+        <td>
+          <a class="btn waves-effect waves-light disabled">Download
+            <i class="material-icons right">file_download</i>
+          </a>
+        </td>
+      </tr>
+      <tr>
+        <td>EXCEL</td>
+        <td>
+          <a class="btn waves-effect waves-light disabled">Download
+            <i class="material-icons right">file_download</i>
+          </a>
+        </td>
+      </tr>
+      <tr>
+        <td>HTML</td>
+        <td>
+          <a class="btn waves-effect waves-light disabled">Download
+            <i class="material-icons right">file_download</i>
+          </a>
+        </td>
+      </tr>
+    </table>
+  </div>
+  <div class="modal-footer">
+    <a href="#!" class="modal-close waves-effect waves-green red btn">Close</a>
+  </div>
+</div>
+
+<!-- Loading animation card when query results are being loaded -->
+<div class="row">
+  <div class="col s12 hide" id="getting-query-results">
+    <div class="card">
+      <div class="card-content">
+        <span class="card-title">Fetching your results!</span>
+        <div>
+          <div class="progress">
+            <div class="indeterminate"></div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- table showing the query results -->
+<div class="row">
+  <div class="col s12" id="recieved-query-results">
+    <div class="card">
+      <div class="card-content" id="result-list">
+        <span class="card-title">Query Results</span>
+        <p id="query-results-metadata">
+          <button id="export-query-results" class="waves-effect waves-light btn-small right hide" type="submit">Export Results<i class="material-icons right">file_download</i></button>
+        </p>
+        <ul class="pagination paginationTop"></ul>
+        <table class="responsive-table highlight">
+          <thead>
+            <tr>
+              <th style="width: 5%">Title</th>
+              <th style="width: 25%">Left context</th>
+              <th style="width: 45%">Match</th>
+              <th style="width: 25%">Right Context</th>
+            </tr>
+          </thead>
+          <tbody class="list" id="query-results"></tbody>
+        </table>
+        <ul class="pagination paginationBottom"></ul>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Loding animation modal that waits for the CQP server container to be ready
+-->
+<div id="loading-modal" class="modal no-autoinit">
+  <div class="modal-content">
+    <h4>Waiting for analysis software</h4>
+    <div class="progress">
+      <div class="indeterminate"></div>
+    </div>
+  </div>
+</div>
+
+<!-- Context modal used for detailed information about one match -->
+<div id="context-modal" class="modal modal-fixed-footer">
+  <div class="modal-content">
+    <h4>Match context and information</h4>
+    <div id="context-modal-loading">
+      <div class="progress">
+        <div class="indeterminate"></div>
+      </div>
+    </div>
+    <div id="context-modal-ready" class="hide">
+      <div id="context-results"></div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <a href="#!" class="left waves-effect waves-green btn">Export</a>
+    <a href="#!" class="modal-close waves-effect waves-green red btn">Close</a>
+  </div>
+</div>
+
+<script>
+  var contextModal;
+  var loadingModal;
+  var exportModal;
+  document.addEventListener("DOMContentLoaded", function() {
+    contextModal = M.Modal.init(document.getElementById("context-modal"),
+                                {"onCloseEnd": function() {
+                                   document.getElementById("context-modal-loading").classList.remove("hide");
+                                   document.getElementById("context-modal-ready").classList.add("hide");}});
+    loadingModal = M.Modal.init(document.getElementById("loading-modal"),
+                                {"dismissible": false});
+    exportModal = M.Modal.init(document.getElementById("export-query-results-modal"),
+                                {"dismissible": true});
+    M.Collapsible.init(elem, {accordion: false});
+    loadingModal.open();
+    nopaque.socket.emit("request_corpus_analysis", {{ corpus_id }});
+  });
+
+  nopaque.socket.on("request_corpus_analysis", function(msg) {
+    if (msg === "[201]: Created") {loadingModal.close();}
+  });
+
+  var expertModeSwitchElement = document.getElementById("expert-mode-switch");
+  var lookup = {"cpos": null, "s": null, "text": null};
+  var matches = null;
+  var tokenElements = null;
+
+  expertModeSwitchElement.addEventListener('change', function(event) {
+    if (event.target.checked) {
+      tokenElements.forEach(function(tokenElement) {
+        tokenElement.classList.add("chip");
+        token = lookup["cpos"][tokenElement.dataset.cpos];
+        addToolTipToTokenElement(tokenElement, token);
+      });
+    } else {
+      tokenElements.forEach(function(tokenElement) {
+        tokenElement.classList.remove("chip");
+        tokenElement.M_Tooltip.destroy()
+      });
+    }
+  })
+
+  var queryFormElement = document.getElementById("query-form");
+  var queryResultsElement = document.getElementById("query-results");
+  var queryResultsMetadataElement = document.getElementById("query-results-metadata");
+  var exportQueryResults = document.getElementById("export-query-results");
+  exportQueryResults.onclick = function() {
+    exportModal.open();
+  };
+  var contextResultsElement = document.getElementById("context-results");
+  var queryLoadingElement = document.getElementById("getting-query-results");
+  var queryResultsTableElement = document.getElementById("recieved-query-results");
+  var hitsPerPage;
+  queryFormElement.addEventListener("submit", function(event) {
+    event.preventDefault();
+    let formData = new FormData(queryFormElement);
+    let queryData = {"context": formData.get("context"),
+                     "hits_per_page": formData.get("hits_per_page"),
+                     "query": formData.get("query")};
+    hitsPerPage = formData.get("hits_per_page");
+    nopaque.socket.emit("corpus_analysis", queryData);
+    queryLoadingElement.classList.remove("hide");
+    queryResultsTableElement.classList.add("hide");
+    nopaque.toast("Query has been sent!")
+  });
+
+  nopaque.socket.on("corpus_analysis", function(message) {
+    console.log("### corpus_analysis ###");
+    console.log(message);
+    queryLoadingElement.classList.add("hide");
+    queryResultsTableElement.classList.remove("hide");
+    queryResultsElement.innerHTML = "";
+
+    // Reset related global values
+    lookup = {"cpos": null, "s": null, "text": null};
+    matches = null;
+    tokenElements = null;
+
+    if (message === null) {
+      queryResultsTableElement.classList.add("hide");
+      nopaque.toast("No results for this query!")
+      return;
+    } else if (message === "CQI_CQP_ERROR_GENERAL") {
+        queryResultsTableElement.classList.add("hide");
+        nopaque.toast("Invalid query entered!", "red");
+        return;
+      } else {
+        total_nr_matches = message["total_nr_matches"];
+        let count_corpus_files = Object.keys(message["text_lookup"]).length;
+        queryResultsMetadataElement.innerHTML = message["total_nr_matches"] + " matches in " + count_corpus_files + " corpus files.";
+        queryResultsMetadataElement.appendChild(exportQueryResults);
+        exportQueryResults.classList.remove("hide");
+
+        var matchElement;
+        var matchTextTitlesElement;
+        var matchLeftContextElement;
+        var matchHitElement;
+        var matchRightContextElement;
+        var textTitles;
+        var token;
+        var tokenElement;
+
+        // Set related global values
+        lookup["cpos"] = message["cpos_lookup"];
+        lookup["text"] = message["text_lookup"];
+        matches = message["matches"];
+        tokenElements = new Set();
+
+        for (let [index, match] of matches.entries()) {
+          matchElement = document.createElement("tr");
+          matchElement.classList.add("match");
+          matchElement.dataset.index = index;
+          matchTextTitlesElement = document.createElement("td");
+          matchTextTitlesElement.classList.add("text-titles");
+          matchElement.append(matchTextTitlesElement);
+          matchLeftContextElement = document.createElement("td");
+          matchLeftContextElement.classList.add("left-context");
+          for (cpos of match["lc"]) {
+            token = lookup["cpos"][cpos];
+            tokenElement = document.createElement("span");
+            tokenElement.classList.add("token");
+            tokenElement.dataset.cpos = cpos;
+            tokenElement.innerText = token["word"];
+            if (expertModeSwitchElement.checked) {
+              tokenElement.classList.add("chip");
+              addToolTipToTokenElement(tokenElement, token);
+            }
+            matchLeftContextElement.append(tokenElement);
+            matchLeftContextElement.append(document.createTextNode(" "));
+            tokenElements.add(tokenElement);
+          }
+          matchElement.append(matchLeftContextElement);
+          matchHitElement = document.createElement("td");
+          matchHitElement.classList.add("hit");
+          textTitles = new Set();
+          for (cpos of match["hit"]) {
+            token = lookup["cpos"][cpos];
+            tokenElement = document.createElement("span");
+            tokenElement.classList.add("token");
+            tokenElement.dataset.cpos = cpos;
+            tokenElement.innerText = token["word"];
+            if (expertModeSwitchElement.checked) {
+              tokenElement.classList.add("chip");
+              addToolTipToTokenElement(tokenElement, token);
+            }
+            matchHitElement.append(tokenElement);
+            matchHitElement.append(document.createTextNode(" "));
+            tokenElements.add(tokenElement);
+            textTitles.add(lookup["text"][token["text"]]["title"]);
+          }
+          var moreContextBtn = document.createElement("a");
+          moreContextBtn.setAttribute("class", "btn-floating btn waves-effect waves-light teal right inspect");
+          moreContextBtn.innerHTML = '<i class="material-icons">search</i>';
+          matchHitElement.append(document.createElement("br"), document.createElement("br"));
+          matchHitElement.append(moreContextBtn);
+
+          matchTextTitlesElement.innerText = [...textTitles].join(",");
+          matchElement.append(matchHitElement);
+          matchRightContextElement = document.createElement("td");
+          matchRightContextElement.classList.add("right-context");
+          for (cpos of match["rc"]) {
+            token = lookup["cpos"][cpos];
+            tokenElement = document.createElement("span");
+            tokenElement.classList.add("token");
+            tokenElement.dataset.cpos = cpos;
+            tokenElement.innerText = token["word"];
+            if (expertModeSwitchElement.checked) {
+              tokenElement.classList.add("chip");
+              addToolTipToTokenElement(tokenElement, token);
+            }
+            matchRightContextElement.append(tokenElement);
+            matchRightContextElement.append(document.createTextNode(" "));
+            tokenElements.add(tokenElement);
+          }
+          matchElement.append(matchRightContextElement);
+          queryResultsElement.append(matchElement);
+        }
+      }
+      var options = {page: hitsPerPage,
+                     pagination: [{
+                                   name: "paginationTop",
+                                   paginationClass: "paginationTop",
+                                   innerWindow: 8,
+                                   outerWindow: 1
+                                 }, {
+                                   paginationClass: "paginationBottom",
+                                   innerWindow: 8,
+                                   outerWindow: 1
+                                 }],
+                     valueNames: ["text-titles", "left-context", "hit", "right-context"]};
+      var userList = new List('result-list', options);
+
+      var inspectBtns = document.getElementsByClassName("inspect");
+      for(var i = 0; i < inspectBtns.length; i++) {
+        var inspectBtn = inspectBtns[i];
+        var dataIndex = inspectBtn.parentNode.parentNode.getAttribute("data-index");
+        inspectBtn.onclick = function() {
+          contextModal.open();
+          nopaque.socket.emit("inspect_match", {"cpos": matches[dataIndex]["hit"]});
+        };
+      }
+
+    // Function to download data to a file
+    function download(downloadElem, data, filename, type) {
+        var file = new Blob([data], {type: type});
+        if (window.navigator.msSaveOrOpenBlob) // IE10+
+            window.navigator.msSaveOrOpenBlob(file, filename);
+        else { // Others
+            var url = URL.createObjectURL(file);
+            downloadElem.href = url;
+            downloadElem.download = filename;
+        }
+    }
+    // create json filename for download
+    var today = new Date();
+    var currentDate = today.getUTCFullYear() + '-' + (today.getUTCMonth() +1) + '-' + today.getUTCDate();
+    var currentTime = today.getUTCHours() + ":" + today.getUTCMinutes() + ":" + today.getUTCSeconds();
+    var safeFilename = message['query'].replace(/[^a-z0-9_-]/gi, "_");
+    var resultFilename = "UTC-" + currentDate + "_" + currentTime + "_" + safeFilename + ".json";
+    // get a where download is served
+    var downloadResults = document.getElementById("download-results");
+    // stringify JSON object for json download
+    var dataStr = JSON.stringify(message, undefined, 2);
+    downloadResults.onclick = download(downloadResults,
+                                       dataStr,
+                                       resultFilename,
+                                       "text/json");
+  });
+
+
+  function addToolTipToTokenElement(tokenElement, token) {
+    M.Tooltip.init(tokenElement,
+                   {"html": `<table>
+                               <tr>
+                                 <th>Token information</th>
+                                 <th>Source information</th>
+                               </tr>
+                               <tr>
+                                 <td class="left-align">
+                                   Word: ${token["word"]}<br>
+                                   Lemma: ${token["lemma"]}<br>
+                                   POS: ${token["pos"]}<br>
+                                   Simple POS: ${token["simple_pos"]}<br>
+                                   NER: ${token["ner"]}
+                                 </td>
+                                 <td class="left-align">
+                                   Title: ${lookup["text"][token["text"]]["title"]}<br>
+                                   Author: ${lookup["text"][token["text"]]["author"]}<br>
+                                   Publishing year: ${lookup["text"][token["text"]]["publishing_year"]}
+                                 </td>
+                               </tr>
+                             </table>`,
+                   "inDuration": 1500,
+                   "margin": 15,
+                   "position": "top",
+                   "transitionMovement": 0});
+  }
+
+
+  nopaque.socket.on("match_context", function(message) {
+    console.log("### match_context ###");
+    console.log(message);
+    contextResultsElement.innerHTML = "<p>&nbsp;</p>";
+    document.getElementById("context-modal-loading").classList.add("hide");
+    document.getElementById("context-modal-ready").classList.remove("hide");
+
+    var sentenceElement;
+    var token;
+    var tokenElement;
+
+    lookup["cpos"] = {...lookup["cpos"], ...message["cpos_lookup"]};
+    lookup["s"] = message["context_s_cpos"];
+    lookup["text"] = {...lookup["text"], ...message["text_lookup"]};
+
+    for (let [key, value] of Object.entries(message['context_s_cpos'])) {
+      sentenceElement = document.createElement("p");
+      for (cpos of value) {
+        token = lookup["cpos"][cpos];
+        tokenElement = document.createElement("span");
+        tokenElement.classList.add("token");
+        if (message["match_cpos_list"].includes(cpos)) {
+          tokenElement.classList.add("bold");
+        }
+        tokenElement.dataset.cpos = cpos;
+        tokenElement.innerText = token["word"];
+        if (expertModeSwitchElement.checked) {
+          tokenElement.classList.add("chip");
+          addToolTipToTokenElement(tokenElement, token);
+        }
+        tokenElements.add(tokenElement);
+        sentenceElement.append(tokenElement);
+        sentenceElement.append(document.createTextNode(" "));
+      }
+      contextResultsElement.append(sentenceElement);
+    }
+  });
+  // collapsible display options
+  var elem = document.querySelector('.collapsible.expandable');
+</script>
+{% endblock %}

app/templates/corpora/analyse_corpus.html.j2

@@ -1,40 +1,6 @@
 {% extends "full_width.html.j2" %}
 
 {% block page_content %}
-<style>
-  .token.chip.bold {
-    background-color: #48a54b;
-  }
-  .token.bold {
-    font-weight: bolder;
-  }
-  .token.chip:hover {
-    background-color: #9e9e9e;
-    cursor: pointer;
-  }
-  .material-tooltip {
-      padding: 5px 20px 20px;
-      border-radius: 10px;
-      background: rgba(37, 36, 36);
-      overflow: visible;
-  }
-
-  .material-tooltip .backdrop {
-      display: none !important;
-  }
-  .material-tooltip:after {
-    position: absolute;
-    content: "";
-    top: 100%;
-    left: 50%;
-    margin-left: -8px;
-    width: 0;
-    height: 0;
-    border-style: solid;
-    border-width: 15px 13px 0 13px;
-    border-color: rgba(37, 36, 36) transparent transparent transparent;
-}
-</style>
 
 <!-- Query Form -->
 <form id="query-form">
@@ -203,7 +169,8 @@
               <th style="width: 25%">Right Context</th>
             </tr>
           </thead>
-          <tbody class="list" id="query-results"></tbody>
+          <tbody class="list" id="query-results">
+          </tbody>
         </table>
         <ul class="pagination paginationBottom"></ul>
       </div>
@@ -242,6 +209,7 @@
 </div>
 
 <script>
+ // Initialising some modals for later usage
   var contextModal;
   var loadingModal;
   var exportModal;
@@ -259,10 +227,12 @@
     nopaque.socket.emit("request_corpus_analysis", {{ corpus_id }});
   });
 
+  // close loading modal if container for analysis has started
   nopaque.socket.on("request_corpus_analysis", function(msg) {
     if (msg === "[201]: Created") {loadingModal.close();}
   });
 
+  // exper view stuff reuse maybe and REMOVE later
   var expertModeSwitchElement = document.getElementById("expert-mode-switch");
   var lookup = {"cpos": null, "s": null, "text": null};
   var matches = null;
@@ -283,7 +253,7 @@
     }
   })
 
-  var queryFormElement = document.getElementById("query-form");
+  // getting some HTML-elements to use/hide/remove/show or add some other elements to them
   var queryResultsElement = document.getElementById("query-results");
   var queryResultsMetadataElement = document.getElementById("query-results-metadata");
   var exportQueryResults = document.getElementById("export-query-results");
@@ -293,7 +263,15 @@
   var contextResultsElement = document.getElementById("context-results");
   var queryLoadingElement = document.getElementById("getting-query-results");
   var queryResultsTableElement = document.getElementById("recieved-query-results");
+  ;
+  // create some global variables
   var hitsPerPage;
+  var full_results;
+  var resultList;
+
+  // Get query form element and save its data on submit and send this data via
+  // socket.io to the CQP server
+  var queryFormElement = document.getElementById("query-form");
   queryFormElement.addEventListener("submit", function(event) {
     event.preventDefault();
     let formData = new FormData(queryFormElement);
@@ -302,146 +280,110 @@
                      "query": formData.get("query")};
     hitsPerPage = formData.get("hits_per_page");
     nopaque.socket.emit("corpus_analysis", queryData);
+    // full results object declaration, kind of global maybe store it later?
+    // will always be reset if a query is sent, so that only the chunks of the
+    // current query will be saved in it
+    full_results = {};
+    full_results["matches"] = [];
+    full_results["cpos_lookup"] = {};
+    full_results["text_lookup"] = {};
+    full_results["total_nr_matches"] = 0;
+    full_results["query"] = "";
+    // some hiding/showing for loading animation
     queryLoadingElement.classList.remove("hide");
     queryResultsTableElement.classList.add("hide");
-    nopaque.toast("Query has been sent!")
+    nopaque.toast("Query has been sent!");
+
+    var options = {page: hitsPerPage,
+      pagination: [{
+        name: "paginationTop",
+        paginationClass: "paginationTop",
+        innerWindow: 8,
+        outerWindow: 1
+      }, {
+        paginationClass: "paginationBottom",
+        innerWindow: 8,
+        outerWindow: 1
+      }],
+      valueNames: ["titles", "lc", "hit", "rc"],
+      item: `<tr>
+      <td class="titles"></td>
+      <td class="lc"></td>
+      <td class="hit"></td>
+      <td class="rc"></td>
+      </tr>`};
+      resultList = new List('result-list', options);
+
   });
 
-  nopaque.socket.on("corpus_analysis", function(message) {
+
-    console.log("### corpus_analysis ###");
+  // socket.on triggered when result chunks are recieved
-    console.log(message);
+  nopaque.socket.on("corpus_analysis", function(chunk) {
+    // logs the current recieved chunk
+    console.log("### corpus_analysis chunk ###");
+    console.log(chunk);
+    // logs and extends/push/update the current recieved chunk to the
+    // full_results Object
+    console.log("### corpus analysis updated full_results json ###");
+    full_results["matches"].push(...chunk["matches"]);
+    Object.assign(full_results["cpos_lookup"], chunk["cpos_lookup"]);
+    Object.assign(full_results["text_lookup"], chunk["text_lookup"]);
+    full_results["total_nr_matches"] = chunk["total_nr_matches"];
+    full_results["query"] = chunk["query"];
+    console.log(full_results);
+    // Some hiding and showing of loading animations
     queryLoadingElement.classList.add("hide");
     queryResultsTableElement.classList.remove("hide");
     queryResultsElement.innerHTML = "";
 
-    // Reset related global values
+    // some checks for erroneous or empty query results
-    lookup = {"cpos": null, "s": null, "text": null};
+    // No results for this query
-    matches = null;
+    if (chunk === null) {
-    tokenElements = null;
-
-    if (message === null) {
       queryResultsTableElement.classList.add("hide");
-      nopaque.toast("No results for this query!")
+      nopaque.toast("No results for this query!");
       return;
-    } else if (message === "CQI_CQP_ERROR_GENERAL") {
+    // Query was invalid
+    } else if (chunk === "CQI_CQP_ERROR_GENERAL") {
         queryResultsTableElement.classList.add("hide");
         nopaque.toast("Invalid query entered!", "red");
         return;
+    // List building/appending the chunks when query had results
       } else {
-        total_nr_matches = message["total_nr_matches"];
+        // write metadata query information into HTML elements
-        let count_corpus_files = Object.keys(message["text_lookup"]).length;
+        // like nr. of all matches in how many files etc.
-        queryResultsMetadataElement.innerHTML = message["total_nr_matches"] + " matches in " + count_corpus_files + " corpus files.";
+        // TODO: count_corpus_files müssen aus full results genommen werden.
+        total_nr_matches = chunk["total_nr_matches"];
+        let count_corpus_files = Object.keys(chunk["text_lookup"]).length;
+        queryResultsMetadataElement.innerHTML = chunk["total_nr_matches"] + " matches in " + count_corpus_files + " corpus files.";
         queryResultsMetadataElement.appendChild(exportQueryResults);
         exportQueryResults.classList.remove("hide");
 
-        var matchElement;
+        var toAdd = [];
-        var matchTextTitlesElement;
+        for (let [index, match] of chunk["matches"].entries()) {
-        var matchLeftContextElement;
+          lc_tokens = "";
-        var matchHitElement;
-        var matchRightContextElement;
-        var textTitles;
-        var token;
-        var tokenElement;
-
-        // Set related global values
-        lookup["cpos"] = message["cpos_lookup"];
-        lookup["text"] = message["text_lookup"];
-        matches = message["matches"];
-        tokenElements = new Set();
-
-        for (let [index, match] of matches.entries()) {
-          matchElement = document.createElement("tr");
-          matchElement.classList.add("match");
-          matchElement.dataset.index = index;
-          matchTextTitlesElement = document.createElement("td");
-          matchTextTitlesElement.classList.add("text-titles");
-          matchElement.append(matchTextTitlesElement);
-          matchLeftContextElement = document.createElement("td");
-          matchLeftContextElement.classList.add("left-context");
           for (cpos of match["lc"]) {
-            token = lookup["cpos"][cpos];
+            word = chunk["cpos_lookup"][cpos]["word"];
-            tokenElement = document.createElement("span");
+            lc_tokens += " " + word;
-            tokenElement.classList.add("token");
-            tokenElement.dataset.cpos = cpos;
-            tokenElement.innerText = token["word"];
-            if (expertModeSwitchElement.checked) {
-              tokenElement.classList.add("chip");
-              addToolTipToTokenElement(tokenElement, token);
-            }
-            matchLeftContextElement.append(tokenElement);
-            matchLeftContextElement.append(document.createTextNode(" "));
-            tokenElements.add(tokenElement);
           }
-          matchElement.append(matchLeftContextElement);
+          // console.log(lc_tokens);
-          matchHitElement = document.createElement("td");
+          hit_tokens = "";
-          matchHitElement.classList.add("hit");
-          textTitles = new Set();
           for (cpos of match["hit"]) {
-            token = lookup["cpos"][cpos];
+            word = chunk["cpos_lookup"][cpos]["word"];
-            tokenElement = document.createElement("span");
+            hit_tokens += " " + word;
-            tokenElement.classList.add("token");
-            tokenElement.dataset.cpos = cpos;
-            tokenElement.innerText = token["word"];
-            if (expertModeSwitchElement.checked) {
-              tokenElement.classList.add("chip");
-              addToolTipToTokenElement(tokenElement, token);
-            }
-            matchHitElement.append(tokenElement);
-            matchHitElement.append(document.createTextNode(" "));
-            tokenElements.add(tokenElement);
-            textTitles.add(lookup["text"][token["text"]]["title"]);
           }
-          var moreContextBtn = document.createElement("a");
+          // console.log(hit_tokens);
-          moreContextBtn.setAttribute("class", "btn-floating btn waves-effect waves-light teal right inspect");
+          rc_tokens = "";
-          moreContextBtn.innerHTML = '<i class="material-icons">search</i>';
-          matchHitElement.append(document.createElement("br"), document.createElement("br"));
-          matchHitElement.append(moreContextBtn);
-
-          matchTextTitlesElement.innerText = [...textTitles].join(",");
-          matchElement.append(matchHitElement);
-          matchRightContextElement = document.createElement("td");
-          matchRightContextElement.classList.add("right-context");
           for (cpos of match["rc"]) {
-            token = lookup["cpos"][cpos];
+            word = chunk["cpos_lookup"][cpos]["word"];
-            tokenElement = document.createElement("span");
+            rc_tokens += " " + word;
-            tokenElement.classList.add("token");
-            tokenElement.dataset.cpos = cpos;
-            tokenElement.innerText = token["word"];
-            if (expertModeSwitchElement.checked) {
-              tokenElement.classList.add("chip");
-              addToolTipToTokenElement(tokenElement, token);
-            }
-            matchRightContextElement.append(tokenElement);
-            matchRightContextElement.append(document.createTextNode(" "));
-            tokenElements.add(tokenElement);
           }
-          matchElement.append(matchRightContextElement);
+          // console.log(rc_tokens);
-          queryResultsElement.append(matchElement);
+          item = { titles: "test", lc: lc_tokens, hit: hit_tokens, rc: rc_tokens };
+          toAdd.push(item);
         }
-      }
+        resultList.add(toAdd, function(toAdd) {console.log('All ' + toAdd.length + 'results were added!')});
-      var options = {page: hitsPerPage,
+        }
-                     pagination: [{
+      });
-                                   name: "paginationTop",
-                                   paginationClass: "paginationTop",
-                                   innerWindow: 8,
-                                   outerWindow: 1
-                                 }, {
-                                   paginationClass: "paginationBottom",
-                                   innerWindow: 8,
-                                   outerWindow: 1
-                                 }],
-                     valueNames: ["text-titles", "left-context", "hit", "right-context"]};
-      var userList = new List('result-list', options);
-
-      var inspectBtns = document.getElementsByClassName("inspect");
-      for(var i = 0; i < inspectBtns.length; i++) {
-        var inspectBtn = inspectBtns[i];
-        var dataIndex = inspectBtn.parentNode.parentNode.getAttribute("data-index");
-        inspectBtn.onclick = function() {
-          contextModal.open();
-          nopaque.socket.emit("inspect_match", {"cpos": matches[dataIndex]["hit"]});
-        };
-      }
 
     // Function to download data to a file
     function download(downloadElem, data, filename, type) {
@@ -453,7 +395,6 @@
             downloadElem.href = url;
             downloadElem.download = filename;
         }
-    }
     // create json filename for download
     var today = new Date();
     var currentDate = today.getUTCFullYear() + '-' + (today.getUTCMonth() +1) + '-' + today.getUTCDate();
@@ -468,7 +409,7 @@
                                        dataStr,
                                        resultFilename,
                                        "text/json");
-  });
+  };
 
 
   function addToolTipToTokenElement(tokenElement, token) {