sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-04-06 01:04:22 +00:00
Code Issues Projects Releases Wiki Activity

Add tmp List.js implemetation

Browse Source
This commit is contained in:
Stephan Porada 2020-01-17 14:31:14 +01:00
parent e04b31d282
commit e4b794d37c
4 changed files with 162 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/corpora/CQiWrapper/CQiWrapper.py
View File

@ -99,9 +99,9 @@ class CQiWrapper(CQiClient):
+ ':' + ':'
+ result_subcorpus_name) + result_subcorpus_name)
self.SUBCORPUS_NAMES.append(self.result_subcorpus) self.SUBCORPUS_NAMES.append(self.result_subcorpus)
self.nr_matches = self.cqp_subcorpus_size(self.result_subcorpus) self.total_nr_matches = self.cqp_subcorpus_size(self.result_subcorpus)
print('Nr of all matches is:', self.nr_matches) logger.warning('Nr of all matches is: {}'.format(self.total_nr_matches))
logger.warning('Nr of all matches is: {}'.format(self.nr_matches)) logger.warning('Nr of all matches is: {}'.format(self.total_nr_matches))
def show_subcorpora(self): def show_subcorpora(self):
''' '''
@ -135,7 +135,7 @@ class CQiWrapper(CQiClient):
self.corpus_max_len = self.cl_attribute_size( self.corpus_max_len = self.cl_attribute_size(
self.attr_strings['positional_attrs']['word'] self.attr_strings['positional_attrs']['word']
) )
self.nr_matches = min(result_len, self.nr_matches) self.nr_matches = min(result_len, self.total_nr_matches)
if self.nr_matches == 0: if self.nr_matches == 0:
logger.warning('Query resulted in 0 matches.') logger.warning('Query resulted in 0 matches.')
return None return None
@ -145,7 +145,9 @@ class CQiWrapper(CQiClient):
# pair of cpositions # pair of cpositions
# [(1355, 1357), (1477, 1479)] Example for two boundry pairs # [(1355, 1357), (1477, 1479)] Example for two boundry pairs
offset_start = 0 if result_offset == 0 else result_offset offset_start = 0 if result_offset == 0 else result_offset
offset_end = self.nr_matches + result_offset - 1 logger.warning('Offset start is: {}'.format(offset_start))
offset_end = min((self.nr_matches + result_offset - 1), self.total_nr_matches - 1)
logger.warning('Offset end is: {}'.format(offset_end))
match_boundaries = zip(self.cqp_dump_subcorpus(self.result_subcorpus, match_boundaries = zip(self.cqp_dump_subcorpus(self.result_subcorpus,
CONST_FIELD_MATCH, CONST_FIELD_MATCH,
offset_start, offset_start,
@ -182,7 +184,7 @@ class CQiWrapper(CQiClient):
t1 = time.time() t1 = time.time()
t_total = t1 - t0 t_total = t1 - t0
logger.warning('Time to create all CPOS for query: {}'.format(t_total)) logger.warning('Time to create all CPOS for query: {}'.format(t_total))
print('Requesting {} CPOS with one query.'.format(len_all_cpos)) logger.warning('Requesting {} CPOS with one query.'.format(len_all_cpos))
# Get cpos informations like CORPUS_NAME.word or CORPUS_NAME.lemma for # Get cpos informations like CORPUS_NAME.word or CORPUS_NAME.lemma for
# all cpos entries in all_cpos_list # all cpos entries in all_cpos_list
@ -191,12 +193,12 @@ class CQiWrapper(CQiClient):
all_cpos_infos, text_lookup = self.get_cpos_infos(all_cpos) all_cpos_infos, text_lookup = self.get_cpos_infos(all_cpos)
t3 = time.time() t3 = time.time()
t_final = t3 - t2 t_final = t3 - t2
print('Got infos for {} CPOS in {} seconds:'.format(len_all_cpos, logger.warning('Got infos for {} CPOS in {} seconds:'.format(len_all_cpos, t_final))
t_final))
self.results = {'matches': all_matches, self.results = {'matches': all_matches,
'cpos_lookup': all_cpos_infos, 'cpos_lookup': all_cpos_infos,
'text_lookup': text_lookup, 'text_lookup': text_lookup,
'nr_matches': self.nr_matches} 'nr_matches': self.nr_matches,
'total_nr_matches': self.total_nr_matches}
return self.results return self.results
def get_cpos_infos(self, all_cpos): def get_cpos_infos(self, all_cpos):
@ -235,7 +237,10 @@ class CQiWrapper(CQiClient):
struct_values = self.cl_struc2str(s_key[0], text_lookup_ids) struct_values = self.cl_struc2str(s_key[0], text_lookup_ids)
zipped = dict(zip(text_lookup_ids, struct_values)) zipped = dict(zip(text_lookup_ids, struct_values))
for zip_key, zip_value in zipped.items(): for zip_key, zip_value in zipped.items():
logger.warning('key: {}'.format(zip_key))
logger.warning('value: {}'.format(zip_value))
check = text_lookup.get(zip_key) check = text_lookup.get(zip_key)
logger.warning('check: {}'.format(check))
if check is None: if check is None:
text_lookup[zip_key] = {s_value: zip_value} text_lookup[zip_key] = {s_value: zip_value}
else: else:

16
app/corpora/events.py
View File

@ -45,7 +45,7 @@ def corpus_analysis(message):
# Prepare and execute a query # Prepare and execute a query
corpus_name = 'CORPUS' corpus_name = 'CORPUS'
query = str(message['query']) query = str(message['query'])
result_len = int(message['hits_per_page']) result_len = 500
context_len = int(message['context']) context_len = int(message['context'])
result_offset = 0 result_offset = 0
client.select_corpus(corpus_name) client.select_corpus(corpus_name)
@ -58,7 +58,19 @@ def corpus_analysis(message):
results = client.show_query_results(result_len=result_len, results = client.show_query_results(result_len=result_len,
context_len=context_len, context_len=context_len,
result_offset=result_offset) result_offset=result_offset)
result_offset += result_len # initial offfset is plus result len because client.show_query_results has been already executed once
while result_offset < client.total_nr_matches:
logger.warning('===While loop start.===')
logger.warning('result_offset: {}'.format(result_offset))
results_append = client.show_query_results(result_len=result_len,
context_len=context_len,
result_offset=result_offset)
results['matches'].extend(results_append['matches'])
results['cpos_lookup'].update(results_append['cpos_lookup'])
results['text_lookup'].update(results_append['text_lookup'])
result_offset += result_len
result_offset = min(result_offset, client.total_nr_matches)
logger.warning('result_offset end of while loop: {}'.format(result_offset))
socketio.emit('corpus_analysis', results, room=request.sid) socketio.emit('corpus_analysis', results, room=request.sid)

11
app/templates/admin/index.html.j2
View File

@ -11,15 +11,22 @@
<input id="search-user" class="search" type="text"></input> <input id="search-user" class="search" type="text"></input>
<label for="search-user">Search user</label> <label for="search-user">Search user</label>
</div> </div>
<ul class="pagination paginationTop"></ul>
{{ table }} {{ table }}
<ul class="pagination"></ul> <ul class="pagination paginationBottom"></ul>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<script> <script>
var options = {page: 10, pagination: true, var options = {page: 10,
pagination: [{
name: "paginationTop",
paginationClass: "paginationTop",
}, {
paginationClass: "paginationBottom",
}],
valueNames: ['username', 'email', 'role', 'confirmed', 'id']}; valueNames: ['username', 'email', 'role', 'confirmed', 'id']};
var userList = new List('user-list', options); var userList = new List('user-list', options);
</script> </script>

220
app/templates/corpora/analyse_corpus.html.j2
View File

@ -129,19 +129,22 @@
<div class="col s12 m9" id="recieved-query-results"> <div class="col s12 m9" id="recieved-query-results">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content" id="result-list">
<span class="card-title">Query Results</span> <span class="card-title">Query Results</span>
<table class="responsive-table highlight"> <p id="query-results-metadata"></p>
<thead> <ul class="pagination paginationTop"></ul>
<tr> <table class="responsive-table highlight">
<th style="width: 5%">Title</th> <thead>
<th style="width: 25%">Left context</th> <tr>
<th style="width: 45%">Match</th> <th style="width: 5%">Title</th>
<th style="width: 25%">Right Context</th> <th style="width: 25%">Left context</th>
</tr> <th style="width: 45%">Match</th>
</thead> <th style="width: 25%">Right Context</th>
<tbody id="query-results"></tbody> </tr>
</table> </thead>
<tbody class="list" id="query-results"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
</div> </div>
</div> </div>
</div> </div>
@ -215,15 +218,18 @@
var queryFormElement = document.getElementById("query-form"); var queryFormElement = document.getElementById("query-form");
var queryResultsElement = document.getElementById("query-results"); var queryResultsElement = document.getElementById("query-results");
var queryResultsMetadataElement = document.getElementById("query-results-metadata")
var contextResultsElement = document.getElementById("context-results"); var contextResultsElement = document.getElementById("context-results");
var queryLoadingElement = document.getElementById("getting-query-results"); var queryLoadingElement = document.getElementById("getting-query-results");
var queryResultsTableElement = document.getElementById("recieved-query-results"); var queryResultsTableElement = document.getElementById("recieved-query-results");
var hitsPerPage;
queryFormElement.addEventListener("submit", function(event) { queryFormElement.addEventListener("submit", function(event) {
event.preventDefault(); event.preventDefault();
let formData = new FormData(queryFormElement); let formData = new FormData(queryFormElement);
let queryData = {"context": formData.get("context"), let queryData = {"context": formData.get("context"),
"hits_per_page": formData.get("hits_per_page"), "hits_per_page": formData.get("hits_per_page"),
"query": formData.get("query")}; "query": formData.get("query")};
hitsPerPage = formData.get("hits_per_page");
socket.emit("corpus_analysis", queryData); socket.emit("corpus_analysis", queryData);
queryLoadingElement.classList.remove("hide"); queryLoadingElement.classList.remove("hide");
queryResultsTableElement.classList.add("hide"); queryResultsTableElement.classList.add("hide");
@ -268,95 +274,118 @@
}); });
} }
return; return;
} } else {
total_nr_matches = message["total_nr_matches"];
let count_corpus_files = Object.keys(message["text_lookup"]).length;
queryResultsMetadataElement.innerText = message["total_nr_matches"] + " matches in " + count_corpus_files + " corpus files.";
var matchElement; var matchElement;
var matchTextTitlesElement; var matchTextTitlesElement;
var matchLeftContextElement; var matchLeftContextElement;
var matchHitElement; var matchHitElement;
var matchRightContextElement; var matchRightContextElement;
var textTitles; var textTitles;
var token; var token;
var tokenElement; var tokenElement;
// Set related global values // Set related global values
lookup["cpos"] = message["cpos_lookup"]; lookup["cpos"] = message["cpos_lookup"];
lookup["text"] = message["text_lookup"]; lookup["text"] = message["text_lookup"];
matches = message["matches"]; matches = message["matches"];
tokenElements = new Set(); tokenElements = new Set();
for (let [index, match] of matches.entries()) { for (let [index, match] of matches.entries()) {
matchElement = document.createElement("tr"); matchElement = document.createElement("tr");
matchElement.classList.add("match"); matchElement.classList.add("match");
matchElement.dataset.index = index; matchElement.dataset.index = index;
matchTextTitlesElement = document.createElement("td"); matchTextTitlesElement = document.createElement("td");
matchTextTitlesElement.classList.add("text-titles"); matchTextTitlesElement.classList.add("text-titles");
matchElement.append(matchTextTitlesElement); matchElement.append(matchTextTitlesElement);
matchLeftContextElement = document.createElement("td"); matchLeftContextElement = document.createElement("td");
matchLeftContextElement.classList.add("left-context"); matchLeftContextElement.classList.add("left-context");
for (cpos of match["lc"]) { for (cpos of match["lc"]) {
token = lookup["cpos"][cpos]; token = lookup["cpos"][cpos];
tokenElement = document.createElement("span"); tokenElement = document.createElement("span");
tokenElement.classList.add("token"); tokenElement.classList.add("token");
tokenElement.dataset.cpos = cpos; tokenElement.dataset.cpos = cpos;
tokenElement.innerText = token["word"]; tokenElement.innerText = token["word"];
if (expertModeSwitchElement.checked) { if (expertModeSwitchElement.checked) {
tokenElement.classList.add("chip"); tokenElement.classList.add("chip");
addToolTipToTokenElement(tokenElement, token); addToolTipToTokenElement(tokenElement, token);
}
matchLeftContextElement.append(tokenElement);
matchLeftContextElement.append(document.createTextNode(" "));
tokenElements.add(tokenElement);
}
matchElement.append(matchLeftContextElement);
matchHitElement = document.createElement("td");
matchHitElement.classList.add("hit");
textTitles = new Set();
for (cpos of match["hit"]) {
token = lookup["cpos"][cpos];
tokenElement = document.createElement("span");
tokenElement.classList.add("token");
tokenElement.dataset.cpos = cpos;
tokenElement.innerText = token["word"];
if (expertModeSwitchElement.checked) {
tokenElement.classList.add("chip");
addToolTipToTokenElement(tokenElement, token);
}
matchHitElement.append(tokenElement);
matchHitElement.append(document.createTextNode(" "));
tokenElements.add(tokenElement);
textTitles.add(lookup["text"][token["text"]]["title"]);
}
var moreContextBtn = document.createElement("a");
moreContextBtn.setAttribute("class", "btn-floating btn waves-effect waves-light teal right inspect");
moreContextBtn.innerHTML = '<i class="material-icons">search</i>';
matchHitElement.append(document.createElement("br"), document.createElement("br"));
matchHitElement.append(moreContextBtn);
matchTextTitlesElement.innerText = [...textTitles].join(",");
matchElement.append(matchHitElement);
matchRightContextElement = document.createElement("td");
matchRightContextElement.classList.add("right-context");
for (cpos of match["rc"]) {
token = lookup["cpos"][cpos];
tokenElement = document.createElement("span");
tokenElement.classList.add("token");
tokenElement.dataset.cpos = cpos;
tokenElement.innerText = token["word"];
if (expertModeSwitchElement.checked) {
tokenElement.classList.add("chip");
addToolTipToTokenElement(tokenElement, token);
}
matchRightContextElement.append(tokenElement);
matchRightContextElement.append(document.createTextNode(" "));
tokenElements.add(tokenElement);
}
matchElement.append(matchRightContextElement);
queryResultsElement.append(matchElement);
} }
matchLeftContextElement.append(tokenElement);
matchLeftContextElement.append(document.createTextNode(" "));
tokenElements.add(tokenElement);
} }
matchElement.append(matchLeftContextElement); var options = {page: hitsPerPage,
matchHitElement = document.createElement("td"); pagination: [{
matchHitElement.classList.add("hit"); name: "paginationTop",
textTitles = new Set(); paginationClass: "paginationTop",
for (cpos of match["hit"]) { innerWindow: 8,
token = lookup["cpos"][cpos]; outerWindow: 1
tokenElement = document.createElement("span"); }, {
tokenElement.classList.add("token"); paginationClass: "paginationBottom",
tokenElement.dataset.cpos = cpos; innerWindow: 8,
tokenElement.innerText = token["word"]; outerWindow: 1
if (expertModeSwitchElement.checked) { }],
tokenElement.classList.add("chip"); valueNames: ["text-titles", "left-context", "hit", "right-context"]};
addToolTipToTokenElement(tokenElement, token); var userList = new List('result-list', options);
} var inspectBtns = document.getElementsByClassName("inspect");
matchHitElement.append(tokenElement); for(var i = 0; i < inspectBtns.length; i++) {
matchHitElement.append(document.createTextNode(" ")); var inspectBtn = inspectBtns[i];
tokenElements.add(tokenElement); var dataIndex = inspectBtn.parentNode.parentNode.getAttribute("data-index");
textTitles.add(lookup["text"][token["text"]]["title"]); inspectBtn.onclick = function() {
contextModal.open();
socket.emit("inspect_match", {"cpos": matches[dataIndex]["hit"]});
};
} }
var moreContextBtn = document.createElement("a");
moreContextBtn.setAttribute("class", "btn-floating btn waves-effect waves-light teal right");
moreContextBtn.innerHTML = '<i class="material-icons">search</i>';
matchHitElement.append(document.createElement("br"), document.createElement("br"));
matchHitElement.append(moreContextBtn);
moreContextBtn.onclick = function() {
contextModal.open();
socket.emit("inspect_match", {"cpos": match["hit"]});
};
matchTextTitlesElement.innerText = [...textTitles].join(",");
matchElement.append(matchHitElement);
matchRightContextElement = document.createElement("td");
matchRightContextElement.classList.add("right-context");
for (cpos of match["rc"]) {
token = lookup["cpos"][cpos];
tokenElement = document.createElement("span");
tokenElement.classList.add("token");
tokenElement.dataset.cpos = cpos;
tokenElement.innerText = token["word"];
if (expertModeSwitchElement.checked) {
tokenElement.classList.add("chip");
addToolTipToTokenElement(tokenElement, token);
}
matchRightContextElement.append(tokenElement);
matchRightContextElement.append(document.createTextNode(" "));
tokenElements.add(tokenElement);
}
matchElement.append(matchRightContextElement);
queryResultsElement.append(matchElement);
}
}); });
function addToolTipToTokenElement(tokenElement, token) { function addToolTipToTokenElement(tokenElement, token) {
@ -425,5 +454,6 @@
contextResultsElement.append(sentenceElement); contextResultsElement.append(sentenceElement);
} }
}); });
</script> </script>
{% endblock %} {% endblock %}