From 691d4757ff9b9b4cef3c6b88545d4c577fd2f477 Mon Sep 17 00:00:00 2001 From: Inga Kirschnick Date: Tue, 18 Jul 2023 16:01:31 +0200 Subject: [PATCH] Token list first implementation+ query builder fix --- .../js/CorpusAnalysis/CorpusAnalysisApp.js | 5 - .../CorpusAnalysisStaticVisualization.js | 256 ++++++++------ .../js/ResourceLists/CorpusTextInfoList.js | 14 +- .../js/ResourceLists/CorpusTokenList.js | 121 +++++++ app/static/js/ResourceLists/ResourceList.js | 1 + app/templates/_scripts.html.j2 | 3 +- .../_analysis/static_visualization.html.j2 | 70 ++-- app/templates/corpora/analysis.html.j2 | 326 +----------------- 8 files changed, 322 insertions(+), 474 deletions(-) create mode 100644 app/static/js/ResourceLists/CorpusTokenList.js diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js b/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js index 25f58a65..ea7aed3a 100644 --- a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js +++ b/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js @@ -34,11 +34,6 @@ class CorpusAnalysisApp { .then((cqiCorpora) => { this.data.corpus = {o: cqiCorpora[0]}; console.log(this.data.corpus.o.staticData); - // this.renderGeneralCorpusInfo(); - // this.renderTextInfoList(); - // this.renderTextProportionsGraphic() - // this.renderFrequenciesGraphic(); - // this.renderBoundsGraphic(); // TODO: Don't do this hgere this.data.corpus.o.updateDb(); diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js b/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js index 81e1d466..2d8f9002 100644 --- a/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js +++ b/app/static/js/CorpusAnalysis/CorpusAnalysisStaticVisualization.js @@ -18,9 +18,10 @@ class CorpusAnalysisStaticVisualization { this.data.corpus = this.app.data.corpus; this.renderGeneralCorpusInfo(); this.renderTextInfoList(); - this.renderTextProportionsGraphic() + this.renderTextProportionsGraphic(); + this.renderTokenList(); this.renderFrequenciesGraphic(); - this.renderBoundsGraphic(); + // Add event listeners let frequenciesStopwordSettingModal = document.querySelector('#frequencies-stopwords-setting-modal'); let frequenciesStopwordSettingModalButton = document.querySelector('#frequencies-stopwords-setting-modal-button'); @@ -30,6 +31,35 @@ class CorpusAnalysisStaticVisualization { M.Modal.init(frequenciesStopwordSettingModal, {dismissible: false}); }); + let textProportionsGraphModeButtons = document.querySelectorAll('.text-proportions-graph-mode-button'); + textProportionsGraphModeButtons.forEach(graphModeButton => { + graphModeButton.addEventListener('click', (event) => { + textProportionsGraphModeButtons.forEach(btn => { + btn.classList.remove('disabled'); + }); + event.target.closest('.text-proportions-graph-mode-button').classList.add('disabled'); + this.renderTextProportionsGraphic(); + }); + }); + + let frequenciesTokenCategoryDropdownElement = document.querySelector('[data-target="frequencies-token-category-dropdown"]'); + let frequenciesTokenCategoryDropdownListElement = document.querySelector("#frequencies-token-category-dropdown"); + frequenciesTokenCategoryDropdownListElement.addEventListener('click', (event) => { + frequenciesTokenCategoryDropdownElement.firstChild.textContent = event.target.innerHTML; + this.renderFrequenciesGraphic(); + }); + + let frequenciesGraphModeButtons = document.querySelectorAll('.frequencies-graph-mode-button'); + frequenciesGraphModeButtons.forEach(graphModeButton => { + graphModeButton.addEventListener('click', (event) => { + frequenciesGraphModeButtons.forEach(btn => { + btn.classList.remove('disabled'); + }); + event.target.closest('.frequencies-graph-mode-button').classList.add('disabled'); + this.renderFrequenciesGraphic(); + }); + }); + for (let actionButton of document.querySelectorAll('.frequencies-stopword-setting-modal-action-buttons')) { actionButton.addEventListener('click', (event) => { let action = event.target.closest('.frequencies-stopword-setting-modal-action-buttons').dataset.action; @@ -101,70 +131,121 @@ class CorpusAnalysisStaticVisualization { let corpusData = this.data.corpus.o.staticData; let textProportionsGraphicElement = document.querySelector('#text-proportions-graphic'); let texts = Object.entries(corpusData.s_attrs.text.lexicon); - let graphData = [ - { - values: texts.map(text => text[1].counts.token), - labels: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), - type: 'pie' - } - ]; + let graphtype = document.querySelector('.text-proportions-graph-mode-button.disabled').dataset.graphType; + let textProportionsTitleElement = document.querySelector('#text-proportions-title-element'); + + if (graphtype === 'bar') { + textProportionsTitleElement.innerHTML = 'Bounds'; + } else if (graphtype === 'pie') { + textProportionsTitleElement.innerHTML = 'Proportions'; + } + + let graphData = this.createTextProportionsGraphData(texts, graphtype); let graphLayout = { - showlegend: true, - height: 486, + barmode: graphtype === 'bar' ? 'relative' : '', + type: graphtype, + showgrid: false, + height: 447, margin: { l: 10, r: 10, - b: 10, - t: 10 + b: graphtype === 'bar' ? 80 : 10, + t: graphtype === 'bar' ? 80 : 10, }, legend: { "orientation": "h", font: { size: 10 } + }, + xaxis: { + rangemode: 'nonnegative', + autorange: true + }, + yaxis: { + autorange: true, + showticklabels: false } }; let config = { responsive: true, + modeBarButtonsToRemove: ['zoom2d', 'select2d', 'lasso2d', 'zoomIn2d', 'zoomOut2d', 'autoScale2d', 'resetScale2d'], displaylogo: false }; Plotly.newPlot(textProportionsGraphicElement, graphData, graphLayout, config); } + createTextProportionsGraphData(texts, graphtype) { + let corpusData = this.data.corpus.o.staticData; + let graphData = []; + switch (graphtype) { + case 'bar': + for (let text of texts) { + let textData = { + type: 'bar', + orientation: 'h', + x: [text[1].bounds[1] - text[1].bounds[0]], + y: [0.5], + text: [`${text[1].bounds[0]} - ${text[1].bounds[1]}`], + name: `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`, + hovertemplate: `${text[1].bounds[0]} - ${text[1].bounds[1]}`, + }; + graphData.push(textData); + } + break; + default: + graphData = [ + { + values: texts.map(text => text[1].counts.token), + labels: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), + type: graphtype + } + ]; + break; + } + return graphData; + } + + async renderTokenList() { + let corpusData = this.data.corpus.o.staticData; + let corpusTokenListElement = document.querySelector('.corpus-token-list'); + let corpusTokenList = new CorpusTokenList(corpusTokenListElement); + let stopwords = this.data.stopwords; + if (this.data.stopwords === undefined) { + stopwords = await this.getStopwords(); + } + stopwords = Object.values(stopwords).flat(); + let mostFrequent = Object.entries(corpusData.corpus.freqs.word) + .sort((a, b) => b[1] - a[1]) + .filter(item => !stopwords.includes(corpusData.values.p_attrs.word[item[0]].toLowerCase())) + .slice(0, 4) + .map(item => parseInt(item[0])); + let tokenData = []; + for (let i = 0; i < Object.values(corpusData.corpus.freqs.word).length; i++) { + let resource = { + term: corpusData.values.p_attrs.word[i].toLowerCase(), + count: corpusData.corpus.freqs.word[i], + mostFrequent: mostFrequent.includes(i) + }; + if (!Object.values(stopwords).includes(resource.term)) { + tokenData.push(resource); + } + } + corpusTokenList.add(tokenData); + } + async renderFrequenciesGraphic() { let corpusData = this.data.corpus.o.staticData; let frequenciesTokenCategoryDropdownElement = document.querySelector('[data-target="frequencies-token-category-dropdown"]'); - let frequenciesTokenCategoryDropdownListElement = document.querySelector("#frequencies-token-category-dropdown"); let frequenciesGraphicElement = document.querySelector('#frequencies-graphic'); let texts = Object.entries(corpusData.s_attrs.text.lexicon); let graphtype = document.querySelector('.frequencies-graph-mode-button.disabled').dataset.graphType; - let graphModeButtons = document.querySelectorAll('.frequencies-graph-mode-button'); - - frequenciesTokenCategoryDropdownListElement.addEventListener('click', (event) => { - frequenciesTokenCategoryDropdownElement.firstChild.textContent = event.target.innerHTML; - this.renderFrequenciesGraphic(corpusData); - }); - - graphModeButtons.forEach(graphModeButton => { - graphModeButton.addEventListener('click', (event) => { - graphModeButtons.forEach(btn => { - btn.classList.remove('disabled'); - }); - event.target.closest('.frequencies-graph-mode-button').classList.add('disabled'); - this.renderFrequenciesGraphic(corpusData); - }); - }); - let tokenCategory = frequenciesTokenCategoryDropdownElement.firstChild.textContent.toLowerCase(); - let graphData = await this.createFrequenciesGraphData(tokenCategory, texts, corpusData, graphtype); + let graphData = await this.createFrequenciesGraphData(tokenCategory, texts, graphtype); let graphLayout = { barmode: graphtype === 'bar' ? 'stack' : '', - margin: { - t: 20, - l: 50 - }, yaxis: { showticklabels: graphtype === 'markers' ? false : true }, @@ -177,44 +258,48 @@ class CorpusAnalysisStaticVisualization { Plotly.newPlot(frequenciesGraphicElement, graphData, graphLayout, config); } - async createFrequenciesGraphData(category, texts, corpusData, graphtype) { + async createFrequenciesGraphData(tokenCategory, texts, graphtype) { + let corpusData = this.data.corpus.o.staticData; let stopwords = this.data.stopwords; if (this.data.stopwords === undefined) { stopwords = await this.getStopwords(); } let stopwordList = Object.values(stopwords).flat(); let graphData = []; - let filteredData = Object.entries(corpusData.corpus.freqs[category]) + let filteredData = Object.entries(corpusData.corpus.freqs[tokenCategory]) .sort((a, b) => b[1] - a[1]) - .filter(item => !stopwordList.includes(corpusData.values.p_attrs[category][item[0]].toLowerCase())) + .filter(item => !stopwordList.includes(corpusData.values.p_attrs[tokenCategory][item[0]].toLowerCase())) .slice(0, 5); - if (graphtype !== 'markers') { - for (let item of filteredData) { - let data = { - x: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), - y: texts.map(text => text[1].freqs[category][item[0]] || 0), - name: corpusData.values.p_attrs[category][item[0]], - type: graphtype - }; - graphData.push(data); - } - } else { - for (let item of filteredData) { - let size = texts.map(text => text[1].freqs[category][item[0]] || 0); - let data = { - x: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), - y: texts.map(text => corpusData.values.p_attrs[category][item[0]]), - name: corpusData.values.p_attrs[category][item[0]], - text: texts.map(text => `${corpusData.values.p_attrs[category][item[0]]}
${text[1].freqs[category][item[0]] || 0}`), - mode: 'markers', - marker: { - size: size, - sizeref: 0.4 - } - }; - graphData.push(data); - } + switch (graphtype) { + case 'markers': + for (let item of filteredData) { + let size = texts.map(text => text[1].freqs[tokenCategory][item[0]] || 0); + let data = { + x: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), + y: texts.map(text => corpusData.values.p_attrs[tokenCategory][item[0]]), + name: corpusData.values.p_attrs[tokenCategory][item[0]], + text: texts.map(text => `${corpusData.values.p_attrs[tokenCategory][item[0]]}
${text[1].freqs[tokenCategory][item[0]] || 0}`), + mode: 'markers', + marker: { + size: size, + sizeref: 0.4 + } + }; + graphData.push(data); + } + break; + default: + for (let item of filteredData) { + let data = { + x: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), + y: texts.map(text => text[1].freqs[tokenCategory][item[0]] || 0), + name: corpusData.values.p_attrs[tokenCategory][item[0]], + type: graphtype + }; + graphData.push(data); + } + break; } return graphData; } @@ -320,45 +405,4 @@ class CorpusAnalysisStaticVisualization { } this.buttonRendering(); } - - renderBoundsGraphic() { - let corpusData = this.data.corpus.o.staticData; - let boundsGraphicElement = document.querySelector('#bounds-graphic'); - - let graphData = []; - let texts = Object.entries(corpusData.s_attrs.text.lexicon); - - graphData = [{ - type: 'bar', - x: texts.map(text => text[1].bounds[1] - text[1].bounds[0]), - y: texts.map(text => corpusData.values.s_attrs.text[text[0]].title), - base: texts.map(text => text[1].bounds[0]), - text: texts.map(text => `${corpusData.values.s_attrs.text[text[0]].title} (${corpusData.values.s_attrs.text[text[0]].publishing_year})`), - orientation: 'h', - hovertemplate: '%{base} - %{x}
%{y}', - showlegend: false - }]; - - let graphLayout = { - barmode: 'stack', - type: 'bar', - showgrid: false, - xaxis: { - rangemode: 'nonnegative', - autorange: true - }, - yaxis: { - autorange: true, - showticklabels: false - } - }; - - let config = { - responsive: true, - modeBarButtonsToRemove: ['zoom2d', 'select2d', 'lasso2d', 'zoomIn2d', 'zoomOut2d', 'autoScale2d', 'resetScale2d'], - displaylogo: false - }; - - Plotly.newPlot(boundsGraphicElement, graphData, graphLayout, config); - } } diff --git a/app/static/js/ResourceLists/CorpusTextInfoList.js b/app/static/js/ResourceLists/CorpusTextInfoList.js index 8338c70a..f1545d70 100644 --- a/app/static/js/ResourceLists/CorpusTextInfoList.js +++ b/app/static/js/ResourceLists/CorpusTextInfoList.js @@ -7,7 +7,7 @@ class CorpusTextInfoList extends ResourceList { } static defaultOptions = { - page: 4 + page: 5 }; constructor(listContainerElement, options = {}) { @@ -67,12 +67,12 @@ class CorpusTextInfoList extends ResourceList { Textarrow_drop_down - Number of tokensarrow_drop_down - Number of sentencesarrow_drop_down - Number of unique wordsarrow_drop_down - Number of unique lemmasarrow_drop_down - Number of unique posarrow_drop_down - Number of unique simple posarrow_drop_down + Tokensarrow_drop_down + Sentencesarrow_drop_down + Unique wordsarrow_drop_down + Unique lemmasarrow_drop_down + Unique posarrow_drop_down + Unique simple posarrow_drop_down diff --git a/app/static/js/ResourceLists/CorpusTokenList.js b/app/static/js/ResourceLists/CorpusTokenList.js new file mode 100644 index 00000000..1b992038 --- /dev/null +++ b/app/static/js/ResourceLists/CorpusTokenList.js @@ -0,0 +1,121 @@ +class CorpusTokenList extends ResourceList { + static autoInit() { + for (let corpusTokenListElement of document.querySelectorAll('.corpus-token-list:not(.no-autoinit)')) { + new CorpusTokenList(corpusTokenListElement); + } + } + + static defaultOptions = { + page: 100 + }; + + constructor(listContainerElement, options = {}) { + let _options = Utils.mergeObjectsDeep( + CorpusTokenList.defaultOptions, + options + ); + super(listContainerElement, _options); + this.listjs.list.addEventListener('click', (event) => {this.onClick(event)}); + this.selectedItemIds = new Set(); + + } + + get item() { + return (values) => { + return ` + + + + + + + + + `.trim(); + } + } + + get valueNames() { + return [ + 'term', + 'count', + 'mostFrequent', + 'frequency' + ]; + } + + initListContainerElement() { + if (!this.listContainerElement.hasAttribute('id')) { + this.listContainerElement.id = Utils.generateElementId('corpus-token-list-'); + } + let listSearchElementId = Utils.generateElementId(`${this.listContainerElement.id}-search-`); + this.listContainerElement.innerHTML = ` +
+ search + + +
+
+
+ + + + + + + + + + +
TermCountFrequency
+
+
+ + `.trim(); + this.listContainerElement.style.padding = '30px'; + } + + mapResourceToValue(corpusTokenData) { + return { + term: corpusTokenData.term, + count: corpusTokenData.count, + mostFrequent: corpusTokenData.mostFrequent, + frequency: '-' + }; + } + + sort() { + this.listjs.sort('count', {order: 'desc'}); + } + + onClick(event) { + let listItemElement = event.target.closest('.list-item[data-id]'); + if (listItemElement === null) {return;} + let itemId = listItemElement.dataset.id; + let listActionElement = event.target.closest('.list-action-trigger[data-list-action]'); + let listAction = listActionElement === null ? '' : listActionElement.dataset.listAction; + switch (listAction) { + case 'select': { + if (event.target.checked) { + this.selectedItemIds.add(itemId); + } else { + this.selectedItemIds.delete(itemId); + } + this.renderingItemSelection(); + break; + } + default: { + break; + } + } + } + + renderingItemSelection() { + + + } + +} diff --git a/app/static/js/ResourceLists/ResourceList.js b/app/static/js/ResourceLists/ResourceList.js index 959a5fe1..6bc6ac1f 100644 --- a/app/static/js/ResourceLists/ResourceList.js +++ b/app/static/js/ResourceLists/ResourceList.js @@ -16,6 +16,7 @@ class ResourceList { AdminUserList.autoInit(); CorpusFollowerList.autoInit(); CorpusTextInfoList.autoInit(); + CorpusTokenList.autoInit(); } static defaultOptions = { diff --git a/app/templates/_scripts.html.j2 b/app/templates/_scripts.html.j2 index 772a0596..dcf37513 100644 --- a/app/templates/_scripts.html.j2 +++ b/app/templates/_scripts.html.j2 @@ -70,7 +70,8 @@ 'js/ResourceLists/AdminUserList.js', 'js/ResourceLists/CorpusFollowerList.js', 'js/ResourceLists/CorpusTextInfoList.js', - 'js/ResourceLists/DetailledPublicCorpusList.js' + 'js/ResourceLists/DetailledPublicCorpusList.js', + 'js/ResourceLists/CorpusTokenList.js' %} {%- endassets %} diff --git a/app/templates/corpora/_analysis/static_visualization.html.j2 b/app/templates/corpora/_analysis/static_visualization.html.j2 index 3e6cfcff..70a0b605 100644 --- a/app/templates/corpora/_analysis/static_visualization.html.j2 +++ b/app/templates/corpora/_analysis/static_visualization.html.j2 @@ -63,7 +63,18 @@
-
+
+
+
+ Proportions +

of texts within the corpus

+
+ incomplete_circle + sort +
+
+
+
Text Information Overview @@ -74,46 +85,39 @@
-
-
-
- Proportions -

of texts within the corpus

-
-
-
-
-
+
Frequencies - -

within the texts of the 5 most frequent words in the corpus

-
- Wordarrow_drop_down - equalizer - show_chart - bubble_chart - settings +
+ {#
#} +
+
+ Wordarrow_drop_down + + settings + + +
+ {#
#} +
+
+ stacked_bar_chart + show_chart + bubble_chart +
+
-
-
-
-
- Text Bounds -
-
-
-
+
{% endset %} diff --git a/app/templates/corpora/analysis.html.j2 b/app/templates/corpora/analysis.html.j2 index 6d7e9622..ee39be59 100644 --- a/app/templates/corpora/analysis.html.j2 +++ b/app/templates/corpora/analysis.html.j2 @@ -36,14 +36,18 @@ {% endif %} {% endfor %} + {{ static_visualization_extension.container_content }} +
{% for extension in extensions %} +{% if extension.name != 'Static Visualization'%}
{{ extension.container_content }}
+{% endif %} {% endfor %} {% endblock page_content %} @@ -66,328 +70,6 @@ {{ extension.modals }} {% endfor %} - {% endblock modals %} {% block scripts %}