move blueprints in dedicated folder

app/blueprints/corpora/__init__.py

@@ -0,0 +1,19 @@
+from flask import Blueprint
+from flask_login import login_required
+
+
+bp = Blueprint('corpora', __name__)
+bp.cli.short_help = 'Corpus commands.'
+
+
+@bp.before_request
+@login_required
+def before_request():
+    '''
+    Ensures that the routes in this package can only be visited by users that
+    are logged in.
+    '''
+    pass
+
+
+from . import cli, files, followers, routes, json_routes

app/blueprints/corpora/cli.py

@@ -0,0 +1,34 @@
+from flask import current_app
+import shutil
+from app import db
+from app.models import Corpus, CorpusStatus
+from . import bp
+
+
+@bp.cli.command('reset')
+def reset():
+    ''' Reset built corpora. '''
+    status = [
+        CorpusStatus.QUEUED,
+        CorpusStatus.BUILDING,
+        CorpusStatus.BUILT,
+        CorpusStatus.STARTING_ANALYSIS_SESSION,
+        CorpusStatus.RUNNING_ANALYSIS_SESSION,
+        CorpusStatus.CANCELING_ANALYSIS_SESSION
+    ]
+    for corpus in [x for x in Corpus.query.all() if x.status in status]:
+        print(f'Resetting corpus {corpus}')
+        corpus_cwb_dir = corpus.path / 'cwb'
+        corpus_cwb_data_dir = corpus_cwb_dir / 'data'
+        corpus_cwb_registry_dir = corpus_cwb_dir / 'registry'
+        try:
+            shutil.rmtree(corpus.path / 'cwb', ignore_errors=True)
+            corpus_cwb_dir.mkdir()
+            corpus_cwb_data_dir.mkdir()
+            corpus_cwb_registry_dir.mkdir()
+        except OSError as e:
+            current_app.logger.error(e)
+            raise
+        corpus.status = CorpusStatus.UNPREPARED
+        corpus.num_analysis_sessions = 0
+    db.session.commit()

app/blueprints/corpora/cqi_over_sio/__init__.py

@@ -0,0 +1,206 @@
+from cqi import CQiClient
+from cqi.errors import CQiException
+from cqi.status import CQiStatus
+from docker.models.containers import Container
+from flask import current_app, session
+from flask_login import current_user
+from flask_socketio import Namespace
+from inspect import signature
+from threading import Lock
+from typing import Callable
+from app import db, docker_client, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus, CorpusStatus
+from . import extensions
+
+
+'''
+This package tunnels the Corpus Query interface (CQi) protocol through
+Socket.IO (SIO) by tunneling CQi API calls through an event called "exec".
+
+Basic concept:
+1. A client connects to the namespace.
+2. The client emits the "init" event and provides a corpus id for the corpus
+   that should be analysed in this session.
+     1.1 The analysis session counter of the corpus is incremented.
+     1.2 A CQiClient and a (Mutex) Lock belonging to it is created.
+     1.3 Wait until the CQP server is running.
+     1.4 Connect the CQiClient to the server.
+     1.5 Save the CQiClient, the Lock and the corpus id in the session for
+         subsequential use.
+3. The client emits "exec" events, within which it provides the name of a CQi
+   API function and the corresponding arguments.
+     3.1 The "exec" event handler will execute the function, make sure that
+         the result is serializable and returns the result back to the client.
+4. The client disconnects from the namespace
+     4.1 The analysis session counter of the corpus is decremented.
+     4.2 The CQiClient and (Mutex) Lock belonging to it are teared down.
+'''
+
+
+CQI_API_FUNCTION_NAMES: list[str] = [
+    'ask_feature_cl_2_3',
+    'ask_feature_cqi_1_0',
+    'ask_feature_cqp_2_3',
+    'cl_alg2cpos',
+    'cl_attribute_size',
+    'cl_cpos2alg',
+    'cl_cpos2id',
+    'cl_cpos2lbound',
+    'cl_cpos2rbound',
+    'cl_cpos2str',
+    'cl_cpos2struc',
+    'cl_drop_attribute',
+    'cl_id2cpos',
+    'cl_id2freq',
+    'cl_id2str',
+    'cl_idlist2cpos',
+    'cl_lexicon_size',
+    'cl_regex2id',
+    'cl_str2id',
+    'cl_struc2cpos',
+    'cl_struc2str',
+    'corpus_alignment_attributes',
+    'corpus_charset',
+    'corpus_drop_corpus',
+    'corpus_full_name',
+    'corpus_info',
+    'corpus_list_corpora',
+    'corpus_positional_attributes',
+    'corpus_properties',
+    'corpus_structural_attribute_has_values',
+    'corpus_structural_attributes',
+    'cqp_drop_subcorpus',
+    'cqp_dump_subcorpus',
+    'cqp_fdist_1',
+    'cqp_fdist_2',
+    'cqp_list_subcorpora',
+    'cqp_query',
+    'cqp_subcorpus_has_field',
+    'cqp_subcorpus_size',
+    'ctrl_bye',
+    'ctrl_connect',
+    'ctrl_last_general_error',
+    'ctrl_ping',
+    'ctrl_user_abort'
+]
+
+
+class CQiOverSocketIO(Namespace):
+    @socketio_login_required
+    def on_connect(self):
+        pass
+
+    @socketio_login_required
+    def on_init(self, db_corpus_hashid: str):
+        db_corpus_id: int = hashids.decode(db_corpus_hashid)
+        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
+        if db_corpus is None:
+            return {'code': 404, 'msg': 'Not Found'}
+        if not (db_corpus.user == current_user
+                or current_user.is_following_corpus(db_corpus)
+                or current_user.is_administrator):
+            return {'code': 403, 'msg': 'Forbidden'}
+        if db_corpus.status not in [
+            CorpusStatus.BUILT,
+            CorpusStatus.STARTING_ANALYSIS_SESSION,
+            CorpusStatus.RUNNING_ANALYSIS_SESSION,
+            CorpusStatus.CANCELING_ANALYSIS_SESSION
+        ]:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if db_corpus.num_analysis_sessions is None:
+            db_corpus.num_analysis_sessions = 0
+            db.session.commit()
+        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1
+        db.session.commit()
+        retry_counter: int = 20
+        while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION:
+            if retry_counter == 0:
+                db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+                db.session.commit()
+                return {'code': 408, 'msg': 'Request Timeout'}
+            socketio.sleep(3)
+            retry_counter -= 1
+            db.session.refresh(db_corpus)
+        # cqi_client: CQiClient = CQiClient(f'cqpserver_{db_corpus_id}')
+        cqpserver_container_name: str = f'cqpserver_{db_corpus_id}'
+        cqpserver_container: Container = docker_client.containers.get(cqpserver_container_name)
+        cqpserver_host: str = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress']
+        cqi_client: CQiClient = CQiClient(cqpserver_host)
+        session['cqi_over_sio'] = {
+            'cqi_client': cqi_client,
+            'cqi_client_lock': Lock(),
+            'db_corpus_id': db_corpus_id
+        }
+        return {'code': 200, 'msg': 'OK'}
+
+    @socketio_login_required
+    def on_exec(self, fn_name: str, fn_args: dict = {}):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+        except KeyError:
+            return {'code': 424, 'msg': 'Failed Dependency'}
+        if fn_name in CQI_API_FUNCTION_NAMES:
+            fn: Callable = getattr(cqi_client.api, fn_name)
+        elif fn_name in extensions.CQI_EXTENSION_FUNCTION_NAMES:
+            fn: Callable = getattr(extensions, fn_name)
+        else:
+            return {'code': 400, 'msg': 'Bad Request'}
+        for param in signature(fn).parameters.values():
+            if param.default is param.empty:
+                if param.name not in fn_args:
+                    return {'code': 400, 'msg': 'Bad Request'}
+            else:
+                if param.name not in fn_args:
+                    continue
+            if type(fn_args[param.name]) is not param.annotation:
+                return {'code': 400, 'msg': 'Bad Request'}
+        cqi_client_lock.acquire()
+        try:
+            fn_return_value = fn(**fn_args)
+        except BrokenPipeError as e:
+            return {'code': 500, 'msg': 'Internal Server Error'}
+        except CQiException as e:
+            return {
+                'code': 502,
+                'msg': 'Bad Gateway',
+                'payload': {
+                    'code': e.code,
+                    'desc': e.description,
+                    'msg': e.__class__.__name__
+                }
+            }
+        finally:
+            cqi_client_lock.release()
+        if isinstance(fn_return_value, CQiStatus):
+            payload = {
+                'code': fn_return_value.code,
+                'msg': fn_return_value.__class__.__name__
+            }
+        else:
+            payload = fn_return_value
+        return {'code': 200, 'msg': 'OK', 'payload': payload}
+
+    def on_disconnect(self):
+        try:
+            cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+            cqi_client_lock: Lock = session['cqi_over_sio']['cqi_client_lock']
+            db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+        except KeyError:
+            return
+        cqi_client_lock.acquire()
+        try:
+            session.pop('cqi_over_sio')
+        except KeyError:
+            pass
+        try:
+            cqi_client.api.ctrl_bye()
+        except (BrokenPipeError, CQiException):
+            pass
+        cqi_client_lock.release()
+        db_corpus: Corpus | None = Corpus.query.get(db_corpus_id)
+        if db_corpus is None:
+            return
+        db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1
+        db.session.commit()

app/blueprints/corpora/cqi_over_sio/extensions.py

@@ -0,0 +1,286 @@
+from collections import Counter
+from cqi import CQiClient
+from cqi.models.corpora import Corpus as CQiCorpus
+from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
+from cqi.models.attributes import (
+    PositionalAttribute as CQiPositionalAttribute,
+    StructuralAttribute as CQiStructuralAttribute
+)
+from cqi.status import StatusOk as CQiStatusOk
+from flask import session
+import gzip
+import json
+import math
+from app import db
+from app.models import Corpus
+from .utils import lookups_by_cpos, partial_export_subcorpus, export_subcorpus
+
+
+CQI_EXTENSION_FUNCTION_NAMES: list[str] = [
+    'ext_corpus_update_db',
+    'ext_corpus_static_data',
+    'ext_corpus_paginate_corpus',
+    'ext_cqp_paginate_subcorpus',
+    'ext_cqp_partial_export_subcorpus',
+    'ext_cqp_export_subcorpus',
+]
+
+
+def ext_corpus_update_db(corpus: str) -> CQiStatusOk:
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    db_corpus.num_tokens = cqi_corpus.size
+    db.session.commit()
+    return CQiStatusOk()
+
+
+def ext_corpus_static_data(corpus: str) -> dict:
+    db_corpus_id: int = session['cqi_over_sio']['db_corpus_id']
+    db_corpus: Corpus = Corpus.query.get(db_corpus_id)
+
+    static_data_file_path = db_corpus.path / 'cwb' / 'static.json.gz'
+    if static_data_file_path.exists():
+        with static_data_file_path.open('rb') as f:
+            return f.read()
+
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus)
+    cqi_p_attrs: list[CQiPositionalAttribute] = cqi_corpus.positional_attributes.list()
+    cqi_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list()
+
+    static_data = {
+        'corpus': {
+            'bounds': [0, cqi_corpus.size - 1],
+            'freqs': {}
+        },
+        'p_attrs': {},
+        's_attrs': {},
+        'values': {'p_attrs': {}, 's_attrs': {}}
+    }
+
+    for p_attr in cqi_p_attrs:
+        print(f'corpus.freqs.{p_attr.name}')
+        static_data['corpus']['freqs'][p_attr.name] = []
+        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        static_data['corpus']['freqs'][p_attr.name].extend(p_attr.freqs_by_ids(p_attr_id_list))
+        del p_attr_id_list
+
+        print(f'p_attrs.{p_attr.name}')
+        static_data['p_attrs'][p_attr.name] = []
+        cpos_list: list[int] = list(range(cqi_corpus.size))
+        static_data['p_attrs'][p_attr.name].extend(p_attr.ids_by_cpos(cpos_list))
+        del cpos_list
+
+        print(f'values.p_attrs.{p_attr.name}')
+        static_data['values']['p_attrs'][p_attr.name] = []
+        p_attr_id_list: list[int] = list(range(p_attr.lexicon_size))
+        static_data['values']['p_attrs'][p_attr.name].extend(p_attr.values_by_ids(p_attr_id_list))
+        del p_attr_id_list
+
+    for s_attr in cqi_s_attrs:
+        if s_attr.has_values:
+            continue
+
+        static_data['s_attrs'][s_attr.name] = {'lexicon': [], 'values': None}
+
+        if s_attr.name in ['s', 'ent']:
+            ##############################################################
+            # A faster way to get cpos boundaries for smaller s_attrs    #
+            # Note: Needs more testing, don't use it in production       #
+            ##############################################################
+            cqi_corpus.query('Last', f'<{s_attr.name}> []* </{s_attr.name}>;')
+            cqi_subcorpus: CQiSubcorpus = cqi_corpus.subcorpora.get('Last')
+            first_match: int = 0
+            last_match: int = cqi_subcorpus.size - 1
+            match_boundaries = zip(
+                range(first_match, last_match + 1),
+                cqi_subcorpus.dump(
+                    cqi_subcorpus.fields['match'],
+                    first_match,
+                    last_match
+                ),
+                cqi_subcorpus.dump(
+                    cqi_subcorpus.fields['matchend'],
+                    first_match,
+                    last_match
+                )
+            )
+            cqi_subcorpus.drop()
+            del cqi_subcorpus, first_match, last_match
+            for id, lbound, rbound in match_boundaries:
+                static_data['s_attrs'][s_attr.name]['lexicon'].append({})
+                print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+                static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
+            del match_boundaries
+
+        if s_attr.name != 'text':
+            continue
+
+        for id in range(0, s_attr.size):
+            static_data['s_attrs'][s_attr.name]['lexicon'].append({})
+            # This is a very slow operation, thats why we only use it for
+            # the text attribute
+            lbound, rbound = s_attr.cpos_by_id(id)
+            print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds')
+            static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound]
+            static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {}
+            cpos_list: list[int] = list(range(lbound, rbound + 1))
+            for p_attr in cqi_p_attrs:
+                p_attr_ids: list[int] = []
+                p_attr_ids.extend(p_attr.ids_by_cpos(cpos_list))
+                print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}')
+                static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids))
+                del p_attr_ids
+            del cpos_list
+
+        sub_s_attrs: list[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr})
+        print(f's_attrs.{s_attr.name}.values')
+        static_data['s_attrs'][s_attr.name]['values'] = [
+            sub_s_attr.name[(len(s_attr.name) + 1):]
+            for sub_s_attr in sub_s_attrs
+        ]
+        s_attr_id_list: list[int] = list(range(s_attr.size))
+        sub_s_attr_values: list[str] = []
+        for sub_s_attr in sub_s_attrs:
+            tmp = []
+            tmp.extend(sub_s_attr.values_by_ids(s_attr_id_list))
+            sub_s_attr_values.append(tmp)
+            del tmp
+        del s_attr_id_list
+        print(f'values.s_attrs.{s_attr.name}')
+        static_data['values']['s_attrs'][s_attr.name] = [
+            {
+                s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id]
+                for s_attr_value_name_idx, s_attr_value_name in enumerate(
+                    static_data['s_attrs'][s_attr.name]['values']
+                )
+            } for s_attr_id in range(0, s_attr.size)
+        ]
+        del sub_s_attr_values
+    print('Saving static data to file')
+    with gzip.open(static_data_file_path, 'wt') as f:
+        json.dump(static_data, f)
+    del static_data
+    print('Sending static data to client')
+    with open(static_data_file_path, 'rb') as f:
+        return f.read()
+
+
+def ext_corpus_paginate_corpus(
+    corpus: str,
+    page: int = 1,
+    per_page: int = 20
+) -> dict:
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_corpus.size > 0
+            and page > math.ceil(cqi_corpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    first_cpos = (page - 1) * per_page
+    last_cpos = min(cqi_corpus.size, first_cpos + per_page)
+    cpos_list = [*range(first_cpos, last_cpos)]
+    lookups = lookups_by_cpos(cqi_corpus, cpos_list)
+    payload = {}
+    # the items for the current page
+    payload['items'] = [cpos_list]
+    # the lookups for the items
+    payload['lookups'] = lookups
+    # the total number of items matching the query
+    payload['total'] = cqi_corpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_paginate_subcorpus(
+    subcorpus: str,
+    context: int = 50,
+    page: int = 1,
+    per_page: int = 20
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    # Sanity checks
+    if (
+        per_page < 1
+        or page < 1
+        or (
+            cqi_subcorpus.size > 0
+            and page > math.ceil(cqi_subcorpus.size / per_page)
+        )
+    ):
+        return {'code': 416, 'msg': 'Range Not Satisfiable'}
+    offset = (page - 1) * per_page
+    cutoff = per_page
+    cqi_results_export = export_subcorpus(
+        cqi_subcorpus, context=context, cutoff=cutoff, offset=offset)
+    payload = {}
+    # the items for the current page
+    payload['items'] = cqi_results_export.pop('matches')
+    # the lookups for the items
+    payload['lookups'] = cqi_results_export
+    # the total number of items matching the query
+    payload['total'] = cqi_subcorpus.size
+    # the number of items to be displayed on a page.
+    payload['per_page'] = per_page
+    # The total number of pages
+    payload['pages'] = math.ceil(payload['total'] / payload['per_page'])
+    # the current page number (1 indexed)
+    payload['page'] = page if payload['pages'] > 0 else None
+    # True if a previous page exists
+    payload['has_prev'] = payload['page'] > 1 if payload['page'] else False
+    # True if a next page exists.
+    payload['has_next'] = payload['page'] < payload['pages'] if payload['page'] else False  # noqa
+    # Number of the previous page.
+    payload['prev_num'] = payload['page'] - 1 if payload['has_prev'] else None
+    # Number of the next page
+    payload['next_num'] = payload['page'] + 1 if payload['has_next'] else None
+    return payload
+
+
+def ext_cqp_partial_export_subcorpus(
+    subcorpus: str,
+    match_id_list: list,
+    context: int = 50
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_partial_export = partial_export_subcorpus(cqi_subcorpus, match_id_list, context=context)
+    return cqi_subcorpus_partial_export
+
+
+def ext_cqp_export_subcorpus(
+    subcorpus: str,
+    context: int = 50
+) -> dict:
+    corpus_name, subcorpus_name = subcorpus.split(':', 1)
+    cqi_client: CQiClient = session['cqi_over_sio']['cqi_client']
+    cqi_corpus = cqi_client.corpora.get(corpus_name)
+    cqi_subcorpus = cqi_corpus.subcorpora.get(subcorpus_name)
+    cqi_subcorpus_export = export_subcorpus(cqi_subcorpus, context=context)
+    return cqi_subcorpus_export

app/blueprints/corpora/cqi_over_sio/utils.py

@@ -0,0 +1,130 @@
+from cqi.models.corpora import Corpus as CQiCorpus
+from cqi.models.subcorpora import Subcorpus as CQiSubcorpus
+
+
+def lookups_by_cpos(corpus: CQiCorpus, cpos_list: list[int]) -> dict:
+    lookups = {}
+    lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list}
+    for attr in corpus.positional_attributes.list():
+        cpos_attr_values: list[str] = attr.values_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i]
+    for attr in corpus.structural_attributes.list():
+        # We only want to iterate over non subattributes, identifiable by
+        # attr.has_values == False
+        if attr.has_values:
+            continue
+        cpos_attr_ids: list[int] = attr.ids_by_cpos(cpos_list)
+        for i, cpos in enumerate(cpos_list):
+            if cpos_attr_ids[i] == -1:
+                continue
+            lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i]
+        occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1]
+        if len(occured_attr_ids) == 0:
+            continue
+        subattrs = corpus.structural_attributes.list(filters={'part_of': attr})
+        if len(subattrs) == 0:
+            continue
+        lookup_name: str = f'{attr.name}_lookup'
+        lookups[lookup_name] = {}
+        for attr_id in occured_attr_ids:
+            lookups[lookup_name][attr_id] = {}
+        for subattr in subattrs:
+            subattr_name = subattr.name[(len(attr.name) + 1):]  # noqa
+            for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)):  # noqa
+                lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value  # noqa
+    return lookups
+
+
+def partial_export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    match_id_list: list[int],
+    context: int = 25
+) -> dict:
+    if subcorpus.size == 0:
+        return {"matches": []}
+    match_boundaries = []
+    for match_id in match_id_list:
+        if match_id < 0 or match_id >= subcorpus.size:
+            continue
+        match_boundaries.append(
+            (
+                match_id,
+                subcorpus.dump(subcorpus.fields['match'], match_id, match_id)[0],
+                subcorpus.dump(subcorpus.fields['matchend'], match_id, match_id)[0]
+            )
+        )
+    cpos_set = set()
+    matches = []
+    for match_boundary in match_boundaries:
+        match_num, match_start, match_end = match_boundary
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}
+
+
+def export_subcorpus(
+    subcorpus: CQiSubcorpus,
+    context: int = 25,
+    cutoff: float = float('inf'),
+    offset: int = 0
+) -> dict:
+    if subcorpus.size == 0:
+        return {"matches": []}
+    first_match = max(0, offset)
+    last_match = min((offset + cutoff - 1), (subcorpus.size - 1))
+    match_boundaries = zip(
+        range(first_match, last_match + 1),
+        subcorpus.dump(subcorpus.fields['match'], first_match, last_match),
+        subcorpus.dump(subcorpus.fields['matchend'], first_match, last_match)
+    )
+    cpos_set = set()
+    matches = []
+    for match_num, match_start, match_end in match_boundaries:
+        c = (match_start, match_end)
+        if match_start == 0 or context == 0:
+            lc = None
+            cpos_list_lbound = match_start
+        else:
+            lc_lbound = max(0, (match_start - context))
+            lc_rbound = match_start - 1
+            lc = (lc_lbound, lc_rbound)
+            cpos_list_lbound = lc_lbound
+        if match_end == (subcorpus.collection.corpus.size - 1) or context == 0:
+            rc = None
+            cpos_list_rbound = match_end
+        else:
+            rc_lbound = match_end + 1
+            rc_rbound = min(
+                (match_end + context),
+                (subcorpus.collection.corpus.size - 1)
+            )
+            rc = (rc_lbound, rc_rbound)
+            cpos_list_rbound = rc_rbound
+        match = {'num': match_num, 'lc': lc, 'c': c, 'rc': rc}
+        matches.append(match)
+        cpos_set.update(range(cpos_list_lbound, cpos_list_rbound + 1))
+    lookups = lookups_by_cpos(subcorpus.collection.corpus, list(cpos_set))
+    return {'matches': matches, **lookups}

app/blueprints/corpora/decorators.py

@@ -0,0 +1,33 @@
+from flask import abort
+from flask_login import current_user
+from functools import wraps
+from app.models import Corpus, CorpusFollowerAssociation
+
+
+def corpus_follower_permission_required(*permissions):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            corpus_id = kwargs.get('corpus_id')
+            corpus = Corpus.query.get_or_404(corpus_id)
+            if not (corpus.user == current_user or current_user.is_administrator):
+                cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+                if cfa is None:
+                    abort(403)
+                if not all([cfa.role.has_permission(p) for p in permissions]):
+                    abort(403)
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator
+
+
+def corpus_owner_or_admin_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        corpus_id = kwargs.get('corpus_id')
+        corpus = Corpus.query.get_or_404(corpus_id)
+        if not (corpus.user == current_user or current_user.is_administrator):
+            abort(403)
+        return f(*args, **kwargs)
+    return decorated_function
+

app/blueprints/corpora/events.py

@@ -0,0 +1,45 @@
+from flask_login import current_user
+from flask_socketio import join_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus
+
+
+@socketio.on('GET /corpora/<corpus_id>')
+@socketio_login_required
+def get_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    return {
+        'body': corpus.to_json_serializable(),
+        'options': {
+            'status': 200,
+            'statusText': 'OK',
+            'headers': {'Content-Type: application/json'}
+        }
+    }
+
+
+@socketio.on('SUBSCRIBE /corpora/<corpus_id>')
+@socketio_login_required
+def subscribe_corpus(corpus_hashid):
+    corpus_id = hashids.decode(corpus_hashid)
+    corpus = Corpus.query.get(corpus_id)
+    if corpus is None:
+        return {'options': {'status': 404, 'statusText': 'Not found'}}
+    if not (
+        corpus.is_public
+        or corpus.user == current_user
+        or current_user.is_administrator
+    ):
+        return {'options': {'status': 403, 'statusText': 'Forbidden'}}
+    join_room(f'/corpora/{corpus.hashid}')
+    return {'options': {'status': 200, 'statusText': 'OK'}}

app/blueprints/corpora/files/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes, routes

app/blueprints/corpora/files/forms.py

@@ -0,0 +1,54 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired
+from wtforms import (
+    StringField,
+    SubmitField,
+    ValidationError,
+    IntegerField
+)
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusFileBaseForm(FlaskForm):
+    author = StringField(
+        'Author',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    publishing_year = IntegerField(
+        'Publishing year',
+        validators=[InputRequired()]
+    )
+    title = StringField(
+        'Title',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    address = StringField('Adress', validators=[Length(max=255)])
+    booktitle = StringField('Booktitle', validators=[Length(max=255)])
+    chapter = StringField('Chapter', validators=[Length(max=255)])
+    editor = StringField('Editor', validators=[Length(max=255)])
+    institution = StringField('Institution', validators=[Length(max=255)])
+    journal = StringField('Journal', validators=[Length(max=255)])
+    pages = StringField('Pages', validators=[Length(max=255)])
+    publisher = StringField('Publisher', validators=[Length(max=255)])
+    school = StringField('School', validators=[Length(max=255)])
+    submit = SubmitField()
+
+
+class CreateCorpusFileForm(CorpusFileBaseForm):
+    vrt = FileField('File', validators=[FileRequired()])
+
+    def validate_vrt(self, field):
+        if not field.data.filename.lower().endswith('.vrt'):
+            raise ValidationError('VRT files only!')
+
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-file-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusFileForm(CorpusFileBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-file-form'
+        super().__init__(*args, **kwargs)

app/blueprints/corpora/files/json_routes.py

@@ -0,0 +1,30 @@
+from flask import current_app
+from threading import Thread
+from app.decorators import content_negotiation
+from app import db
+from app.models import CorpusFile
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def delete_corpus_file(corpus_id, corpus_file_id):
+    def _delete_corpus_file(app, corpus_file_id):
+        with app.app_context():
+            corpus_file = CorpusFile.query.get(corpus_file_id)
+            corpus_file.delete()
+            db.session.commit()
+
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    thread = Thread(
+        target=_delete_corpus_file,
+        args=(current_app._get_current_object(), corpus_file.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus File "{corpus_file.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 202

app/blueprints/corpora/files/routes.py

@@ -0,0 +1,93 @@
+from flask import (
+    abort,
+    flash,
+    redirect,
+    render_template,
+    send_from_directory,
+    url_for
+)
+from app import db
+from app.models import Corpus, CorpusFile, CorpusStatus
+from ..decorators import corpus_follower_permission_required
+from . import bp
+from .forms import CreateCorpusFileForm, UpdateCorpusFileForm
+
+
+@bp.route('/<hashid:corpus_id>/files')
+def corpus_files(corpus_id):
+    return redirect(url_for('.corpus', _anchor='files', corpus_id=corpus_id))
+
+
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def create_corpus_file(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    form = CreateCorpusFileForm()
+    if form.is_submitted():
+        if not form.validate():
+            response = {'errors': form.errors}
+            return response, 400
+        try:
+            corpus_file = CorpusFile.create(
+                form.vrt.data,
+                address=form.address.data,
+                author=form.author.data,
+                booktitle=form.booktitle.data,
+                chapter=form.chapter.data,
+                editor=form.editor.data,
+                institution=form.institution.data,
+                journal=form.journal.data,
+                pages=form.pages.data,
+                publisher=form.publisher.data,
+                publishing_year=form.publishing_year.data,
+                school=form.school.data,
+                title=form.title.data,
+                mimetype='application/vrt+xml',
+                corpus=corpus
+            )
+        except (AttributeError, OSError):
+            abort(500)
+        corpus.status = CorpusStatus.UNPREPARED
+        db.session.commit()
+        flash(f'Corpus File "{corpus_file.filename}" added', category='corpus')
+        return '', 201, {'Location': corpus.url}
+    return render_template(
+        'corpora/files/create.html.j2',
+        title='Add corpus file',
+        form=form,
+        corpus=corpus
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+def corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    form = UpdateCorpusFileForm(data=corpus_file.to_json_serializeable())
+    if form.validate_on_submit():
+        form.populate_obj(corpus_file)
+        if db.session.is_modified(corpus_file):
+            corpus_file.corpus.status = CorpusStatus.UNPREPARED
+            db.session.commit()
+            flash(f'Corpus file "{corpus_file.filename}" updated', category='corpus')
+        return redirect(corpus_file.corpus.url)
+    return render_template(
+        'corpora/files/corpus_file.html.j2',
+        title='Edit corpus file',
+        form=form,
+        corpus=corpus_file.corpus,
+        corpus_file=corpus_file
+    )
+
+
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
+@corpus_follower_permission_required('VIEW')
+def download_corpus_file(corpus_id, corpus_file_id):
+    corpus_file = CorpusFile.query.filter_by(corpus_id=corpus_id, id=corpus_file_id).first_or_404()
+    return send_from_directory(
+        corpus_file.path.parent,
+        corpus_file.path.name,
+        as_attachment=True,
+        download_name=corpus_file.filename,
+        mimetype=corpus_file.mimetype
+    )

app/blueprints/corpora/followers/__init__.py

@@ -0,0 +1,2 @@
+from .. import bp
+from . import json_routes

app/blueprints/corpora/followers/json_routes.py

@@ -0,0 +1,76 @@
+from flask import abort, flash, jsonify, make_response, request
+from flask_login import current_user
+from app import db
+from app.decorators import content_negotiation
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from ..decorators import corpus_follower_permission_required
+from . import bp
+
+
+@bp.route('/<hashid:corpus_id>/followers', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def create_corpus_followers(corpus_id):
+    usernames = request.json
+    if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    for username in usernames:
+        user = User.query.filter_by(username=username, is_public=True).first_or_404()
+        user.follow_corpus(corpus)
+    db.session.commit()
+    response_data = {
+        'message': f'Users are now following "{corpus.title}"',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>/role', methods=['PUT'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_follower_role(corpus_id, follower_id):
+    role_name = request.json
+    if not isinstance(role_name, str):
+        abort(400)
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    cfa.role = cfr
+    db.session.commit()
+    response_data = {
+        'message': f'User "{cfa.follower.username}" is now {cfa.role.name}',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/followers/<hashid:follower_id>', methods=['DELETE'])
+def delete_corpus_follower(corpus_id, follower_id):
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404()
+    if not (
+        current_user.id == follower_id
+        or current_user == cfa.corpus.user 
+        or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS')
+        or current_user.is_administrator):
+        abort(403)
+    if current_user.id == follower_id:
+        flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus')
+        response = make_response()
+        response.status_code = 204
+    else:
+        response_data = {
+            'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore',
+            'category': 'corpus'
+        }
+        response = jsonify(response_data)
+        response.status_code = 200
+    cfa.follower.unfollow_corpus(cfa.corpus)
+    db.session.commit()
+    return response

app/blueprints/corpora/forms.py

@@ -0,0 +1,33 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, SubmitField, TextAreaField
+from wtforms.validators import InputRequired, Length
+
+
+class CorpusBaseForm(FlaskForm):
+    description = TextAreaField(
+        'Description',
+        validators=[InputRequired(), Length(max=255)]
+    )
+    title = StringField('Title', validators=[InputRequired(), Length(max=32)])
+    submit = SubmitField()
+
+
+class CreateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'create-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class UpdateCorpusForm(CorpusBaseForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'update-corpus-form'
+        super().__init__(*args, **kwargs)
+
+
+class ImportCorpusForm(FlaskForm):
+    def __init__(self, *args, **kwargs):
+        if 'prefix' not in kwargs:
+            kwargs['prefix'] = 'import-corpus-form'
+        super().__init__(*args, **kwargs)

app/blueprints/corpora/json_routes.py

@@ -0,0 +1,125 @@
+from datetime import datetime
+from flask import abort, current_app, request, url_for
+from flask_login import current_user
+from threading import Thread
+from app import db
+from app.decorators import content_negotiation
+from app.models import Corpus, CorpusFollowerRole
+from . import bp
+from .decorators import corpus_follower_permission_required, corpus_owner_or_admin_required
+import nltk
+from string import punctuation
+
+
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+@corpus_owner_or_admin_required
+@content_negotiation(produces='application/json')
+def delete_corpus(corpus_id):
+    def _delete_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.delete()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    thread = Thread(
+        target=_delete_corpus,
+        args=(current_app._get_current_object(), corpus.id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for deletion',
+        'category': 'corpus'
+    }
+    return response_data, 200
+
+
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FILES')
+@content_negotiation(produces='application/json')
+def build_corpus(corpus_id):
+    def _build_corpus(app, corpus_id):
+        with app.app_context():
+            corpus = Corpus.query.get(corpus_id)
+            corpus.build()
+            db.session.commit()
+
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if len(corpus.files.all()) == 0:
+        abort(409)
+    thread = Thread(
+        target=_build_corpus,
+        args=(current_app._get_current_object(), corpus_id)
+    )
+    thread.start()
+    response_data = {
+        'message': f'Corpus "{corpus.title}" marked for building',
+        'category': 'corpus'
+    }
+    return response_data, 202
+
+@bp.route('/stopwords')
+@content_negotiation(produces='application/json')
+def get_stopwords():
+    nltk.download('stopwords', quiet=True)
+    languages = ["german", "english", "catalan", "greek", "spanish", "french", "italian", "russian", "chinese"]
+    stopwords = {}
+    for language in languages:
+        stopwords[language] = nltk.corpus.stopwords.words(language)
+    stopwords['punctuation'] = list(punctuation) + ['—', '|', '–', '“', '„', '--']
+    stopwords['user_stopwords'] = []
+    response_data = stopwords
+    return response_data, 202
+
+@bp.route('/<hashid:corpus_id>/generate-share-link', methods=['POST'])
+@corpus_follower_permission_required('MANAGE_FOLLOWERS')
+@content_negotiation(consumes='application/json', produces='application/json')
+def generate_corpus_share_link(corpus_id):
+    data = request.json
+    if not isinstance(data, dict):
+        abort(400)
+    expiration = data.get('expiration')
+    if not isinstance(expiration, str):
+        abort(400)
+    role_name = data.get('role')
+    if not isinstance(role_name, str):
+        abort(400)
+    expiration_date = datetime.strptime(expiration, '%b %d, %Y')
+    cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
+    if cfr is None:
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date)
+    corpus_share_link = url_for(
+        'corpora.follow_corpus',
+        corpus_id=corpus_id,
+        token=token,
+        _external=True
+    )
+    response_data = {
+        'message': 'Corpus share link generated',
+        'category': 'corpus',
+        'corpusShareLink': corpus_share_link
+    }
+    return response_data, 200
+    
+
+
+@bp.route('/<hashid:corpus_id>/is_public', methods=['PUT'])
+@corpus_owner_or_admin_required
+@content_negotiation(consumes='application/json', produces='application/json')
+def update_corpus_is_public(corpus_id):
+    is_public = request.json
+    if not isinstance(is_public, bool):
+        abort(400)
+    corpus = Corpus.query.get_or_404(corpus_id)
+    corpus.is_public = is_public
+    db.session.commit()
+    response_data = {
+        'message': (
+            f'Corpus "{corpus.title}" is now'
+            f' {"public" if is_public else "private"}'
+        ),
+        'category': 'corpus'
+    }
+    return response_data, 200

app/blueprints/corpora/routes.py

@@ -0,0 +1,109 @@
+from flask import abort, flash, redirect, render_template, url_for
+from flask_login import current_user
+from app import db
+from app.models import (
+    Corpus,
+    CorpusFollowerAssociation,
+    CorpusFollowerRole,
+    User
+)
+from . import bp
+from .decorators import corpus_follower_permission_required
+from .forms import CreateCorpusForm
+
+
+@bp.route('')
+def corpora():
+    return redirect(url_for('main.dashboard', _anchor='corpora'))
+
+
+@bp.route('/create', methods=['GET', 'POST'])
+def create_corpus():
+    form = CreateCorpusForm()
+    if form.validate_on_submit():
+        try:
+            corpus = Corpus.create(
+                title=form.title.data,
+                description=form.description.data,
+                user=current_user
+            )
+        except OSError:
+            abort(500)
+        db.session.commit()
+        flash(f'Corpus "{corpus.title}" created', 'corpus')
+        return redirect(corpus.url)
+    return render_template(
+        'corpora/create.html.j2',
+        title='Create corpus',
+        form=form
+    )
+
+
+@bp.route('/<hashid:corpus_id>')
+def corpus(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    cfrs = CorpusFollowerRole.query.all()
+    # TODO: Better solution for filtering admin
+    users = User.query.filter(User.is_public == True, User.id != current_user.id, User.id != corpus.user.id, User.role_id < 4).all()
+    cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
+    if cfa is None:
+        if corpus.user == current_user or current_user.is_administrator:
+            cfr = CorpusFollowerRole.query.filter_by(name='Administrator').first()
+        else:
+            cfr = CorpusFollowerRole.query.filter_by(name='Anonymous').first()
+    else:
+        cfr = cfa.role
+    if corpus.user == current_user or current_user.is_administrator:
+        return render_template(
+            'corpora/corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfr=cfr,
+            cfrs=cfrs,
+            users=users
+        )
+    if (current_user.is_following_corpus(corpus) or corpus.is_public):
+        cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all()
+        return render_template(
+            'corpora/public_corpus.html.j2',
+            title=corpus.title,
+            corpus=corpus,
+            cfrs=cfrs,
+            cfr=cfr,
+            cfas=cfas,
+            users=users
+        )
+    abort(403)
+
+
+
+@bp.route('/<hashid:corpus_id>/analysis')
+@corpus_follower_permission_required('VIEW')
+def analysis(corpus_id):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    return render_template(
+        'corpora/analysis.html.j2',
+        corpus=corpus,
+        title=f'Analyse Corpus {corpus.title}'
+    )
+
+
+@bp.route('/<hashid:corpus_id>/follow/<token>')
+def follow_corpus(corpus_id, token):
+    corpus = Corpus.query.get_or_404(corpus_id)
+    if current_user.follow_corpus_by_token(token):
+        db.session.commit()
+        flash(f'You are following "{corpus.title}" now', category='corpus')
+        return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+    abort(403)
+
+
+@bp.route('/import', methods=['GET', 'POST'])
+def import_corpus():
+    abort(503)
+
+
+@bp.route('/<hashid:corpus_id>/export')
+@corpus_follower_permission_required('VIEW')
+def export_corpus(corpus_id):
+    abort(503)