app/__init__.py

@@ -13,7 +13,6 @@ from flask_paranoid import Paranoid
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 from flask_hashids import Hashids
-from werkzeug.exceptions import HTTPException
 
 
 apifairy = APIFairy()
@@ -36,7 +35,7 @@ socketio = SocketIO()
 
 def create_app(config: Config = Config) -> Flask:
     ''' Creates an initialized Flask (WSGI Application) object. '''
-    app = Flask(__name__)
+    app: Flask = Flask(__name__)
     app.config.from_object(config)
     config.init_app(app)
     docker_client.login(
@@ -58,6 +57,12 @@ def create_app(config: Config = Config) -> Flask:
     scheduler.init_app(app)
     socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])  # noqa
 
+    from .errors import init_app as init_error_handlers
+    init_error_handlers(app)
+
+    from .cli import init_app as init_cli
+    init_cli(app)
+
     from .admin import bp as admin_blueprint
     default_breadcrumb_root(admin_blueprint, '.admin')
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
@@ -75,10 +80,7 @@ def create_app(config: Config = Config) -> Flask:
 
     from .corpora import bp as corpora_blueprint
     default_breadcrumb_root(corpora_blueprint, '.corpora')
-    app.register_blueprint(corpora_blueprint, cli_group='corpus', url_prefix='/corpora')
+    app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
-
-    from .errors import bp as errors_bp
-    app.register_blueprint(errors_bp)
 
     from .jobs import bp as jobs_blueprint
     default_breadcrumb_root(jobs_blueprint, '.jobs')
@@ -86,7 +88,7 @@ def create_app(config: Config = Config) -> Flask:
 
     from .main import bp as main_blueprint
     default_breadcrumb_root(main_blueprint, '.')
-    app.register_blueprint(main_blueprint, cli_group=None)
+    app.register_blueprint(main_blueprint)
 
     from .services import bp as services_blueprint
     default_breadcrumb_root(services_blueprint, '.services')

app/cli/__init__.py

@@ -0,0 +1,10 @@
+from .converter import init_app as converter_init_app
+from .corpus import init_app as corpus_init_app
+from .main import init_app as main_init_app
+
+
+
+def init_app(app):
+    converter_init_app(app)
+    corpus_init_app(app)
+    main_init_app(app)

app/cli/converter.py

@@ -0,0 +1,21 @@
+import click
+
+
+def init_app(app):
+    @app.cli.group('converter')
+    def converter():
+        ''' Converter commands. '''
+        pass
+
+    @converter.group('sandpaper')
+    def sandpaper_converter():
+        ''' Sandpaper converter commands. '''
+        pass
+
+    @sandpaper_converter.command('run')
+    @click.argument('json_db')
+    @click.argument('data_dir')
+    def run_sandpaper_converter(json_db, data_dir):
+        ''' Run the sandpaper converter. '''
+        from app.converters.sandpaper import convert
+        convert(json_db, data_dir)

app/cli/corpus.py

@@ -0,0 +1,23 @@
+from app.models import Corpus, CorpusStatus
+
+
+def init_app(app):
+    @app.cli.group('corpus')
+    def corpus():
+        ''' Corpus commands. '''
+        pass
+
+    @corpus.command('dismantle')
+    def dismantle():
+        ''' Dismantle built corpora. '''
+        status = [
+            CorpusStatus.QUEUED,
+            CorpusStatus.BUILDING,
+            CorpusStatus.BUILT,
+            CorpusStatus.STARTING_ANALYSIS_SESSION,
+            CorpusStatus.RUNNING_ANALYSIS_SESSION,
+            CorpusStatus.CANCELING_ANALYSIS_SESSION
+        ]
+        for corpus in [x for x in Corpus.query.all() if x.status in status]:
+            corpus.status = CorpusStatus.SUBMITTED
+            corpus.num_analysis_sessions = 0

app/cli/main.py

@@ -0,0 +1,45 @@
+from flask import current_app
+from flask_migrate import upgrade
+import os
+from app.models import (
+    CorpusFollowerRole,
+    Role,
+    SpaCyNLPPipelineModel,
+    TesseractOCRPipelineModel,
+    User
+)
+
+
+def init_app(app):
+    @app.cli.command('deploy')
+    def deploy():
+        ''' Run deployment tasks. '''
+        # Make default directories
+        print('Make default directories')
+        base_dir = current_app.config['NOPAQUE_DATA_DIR']
+        default_dirs = [
+            os.path.join(base_dir, 'tmp'),
+            os.path.join(base_dir, 'users')
+        ]
+        for dir in default_dirs:
+            if os.path.exists(dir):
+                if not os.path.isdir(dir):
+                    raise NotADirectoryError(f'{dir} is not a directory')
+            else:
+                os.mkdir(dir)
+
+        # migrate database to latest revision
+        print('Migrate database to latest revision')
+        upgrade()
+
+        # Insert/Update default database values
+        print('Insert/Update default Roles')
+        Role.insert_defaults()
+        print('Insert/Update default Users')
+        User.insert_defaults()
+        print('Insert/Update default CorpusFollowerRoles')
+        CorpusFollowerRole.insert_defaults()
+        print('Insert/Update default SpaCyNLPPipelineModels')
+        SpaCyNLPPipelineModel.insert_defaults()
+        print('Insert/Update default TesseractOCRPipelineModels')
+        TesseractOCRPipelineModel.insert_defaults()

app/contributions/__init__.py

@@ -15,9 +15,7 @@ def before_request():
     pass
 
 
-from . import (
+from . import routes
-    routes,
+from . import spacy_nlp_pipeline_models
-    spacy_nlp_pipeline_models,
+from . import tesseract_ocr_pipeline_models
-    tesseract_ocr_pipeline_models,
+from . import transkribus_htr_pipeline_models
-    transkribus_htr_pipeline_models
-)

app/converters/cli.py

@@ -1,22 +0,0 @@
-import click
-from . import bp
-from .sandpaper import SandpaperConverter
-
-
-@bp.cli.group('converter')
-def converter():
-    ''' Converter commands. '''
-    pass
-
-@converter.group('sandpaper')
-def sandpaper_converter():
-    ''' Sandpaper converter commands. '''
-    pass
-
-@sandpaper_converter.command('run')
-@click.argument('json_db_file')
-@click.argument('data_dir')
-def run_sandpaper_converter(json_db_file, data_dir):
-    ''' Run the sandpaper converter. '''
-    sandpaper_converter = SandpaperConverter(json_db_file, data_dir)
-    sandpaper_converter.run()

app/converters/sandpaper.py

@@ -7,25 +7,20 @@ import os
 import shutil
 
 
-class SandpaperConverter:
+def convert(json_db_file, data_dir):
-    def __init__(self, json_db_file, data_dir):
+    with open(json_db_file, 'r') as f:
-        self.json_db_file = json_db_file
-        self.data_dir = data_dir
-
-    def run(self):
-        with open(self.json_db_file, 'r') as f:
         json_db = json.loads(f.read())
 
     for json_user in json_db:
         if not json_user['confirmed']:
             current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
             continue
-            user_dir = os.path.join(self.data_dir, str(json_user['id']))
+        user_dir = os.path.join(data_dir, str(json_user['id']))
-            self.convert_user(json_user, user_dir)
+        convert_user(json_user, user_dir)
         db.session.commit()
 
 
-    def convert_user(self, json_user, user_dir):
+def convert_user(json_user, user_dir):
     current_app.logger.info(f'Create User {json_user["username"]}...')
     user = User(
         confirmed=json_user['confirmed'],
@@ -49,11 +44,11 @@ class SandpaperConverter:
             current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
             continue
         corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
-            self.convert_corpus(json_corpus, user, corpus_dir)
+        convert_corpus(json_corpus, user, corpus_dir)
     current_app.logger.info('Done')
 
 
-    def convert_corpus(self, json_corpus, user, corpus_dir):
+def convert_corpus(json_corpus, user, corpus_dir):
     current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
     corpus = Corpus(
         user=user,
@@ -71,11 +66,11 @@ class SandpaperConverter:
         db.session.rollback()
         raise Exception('Internal Server Error')
     for json_corpus_file in json_corpus['files'].values():
-            self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
+        convert_corpus_file(json_corpus_file, corpus, corpus_dir)
     current_app.logger.info('Done')
 
 
-    def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
+def convert_corpus_file(json_corpus_file, corpus, corpus_dir):
     current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
     corpus_file = CorpusFile(
         corpus=corpus,

app/corpora/__init__.py

@@ -3,7 +3,6 @@ from flask_login import login_required
 
 
 bp = Blueprint('corpora', __name__)
-bp.cli.short_help = 'Corpus commands.'
 
 
 @bp.before_request
@@ -16,4 +15,6 @@ def before_request():
     pass
 
 
-from . import cli, cqi_over_socketio, files, followers, routes, json_routes
+from . import cqi_over_socketio, routes, json_routes
+from . import files
+from . import followers

app/corpora/cli.py

@@ -1,24 +0,0 @@
-from app.models import Corpus, CorpusStatus
-import os
-import shutil
-from app import db
-from . import bp
-
-
-@bp.cli.command('reset')
-def reset():
-    ''' Reset built corpora. '''
-    status = [
-        CorpusStatus.QUEUED,
-        CorpusStatus.BUILDING,
-        CorpusStatus.BUILT,
-        CorpusStatus.STARTING_ANALYSIS_SESSION,
-        CorpusStatus.RUNNING_ANALYSIS_SESSION,
-        CorpusStatus.CANCELING_ANALYSIS_SESSION
-    ]
-    for corpus in [x for x in Corpus.query.all() if x.status in status]:
-        print(f'Resetting corpus {corpus}')
-        shutil.rmtree(os.path.join(corpus.path, 'cwb'), ignore_errors=True)
-        corpus.status = CorpusStatus.UNPREPARED
-        corpus.num_analysis_sessions = 0
-    db.session.commit()

app/errors/__init__.py

@@ -1,5 +1,6 @@
-from flask import Blueprint
+from werkzeug.exceptions import HTTPException
+from .handlers import generic
 
 
-bp = Blueprint('errors', __name__)
+def init_app(app):
-from . import handlers
+    app.register_error_handler(HTTPException, generic)

app/errors/handlers.py

@@ -1,14 +1,13 @@
-from flask import jsonify, render_template, request
+from flask import jsonify, render_template, request, Response
 from werkzeug.exceptions import HTTPException
-from . import bp
+from typing import Tuple, Union
 
 
-@bp.app_errorhandler(HTTPException)
+def generic(error: HTTPException) -> Tuple[Union[str, Response], int]:
-def handle_http_exception(error):
+    ''' Generic error handler '''
-    ''' Generic HTTP exception handler '''
+    accent_json: bool = request.accept_mimetypes.accept_json
-    accept_json = request.accept_mimetypes.accept_json
+    accept_html: bool = request.accept_mimetypes.accept_html
-    accept_html = request.accept_mimetypes.accept_html
+    if accent_json and not accept_html:
-    if accept_json and not accept_html:
+        response: Response = jsonify(str(error))
-        response = jsonify(str(error))
         return response, error.code
     return render_template('errors/error.html.j2', error=error), error.code

app/main/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 
 bp = Blueprint('main', __name__, cli_group=None)
-from . import cli, routes
+from . import routes

app/main/cli.py

@@ -1,45 +0,0 @@
-from flask import current_app
-from flask_migrate import upgrade
-import os
-from app.models import (
-    CorpusFollowerRole,
-    Role,
-    SpaCyNLPPipelineModel,
-    TesseractOCRPipelineModel,
-    User
-)
-from . import bp
-
-
-@bp.cli.command('deploy')
-def deploy():
-    ''' Run deployment tasks. '''
-    # Make default directories
-    print('Make default directories')
-    base_dir = current_app.config['NOPAQUE_DATA_DIR']
-    default_dirs = [
-        os.path.join(base_dir, 'tmp'),
-        os.path.join(base_dir, 'users')
-    ]
-    for dir in default_dirs:
-        if os.path.exists(dir):
-            if not os.path.isdir(dir):
-                raise NotADirectoryError(f'{dir} is not a directory')
-        else:
-            os.mkdir(dir)
-
-    # migrate database to latest revision
-    print('Migrate database to latest revision')
-    upgrade()
-
-    # Insert/Update default database values
-    print('Insert/Update default Roles')
-    Role.insert_defaults()
-    print('Insert/Update default Users')
-    User.insert_defaults()
-    print('Insert/Update default CorpusFollowerRoles')
-    CorpusFollowerRole.insert_defaults()
-    print('Insert/Update default SpaCyNLPPipelineModels')
-    SpaCyNLPPipelineModel.insert_defaults()
-    print('Insert/Update default TesseractOCRPipelineModels')
-    TesseractOCRPipelineModel.insert_defaults()

app/tests/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('tests', __name__)
+from . import cli

app/users/__init__.py

@@ -15,4 +15,5 @@ def before_request():
     pass
 
 
-from . import events, json_routes, routes, settings
+from . import events, json_routes, routes
+from . import settings