Add parallel fs changes to migration scripts and cleanup

2025-11-04 04:12:45 +00:00 · 2022-10-26 10:34:07 +02:00
parent cbf3abb424
commit 0b656d3cf1
8 changed files with 223 additions and 210 deletions
--- a/migrations/versions/116b4ab3ef9c_.py
+++ b/migrations/versions/116b4ab3ef9c_.py
@@ -17,8 +17,8 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('tokens',
+    op.create_table(
+        'tokens',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('access_token', sa.String(length=64), nullable=True),
@@ -30,12 +30,9 @@ def upgrade():
    )
    op.create_index(op.f('ix_tokens_access_token'), 'tokens', ['access_token'], unique=False)
    op.create_index(op.f('ix_tokens_refresh_token'), 'tokens', ['refresh_token'], unique=False)
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_tokens_refresh_token'), table_name='tokens')
    op.drop_index(op.f('ix_tokens_access_token'), table_name='tokens')
    op.drop_table('tokens')
-    # ### end Alembic commands ###
--- a/migrations/versions/260b57d5f4e7_.py
+++ b/migrations/versions/260b57d5f4e7_.py
@@ -17,12 +17,10 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('transkribus_htr_models', 'transkribus_name')
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('transkribus_htr_models', sa.Column('transkribus_name', sa.VARCHAR(length=64), autoincrement=False, nullable=True))
-    # ### end Alembic commands ###
+    op.add_column('transkribus_htr_models',
+        sa.Column('transkribus_name', sa.String(length=64), autoincrement=False, nullable=True)
+    )
--- a/migrations/versions/2c4e27331ccb_.py
+++ b/migrations/versions/2c4e27331ccb_.py
@@ -17,13 +17,12 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('corpora', sa.Column('is_public', sa.Boolean(), nullable=True))
+    op.add_column(
+        'corpora',
+        sa.Column('is_public', sa.Boolean(), nullable=True)
+    )
    op.execute('UPDATE corpora SET is_public = false;')
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('corpora', 'is_public')
-    # ### end Alembic commands ###
--- a/migrations/versions/31dd42e5ea6f_.py
+++ b/migrations/versions/31dd42e5ea6f_.py
@@ -20,10 +20,19 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    for user in User.query.all():
-        os.mkdir(os.path.join(user.path, 'spacy_nlp_pipeline_models'))
-    op.create_table('spacy_nlp_pipeline_models',
+        spacy_nlp_pipeline_models_dir = os.path.join(user.path, 'spacy_nlp_pipeline_models')
+        if os.path.exists(spacy_nlp_pipeline_models_dir):
+            if not os.path.isdir(spacy_nlp_pipeline_models_dir):
+                raise OSError(f'Not a directory: {spacy_nlp_pipeline_models_dir}')
+            if not os.listdir(spacy_nlp_pipeline_models_dir):
+                raise OSError(f'Directory not empty: {spacy_nlp_pipeline_models_dir}')
+        else:
+            os.mkdir(spacy_nlp_pipeline_models_dir)
+
+
+    op.create_table(
+        'spacy_nlp_pipeline_models',
        sa.Column('creation_date', sa.DateTime(), nullable=True),
        sa.Column('filename', sa.String(length=255), nullable=True),
        sa.Column('last_edited_date', sa.DateTime(), nullable=True),
@@ -42,12 +51,13 @@ def upgrade():
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    for user in User.query.all():
-        shutil.rmtree(os.path.join(user.path, 'spacy_nlp_pipeline_models'))
+        spacy_nlp_pipeline_models_dir = os.path.join(user.path, 'spacy_nlp_pipeline_models')
+        if os.path.exists(spacy_nlp_pipeline_models_dir):
+            shutil.rmtree(spacy_nlp_pipeline_models_dir)
+
+
    op.drop_table('spacy_nlp_pipeline_models')
-    # ### end Alembic commands ###
--- a/migrations/versions/63b2cc26a01f_.py
+++ b/migrations/versions/63b2cc26a01f_.py
@@ -5,8 +5,8 @@ Revises: 260b57d5f4e7
 Create Date: 2022-10-11 14:32:13.227364

 """
+from genericpath import isdir
 from alembic import op
-from flask import current_app
 import os
 from app.models import User

@@ -18,40 +18,19 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    for user in User.query.all():
-        old_tesseract_ocr_pipeline_model_path = os.path.join(
-            user.path,
-            'tesseract_ocr_models'
-        )
-        new_tesseract_ocr_pipeline_model_path = os.path.join(
-            user.path,
-            'tesseract_ocr_pipeline_models'
-        )
-        os.rename(
-            old_tesseract_ocr_pipeline_model_path,
-            new_tesseract_ocr_pipeline_model_path
-        )
+        old_tesseract_ocr_pipeline_model_path = os.path.join(user.path, 'tesseract_ocr_models')
+        new_tesseract_ocr_pipeline_model_path = os.path.join(user.path, 'tesseract_ocr_pipeline_models')
+        os.rename(old_tesseract_ocr_pipeline_model_path, new_tesseract_ocr_pipeline_model_path)
    op.rename_table('tesseract_ocr_models', 'tesseract_ocr_pipeline_models')
    op.rename_table('transkribus_htr_models', 'transkribus_htr_pipeline_models')
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    for user in User.query.all():
-        old_tesseract_ocr_pipeline_model_path = os.path.join(
-            user.path,
-            'tesseract_ocr_models'
-        )
-        new_tesseract_ocr_pipeline_model_path = os.path.join(
-            user.path,
-            'tesseract_ocr_pipeline_models'
-        )
-        os.rename(
-            new_tesseract_ocr_pipeline_model_path,
-            old_tesseract_ocr_pipeline_model_path
-        )
+        old_tesseract_ocr_pipeline_model_path = os.path.join(user.path, 'tesseract_ocr_models')
+        new_tesseract_ocr_pipeline_model_path = os.path.join(user.path, 'tesseract_ocr_pipeline_models')
+        os.rename(old_tesseract_ocr_pipeline_model_path, new_tesseract_ocr_pipeline_model_path)
+        os.rename(new_tesseract_ocr_pipeline_model_path, old_tesseract_ocr_pipeline_model_path)
    op.rename_table('tesseract_ocr_pipeline_models', 'tesseract_ocr_models')
    op.rename_table('transkribus_htr_pipeline_models', 'transkribus_htr_models')
-    # ### end Alembic commands ###
--- a/migrations/versions/9e8d7d15d950_.py
+++ b/migrations/versions/9e8d7d15d950_.py
@@ -6,7 +6,10 @@ Create Date: 2022-04-22 09:38:49.527498

 """
 from alembic import op
+from flask import current_app
 import sqlalchemy as sa
+import os
+import shutil


 # revision identifiers, used by Alembic.
@@ -17,8 +20,18 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('roles',
+    users_dir = os.path.join(current_app.config['NOPAQUE_DATA_DIR'], 'users')
+    if os.path.exists(users_dir):
+        if not os.path.isdir(users_dir):
+            raise OSError(f'Not a directory: {users_dir}')
+        if not os.listdir(users_dir):
+            raise OSError(f'Directory not empty: {users_dir}')
+    else:
+        os.mkdir(users_dir)
+
+
+    op.create_table(
+        'roles',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('default', sa.Boolean(), nullable=True),
        sa.Column('name', sa.String(length=64), nullable=True),
@@ -27,7 +40,9 @@ def upgrade():
        sa.UniqueConstraint('name')
    )
    op.create_index(op.f('ix_roles_default'), 'roles', ['default'], unique=False)
-    op.create_table('users',
+
+    op.create_table(
+        'users',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('role_id', sa.Integer(), nullable=True),
        sa.Column('confirmed', sa.Boolean(), nullable=True),
@@ -46,7 +61,9 @@ def upgrade():
    op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True)
    op.create_index(op.f('ix_users_token'), 'users', ['token'], unique=True)
    op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True)
-    op.create_table('corpora',
+
+    op.create_table(
+        'corpora',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('creation_date', sa.DateTime(), nullable=True),
@@ -59,7 +76,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('jobs',
+
+    op.create_table(
+        'jobs',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('creation_date', sa.DateTime(), nullable=True),
@@ -73,7 +92,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('tesseract_ocr_models',
+
+    op.create_table(
+        'tesseract_ocr_models',
        sa.Column('creation_date', sa.DateTime(), nullable=True),
        sa.Column('filename', sa.String(length=255), nullable=True),
        sa.Column('last_edited_date', sa.DateTime(), nullable=True),
@@ -92,7 +113,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('transkribus_htr_models',
+
+    op.create_table(
+        'transkribus_htr_models',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=True),
        sa.Column('shared', sa.Boolean(), nullable=True),
@@ -101,7 +124,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('corpus_files',
+
+    op.create_table(
+        'corpus_files',
        sa.Column('creation_date', sa.DateTime(), nullable=True),
        sa.Column('filename', sa.String(length=255), nullable=True),
        sa.Column('last_edited_date', sa.DateTime(), nullable=True),
@@ -123,7 +148,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['corpus_id'], ['corpora.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('job_inputs',
+
+    op.create_table(
+        'job_inputs',
        sa.Column('creation_date', sa.DateTime(), nullable=True),
        sa.Column('filename', sa.String(length=255), nullable=True),
        sa.Column('last_edited_date', sa.DateTime(), nullable=True),
@@ -133,7 +160,9 @@ def upgrade():
        sa.ForeignKeyConstraint(['job_id'], ['jobs.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    op.create_table('job_results',
+
+    op.create_table(
+        'job_results',
        sa.Column('creation_date', sa.DateTime(), nullable=True),
        sa.Column('filename', sa.String(length=255), nullable=True),
        sa.Column('last_edited_date', sa.DateTime(), nullable=True),
@@ -144,11 +173,14 @@ def upgrade():
        sa.ForeignKeyConstraint(['job_id'], ['jobs.id'], ),
        sa.PrimaryKeyConstraint('id')
    )
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
+    users_dir = os.path.join(current_app.config['NOPAQUE_DATA_DIR'], 'users')
+    if os.path.exists(users_dir):
+        shutil.rmtree(users_dir)
+
+
    op.drop_table('job_results')
    op.drop_table('job_inputs')
    op.drop_table('corpus_files')
@@ -162,4 +194,3 @@ def downgrade():
    op.drop_table('users')
    op.drop_index(op.f('ix_roles_default'), table_name='roles')
    op.drop_table('roles')
-    # ### end Alembic commands ###
--- a/migrations/versions/a3b727e3ff71_.py
+++ b/migrations/versions/a3b727e3ff71_.py
@@ -17,19 +17,16 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('transkribus_htr_pipeline_models')
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('transkribus_htr_pipeline_models',
-    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
-    sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=True),
-    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=True),
-    sa.Column('transkribus_model_id', sa.INTEGER(), autoincrement=False, nullable=True),
+    op.create_table(
+        'transkribus_htr_pipeline_models',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('user_id', sa.Integer(), autoincrement=False, nullable=True),
+        sa.Column('shared', sa.Boolean(), autoincrement=False, nullable=True),
+        sa.Column('transkribus_model_id', sa.Integer(), autoincrement=False, nullable=True),
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='transkribus_htr_models_user_id_fkey'),
        sa.PrimaryKeyConstraint('id', name='transkribus_htr_models_pkey')
    )
-    # ### end Alembic commands ###
--- a/migrations/versions/f9070ff1fa4a_.py
+++ b/migrations/versions/f9070ff1fa4a_.py
@@ -16,16 +16,18 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('ix_users_token', table_name='users')
    op.drop_column('users', 'token')
    op.drop_column('users', 'token_expiration')
-    # ### end Alembic commands ###


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('users', sa.Column('token_expiration', sa.DateTime(), autoincrement=False, nullable=True))
-    op.add_column('users', sa.Column('token', sa.VARCHAR(length=32), autoincrement=False, nullable=True))
+    op.add_column(
+        'users',
+        sa.Column('token_expiration', sa.DateTime(), autoincrement=False, nullable=True)
+    )
+    op.add_column(
+        'users',
+        sa.Column('token', sa.String(length=32), autoincrement=False, nullable=True)
+    )
    op.create_index('ix_users_token', 'users', ['token'], unique=False)
-    # ### end Alembic commands ###