sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-11-03 20:02:47 +00:00
Code Issues Projects Releases Wiki Activity

Use pathlib where possible

Browse Source
This commit is contained in:
Patrick Jentsch
2024-03-07 15:49:04 +01:00
parent ec23bd94ee
commit 9da74c1c6f
21 changed files with 164 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
app/converters/sandpaper.py
View File

@@ -2,80 +2,69 @@ from flask import current_app
from app import db
from app.models import User, Corpus, CorpusFile
from datetime import datetime
from pathlib import Path
from typing import Dict, List
import json
import os
import shutil
class SandpaperConverter:
def __init__(self, json_db_file, data_dir):
def __init__(self, json_db_file: Path, data_dir: Path):
self.json_db_file = json_db_file
self.data_dir = data_dir
def run(self):
with open(self.json_db_file, 'r') as f:
json_db = json.loads(f.read())
with self.json_db_file.open('r') as f:
json_db: List[Dict] = json.load(f)
for json_user in json_db:
if not json_user['confirmed']:
current_app.logger.info(f'Skip unconfirmed user {json_user["username"]}')
continue
user_dir = os.path.join(self.data_dir, str(json_user['id']))
user_dir = self.data_dir / f'{json_user["id"]}'
self.convert_user(json_user, user_dir)
db.session.commit()
def convert_user(self, json_user, user_dir):
def convert_user(self, json_user: Dict, user_dir: Path):
current_app.logger.info(f'Create User {json_user["username"]}...')
user = User(
confirmed=json_user['confirmed'],
email=json_user['email'],
last_seen=datetime.fromtimestamp(json_user['last_seen']),
member_since=datetime.fromtimestamp(json_user['member_since']),
password_hash=json_user['password_hash'], # TODO: Needs to be added manually
username=json_user['username']
)
db.session.add(user)
db.session.flush(objects=[user])
db.session.refresh(user)
try:
user.makedirs()
except OSError as e:
current_app.logger.error(e)
db.session.rollback()
user = User.create(
confirmed=json_user['confirmed'],
email=json_user['email'],
last_seen=datetime.fromtimestamp(json_user['last_seen']),
member_since=datetime.fromtimestamp(json_user['member_since']),
password_hash=json_user['password_hash'], # TODO: Needs to be added manually
username=json_user['username']
)
except OSError:
raise Exception('Internal Server Error')
for json_corpus in json_user['corpora'].values():
if not json_corpus['files'].values():
current_app.logger.info(f'Skip empty corpus {json_corpus["title"]}')
continue
corpus_dir = os.path.join(user_dir, 'corpora', str(json_corpus['id']))
corpus_dir = user_dir / 'corpora' / f'{json_corpus["id"]}'
self.convert_corpus(json_corpus, user, corpus_dir)
current_app.logger.info('Done')
def convert_corpus(self, json_corpus, user, corpus_dir):
def convert_corpus(self, json_corpus: Dict, user: User, corpus_dir: Path):
current_app.logger.info(f'Create Corpus {json_corpus["title"]}...')
corpus = Corpus(
user=user,
creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
description=json_corpus['description'],
title=json_corpus['title']
)
db.session.add(corpus)
db.session.flush(objects=[corpus])
db.session.refresh(corpus)
try:
corpus.makedirs()
except OSError as e:
current_app.logger.error(e)
db.session.rollback()
corpus = Corpus.create(
user=user,
creation_date=datetime.fromtimestamp(json_corpus['creation_date']),
description=json_corpus['description'],
title=json_corpus['title']
)
except OSError:
raise Exception('Internal Server Error')
for json_corpus_file in json_corpus['files'].values():
self.convert_corpus_file(json_corpus_file, corpus, corpus_dir)
current_app.logger.info('Done')
def convert_corpus_file(self, json_corpus_file, corpus, corpus_dir):
def convert_corpus_file(self, json_corpus_file: Dict, corpus: Corpus, corpus_dir: Path):
current_app.logger.info(f'Create CorpusFile {json_corpus_file["title"]}...')
corpus_file = CorpusFile(
corpus=corpus,
@@ -99,13 +88,13 @@ class SandpaperConverter:
db.session.refresh(corpus_file)
try:
shutil.copy2(
os.path.join(corpus_dir, json_corpus_file['filename']),
corpus_dir / json_corpus_file['filename'],
corpus_file.path
)
except:
current_app.logger.warning(
'Can not convert corpus file: '
f'{os.path.join(corpus_dir, json_corpus_file["filename"])}'
f'{corpus_dir / json_corpus_file["filename"]}'
' -> '
f'{corpus_file.path}'
)