mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2024-11-15 01:05:42 +00:00
Remove old scheduler code.
This commit is contained in:
parent
8f3c53dad2
commit
7d4a1e6d37
@ -1,6 +1,5 @@
|
|||||||
from config import config
|
from config import config
|
||||||
from flask import Flask
|
from flask import Flask
|
||||||
# from flask_apscheduler import APScheduler
|
|
||||||
from flask_login import LoginManager
|
from flask_login import LoginManager
|
||||||
from flask_mail import Mail
|
from flask_mail import Mail
|
||||||
from flask_socketio import SocketIO
|
from flask_socketio import SocketIO
|
||||||
@ -11,7 +10,6 @@ db = SQLAlchemy()
|
|||||||
login_manager = LoginManager()
|
login_manager = LoginManager()
|
||||||
login_manager.login_view = 'auth.login'
|
login_manager.login_view = 'auth.login'
|
||||||
mail = Mail()
|
mail = Mail()
|
||||||
# scheduler = APScheduler()
|
|
||||||
socketio = SocketIO()
|
socketio = SocketIO()
|
||||||
|
|
||||||
|
|
||||||
@ -27,8 +25,6 @@ def create_app(config_name, main=True):
|
|||||||
|
|
||||||
login_manager.init_app(app)
|
login_manager.init_app(app)
|
||||||
mail.init_app(app)
|
mail.init_app(app)
|
||||||
# scheduler.init_app(app)
|
|
||||||
# scheduler.start()
|
|
||||||
socketio.init_app(app, message_qeue='redis://')
|
socketio.init_app(app, message_qeue='redis://')
|
||||||
|
|
||||||
from .auth import auth as auth_blueprint
|
from .auth import auth as auth_blueprint
|
||||||
|
@ -1,82 +0,0 @@
|
|||||||
from datetime import datetime
|
|
||||||
from . import db, scheduler
|
|
||||||
from .models import Job
|
|
||||||
import docker
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
def checkout_jobs():
|
|
||||||
with scheduler.app.app_context():
|
|
||||||
client = docker.from_env()
|
|
||||||
jobs = db.session.query(Job)
|
|
||||||
for job in jobs.filter_by(status='submitted').all():
|
|
||||||
_command = (job.service
|
|
||||||
+ ' -i /files'
|
|
||||||
+ ' -o /files/output'
|
|
||||||
+ ' ' + ' '.join(json.loads(job.service_args)))
|
|
||||||
_constraints = ['node.role==worker']
|
|
||||||
_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
|
|
||||||
job.service,
|
|
||||||
job.service_version
|
|
||||||
)
|
|
||||||
_labels = {'service': job.service}
|
|
||||||
_mounts = [os.path.join('/home/compute/mnt/opaque',
|
|
||||||
str(job.user_id),
|
|
||||||
'jobs',
|
|
||||||
str(job.id))
|
|
||||||
+ ':/files:rw']
|
|
||||||
_name = str(job.id)
|
|
||||||
'''
|
|
||||||
' The Docker SDK for Python expects the cpu_reservation value to be
|
|
||||||
' scaled to nanos (10^9). Because the job object contains unscaled
|
|
||||||
' (10^0) values, it must be conveted.
|
|
||||||
'
|
|
||||||
' While the cpu_reservation value has to be in nanos, the
|
|
||||||
' mem_reservation value must be presented in an unscaled form
|
|
||||||
' (intuitive right?). Bacause the job object provides the memory
|
|
||||||
' value in megabytes, it is also necessary to convert the value.
|
|
||||||
'''
|
|
||||||
_resources = docker.types.Resources(
|
|
||||||
cpu_reservation=job.n_cores * (10 ** 9),
|
|
||||||
mem_reservation=job.mem_mb * (10 ** 6)
|
|
||||||
)
|
|
||||||
_restart_policy = docker.types.RestartPolicy(condition='none')
|
|
||||||
try:
|
|
||||||
service = client.services.create(
|
|
||||||
_image,
|
|
||||||
command=_command,
|
|
||||||
constraints=_constraints,
|
|
||||||
labels=_labels,
|
|
||||||
mounts=_mounts,
|
|
||||||
name=_name,
|
|
||||||
resources=_resources,
|
|
||||||
restart_policy=_restart_policy
|
|
||||||
)
|
|
||||||
job.status = 'scheduled'
|
|
||||||
except docker.errors.APIError:
|
|
||||||
job.status = 'failed'
|
|
||||||
print('[ERROR] {}: client.services.create raised APIError'
|
|
||||||
.format(job.id))
|
|
||||||
for job in jobs.filter(Job.status != 'complete',
|
|
||||||
Job.status != 'failed',
|
|
||||||
Job.status != 'submitted').all():
|
|
||||||
try:
|
|
||||||
service = client.services.get(str(job.id))
|
|
||||||
job.status = service.tasks()[0].get('Status').get('State')
|
|
||||||
if job.status == 'complete' or job.status == 'failed':
|
|
||||||
job.end_date = datetime.utcnow()
|
|
||||||
service.remove()
|
|
||||||
except docker.errors.APIError:
|
|
||||||
job.status = 'failed'
|
|
||||||
print('[ERROR] {}: client.services.get raised APIError'
|
|
||||||
.format(job.id))
|
|
||||||
except docker.errors.NotFound:
|
|
||||||
job.status = 'failed'
|
|
||||||
print('[ERROR] {}: client.services.get raised NotFound'
|
|
||||||
.format(job.id))
|
|
||||||
except docker.errors.InvalidVersion:
|
|
||||||
job.status = 'failed'
|
|
||||||
print('[ERROR] {}: client.services.get raised InvalidVersion'
|
|
||||||
.format(job.id))
|
|
||||||
db.session.commit()
|
|
@ -10,6 +10,7 @@ class CorpusList extends List {
|
|||||||
this.addCorpus(corpus);
|
this.addCorpus(corpus);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.update()
|
||||||
List.updatePagination(this);
|
List.updatePagination(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -22,10 +23,13 @@ class CorpusList extends List {
|
|||||||
switch(operation.op) {
|
switch(operation.op) {
|
||||||
case "add":
|
case "add":
|
||||||
this.addCorpus(operation.value);
|
this.addCorpus(operation.value);
|
||||||
|
this.update()
|
||||||
|
List.updatePagination(this);
|
||||||
break;
|
break;
|
||||||
case "remove":
|
case "remove":
|
||||||
if (pathArray.length != 1) {break;}
|
if (pathArray.length != 1) {break;}
|
||||||
this.remove("id", pathArray[0]);
|
this.remove("id", pathArray[0]);
|
||||||
|
List.updatePagination(this);
|
||||||
break;
|
break;
|
||||||
case "replace":
|
case "replace":
|
||||||
if (pathArray.length != 2) {break;}
|
if (pathArray.length != 2) {break;}
|
||||||
|
@ -10,6 +10,7 @@ class JobList extends List {
|
|||||||
this.addJob(job);
|
this.addJob(job);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.update()
|
||||||
List.updatePagination(this);
|
List.updatePagination(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -23,10 +24,13 @@ class JobList extends List {
|
|||||||
switch(operation.op) {
|
switch(operation.op) {
|
||||||
case "add":
|
case "add":
|
||||||
this.addJob(operation.value);
|
this.addJob(operation.value);
|
||||||
|
this.update();
|
||||||
|
List.updatePagination(this);
|
||||||
break;
|
break;
|
||||||
case "remove":
|
case "remove":
|
||||||
if (pathArray.length != 1) {break;}
|
if (pathArray.length != 1) {break;}
|
||||||
this.remove("id", pathArray[0]);
|
this.remove("id", pathArray[0]);
|
||||||
|
List.updatePagination(this);
|
||||||
break;
|
break;
|
||||||
case "replace":
|
case "replace":
|
||||||
if (pathArray.length != 2) {break;}
|
if (pathArray.length != 2) {break;}
|
||||||
@ -52,6 +56,7 @@ class JobList extends List {
|
|||||||
jobStatusElement.classList.remove(statusColor);
|
jobStatusElement.classList.remove(statusColor);
|
||||||
jobStatusElement.classList.add(newStatusColor);
|
jobStatusElement.classList.add(newStatusColor);
|
||||||
jobStatusElement.innerHTML = operation.value;
|
jobStatusElement.innerHTML = operation.value;
|
||||||
|
break;
|
||||||
case "title":
|
case "title":
|
||||||
item.values({"title": operation.value});
|
item.values({"title": operation.value});
|
||||||
break;
|
break;
|
||||||
@ -109,7 +114,9 @@ class JobList extends List {
|
|||||||
|
|
||||||
this.add(
|
this.add(
|
||||||
[{description: job.description, id: job.id, title: job.title}],
|
[{description: job.description, id: job.id, title: job.title}],
|
||||||
function(items) {items[0].elm = jobElement;}
|
(items) => {
|
||||||
|
items[0].elm = jobElement;
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,5 @@
|
|||||||
docker
|
|
||||||
eventlet
|
eventlet
|
||||||
Flask
|
Flask
|
||||||
Flask-APScheduler
|
|
||||||
Flask-Login
|
Flask-Login
|
||||||
Flask-Mail
|
Flask-Mail
|
||||||
Flask-Migrate
|
Flask-Migrate
|
||||||
|
Loading…
Reference in New Issue
Block a user