sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-04-06 01:04:22 +00:00
Code Issues Projects Releases Wiki Activity

Remove ugly workaround for scheduler function. Now the current app_context is used, instead of creating a new app instance.

Browse Source
This commit is contained in:
Patrick Jentsch 2019-08-20 15:57:58 +02:00
parent 5ff2ef9301
commit 6975076fc3
2 changed files with 69 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/__init__.py
View File

@ -40,13 +40,3 @@ def create_app(config_name):
app.register_blueprint(admin_blueprint, url_prefix='/admin') app.register_blueprint(admin_blueprint, url_prefix='/admin')
return app return app
def create_minimal_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config[config_name].init_app(app)
db.init_app(app)
return app

139
app/scheduler_functions.py
View File

@ -1,5 +1,5 @@
from datetime import datetime from datetime import datetime
from . import create_minimal_app, db from . import db, scheduler
from .models import Job from .models import Job
import docker import docker
import json import json
@ -7,72 +7,71 @@ import os
def checkout_jobs(): def checkout_jobs():
app = create_minimal_app(os.getenv('FLASK_CONFIG') or 'default') with scheduler.app.app_context():
app.app_context().push() client = docker.from_env()
client = docker.from_env() jobs = db.session.query(Job)
jobs = db.session.query(Job) submitted_jobs = jobs.filter_by(status='submitted').all()
submitted_jobs = jobs.filter_by(status='submitted').all() foo_jobs = jobs.filter(Job.status != 'complete',
foo_jobs = jobs.filter(Job.status != 'complete', Job.status != 'failed',
Job.status != 'failed', Job.status != 'submitted').all()
Job.status != 'submitted').all() for job in submitted_jobs:
for job in submitted_jobs: _command = (job.service
_command = (job.service + ' -i /files'
+ ' -i /files' + ' -o /files/output'
+ ' -o /files/output' + ' ' + ' '.join(json.loads(job.service_args)))
+ ' ' + ' '.join(json.loads(job.service_args))) _constraints = ['node.role==worker']
_constraints = ['node.role==worker'] _image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format( job.service,
job.service, job.service_version
job.service_version )
) _labels = {'service': job.service}
_labels = {'service': job.service} _mounts = [os.path.join('/home/compute/mnt/opaque',
_mounts = [os.path.join('/home/compute/mnt/opaque', str(job.user_id),
str(job.user_id), 'jobs',
'jobs', str(job.id))
str(job.id)) + ':/files:rw']
+ ':/files:rw'] _name = str(job.id)
_name = str(job.id) '''
''' ' The Docker SDK for Python expects the cpu_reservation value to be
' The Docker SDK for Python expects the cpu_reservation value to be ' scaled to nanos (10^9). Because the job object contains unscaled
' scaled to nanos (10^9). Because the job object contains unscaled ' (10^0) values, it must be conveted.
' (10^0) values, it must be conveted. '
' ' While the cpu_reservation value has to be in nanos, the
' While the cpu_reservation value has to be in nanos, the ' mem_reservation value must be presented in an unscaled form
' mem_reservation value must be presented in an unscaled form ' (intuitive right?). Bacause the job object provides the memory
' (intuitive right?). Bacause the job object provides the memory value ' value in megabytes, it is also necessary to convert the value.
' in megabytes, it is also necessary to convert the value. '''
''' _resources = docker.types.Resources(
_resources = docker.types.Resources( cpu_reservation=job.n_cores * (10 ** 9),
cpu_reservation=job.n_cores * (10 ** 9), mem_reservation=job.mem_mb * (10 ** 6)
mem_reservation=job.mem_mb * (10 ** 6) )
) _restart_policy = docker.types.RestartPolicy(condition='none')
_restart_policy = docker.types.RestartPolicy(condition='none') '''
''' ' Create the service with the prepared values.
' Create the service with the prepared values. '
' ' Note: A service reserves hardware ressources. In case no worker
' Note: A service reserves hardware ressources. In case no worker node ' node has the required ressources available (not reserved),
' has the required ressources available (not reserved), the ' the service gets queued by the Docker engine until a node
' service gets queued by the Docker engine until a node is able ' is able to meet the requirements.
' to meet the requirements. '''
''' service = client.services.create(
service = client.services.create( _image,
_image, command=_command,
command=_command, constraints=_constraints,
constraints=_constraints, labels=_labels,
labels=_labels, mounts=_mounts,
mounts=_mounts, name=_name,
name=_name, resources=_resources,
resources=_resources, restart_policy=_restart_policy
restart_policy=_restart_policy )
) job.status = 'scheduled'
job.status = 'scheduled' for job in foo_jobs:
for job in foo_jobs: '''
''' ' TODO: Handle service not found error.
' TODO: Handle service not found error. '''
''' service = client.services.get(str(job.id))
service = client.services.get(str(job.id)) job.status = service.tasks()[0].get('Status').get('State')
job.status = service.tasks()[0].get('Status').get('State') if job.status == 'complete' or job.status == 'failed':
if job.status == 'complete' or job.status == 'failed': job.end_date = datetime.utcnow()
job.end_date = datetime.utcnow() service.remove()
service.remove() db.session.commit()
db.session.commit()