mirror of
				https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
				synced 2025-10-31 10:42:43 +00:00 
			
		
		
		
	Remove old scheduler code.
This commit is contained in:
		| @@ -1,6 +1,5 @@ | |||||||
| from config import config | from config import config | ||||||
| from flask import Flask | from flask import Flask | ||||||
| # from flask_apscheduler import APScheduler |  | ||||||
| from flask_login import LoginManager | from flask_login import LoginManager | ||||||
| from flask_mail import Mail | from flask_mail import Mail | ||||||
| from flask_socketio import SocketIO | from flask_socketio import SocketIO | ||||||
| @@ -11,7 +10,6 @@ db = SQLAlchemy() | |||||||
| login_manager = LoginManager() | login_manager = LoginManager() | ||||||
| login_manager.login_view = 'auth.login' | login_manager.login_view = 'auth.login' | ||||||
| mail = Mail() | mail = Mail() | ||||||
| # scheduler = APScheduler() |  | ||||||
| socketio = SocketIO() | socketio = SocketIO() | ||||||
|  |  | ||||||
|  |  | ||||||
| @@ -27,8 +25,6 @@ def create_app(config_name, main=True): | |||||||
|  |  | ||||||
|     login_manager.init_app(app) |     login_manager.init_app(app) | ||||||
|     mail.init_app(app) |     mail.init_app(app) | ||||||
|     # scheduler.init_app(app) |  | ||||||
|     # scheduler.start() |  | ||||||
|     socketio.init_app(app, message_qeue='redis://') |     socketio.init_app(app, message_qeue='redis://') | ||||||
|  |  | ||||||
|     from .auth import auth as auth_blueprint |     from .auth import auth as auth_blueprint | ||||||
|   | |||||||
| @@ -1,82 +0,0 @@ | |||||||
| from datetime import datetime |  | ||||||
| from . import db, scheduler |  | ||||||
| from .models import Job |  | ||||||
| import docker |  | ||||||
| import json |  | ||||||
| import os |  | ||||||
|  |  | ||||||
|  |  | ||||||
| def checkout_jobs(): |  | ||||||
|     with scheduler.app.app_context(): |  | ||||||
|         client = docker.from_env() |  | ||||||
|         jobs = db.session.query(Job) |  | ||||||
|         for job in jobs.filter_by(status='submitted').all(): |  | ||||||
|             _command = (job.service |  | ||||||
|                         + ' -i /files' |  | ||||||
|                         + ' -o /files/output' |  | ||||||
|                         + ' ' + ' '.join(json.loads(job.service_args))) |  | ||||||
|             _constraints = ['node.role==worker'] |  | ||||||
|             _image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format( |  | ||||||
|                 job.service, |  | ||||||
|                 job.service_version |  | ||||||
|             ) |  | ||||||
|             _labels = {'service': job.service} |  | ||||||
|             _mounts = [os.path.join('/home/compute/mnt/opaque', |  | ||||||
|                                     str(job.user_id), |  | ||||||
|                                     'jobs', |  | ||||||
|                                     str(job.id)) |  | ||||||
|                        + ':/files:rw'] |  | ||||||
|             _name = str(job.id) |  | ||||||
|             ''' |  | ||||||
|             ' The Docker SDK for Python expects the cpu_reservation value to be |  | ||||||
|             ' scaled to nanos (10^9). Because the job object contains unscaled |  | ||||||
|             ' (10^0) values, it must be conveted. |  | ||||||
|             ' |  | ||||||
|             ' While the cpu_reservation value has to be in nanos, the |  | ||||||
|             ' mem_reservation value must be presented in an unscaled form |  | ||||||
|             ' (intuitive right?). Bacause the job object provides the memory |  | ||||||
|             ' value in megabytes, it is also necessary to convert the value. |  | ||||||
|             ''' |  | ||||||
|             _resources = docker.types.Resources( |  | ||||||
|                 cpu_reservation=job.n_cores * (10 ** 9), |  | ||||||
|                 mem_reservation=job.mem_mb * (10 ** 6) |  | ||||||
|             ) |  | ||||||
|             _restart_policy = docker.types.RestartPolicy(condition='none') |  | ||||||
|             try: |  | ||||||
|                 service = client.services.create( |  | ||||||
|                     _image, |  | ||||||
|                     command=_command, |  | ||||||
|                     constraints=_constraints, |  | ||||||
|                     labels=_labels, |  | ||||||
|                     mounts=_mounts, |  | ||||||
|                     name=_name, |  | ||||||
|                     resources=_resources, |  | ||||||
|                     restart_policy=_restart_policy |  | ||||||
|                 ) |  | ||||||
|                 job.status = 'scheduled' |  | ||||||
|             except docker.errors.APIError: |  | ||||||
|                 job.status = 'failed' |  | ||||||
|                 print('[ERROR] {}: client.services.create raised APIError' |  | ||||||
|                       .format(job.id)) |  | ||||||
|         for job in jobs.filter(Job.status != 'complete', |  | ||||||
|                                Job.status != 'failed', |  | ||||||
|                                Job.status != 'submitted').all(): |  | ||||||
|             try: |  | ||||||
|                 service = client.services.get(str(job.id)) |  | ||||||
|                 job.status = service.tasks()[0].get('Status').get('State') |  | ||||||
|                 if job.status == 'complete' or job.status == 'failed': |  | ||||||
|                     job.end_date = datetime.utcnow() |  | ||||||
|                     service.remove() |  | ||||||
|             except docker.errors.APIError: |  | ||||||
|                 job.status = 'failed' |  | ||||||
|                 print('[ERROR] {}: client.services.get raised APIError' |  | ||||||
|                       .format(job.id)) |  | ||||||
|             except docker.errors.NotFound: |  | ||||||
|                 job.status = 'failed' |  | ||||||
|                 print('[ERROR] {}: client.services.get raised NotFound' |  | ||||||
|                       .format(job.id)) |  | ||||||
|             except docker.errors.InvalidVersion: |  | ||||||
|                 job.status = 'failed' |  | ||||||
|                 print('[ERROR] {}: client.services.get raised InvalidVersion' |  | ||||||
|                       .format(job.id)) |  | ||||||
|         db.session.commit() |  | ||||||
| @@ -10,6 +10,7 @@ class CorpusList extends List { | |||||||
|       this.addCorpus(corpus); |       this.addCorpus(corpus); | ||||||
|     } |     } | ||||||
|  |  | ||||||
|  |     this.update() | ||||||
|     List.updatePagination(this); |     List.updatePagination(this); | ||||||
|   } |   } | ||||||
|  |  | ||||||
| @@ -22,10 +23,13 @@ class CorpusList extends List { | |||||||
|       switch(operation.op) { |       switch(operation.op) { | ||||||
|         case "add": |         case "add": | ||||||
|           this.addCorpus(operation.value); |           this.addCorpus(operation.value); | ||||||
|  |           this.update() | ||||||
|  |           List.updatePagination(this); | ||||||
|           break; |           break; | ||||||
|         case "remove": |         case "remove": | ||||||
|           if (pathArray.length != 1) {break;} |           if (pathArray.length != 1) {break;} | ||||||
|           this.remove("id", pathArray[0]); |           this.remove("id", pathArray[0]); | ||||||
|  |           List.updatePagination(this); | ||||||
|           break; |           break; | ||||||
|         case "replace": |         case "replace": | ||||||
|           if (pathArray.length != 2) {break;} |           if (pathArray.length != 2) {break;} | ||||||
|   | |||||||
| @@ -10,6 +10,7 @@ class JobList extends List { | |||||||
|       this.addJob(job); |       this.addJob(job); | ||||||
|     } |     } | ||||||
|  |  | ||||||
|  |     this.update() | ||||||
|     List.updatePagination(this); |     List.updatePagination(this); | ||||||
|   } |   } | ||||||
|  |  | ||||||
| @@ -23,10 +24,13 @@ class JobList extends List { | |||||||
|       switch(operation.op) { |       switch(operation.op) { | ||||||
|         case "add": |         case "add": | ||||||
|           this.addJob(operation.value); |           this.addJob(operation.value); | ||||||
|  |           this.update(); | ||||||
|  |           List.updatePagination(this); | ||||||
|           break; |           break; | ||||||
|         case "remove": |         case "remove": | ||||||
|           if (pathArray.length != 1) {break;} |           if (pathArray.length != 1) {break;} | ||||||
|           this.remove("id", pathArray[0]); |           this.remove("id", pathArray[0]); | ||||||
|  |           List.updatePagination(this); | ||||||
|           break; |           break; | ||||||
|         case "replace": |         case "replace": | ||||||
|           if (pathArray.length != 2) {break;} |           if (pathArray.length != 2) {break;} | ||||||
| @@ -52,6 +56,7 @@ class JobList extends List { | |||||||
|               jobStatusElement.classList.remove(statusColor); |               jobStatusElement.classList.remove(statusColor); | ||||||
|               jobStatusElement.classList.add(newStatusColor); |               jobStatusElement.classList.add(newStatusColor); | ||||||
|               jobStatusElement.innerHTML = operation.value; |               jobStatusElement.innerHTML = operation.value; | ||||||
|  |               break; | ||||||
|             case "title": |             case "title": | ||||||
|               item.values({"title": operation.value}); |               item.values({"title": operation.value}); | ||||||
|               break; |               break; | ||||||
| @@ -109,7 +114,9 @@ class JobList extends List { | |||||||
|  |  | ||||||
|     this.add( |     this.add( | ||||||
|       [{description: job.description, id: job.id, title: job.title}], |       [{description: job.description, id: job.id, title: job.title}], | ||||||
|       function(items) {items[0].elm = jobElement;} |       (items) => { | ||||||
|  |         items[0].elm = jobElement; | ||||||
|  |       } | ||||||
|     ); |     ); | ||||||
|   } |   } | ||||||
| } | } | ||||||
|   | |||||||
| @@ -1,7 +1,5 @@ | |||||||
| docker |  | ||||||
| eventlet | eventlet | ||||||
| Flask | Flask | ||||||
| Flask-APScheduler |  | ||||||
| Flask-Login | Flask-Login | ||||||
| Flask-Mail | Flask-Mail | ||||||
| Flask-Migrate | Flask-Migrate | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user