From c101141ec6a1c32fe158b2f4ed7d8fe68e702cf6 Mon Sep 17 00:00:00 2001 From: Maciej Tronowski Date: Tue, 28 Apr 2015 15:36:33 +0200 Subject: [PATCH] submitting jobs: handling user defined script --- qcg/constants.py | 2 ++ qcg/forms.py | 2 +- qcg/service.py | 22 ++++++++++++++++------ qcg/utils.py | 11 +++++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/qcg/constants.py b/qcg/constants.py index b9d55bb..c2f9ea4 100644 --- a/qcg/constants.py +++ b/qcg/constants.py @@ -1 +1,3 @@ PER_PAGE = 10 + +QCG_DATA_URL = 'gsiftp://qcg.man.poznan.pl/~/.qcg/data/portal' diff --git a/qcg/forms.py b/qcg/forms.py index a08a9e5..ecfb154 100644 --- a/qcg/forms.py +++ b/qcg/forms.py @@ -119,7 +119,7 @@ class JobDescriptionForm(forms.Form): application = forms.ChoiceField(choices=APPLICATION_CHOICES, label=u"Aplikacja", required=False) # TODO choices master_file = forms.CharField(label=u"Plik główny", max_length=500, required=False) executable = forms.CharField(label=u"Plik wykonywalny", max_length=500, required=False) - script = forms.CharField(label=u"Skrypt", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) # TODO saving to grid ftp + script = forms.CharField(label=u"Skrypt", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) arguments = forms.MultipleChoiceField(label=u"Argumenty", required=False) note = forms.CharField(label=u"Opis", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) grant = forms.CharField(label=u"Grant", max_length=100, required=False) diff --git a/qcg/service.py b/qcg/service.py index 44cb42e..972592f 100644 --- a/qcg/service.py +++ b/qcg/service.py @@ -10,7 +10,10 @@ from pyqcg.description import JobDescription from pyqcg.service import Registry from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus +from filex.ftp import FTPOperation +from qcg.constants import QCG_DATA_URL from qcg.models import User, Job, Task, Allocation, NodeInfo +from qcg.utils import random_id, chunks logger = logging.getLogger(__name__) @@ -150,20 +153,27 @@ def submit_job(params, proxy): desc.set_application(*params['application']) desc.stage_in += [params['master_file']] desc.arguments.insert(0, os.path.basename(params['master_file'])) + if params['script']: + ftp = FTPOperation(proxy) + + ftp.mkdir(QCG_DATA_URL, parents=True) + url = os.path.join(QCG_DATA_URL, 'script.{}.sh'.format(random_id())) + ftp.put(url) + + for chunk in chunks(params['script'], 4096): + ftp.stream.put(chunk) + ftp.stream.put(None) + + ftp.wait() + desc.executable = url if params['nodes']: desc.set_nodes(*params['nodes']) if params['reservation']: desc.set_reservation(params['reservation']) if params['watch_output']: desc.set_watch_output(params['watch_output'], params['watch_output_pattern']) - # TODO script # TODO monitoring - # for prop in direct_map + ('application', 'nodes', 'env_variables', 'reservation', 'watch_output'): - # print prop, type(getattr(desc, prop)), repr(getattr(desc, prop)) - - # print desc.xml_description - job = desc.submit() return job.job_id diff --git a/qcg/utils.py b/qcg/utils.py index 24a5a24..29f631b 100644 --- a/qcg/utils.py +++ b/qcg/utils.py @@ -1,3 +1,6 @@ +import string +import random + from django.core.paginator import Paginator from django.utils.formats import date_format from django.utils.timezone import localtime @@ -37,3 +40,11 @@ def paginator_context(request, objects, per_page=constants.PER_PAGE): def localtime_str(datetime): return date_format(localtime(datetime), 'DATETIME_FORMAT') + + +def random_id(size=8, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + +def chunks(seq, size): + return (seq[pos:pos + size] for pos in xrange(0, len(seq), size)) -- 1.7.9.5