From: Maciej Tronowski Date: Mon, 4 May 2015 16:27:06 +0000 (+0200) Subject: job templates: populating submit form with template's attributes X-Git-Tag: v1.0~21 X-Git-Url: http://mmka.chem.univ.gda.pl/gitweb/?a=commitdiff_plain;h=dd8e6b0785c78c42c3866f161dd853a54e47275a;p=qcg-portal.git job templates: populating submit form with template's attributes --- diff --git a/pkgs/pyqcg-0.5.tar.gz b/pkgs/pyqcg-0.5.tar.gz deleted file mode 100644 index 5bbb3ff..0000000 Binary files a/pkgs/pyqcg-0.5.tar.gz and /dev/null differ diff --git a/pkgs/pyqcg-0.6.tar.gz b/pkgs/pyqcg-0.6.tar.gz new file mode 100644 index 0000000..ba13a7a Binary files /dev/null and b/pkgs/pyqcg-0.6.tar.gz differ diff --git a/qcg/forms.py b/qcg/forms.py index 93f9c29..c67f40e 100644 --- a/qcg/forms.py +++ b/qcg/forms.py @@ -160,15 +160,14 @@ class JobDescriptionForm(forms.Form): native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False) persistent = forms.BooleanField(label=u"Trwałe", required=False) - def __init__(self, data=None, *args, **kwargs): - super(JobDescriptionForm, self).__init__(data, *args, **kwargs) + def __init__(self, data=None, initial=None, *args, **kwargs): + super(JobDescriptionForm, self).__init__(data, initial=initial, *args, **kwargs) - if data is not None: - # accept user defined choices - self.fields['queue'].choices += ((data.get('queue'), data.get('queue')), ) - self.fields['arguments'].choices += ((v, v) for v in data.getlist('arguments')) - self.fields['native'].choices += ((v, v) for v in data.getlist('native')) - self.fields['stage_in'].choices += ((v, v) for v in data.getlist('stage_in')) + if data or initial: + self._init_user_choices('queue', data, initial) + self._init_user_choices('arguments', data, initial) + self._init_user_choices('native', data, initial) + self._init_user_choices('stage_in', data, initial) def clean(self): data = super(JobDescriptionForm, self).clean() @@ -176,6 +175,9 @@ class JobDescriptionForm(forms.Form): if data['application'] and not data['master_file']: self.add_error('master_file', u"W trybie uruchamiania aplikacji należy podać plik główny") + if data['procs'] and data['nodes']: + self.add_error(None, u"Zdefiniuj tylko jedno z pól: liczbę procesów lub topologię węzłów") + notify_type = data.get('notify_type') data['notify'] = u'{}:{}'.format(notify_type, data['notify_address']) if notify_type else '' @@ -203,20 +205,20 @@ class JobDescriptionForm(forms.Form): def clean_application(self): return self.cleaned_data['application'].split('/', 1) if self.cleaned_data['application'] else '' + def clean_nodes(self): + return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else '' + def clean_executable(self): return self._gsiftp_suffix(self.cleaned_data['executable']) def clean_master_file(self): return self._gsiftp_suffix(self.cleaned_data['master_file']) - def clean_nodes(self): - return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else '' - def clean_input(self): return self._gsiftp_suffix(self.cleaned_data['input']) def clean_stage_in(self): - return ['gsiftp://' + item for item in self.cleaned_data['stage_in']] + return [self._gsiftp_suffix(item) for item in self.cleaned_data['stage_in']] def clean_preprocess_script(self): return self._gsiftp_suffix(self.cleaned_data['preprocess_script']) @@ -226,7 +228,19 @@ class JobDescriptionForm(forms.Form): @staticmethod def _gsiftp_suffix(url): - return 'gsiftp://' + url if url else '' + if url: + return url if url.startswith('gsiftp://') else 'gsiftp://' + url + + def _init_user_choices(self, name, data, initial): + initial = initial.get(name) if initial is not None else None + choices = data.getlist(name) if data is not None else [] + + if initial: + choices += initial if isinstance(initial, list) else [initial] + self.fields[name].initial = initial + + if choices: + self.fields[name].choices += ((v, v) for v in choices) class EnvForm(forms.Form): diff --git a/qcg/service.py b/qcg/service.py index 15d939c..c69eee0 100644 --- a/qcg/service.py +++ b/qcg/service.py @@ -1,19 +1,14 @@ import logging -import os import time from django.db import transaction from django.utils.functional import SimpleLazyObject from django.utils.timezone import now from pyqcg import QCG -from pyqcg.description import JobDescription from pyqcg.service import Registry from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus -from filex.ftp import FTPOperation -from qcg.constants import QCG_DATA_URL from qcg.models import User, Job, Task, Allocation, NodeInfo -from qcg.utils import random_id, chunks logger = logging.getLogger(__name__) @@ -137,46 +132,6 @@ def update_job(job, proxy): elapsed, job.job_id, elapsed_job, job.tasks.count(), elapsed_tasks, elapsed_py) -def make_job_desc(params, proxy): - QCG.start() - desc = JobDescription(Credential(proxy)) - - direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs', - 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify', - 'preprocess', 'postprocess', 'persistent') - - for name in direct_map: - if params[name]: - setattr(desc, name, params[name]) - - if params['application']: - desc.set_application(*params['application']) - desc.stage_in += [params['master_file']] - desc.arguments.insert(0, os.path.basename(params['master_file'])) - if params['script']: - ftp = FTPOperation(proxy) - - ftp.mkdir(QCG_DATA_URL, parents=True) - url = os.path.join(QCG_DATA_URL, 'script.{}.sh'.format(random_id())) - ftp.put(url) - - for chunk in chunks(params['script'], 4096): - ftp.stream.put(chunk) - ftp.stream.put(None) - - ftp.wait() - desc.executable = url - if params['nodes']: - desc.set_nodes(*params['nodes']) - if params['reservation']: - desc.set_reservation(params['reservation']) - if params['watch_output']: - desc.set_watch_output(params['watch_output'], params['watch_output_pattern']) - # TODO monitoring - - return desc - - def cancel(obj, proxy): ts = time.time() QCG.start() diff --git a/qcg/utils.py b/qcg/utils.py index 29f631b..5555e16 100644 --- a/qcg/utils.py +++ b/qcg/utils.py @@ -1,10 +1,15 @@ +import os import string import random from django.core.paginator import Paginator from django.utils.formats import date_format from django.utils.timezone import localtime +from pyqcg import QCG +from pyqcg.utils import Credential +from pyqcg.description import JobDescription +from filex.ftp import FTPOperation from qcg import constants @@ -48,3 +53,94 @@ def random_id(size=8, chars=string.ascii_uppercase + string.digits): def chunks(seq, size): return (seq[pos:pos + size] for pos in xrange(0, len(seq), size)) + + +def to_job_desc(params, proxy): + QCG.start() + desc = JobDescription(Credential(proxy)) + + direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs', + 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify', + 'preprocess', 'postprocess', 'persistent') + + for name in direct_map: + if params[name]: + setattr(desc, name, params[name]) + + if params['application']: + desc.set_application(*params['application']) + desc.stage_in += [params['master_file']] + desc.arguments.insert(0, os.path.basename(params['master_file'])) + if params['script']: + ftp = FTPOperation(proxy) + + ftp.mkdir(constants.QCG_DATA_URL, parents=True) + url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id())) + ftp.put(url) + + for chunk in chunks(params['script'], 4096): + ftp.stream.put(chunk) + ftp.stream.put(None) + ftp.wait() + + desc.executable = url + if params['nodes']: + desc.set_nodes(*params['nodes']) + if params['reservation']: + desc.set_reservation(params['reservation']) + if params['watch_output']: + desc.set_watch_output(params['watch_output'], params['watch_output_pattern']) + # TODO monitoring + + return desc + + +def to_form_data(xml): + # prevent circular import errors + from forms import JobDescriptionForm + + QCG.start() + desc = JobDescription() + desc.xml_description = xml + + direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs', + 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'persistent') + + params = {} + for name in direct_map: + attr = getattr(desc, name) + if isinstance(attr, bool) or attr: + params[name] = attr + + if desc.application is not None: + app_name, app_ver = desc.application + params['application'] = app_name if app_ver is None else app_name + '/' + app_name + stage_in = params['stage_in'] + params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1] + params['arguments'] = params['arguments'][1:] + if desc.nodes is not None: + params['nodes'] = ':'.join(map(str, desc.nodes)) + if desc.reservation is not None: + res_id, res_type = desc.reservation + params['reservation'] = res_id + if desc.notify is not None: + params['notify_type'], params['notify_address'] = desc.notify.split(':') + if desc.watch_output is not None: + watch_output, params['watch_output_pattern'] = desc.watch_output + params['watch_output_type'], params['watch_output_address'] = watch_output.split(':') + if desc.preprocess is not None: + if desc.preprocess.startswith('gsiftp://'): + params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT + params['preprocess_script'] = desc.preprocess + else: + params['preprocess_type'] = JobDescriptionForm.Process.CMD + params['preprocess_cmd'] = desc.preprocess + if desc.postprocess is not None: + if desc.postprocess.startswith('gsiftp://'): + params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT + params['postprocess_script'] = desc.postprocess + else: + params['postprocess_type'] = JobDescriptionForm.Process.CMD + params['postprocess_cmd'] = desc.postprocess + + return params diff --git a/qcg/views.py b/qcg/views.py index a627524..2dc5e8f 100644 --- a/qcg/views.py +++ b/qcg/views.py @@ -16,14 +16,14 @@ from django.utils.timezone import UTC from django.views.decorators.http import require_POST from django_openid_auth.views import make_consumer from openid.extensions import ax -from pyqcg.utils.qcg_types import PyqcgException +from pyqcg.utils import PyqcgException from filex.forms import HostPathNameForm, RenameForm, ArchiveForm, HostPathForm from filex.ftp import FTPOperation, FTPError from filex.views import make_url from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm -from qcg.utils import paginator_context -from qcg.service import update_user_data, make_job_desc, update_job, cancel, clean +from qcg.utils import paginator_context, to_job_desc, to_form_data +from qcg.service import update_user_data, update_job, cancel, clean def index(request): @@ -174,9 +174,15 @@ def job_submit(request, template_id=None): save_template = 'save-template' in request.POST template = get_object_or_404(request.user.templates, id=template_id) if template_id is not None else None + if template is not None: + form_data = to_form_data(template.description) + env_formset_data = [{'name': name, 'value': value} for name, value in form_data.pop('env_variables', ())] + else: + form_data, env_formset_data = None, None + if request.method == 'POST': - form = JobDescriptionForm(request.POST) - env_formset = EnvFormSet(request.POST) + form = JobDescriptionForm(request.POST, initial=form_data) + env_formset = EnvFormSet(request.POST, initial=env_formset_data) template_form = JobTemplateForm(request.POST, prefix='template', instance=template) if form.is_valid() and env_formset.is_valid() and (not save_template or template_form.is_valid): @@ -184,37 +190,27 @@ def job_submit(request, template_id=None): params['env_variables'] = [(env['name'], env['value']) for env in env_formset.cleaned_data if env and not env['DELETE']] - job_desc = make_job_desc(params, request.session['proxy']) + job_desc = to_job_desc(params, request.session['proxy']) if save_template: - if template is not None: - # save already existing template... - instance = template_form.save() - else: - # or set attributes for new template - instance = template_form.save(commit=False) + template = template_form.save(commit=False) - instance.owner = request.user - instance.description = job_desc.xml_description - instance.save() + template.owner = request.user + template.description = job_desc.xml_description + template.save() - return redirect(instance) - - else: - job = job_desc.submit() + return redirect(template) - messages.success(request, - format_html(' ' - 'Zlecono zadanie {}.', job.job_id)) + job = job_desc.submit() - return redirect('jobs') + messages.success(request, + format_html(' ' + 'Zlecono zadanie {}.', job.job_id)) - print 'form', repr(form.errors) - print 'env_formset', repr(env_formset.errors) - print 'template_form', repr(template_form.errors) + return redirect('jobs') else: - form = JobDescriptionForm() - env_formset = EnvFormSet() + form = JobDescriptionForm(initial=form_data) + env_formset = EnvFormSet(initial=env_formset_data) template_form = JobTemplateForm(prefix='template', instance=template) errors = form.errors or (env_formset.is_bound and not env_formset.is_valid) or ( diff --git a/requirements.txt b/requirements.txt index 94386ea..41e2068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Django<1.8 django-grappelli django-bootstrap3 pkgs/django-openid-auth-0.5.1.tar.gz -pkgs/pyqcg-0.5.tar.gz +pkgs/pyqcg-0.6.tar.gz pkgs/python-gridftp-1.3.4.tar.gz python-openid pytz