job templates: populating submit form with template's attributes

diff --git a/pkgs/pyqcg-0.5.tar.gz b/pkgs/pyqcg-0.5.tar.gz
deleted file mode 100644 (file)
index 5bbb3ff..0000000
Binary files a/pkgs/pyqcg-0.5.tar.gz and /dev/null differ

diff --git a/pkgs/pyqcg-0.6.tar.gz b/pkgs/pyqcg-0.6.tar.gz
new file mode 100644 (file)
index 0000000..ba13a7a
Binary files /dev/null and b/pkgs/pyqcg-0.6.tar.gz differ

diff --git a/qcg/forms.py b/qcg/forms.py
index 93f9c29..c67f40e 100644 (file)
--- a/qcg/forms.py
+++ b/qcg/forms.py
@@ -160,15 +160,14 @@ class JobDescriptionForm(forms.Form):
     native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False)
     persistent = forms.BooleanField(label=u"Trwałe", required=False)
 
     native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False)
     persistent = forms.BooleanField(label=u"Trwałe", required=False)
 

-    def __init__(self, data=None, *args, **kwargs):
-        super(JobDescriptionForm, self).__init__(data, *args, **kwargs)
+    def __init__(self, data=None, initial=None, *args, **kwargs):
+        super(JobDescriptionForm, self).__init__(data, initial=initial, *args, **kwargs)

 
 

-        if data is not None:
-            # accept user defined choices
-            self.fields['queue'].choices += ((data.get('queue'), data.get('queue')), )
-            self.fields['arguments'].choices += ((v, v) for v in data.getlist('arguments'))
-            self.fields['native'].choices += ((v, v) for v in data.getlist('native'))
-            self.fields['stage_in'].choices += ((v, v) for v in data.getlist('stage_in'))
+        if data or initial:
+            self._init_user_choices('queue', data, initial)
+            self._init_user_choices('arguments', data, initial)
+            self._init_user_choices('native', data, initial)
+            self._init_user_choices('stage_in', data, initial)

 
     def clean(self):
         data = super(JobDescriptionForm, self).clean()
 
     def clean(self):
         data = super(JobDescriptionForm, self).clean()

@@ -176,6 +175,9 @@ class JobDescriptionForm(forms.Form):
         if data['application'] and not data['master_file']:
             self.add_error('master_file', u"W trybie uruchamiania aplikacji należy podać plik główny")
 
         if data['application'] and not data['master_file']:
             self.add_error('master_file', u"W trybie uruchamiania aplikacji należy podać plik główny")
 

+        if data['procs'] and data['nodes']:
+            self.add_error(None, u"Zdefiniuj tylko jedno z pól: liczbę procesów lub topologię węzłów")
+

         notify_type = data.get('notify_type')
         data['notify'] = u'{}:{}'.format(notify_type, data['notify_address']) if notify_type else ''
 
         notify_type = data.get('notify_type')
         data['notify'] = u'{}:{}'.format(notify_type, data['notify_address']) if notify_type else ''
 

@@ -203,20 +205,20 @@ class JobDescriptionForm(forms.Form):
     def clean_application(self):
         return self.cleaned_data['application'].split('/', 1) if self.cleaned_data['application'] else ''
 
     def clean_application(self):
         return self.cleaned_data['application'].split('/', 1) if self.cleaned_data['application'] else ''
 

+    def clean_nodes(self):
+        return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else ''
+

     def clean_executable(self):
         return self._gsiftp_suffix(self.cleaned_data['executable'])
 
     def clean_master_file(self):
         return self._gsiftp_suffix(self.cleaned_data['master_file'])
 
     def clean_executable(self):
         return self._gsiftp_suffix(self.cleaned_data['executable'])
 
     def clean_master_file(self):
         return self._gsiftp_suffix(self.cleaned_data['master_file'])
 

-    def clean_nodes(self):
-        return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else ''
-

     def clean_input(self):
         return self._gsiftp_suffix(self.cleaned_data['input'])
 
     def clean_stage_in(self):
     def clean_input(self):
         return self._gsiftp_suffix(self.cleaned_data['input'])
 
     def clean_stage_in(self):

-        return ['gsiftp://' + item for item in self.cleaned_data['stage_in']]
+        return [self._gsiftp_suffix(item) for item in self.cleaned_data['stage_in']]

 
     def clean_preprocess_script(self):
         return self._gsiftp_suffix(self.cleaned_data['preprocess_script'])
 
     def clean_preprocess_script(self):
         return self._gsiftp_suffix(self.cleaned_data['preprocess_script'])

@@ -226,7 +228,19 @@ class JobDescriptionForm(forms.Form):
 
     @staticmethod
     def _gsiftp_suffix(url):
 
     @staticmethod
     def _gsiftp_suffix(url):

-        return 'gsiftp://' + url if url else ''
+        if url:
+            return url if url.startswith('gsiftp://') else 'gsiftp://' + url
+
+    def _init_user_choices(self, name, data, initial):
+        initial = initial.get(name) if initial is not None else None
+        choices = data.getlist(name) if data is not None else []
+
+        if initial:
+            choices += initial if isinstance(initial, list) else [initial]
+            self.fields[name].initial = initial
+
+        if choices:
+            self.fields[name].choices += ((v, v) for v in choices)

 
 
 class EnvForm(forms.Form):
 
 
 class EnvForm(forms.Form):

diff --git a/qcg/service.py b/qcg/service.py
index 15d939c..c69eee0 100644 (file)
--- a/qcg/service.py
+++ b/qcg/service.py
@@ -1,19 +1,14 @@
 import logging
 import logging

-import os

 import time
 
 from django.db import transaction
 from django.utils.functional import SimpleLazyObject
 from django.utils.timezone import now
 from pyqcg import QCG
 import time
 
 from django.db import transaction
 from django.utils.functional import SimpleLazyObject
 from django.utils.timezone import now
 from pyqcg import QCG

-from pyqcg.description import JobDescription

 from pyqcg.service import Registry
 from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus
 
 from pyqcg.service import Registry
 from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus
 

-from filex.ftp import FTPOperation
-from qcg.constants import QCG_DATA_URL

 from qcg.models import User, Job, Task, Allocation, NodeInfo
 from qcg.models import User, Job, Task, Allocation, NodeInfo

-from qcg.utils import random_id, chunks

 
 
 logger = logging.getLogger(__name__)
 
 
 logger = logging.getLogger(__name__)

@@ -137,46 +132,6 @@ def update_job(job, proxy):
                 elapsed, job.job_id, elapsed_job, job.tasks.count(), elapsed_tasks, elapsed_py)
 
 
                 elapsed, job.job_id, elapsed_job, job.tasks.count(), elapsed_tasks, elapsed_py)
 
 

-def make_job_desc(params, proxy):
-    QCG.start()
-    desc = JobDescription(Credential(proxy))
-
-    direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
-                  'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
-                  'preprocess', 'postprocess', 'persistent')
-
-    for name in direct_map:
-        if params[name]:
-            setattr(desc, name, params[name])
-
-    if params['application']:
-        desc.set_application(*params['application'])
-        desc.stage_in += [params['master_file']]
-        desc.arguments.insert(0, os.path.basename(params['master_file']))
-    if params['script']:
-        ftp = FTPOperation(proxy)
-
-        ftp.mkdir(QCG_DATA_URL, parents=True)
-        url = os.path.join(QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
-        ftp.put(url)
-
-        for chunk in chunks(params['script'], 4096):
-            ftp.stream.put(chunk)
-        ftp.stream.put(None)
-
-        ftp.wait()
-        desc.executable = url
-    if params['nodes']:
-        desc.set_nodes(*params['nodes'])
-    if params['reservation']:
-        desc.set_reservation(params['reservation'])
-    if params['watch_output']:
-        desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
-    # TODO monitoring
-
-    return desc
-
-

 def cancel(obj, proxy):
     ts = time.time()
     QCG.start()
 def cancel(obj, proxy):
     ts = time.time()
     QCG.start()

diff --git a/qcg/utils.py b/qcg/utils.py
index 29f631b..5555e16 100644 (file)
--- a/qcg/utils.py
+++ b/qcg/utils.py
@@ -1,10 +1,15 @@
+import os

 import string
 import random
 
 from django.core.paginator import Paginator
 from django.utils.formats import date_format
 from django.utils.timezone import localtime
 import string
 import random
 
 from django.core.paginator import Paginator
 from django.utils.formats import date_format
 from django.utils.timezone import localtime

+from pyqcg import QCG
+from pyqcg.utils import Credential
+from pyqcg.description import JobDescription

 
 

+from filex.ftp import FTPOperation

 from qcg import constants
 
 
 from qcg import constants
 
 

@@ -48,3 +53,94 @@ def random_id(size=8, chars=string.ascii_uppercase + string.digits):
 
 def chunks(seq, size):
     return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
 
 def chunks(seq, size):
     return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))

+
+
+def to_job_desc(params, proxy):
+    QCG.start()
+    desc = JobDescription(Credential(proxy))
+
+    direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
+                  'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
+                  'preprocess', 'postprocess', 'persistent')
+
+    for name in direct_map:
+        if params[name]:
+            setattr(desc, name, params[name])
+
+    if params['application']:
+        desc.set_application(*params['application'])
+        desc.stage_in += [params['master_file']]
+        desc.arguments.insert(0, os.path.basename(params['master_file']))
+    if params['script']:
+        ftp = FTPOperation(proxy)
+
+        ftp.mkdir(constants.QCG_DATA_URL, parents=True)
+        url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
+        ftp.put(url)
+
+        for chunk in chunks(params['script'], 4096):
+            ftp.stream.put(chunk)
+        ftp.stream.put(None)
+        ftp.wait()
+
+        desc.executable = url
+    if params['nodes']:
+        desc.set_nodes(*params['nodes'])
+    if params['reservation']:
+        desc.set_reservation(params['reservation'])
+    if params['watch_output']:
+        desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
+    # TODO monitoring
+
+    return desc
+
+
+def to_form_data(xml):
+    # prevent circular import errors
+    from forms import JobDescriptionForm
+
+    QCG.start()
+    desc = JobDescription()
+    desc.xml_description = xml
+
+    direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
+                  'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'persistent')
+
+    params = {}
+    for name in direct_map:
+        attr = getattr(desc, name)
+        if isinstance(attr, bool) or attr:
+            params[name] = attr
+
+    if desc.application is not None:
+        app_name, app_ver = desc.application
+        params['application'] = app_name if app_ver is None else app_name + '/' + app_name
+        stage_in = params['stage_in']
+        params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1]
+        params['arguments'] = params['arguments'][1:]
+    if desc.nodes is not None:
+        params['nodes'] = ':'.join(map(str, desc.nodes))
+    if desc.reservation is not None:
+        res_id, res_type = desc.reservation
+        params['reservation'] = res_id
+    if desc.notify is not None:
+        params['notify_type'], params['notify_address'] = desc.notify.split(':')
+    if desc.watch_output is not None:
+        watch_output, params['watch_output_pattern'] = desc.watch_output
+        params['watch_output_type'], params['watch_output_address'] = watch_output.split(':')
+    if desc.preprocess is not None:
+        if desc.preprocess.startswith('gsiftp://'):
+            params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT
+            params['preprocess_script'] = desc.preprocess
+        else:
+            params['preprocess_type'] = JobDescriptionForm.Process.CMD
+            params['preprocess_cmd'] = desc.preprocess
+    if desc.postprocess is not None:
+        if desc.postprocess.startswith('gsiftp://'):
+            params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT
+            params['postprocess_script'] = desc.postprocess
+        else:
+            params['postprocess_type'] = JobDescriptionForm.Process.CMD
+            params['postprocess_cmd'] = desc.postprocess
+
+    return params

diff --git a/qcg/views.py b/qcg/views.py
index a627524..2dc5e8f 100644 (file)
--- a/qcg/views.py
+++ b/qcg/views.py
@@ -16,14 +16,14 @@ from django.utils.timezone import UTC
 from django.views.decorators.http import require_POST
 from django_openid_auth.views import make_consumer
 from openid.extensions import ax
 from django.views.decorators.http import require_POST
 from django_openid_auth.views import make_consumer
 from openid.extensions import ax

-from pyqcg.utils.qcg_types import PyqcgException
+from pyqcg.utils import PyqcgException

 
 from filex.forms import HostPathNameForm, RenameForm, ArchiveForm, HostPathForm
 from filex.ftp import FTPOperation, FTPError
 from filex.views import make_url
 from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm
 
 from filex.forms import HostPathNameForm, RenameForm, ArchiveForm, HostPathForm
 from filex.ftp import FTPOperation, FTPError
 from filex.views import make_url
 from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm

-from qcg.utils import paginator_context
-from qcg.service import update_user_data, make_job_desc, update_job, cancel, clean
+from qcg.utils import paginator_context, to_job_desc, to_form_data
+from qcg.service import update_user_data, update_job, cancel, clean

 
 
 def index(request):
 
 
 def index(request):

@@ -174,9 +174,15 @@ def job_submit(request, template_id=None):
     save_template = 'save-template' in request.POST
     template = get_object_or_404(request.user.templates, id=template_id) if template_id is not None else None
 
     save_template = 'save-template' in request.POST
     template = get_object_or_404(request.user.templates, id=template_id) if template_id is not None else None
 

+    if template is not None:
+        form_data = to_form_data(template.description)
+        env_formset_data = [{'name': name, 'value': value} for name, value in form_data.pop('env_variables', ())]
+    else:
+        form_data, env_formset_data = None, None
+

     if request.method == 'POST':
     if request.method == 'POST':

-        form = JobDescriptionForm(request.POST)
-        env_formset = EnvFormSet(request.POST)
+        form = JobDescriptionForm(request.POST, initial=form_data)
+        env_formset = EnvFormSet(request.POST, initial=env_formset_data)

         template_form = JobTemplateForm(request.POST, prefix='template', instance=template)
 
         if form.is_valid() and env_formset.is_valid() and (not save_template or template_form.is_valid):
         template_form = JobTemplateForm(request.POST, prefix='template', instance=template)
 
         if form.is_valid() and env_formset.is_valid() and (not save_template or template_form.is_valid):

@@ -184,37 +190,27 @@ def job_submit(request, template_id=None):
             params['env_variables'] = [(env['name'], env['value'])
                                        for env in env_formset.cleaned_data if env and not env['DELETE']]
 
             params['env_variables'] = [(env['name'], env['value'])
                                        for env in env_formset.cleaned_data if env and not env['DELETE']]
 

-            job_desc = make_job_desc(params, request.session['proxy'])
+            job_desc = to_job_desc(params, request.session['proxy'])

 
             if save_template:
 
             if save_template:

-                if template is not None:
-                    # save already existing template...
-                    instance = template_form.save()
-                else:
-                    # or set attributes for new template
-                    instance = template_form.save(commit=False)
+                template = template_form.save(commit=False)

 
 

-                    instance.owner = request.user
-                    instance.description = job_desc.xml_description
-                    instance.save()
+                template.owner = request.user
+                template.description = job_desc.xml_description
+                template.save()

 
 

-                return redirect(instance)
-
-            else:
-                job = job_desc.submit()
+                return redirect(template)

 
 

-                messages.success(request,
-                                 format_html('<span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> '
-                                             'Zlecono zadanie <em>{}</em>.', job.job_id))
+            job = job_desc.submit()

 
 

-                return redirect('jobs')
+            messages.success(request,
+                             format_html('<span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> '
+                                         'Zlecono zadanie <em>{}</em>.', job.job_id))

 
 

-        print 'form', repr(form.errors)
-        print 'env_formset', repr(env_formset.errors)
-        print 'template_form', repr(template_form.errors)
+            return redirect('jobs')

     else:
     else:

-        form = JobDescriptionForm()
-        env_formset = EnvFormSet()
+        form = JobDescriptionForm(initial=form_data)
+        env_formset = EnvFormSet(initial=env_formset_data)

         template_form = JobTemplateForm(prefix='template', instance=template)
 
     errors = form.errors or (env_formset.is_bound and not env_formset.is_valid) or (
         template_form = JobTemplateForm(prefix='template', instance=template)
 
     errors = form.errors or (env_formset.is_bound and not env_formset.is_valid) or (

diff --git a/requirements.txt b/requirements.txt
index 94386ea..41e2068 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Django<1.8
 django-grappelli
 django-bootstrap3
 pkgs/django-openid-auth-0.5.1.tar.gz
 django-grappelli
 django-bootstrap3
 pkgs/django-openid-auth-0.5.1.tar.gz

-pkgs/pyqcg-0.5.tar.gz
+pkgs/pyqcg-0.6.tar.gz

 pkgs/python-gridftp-1.3.4.tar.gz
 python-openid
 pytz
 pkgs/python-gridftp-1.3.4.tar.gz
 python-openid
 pytz