native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False)
persistent = forms.BooleanField(label=u"Trwałe", required=False)
- def __init__(self, data=None, *args, **kwargs):
- super(JobDescriptionForm, self).__init__(data, *args, **kwargs)
+ def __init__(self, data=None, initial=None, *args, **kwargs):
+ super(JobDescriptionForm, self).__init__(data, initial=initial, *args, **kwargs)
- if data is not None:
- # accept user defined choices
- self.fields['queue'].choices += ((data.get('queue'), data.get('queue')), )
- self.fields['arguments'].choices += ((v, v) for v in data.getlist('arguments'))
- self.fields['native'].choices += ((v, v) for v in data.getlist('native'))
- self.fields['stage_in'].choices += ((v, v) for v in data.getlist('stage_in'))
+ if data or initial:
+ self._init_user_choices('queue', data, initial)
+ self._init_user_choices('arguments', data, initial)
+ self._init_user_choices('native', data, initial)
+ self._init_user_choices('stage_in', data, initial)
def clean(self):
data = super(JobDescriptionForm, self).clean()
if data['application'] and not data['master_file']:
self.add_error('master_file', u"W trybie uruchamiania aplikacji należy podać plik główny")
+ if data['procs'] and data['nodes']:
+ self.add_error(None, u"Zdefiniuj tylko jedno z pól: liczbę procesów lub topologię węzłów")
+
notify_type = data.get('notify_type')
data['notify'] = u'{}:{}'.format(notify_type, data['notify_address']) if notify_type else ''
def clean_application(self):
return self.cleaned_data['application'].split('/', 1) if self.cleaned_data['application'] else ''
+ def clean_nodes(self):
+ return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else ''
+
def clean_executable(self):
return self._gsiftp_suffix(self.cleaned_data['executable'])
def clean_master_file(self):
return self._gsiftp_suffix(self.cleaned_data['master_file'])
- def clean_nodes(self):
- return map(int, self.cleaned_data['nodes'].split(':', 2)) if self.cleaned_data['nodes'] else ''
-
def clean_input(self):
return self._gsiftp_suffix(self.cleaned_data['input'])
def clean_stage_in(self):
- return ['gsiftp://' + item for item in self.cleaned_data['stage_in']]
+ return [self._gsiftp_suffix(item) for item in self.cleaned_data['stage_in']]
def clean_preprocess_script(self):
return self._gsiftp_suffix(self.cleaned_data['preprocess_script'])
@staticmethod
def _gsiftp_suffix(url):
- return 'gsiftp://' + url if url else ''
+ if url:
+ return url if url.startswith('gsiftp://') else 'gsiftp://' + url
+
+ def _init_user_choices(self, name, data, initial):
+ initial = initial.get(name) if initial is not None else None
+ choices = data.getlist(name) if data is not None else []
+
+ if initial:
+ choices += initial if isinstance(initial, list) else [initial]
+ self.fields[name].initial = initial
+
+ if choices:
+ self.fields[name].choices += ((v, v) for v in choices)
class EnvForm(forms.Form):
import logging
-import os
import time
from django.db import transaction
from django.utils.functional import SimpleLazyObject
from django.utils.timezone import now
from pyqcg import QCG
-from pyqcg.description import JobDescription
from pyqcg.service import Registry
from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus
-from filex.ftp import FTPOperation
-from qcg.constants import QCG_DATA_URL
from qcg.models import User, Job, Task, Allocation, NodeInfo
-from qcg.utils import random_id, chunks
logger = logging.getLogger(__name__)
elapsed, job.job_id, elapsed_job, job.tasks.count(), elapsed_tasks, elapsed_py)
-def make_job_desc(params, proxy):
- QCG.start()
- desc = JobDescription(Credential(proxy))
-
- direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
- 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
- 'preprocess', 'postprocess', 'persistent')
-
- for name in direct_map:
- if params[name]:
- setattr(desc, name, params[name])
-
- if params['application']:
- desc.set_application(*params['application'])
- desc.stage_in += [params['master_file']]
- desc.arguments.insert(0, os.path.basename(params['master_file']))
- if params['script']:
- ftp = FTPOperation(proxy)
-
- ftp.mkdir(QCG_DATA_URL, parents=True)
- url = os.path.join(QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
- ftp.put(url)
-
- for chunk in chunks(params['script'], 4096):
- ftp.stream.put(chunk)
- ftp.stream.put(None)
-
- ftp.wait()
- desc.executable = url
- if params['nodes']:
- desc.set_nodes(*params['nodes'])
- if params['reservation']:
- desc.set_reservation(params['reservation'])
- if params['watch_output']:
- desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
- # TODO monitoring
-
- return desc
-
-
def cancel(obj, proxy):
ts = time.time()
QCG.start()
+import os
import string
import random
from django.core.paginator import Paginator
from django.utils.formats import date_format
from django.utils.timezone import localtime
+from pyqcg import QCG
+from pyqcg.utils import Credential
+from pyqcg.description import JobDescription
+from filex.ftp import FTPOperation
from qcg import constants
def chunks(seq, size):
return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
+
+
+def to_job_desc(params, proxy):
+ QCG.start()
+ desc = JobDescription(Credential(proxy))
+
+ direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
+ 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
+ 'preprocess', 'postprocess', 'persistent')
+
+ for name in direct_map:
+ if params[name]:
+ setattr(desc, name, params[name])
+
+ if params['application']:
+ desc.set_application(*params['application'])
+ desc.stage_in += [params['master_file']]
+ desc.arguments.insert(0, os.path.basename(params['master_file']))
+ if params['script']:
+ ftp = FTPOperation(proxy)
+
+ ftp.mkdir(constants.QCG_DATA_URL, parents=True)
+ url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
+ ftp.put(url)
+
+ for chunk in chunks(params['script'], 4096):
+ ftp.stream.put(chunk)
+ ftp.stream.put(None)
+ ftp.wait()
+
+ desc.executable = url
+ if params['nodes']:
+ desc.set_nodes(*params['nodes'])
+ if params['reservation']:
+ desc.set_reservation(params['reservation'])
+ if params['watch_output']:
+ desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
+ # TODO monitoring
+
+ return desc
+
+
+def to_form_data(xml):
+ # prevent circular import errors
+ from forms import JobDescriptionForm
+
+ QCG.start()
+ desc = JobDescription()
+ desc.xml_description = xml
+
+ direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
+ 'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'persistent')
+
+ params = {}
+ for name in direct_map:
+ attr = getattr(desc, name)
+ if isinstance(attr, bool) or attr:
+ params[name] = attr
+
+ if desc.application is not None:
+ app_name, app_ver = desc.application
+ params['application'] = app_name if app_ver is None else app_name + '/' + app_name
+ stage_in = params['stage_in']
+ params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1]
+ params['arguments'] = params['arguments'][1:]
+ if desc.nodes is not None:
+ params['nodes'] = ':'.join(map(str, desc.nodes))
+ if desc.reservation is not None:
+ res_id, res_type = desc.reservation
+ params['reservation'] = res_id
+ if desc.notify is not None:
+ params['notify_type'], params['notify_address'] = desc.notify.split(':')
+ if desc.watch_output is not None:
+ watch_output, params['watch_output_pattern'] = desc.watch_output
+ params['watch_output_type'], params['watch_output_address'] = watch_output.split(':')
+ if desc.preprocess is not None:
+ if desc.preprocess.startswith('gsiftp://'):
+ params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT
+ params['preprocess_script'] = desc.preprocess
+ else:
+ params['preprocess_type'] = JobDescriptionForm.Process.CMD
+ params['preprocess_cmd'] = desc.preprocess
+ if desc.postprocess is not None:
+ if desc.postprocess.startswith('gsiftp://'):
+ params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT
+ params['postprocess_script'] = desc.postprocess
+ else:
+ params['postprocess_type'] = JobDescriptionForm.Process.CMD
+ params['postprocess_cmd'] = desc.postprocess
+
+ return params
from django.views.decorators.http import require_POST
from django_openid_auth.views import make_consumer
from openid.extensions import ax
-from pyqcg.utils.qcg_types import PyqcgException
+from pyqcg.utils import PyqcgException
from filex.forms import HostPathNameForm, RenameForm, ArchiveForm, HostPathForm
from filex.ftp import FTPOperation, FTPError
from filex.views import make_url
from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm
-from qcg.utils import paginator_context
-from qcg.service import update_user_data, make_job_desc, update_job, cancel, clean
+from qcg.utils import paginator_context, to_job_desc, to_form_data
+from qcg.service import update_user_data, update_job, cancel, clean
def index(request):
save_template = 'save-template' in request.POST
template = get_object_or_404(request.user.templates, id=template_id) if template_id is not None else None
+ if template is not None:
+ form_data = to_form_data(template.description)
+ env_formset_data = [{'name': name, 'value': value} for name, value in form_data.pop('env_variables', ())]
+ else:
+ form_data, env_formset_data = None, None
+
if request.method == 'POST':
- form = JobDescriptionForm(request.POST)
- env_formset = EnvFormSet(request.POST)
+ form = JobDescriptionForm(request.POST, initial=form_data)
+ env_formset = EnvFormSet(request.POST, initial=env_formset_data)
template_form = JobTemplateForm(request.POST, prefix='template', instance=template)
if form.is_valid() and env_formset.is_valid() and (not save_template or template_form.is_valid):
params['env_variables'] = [(env['name'], env['value'])
for env in env_formset.cleaned_data if env and not env['DELETE']]
- job_desc = make_job_desc(params, request.session['proxy'])
+ job_desc = to_job_desc(params, request.session['proxy'])
if save_template:
- if template is not None:
- # save already existing template...
- instance = template_form.save()
- else:
- # or set attributes for new template
- instance = template_form.save(commit=False)
+ template = template_form.save(commit=False)
- instance.owner = request.user
- instance.description = job_desc.xml_description
- instance.save()
+ template.owner = request.user
+ template.description = job_desc.xml_description
+ template.save()
- return redirect(instance)
-
- else:
- job = job_desc.submit()
+ return redirect(template)
- messages.success(request,
- format_html('<span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> '
- 'Zlecono zadanie <em>{}</em>.', job.job_id))
+ job = job_desc.submit()
- return redirect('jobs')
+ messages.success(request,
+ format_html('<span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> '
+ 'Zlecono zadanie <em>{}</em>.', job.job_id))
- print 'form', repr(form.errors)
- print 'env_formset', repr(env_formset.errors)
- print 'template_form', repr(template_form.errors)
+ return redirect('jobs')
else:
- form = JobDescriptionForm()
- env_formset = EnvFormSet()
+ form = JobDescriptionForm(initial=form_data)
+ env_formset = EnvFormSet(initial=env_formset_data)
template_form = JobTemplateForm(prefix='template', instance=template)
errors = form.errors or (env_formset.is_bound and not env_formset.is_valid) or (
django-grappelli
django-bootstrap3
pkgs/django-openid-auth-0.5.1.tar.gz
-pkgs/pyqcg-0.5.tar.gz
+pkgs/pyqcg-0.6.tar.gz
pkgs/python-gridftp-1.3.4.tar.gz
python-openid
pytz
projects / qcg-portal.git / commitdiff