From 15df4cc114ef44ce5e1fdf95f861226b04b1a788 Mon Sep 17 00:00:00 2001 From: Maciej Tronowski Date: Thu, 10 Sep 2015 18:24:11 +0200 Subject: [PATCH] fetch resources info and set choices in job submission form --- qcg/forms.py | 34 +++++++++++++--------------------- qcg/service.py | 25 ++++++++++++++++++++++++- qcg/utils.py | 26 ++++++++++++++++++++++++++ qcg/views.py | 7 ++++--- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/qcg/forms.py b/qcg/forms.py index 2798c95..26f901f 100644 --- a/qcg/forms.py +++ b/qcg/forms.py @@ -64,20 +64,6 @@ class FiltersForm(forms.Form): class JobDescriptionForm(forms.Form): - class Host(object): - GALERA = 'galera.task.gda.pl' - HYDRA = 'hydra.icm.edu.pl' - INULA = 'inula.man.poznan.pl' - MOSS = 'moss.man.poznan.pl' - NOVA = 'nova.wcss.wroc.pl' - REEF = 'reef.man.poznan.pl' - ZEUS = 'zeus.cyfronet.pl' - - CHOICES = ( - CHOICES_PLACEHOLDER, - (INULA, u'Inula'), - ) - class Process(object): NONE = '' CMD = 'c' @@ -89,27 +75,28 @@ class JobDescriptionForm(forms.Form): (SCRIPT, u'Skrypt'), ) + APPLICATION_CHOICES = ( CHOICES_PLACEHOLDER, ('unres-gab', 'UNRES GAB'), ('unres-e0ll2y', 'UNRES E0LL2Y'), ) + QUEUE_CHOICES = ( CHOICES_PLACEHOLDER, ('plgrid', 'plgrid'), ('plgrid-long', 'plgrid-long'), ('plgrid-testing', 'plgrid-testing'), ) - MODULES_CHOICES = ( - CHOICES_PLACEHOLDER, - ) + PROTOCOL_CHOICES = ( ('', u'Brak'), ('mailto', u'E-mail'), ('xmpp', u'XMPP'), ) + FORCE_FIELD_CHOICES = ( ('GAB', u'GAB'), ('E0LL2Y', u'E0LL2Y'), @@ -128,15 +115,16 @@ class JobDescriptionForm(forms.Form): sequence = forms.CharField(label=u"Sekwencja", help_text=u"Sekwencja aminokwasów w zapisie jednoliterowym", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) - application = forms.ChoiceField(choices=APPLICATION_CHOICES, label=u"Aplikacja", required=False, initial='unres-gab') # TODO choices + application = forms.ChoiceField(choices=APPLICATION_CHOICES, label=u"Aplikacja", required=False, initial='unres-gab') #master_file = forms.CharField(label=u"Plik główny", max_length=500, required=False) + executable = forms.CharField(label=u"Plik wykonywalny", max_length=500, required=False) script = forms.CharField(label=u"Skrypt", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) arguments = forms.MultipleChoiceField(label=u"Argumenty", required=False) note = forms.CharField(label=u"Opis", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False) grant = forms.CharField(label=u"Grant", max_length=100, required=False) - hosts = forms.MultipleChoiceField(label=u"Host", choices=Host.CHOICES, required=False) + hosts = forms.MultipleChoiceField(label=u"Host", required=False) properties = forms.CharField(label=u"Właściwości węzłów", required=False) queue = forms.ChoiceField(choices=QUEUE_CHOICES, label=u"Kolejka", required=False) procs = forms.IntegerField(label=u"Liczba procesów", min_value=0, required=False) @@ -144,7 +132,7 @@ class JobDescriptionForm(forms.Form): wall_time = TimeRangeField(label=u"Wall time", required=False) memory = forms.IntegerField(label=u"Pamięć (MB)", min_value=0, required=False) memory_per_slot = forms.IntegerField(label=u"Pamięci per proces (MB)", min_value=0, required=False) - modules = forms.MultipleChoiceField(label=u"Moduły", choices=MODULES_CHOICES, required=False) # TODO choices + modules = forms.MultipleChoiceField(label=u"Moduły", required=False) reservation = forms.CharField(label=u"Rezerwacja", max_length=100, required=False) input = forms.CharField(label=u"Standardowe wejście", max_length=500, required=False) @@ -170,9 +158,13 @@ class JobDescriptionForm(forms.Form): native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False) persistent = forms.BooleanField(label=u"Trwałe", required=False) - def __init__(self, data=None, initial=None, *args, **kwargs): + def __init__(self, data=None, initial=None, hosts=(), applications=(), modules=(), *args, **kwargs): super(JobDescriptionForm, self).__init__(data, initial=initial, *args, **kwargs) + self.fields['hosts'].choices = hosts + self.fields['application'].choices = applications + self.fields['modules'].choices = modules + if data or initial: self._init_user_choices('queue', data, initial) self._init_user_choices('arguments', data, initial) diff --git a/qcg/service.py b/qcg/service.py index 9e99d43..e7316a6 100644 --- a/qcg/service.py +++ b/qcg/service.py @@ -5,7 +5,7 @@ from django.db import transaction from django.utils.functional import SimpleLazyObject from django.utils.timezone import now from pyqcg import QCG -from pyqcg.service import Registry +from pyqcg.service import Registry, JobFactory from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus from qcg.models import User, Job, Task, Allocation, NodeInfo @@ -175,3 +175,26 @@ def clean(obj, proxy): elapsed = time.time() - ts elapsed_py = elapsed - elapsed_clean logger.info('(%.3f) OBJ = %s, REMOTE = %.3f, LOCAL = %.3f', elapsed, obj, elapsed_clean, elapsed_py) + + +def fetch_resources(proxy): + ts = time.time() + QCG.start() + cred = Credential(proxy) + + rts = time.time() + resources = list(JobFactory().resources(False, cred)) + elapsed_query = time.time() - rts + + pts = time.time() + hosts = [res.name for res in resources] + storage = [res.storage for res in resources] + applications = {m for res in resources for m in res.applications} + modules = {m for res in resources for m in res.modules if m.startswith('plgrid')} + elapsed_pp = time.time() - pts + + elapsed = time.time() - ts + logger.info('(%.3f) HOSTS = %d, APPS = %d, MODULES = %d, QUERY = %.3f, PROC = %.3f', + elapsed, len(hosts), len(applications), len(modules), elapsed_query, elapsed_pp) + + return hosts, storage, applications, modules diff --git a/qcg/utils.py b/qcg/utils.py index 37a5b1a..da8f83c 100644 --- a/qcg/utils.py +++ b/qcg/utils.py @@ -7,6 +7,7 @@ import string import random from django.contrib.auth.decorators import login_required +from django.core.cache import caches from django.core.paginator import Paginator from django.utils.formats import date_format from django.utils.timezone import localtime @@ -17,8 +18,12 @@ from pyqcg.description import JobDescription from filex.ftp import FTPOperation from qcg import constants + from django.utils import encoding +resources_cache = caches['resources'] + + def get_attributes(obj, attrs): return {name: getattr(obj, name) for name in attrs if getattr(obj, name) is not None} @@ -194,3 +199,24 @@ def to_form_data(xml): def restricted(view): return wraps(view)(cache_control(no_cache=True, must_revalidate=True, no_store=True)(login_required(view))) + + +def cached_resources(proxy): + hosts = resources_cache.get('hosts') + if hosts is None: + # prevent circular import errors + from qcg.service import fetch_resources + + hosts, _, applications, modules = map(make_choices, fetch_resources(proxy)) + resources_cache.set('hosts', hosts) + resources_cache.set('applications', applications) + resources_cache.set('modules', modules) + else: + applications = resources_cache.get('applications') + modules = resources_cache.get('modules') + + return hosts, applications, modules + + +def make_choices(iterable): + return ((None, ''),) + tuple((item, item) for item in sorted(iterable)) diff --git a/qcg/views.py b/qcg/views.py index c52d59c..b5c5315 100644 --- a/qcg/views.py +++ b/qcg/views.py @@ -23,7 +23,7 @@ from filex.ftp import FTPOperation, FTPError from filex.views import make_url from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm from qcg.models import JobTemplate -from qcg.utils import paginator_context, to_job_desc, to_form_data, restricted +from qcg.utils import paginator_context, to_job_desc, to_form_data, restricted, cached_resources from qcg.service import update_user_data, update_job, cancel, clean, submit @@ -208,9 +208,10 @@ def job_submit(request, template_id=None): env_formset_data = [{'name': name, 'value': value} for name, value in form_data.pop('env_variables', ())] else: form_data, env_formset_data = None, None + hosts, applications, modules = cached_resources(request.session['proxy']) if request.method == 'POST': - form = JobDescriptionForm(request.POST, initial=form_data) + form = JobDescriptionForm(request.POST, form_data, hosts, applications, modules) env_formset = EnvFormSet(request.POST, initial=env_formset_data) template_form = JobTemplateForm(request.POST, prefix='template', instance=template) @@ -255,7 +256,7 @@ def job_submit(request, template_id=None): return redirect('jobs') else: - form = JobDescriptionForm(initial=form_data) + form = JobDescriptionForm(initial=form_data, hosts=hosts, applications=applications, modules=modules) env_formset = EnvFormSet(initial=env_formset_data) template_form = JobTemplateForm(prefix='template', instance=template) -- 1.7.9.5