fetch resources info and set choices in job submission form

diff --git a/qcg/forms.py b/qcg/forms.py
index 2798c95..26f901f 100644 (file)
--- a/qcg/forms.py
+++ b/qcg/forms.py
@@ -64,20 +64,6 @@ class FiltersForm(forms.Form):
 
 
 class JobDescriptionForm(forms.Form):
-    class Host(object):
-        GALERA = 'galera.task.gda.pl'
-        HYDRA = 'hydra.icm.edu.pl'
-        INULA = 'inula.man.poznan.pl'
-        MOSS = 'moss.man.poznan.pl'
-        NOVA = 'nova.wcss.wroc.pl'
-        REEF = 'reef.man.poznan.pl'
-        ZEUS = 'zeus.cyfronet.pl'
-
-        CHOICES = (
-            CHOICES_PLACEHOLDER,
-            (INULA, u'Inula'),
-        )
-
     class Process(object):
         NONE = ''
         CMD = 'c'
@@ -89,27 +75,28 @@ class JobDescriptionForm(forms.Form):
             (SCRIPT, u'Skrypt'),
         )
 
+
     APPLICATION_CHOICES = (
         CHOICES_PLACEHOLDER,
         ('unres-gab', 'UNRES GAB'),
         ('unres-e0ll2y', 'UNRES E0LL2Y'),
     )
         
+
     QUEUE_CHOICES = (
         CHOICES_PLACEHOLDER,
         ('plgrid', 'plgrid'),
         ('plgrid-long', 'plgrid-long'),
         ('plgrid-testing', 'plgrid-testing'),
     )
-    MODULES_CHOICES = (
-        CHOICES_PLACEHOLDER,
-    )
+
     PROTOCOL_CHOICES = (
         ('', u'Brak'),
         ('mailto', u'E-mail'),
         ('xmpp', u'XMPP'),
     )
 
+
     FORCE_FIELD_CHOICES = (
         ('GAB', u'GAB'),
         ('E0LL2Y', u'E0LL2Y'),
@@ -128,15 +115,16 @@ class JobDescriptionForm(forms.Form):
     sequence = forms.CharField(label=u"Sekwencja", help_text=u"Sekwencja aminokwasów w zapisie jednoliterowym", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False)
     
     
-    application = forms.ChoiceField(choices=APPLICATION_CHOICES, label=u"Aplikacja", required=False, initial='unres-gab')  # TODO choices
+    application = forms.ChoiceField(choices=APPLICATION_CHOICES, label=u"Aplikacja", required=False, initial='unres-gab') 
     #master_file = forms.CharField(label=u"Plik główny", max_length=500, required=False)
+
     executable = forms.CharField(label=u"Plik wykonywalny", max_length=500, required=False)
     script = forms.CharField(label=u"Skrypt", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False)
     arguments = forms.MultipleChoiceField(label=u"Argumenty", required=False)
     note = forms.CharField(label=u"Opis", widget=forms.Textarea(attrs={'rows': 2, 'cols': 40}), required=False)
     grant = forms.CharField(label=u"Grant", max_length=100, required=False)
 
-    hosts = forms.MultipleChoiceField(label=u"Host", choices=Host.CHOICES, required=False)
+    hosts = forms.MultipleChoiceField(label=u"Host", required=False)
     properties = forms.CharField(label=u"Właściwości węzłów", required=False)
     queue = forms.ChoiceField(choices=QUEUE_CHOICES, label=u"Kolejka", required=False)
     procs = forms.IntegerField(label=u"Liczba procesów", min_value=0, required=False)
@@ -144,7 +132,7 @@ class JobDescriptionForm(forms.Form):
     wall_time = TimeRangeField(label=u"Wall time", required=False)
     memory = forms.IntegerField(label=u"Pamięć (MB)", min_value=0, required=False)
     memory_per_slot = forms.IntegerField(label=u"Pamięci per proces (MB)", min_value=0, required=False)
-    modules = forms.MultipleChoiceField(label=u"Moduły", choices=MODULES_CHOICES, required=False)  # TODO choices
+    modules = forms.MultipleChoiceField(label=u"Moduły", required=False)
     reservation = forms.CharField(label=u"Rezerwacja", max_length=100, required=False)
 
     input = forms.CharField(label=u"Standardowe wejście", max_length=500, required=False)
@@ -170,9 +158,13 @@ class JobDescriptionForm(forms.Form):
     native = forms.MultipleChoiceField(label=u"Opcje systemu kolejkowego", required=False)
     persistent = forms.BooleanField(label=u"Trwałe", required=False)
 
-    def __init__(self, data=None, initial=None, *args, **kwargs):
+    def __init__(self, data=None, initial=None, hosts=(), applications=(), modules=(), *args, **kwargs):
         super(JobDescriptionForm, self).__init__(data, initial=initial, *args, **kwargs)
 
+        self.fields['hosts'].choices = hosts
+        self.fields['application'].choices = applications
+        self.fields['modules'].choices = modules
+
         if data or initial:
             self._init_user_choices('queue', data, initial)
             self._init_user_choices('arguments', data, initial)

diff --git a/qcg/service.py b/qcg/service.py
index 9e99d43..e7316a6 100644 (file)
--- a/qcg/service.py
+++ b/qcg/service.py
@@ -5,7 +5,7 @@ from django.db import transaction
 from django.utils.functional import SimpleLazyObject
 from django.utils.timezone import now
 from pyqcg import QCG
-from pyqcg.service import Registry
+from pyqcg.service import Registry, JobFactory
 from pyqcg.utils import Credential, TimePeriod, JobStatus, TaskStatus
 
 from qcg.models import User, Job, Task, Allocation, NodeInfo
@@ -175,3 +175,26 @@ def clean(obj, proxy):
     elapsed = time.time() - ts
     elapsed_py = elapsed - elapsed_clean
     logger.info('(%.3f) OBJ = %s, REMOTE = %.3f, LOCAL = %.3f', elapsed, obj, elapsed_clean, elapsed_py)
+
+
+def fetch_resources(proxy):
+    ts = time.time()
+    QCG.start()
+    cred = Credential(proxy)
+
+    rts = time.time()
+    resources = list(JobFactory().resources(False, cred))
+    elapsed_query = time.time() - rts
+
+    pts = time.time()
+    hosts = [res.name for res in resources]
+    storage = [res.storage for res in resources]
+    applications = {m for res in resources for m in res.applications}
+    modules = {m for res in resources for m in res.modules if m.startswith('plgrid')}
+    elapsed_pp = time.time() - pts
+
+    elapsed = time.time() - ts
+    logger.info('(%.3f) HOSTS = %d, APPS = %d, MODULES = %d, QUERY = %.3f, PROC = %.3f',
+                elapsed, len(hosts), len(applications), len(modules), elapsed_query, elapsed_pp)
+
+    return hosts, storage, applications, modules

diff --git a/qcg/utils.py b/qcg/utils.py
index 37a5b1a..da8f83c 100644 (file)
--- a/qcg/utils.py
+++ b/qcg/utils.py
@@ -7,6 +7,7 @@ import string
 import random
 
 from django.contrib.auth.decorators import login_required
+from django.core.cache import caches
 from django.core.paginator import Paginator
 from django.utils.formats import date_format
 from django.utils.timezone import localtime
@@ -17,8 +18,12 @@ from pyqcg.description import JobDescription
 from filex.ftp import FTPOperation
 from qcg import constants
 
+
 from django.utils import encoding
 
+resources_cache = caches['resources']
+
+
 def get_attributes(obj, attrs):
     return {name: getattr(obj, name) for name in attrs if getattr(obj, name) is not None}
 
@@ -194,3 +199,24 @@ def to_form_data(xml):
 
 def restricted(view):
     return wraps(view)(cache_control(no_cache=True, must_revalidate=True, no_store=True)(login_required(view)))
+
+
+def cached_resources(proxy):
+    hosts = resources_cache.get('hosts')
+    if hosts is None:
+        # prevent circular import errors
+        from qcg.service import fetch_resources
+
+        hosts, _, applications, modules = map(make_choices, fetch_resources(proxy))
+        resources_cache.set('hosts', hosts)
+        resources_cache.set('applications', applications)
+        resources_cache.set('modules', modules)
+    else:
+        applications = resources_cache.get('applications')
+        modules = resources_cache.get('modules')
+
+    return hosts, applications, modules
+
+
+def make_choices(iterable):
+    return ((None, ''),) + tuple((item, item) for item in sorted(iterable))

diff --git a/qcg/views.py b/qcg/views.py
index c52d59c..b5c5315 100644 (file)
--- a/qcg/views.py
+++ b/qcg/views.py
@@ -23,7 +23,7 @@ from filex.ftp import FTPOperation, FTPError
 from filex.views import make_url
 from qcg.forms import FiltersForm, ColumnsForm, JobDescriptionForm, EnvFormSet, JobTemplateForm
 from qcg.models import JobTemplate
-from qcg.utils import paginator_context, to_job_desc, to_form_data, restricted
+from qcg.utils import paginator_context, to_job_desc, to_form_data, restricted, cached_resources
 from qcg.service import update_user_data, update_job, cancel, clean, submit
 
 
@@ -208,9 +208,10 @@ def job_submit(request, template_id=None):
         env_formset_data = [{'name': name, 'value': value} for name, value in form_data.pop('env_variables', ())]
     else:
         form_data, env_formset_data = None, None
+    hosts, applications, modules = cached_resources(request.session['proxy'])
 
     if request.method == 'POST':
-        form = JobDescriptionForm(request.POST, initial=form_data)
+        form = JobDescriptionForm(request.POST, form_data, hosts, applications, modules)
         env_formset = EnvFormSet(request.POST, initial=env_formset_data)
         template_form = JobTemplateForm(request.POST, prefix='template', instance=template)
 
@@ -255,7 +256,7 @@ def job_submit(request, template_id=None):
             
             return redirect('jobs')
     else:
-        form = JobDescriptionForm(initial=form_data)
+        form = JobDescriptionForm(initial=form_data, hosts=hosts, applications=applications, modules=modules)
         env_formset = EnvFormSet(initial=env_formset_data)
         template_form = JobTemplateForm(prefix='template', instance=template)