qcg/utils.py

   1 # coding=utf-8
   2
   3 from functools import wraps
   4
   5 import os
   6 import string
   7 import random
   8
   9 from django.contrib.auth.decorators import login_required
  10 from django.core.cache import caches
  11 from django.core.paginator import Paginator
  12 from django.utils.formats import date_format
  13 from django.utils.timezone import localtime
  14 from django.views.decorators.cache import cache_control
  15 from pyqcg import QCG
  16 from pyqcg.description import JobDescription
  17
  18 from filex.ftp import FTPOperation
  19 from qcg import constants
  20
  21
  22 from django.utils import encoding
  23
  24 resources_cache = caches['resources']
  25
  26
  27 def get_attributes(obj, attrs):
  28     return {name: getattr(obj, name) for name in attrs if getattr(obj, name) is not None}
  29
  30
  31 def username_from_dn(dn):
  32     _, username = dn.rsplit('=', 1)
  33
  34     return username
  35
  36
  37 def try_parse_int(s, default=None, base=10):
  38     try:
  39         return int(s, base)
  40     except (TypeError, ValueError):
  41         return default
  42
  43
  44 def paginator_context(request, objects, per_page=constants.PER_PAGE):
  45     paginator = Paginator(objects, per_page)
  46
  47     page_num = try_parse_int(request.GET.get('page'), 1)
  48     if not (1 <= page_num <= paginator.num_pages):
  49         page_num = 1
  50
  51     pages_range = range(max(2, min(page_num - 2, paginator.num_pages - 4)),
  52                         min(max(page_num + 2, 5), paginator.num_pages) + 1)
  53
  54     return {'page': paginator.page(page_num), 'num_pages': paginator.num_pages, 'pages_range': pages_range}
  55
  56
  57 def localtime_str(datetime):
  58     return date_format(localtime(datetime), 'DATETIME_FORMAT')
  59
  60
  61 def random_id(size=8, chars=string.ascii_uppercase + string.digits):
  62     return ''.join(random.choice(chars) for _ in range(size))
  63
  64
  65 def chunks(seq, size):
  66     return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
  67
  68 def generate_md_inputfile(params):
  69     md_input = list()
  70     # Opis pliku wyjsciowego
  71     opis=params['note'][:80]
  72     md_input.append(encoding.smart_str(opis, encoding='ascii', errors='ignore'))
  73     # Dane kontrolne obliczen
  74     md_input.append('SEED=-3059743 PDBREF ONE_LETTER MD EXTCONF RESCALE_MODE=2')
  75     ctl_data='nstep='+str(params['nstep'])+' ntwe='+str(params['ntwe'])
  76     ctl_data+=' ntwx='+str(params['ntwx'])+' dt='+str(params['dt'])+' damax='+str(params['damax'])+'lang=1 scal_fric=0.01'
  77     md_input.append('{:<79}&'.format(ctl_data))
  78     md_input.append('tau_bath=1.0 t_bath=300 reset_vel=0 respa ntime_split=1 maxtime_split=512 mdpdb')
  79     # Paramatry pól siłowych
  80     if params['force_field'] == 'GAB':
  81         # Wagi pola GAB
  82         md_input.append('WLONG=1.35279 WSCP=1.59304 WELEC=0.71534 WBOND=1.00000 WANG=1.13873            &')
  83         md_input.append('WSCLOC=0.16258 WTOR=1.98599 WTORD=1.57069 WCORRH=0.42887 WCORR5=0.00000        &')
  84         md_input.append('WCORR6=0.00000 WEL_LOC=0.16036 WTURN3=1.68722 WTURN4=0.66230 WTURN6=0.00000    &')
  85         md_input.append('WVDWPP=0.11371 WHPB=1.00000                                                    &')
  86         md_input.append('CUTOFF=7.00000 WCORR4=0.00000 WSCCOR=0.0')
  87     else:
  88         # Wagi pola E0LLY
  89         md_input.append('WLONG=1.00000 WSCP=1.23315 WELEC=0.84476 WBOND=1.00000 WANG=0.62954            &')
  90         md_input.append('WSCLOC=0.10554 WTOR=1.84316 WTORD=1.26571 WCORRH=0.19212 WCORR5=0.00000        &')
  91         md_input.append('WCORR6=0.00000 WEL_LOC=0.37357 WTURN3=1.40323 WTURN4=0.64673 WTURN6=0.00000    &')
  92         md_input.append('WVDWPP=0.23173 WHPB=1.00000 WSCCOR=0.0                                         &')
  93         md_input.append('CUTOFF=7.00000 WCORR4=0.00000')
  94     # Plik PDB
  95     md_input.append(params['pdb_file'].split('/')[-1])
  96     # Sekwencja aminokwasów
  97     md_input.append(len(params['sequence']))
  98     seq_str=params['sequence']
  99     while seq_str:
 100         md_input.append(seq_str[:80])
 101         seq_str=seq_str[80:]
 102     md_input.append(' 0')
 103     md_input.append(' 0')
 104
 105     return md_input
 106
 107
 108 def to_job_desc(params, proxy):
 109     QCG.start()
 110     desc = JobDescription()
 111
 112     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 113                   'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
 114                   'preprocess', 'postprocess', 'persistent')
 115
 116     for name in direct_map:
 117         if params[name]:
 118             setattr(desc, name, params[name])
 119
 120     if params['application']:
 121         desc.set_application(*params['application'])
 122         desc.stage_in += [params['master_file']]
 123         desc.arguments.insert(0, os.path.basename(params['master_file']))
 124     if params['script']:
 125         ftp = FTPOperation(proxy)
 126
 127         ftp.mkdir(constants.QCG_DATA_URL, parents=True)
 128         url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
 129         ftp.put(url)
 130
 131         for chunk in chunks(params['script'], 4096):
 132             ftp.stream.put(chunk)
 133         ftp.stream.put(None)
 134         ftp.wait()
 135
 136         desc.executable = url
 137     if params['nodes']:
 138         desc.set_nodes(*params['nodes'])
 139     if params['reservation']:
 140         desc.set_reservation(params['reservation'])
 141     if params['watch_output']:
 142         desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
 143
 144     przemytnik = params['env_variables']
 145     if params['force_field']:
 146         przemytnik.append([u'UNRESPORTAL_FORCEFIELD',params['force_field']])
 147     if params['sequence']:
 148         przemytnik.append([u'UNRESPORTAL_SEQUENCE',params['sequence']])
 149     if params['pdb_file']:
 150         przemytnik.append([u'UNRESPORTAL_PDBFILE',params['pdb_file']])
 151     if params['nstep']:
 152         przemytnik.append([u'UNRESPORTAL_NSTEP',params['nstep']])
 153     if params['ntwe']:
 154         przemytnik.append([u'UNRESPORTAL_NTWE',params['ntwe']])
 155     if params['ntwx']:
 156         przemytnik.append([u'UNRESPORTAL_NTWX',params['ntwx']])
 157     if params['dt']:
 158         przemytnik.append([u'UNRESPORTAL_DT',params['dt']])
 159     if params['damax']:
 160         przemytnik.append([u'UNRESPORTAL_DAMAX',params['damax']])
 161     setattr(desc, 'env_variables', przemytnik)
 162     # TODO monitoring
 163
 164     return desc
 165
 166
 167 def to_form_data(xml):
 168     # prevent circular import errors
 169     from forms import JobDescriptionForm
 170
 171     QCG.start()
 172     desc = JobDescription()
 173     desc.xml_description = xml
 174
 175     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 176                   'wall_time', 'modules', 'input', 'stage_in', 'native', 'persistent')
 177
 178     params = {}
 179     for name in direct_map:
 180         attr = getattr(desc, name)
 181         if isinstance(attr, bool) or attr:
 182             params[name] = attr
 183
 184     if desc.application is not None:
 185         app_name, app_ver = desc.application
 186         params['application'] = app_name if app_ver is None else app_name + '/' + app_name
 187         stage_in = params['stage_in']
 188         params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1]
 189         params['arguments'] = params['arguments'][1:]
 190     if desc.nodes is not None:
 191         params['nodes'] = ':'.join(map(str, desc.nodes))
 192     if desc.reservation is not None:
 193         res_id, res_type = desc.reservation
 194         params['reservation'] = res_id
 195     if desc.notify is not None:
 196         params['notify_type'], params['notify_address'] = desc.notify.split(':')
 197     if desc.watch_output is not None:
 198         watch_output, params['watch_output_pattern'] = desc.watch_output
 199         params['watch_output_type'], params['watch_output_address'] = watch_output.split(':')
 200     if desc.preprocess is not None:
 201         if desc.preprocess.startswith('gsiftp://'):
 202             params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT
 203             params['preprocess_script'] = desc.preprocess
 204         else:
 205             params['preprocess_type'] = JobDescriptionForm.Process.CMD
 206             params['preprocess_cmd'] = desc.preprocess
 207     if desc.postprocess is not None:
 208         if desc.postprocess.startswith('gsiftp://'):
 209             params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT
 210             params['postprocess_script'] = desc.postprocess
 211         else:
 212             params['postprocess_type'] = JobDescriptionForm.Process.CMD
 213             params['postprocess_cmd'] = desc.postprocess
 214     if desc.memory:
 215         params['memory'] = int(desc.memory)
 216     if desc.memory_per_slot:
 217         params['memory_per_slot'] = int(desc.memory_per_slot)
 218
 219     for item in desc.env_variables:
 220         if item[0] == u'UNRESPORTAL_FORCEFIELD':
 221             params['force_field'] = item[1]
 222         if item[0] == u'UNRESPORTAL_SEQUENCE':
 223             params['sequence'] = item[1]
 224         if item[0] == u'UNRESPORTAL_PDBFILE':
 225             params['pdb_file'] = item[1]
 226         if item[0] == u'UNRESPORTAL_NSTEP':
 227             params['nstep'] = item[1]
 228         if item[0] == u'UNRESPORTAL_NTWE':
 229             params['ntwe'] = item[1]
 230         if item[0] == u'UNRESPORTAL_NTWX':
 231             params['ntwx'] = item[1]
 232         if item[0] == u'UNRESPORTAL_DT':
 233             params['dt'] = item[1]
 234         if item[0] == u'UNRESPORTAL_DAMAX':
 235             params['damax'] = item[1]
 236     return params
 237
 238
 239 def restricted(view):
 240     return wraps(view)(cache_control(no_cache=True, must_revalidate=True, no_store=True)(login_required(view)))
 241
 242
 243 def cached_resources(proxy):
 244     hosts = resources_cache.get('hosts')
 245     if hosts is None:
 246         # prevent circular import errors
 247         from qcg.service import fetch_resources
 248
 249         hosts, _, applications, modules = map(make_choices, fetch_resources(proxy))
 250         resources_cache.set('hosts', hosts)
 251         resources_cache.set('applications', applications)
 252         resources_cache.set('modules', modules)
 253     else:
 254         applications = resources_cache.get('applications')
 255         modules = resources_cache.get('modules')
 256
 257     return hosts, applications, modules
 258
 259
 260 def make_choices(iterable):
 261     return ((None, ''),) + tuple((item, item) for item in sorted(iterable))