qcg/utils.py

   1 # coding=utf-8
   2
   3 from functools import wraps
   4
   5 import os
   6 import string
   7 import random
   8
   9 from django.contrib.auth.decorators import login_required
  10 from django.core.cache import caches
  11 from django.core.paginator import Paginator
  12 from django.utils.formats import date_format
  13 from django.utils.timezone import localtime
  14 from django.views.decorators.cache import cache_control
  15 from pyqcg import QCG
  16 from pyqcg.description import JobDescription
  17
  18 from filex.ftp import FTPOperation
  19 from qcg import constants
  20
  21
  22 from django.utils import encoding
  23
  24
  25 resources_cache = caches['resources']
  26
  27
  28 def get_attributes(obj, attrs):
  29     return {name: getattr(obj, name) for name in attrs if getattr(obj, name) is not None}
  30
  31
  32 def username_from_dn(dn):
  33     _, username = dn.rsplit('=', 1)
  34
  35     return username
  36
  37
  38 def try_parse_int(s, default=None, base=10):
  39     try:
  40         return int(s, base)
  41     except (TypeError, ValueError):
  42         return default
  43
  44
  45 def paginator_context(request, objects, per_page=constants.PER_PAGE):
  46     paginator = Paginator(objects, per_page)
  47
  48     page_num = try_parse_int(request.GET.get('page'), 1)
  49     if not (1 <= page_num <= paginator.num_pages):
  50         page_num = 1
  51
  52     pages_range = range(max(2, min(page_num - 2, paginator.num_pages - 4)),
  53                         min(max(page_num + 2, 5), paginator.num_pages) + 1)
  54
  55     return {'page': paginator.page(page_num), 'num_pages': paginator.num_pages, 'pages_range': pages_range}
  56
  57
  58 def localtime_str(datetime):
  59     return date_format(localtime(datetime), 'DATETIME_FORMAT')
  60
  61
  62 def random_id(size=8, chars=string.ascii_uppercase + string.digits):
  63     return ''.join(random.choice(chars) for _ in range(size))
  64
  65
  66 def chunks(seq, size):
  67     return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
  68
  69 def generate_md_inputfile(params):
  70     md_input = list()
  71     # Opis pliku wyjsciowego
  72     opis=params['note'][:80]
  73     md_input.append(encoding.smart_str(opis, encoding='ascii', errors='ignore'))
  74     # Dane kontrolne obliczen
  75     md_input.append('SEED=-3059743 PDBREF ONE_LETTER MD EXTCONF RESCALE_MODE=2')
  76     ctl_data='nstep='+str(params['nstep'])+' ntwe='+str(params['ntwe'])
  77     ctl_data+=' ntwx='+str(params['ntwx'])+' dt='+str(params['dt'])+' damax='+str(params['damax'])+'lang=1 scal_fric=0.01'
  78     md_input.append('{:<79}&'.format(ctl_data))
  79     md_input.append('tau_bath=1.0 t_bath=300 reset_vel=0 respa ntime_split=1 maxtime_split=512 mdpdb')
  80     # Paramatry pól siłowych
  81     if params['force_field'] == 'GAB':
  82         # Wagi pola GAB
  83         md_input.append('WLONG=1.35279 WSCP=1.59304 WELEC=0.71534 WBOND=1.00000 WANG=1.13873            &')
  84         md_input.append('WSCLOC=0.16258 WTOR=1.98599 WTORD=1.57069 WCORRH=0.42887 WCORR5=0.00000        &')
  85         md_input.append('WCORR6=0.00000 WEL_LOC=0.16036 WTURN3=1.68722 WTURN4=0.66230 WTURN6=0.00000    &')
  86         md_input.append('WVDWPP=0.11371 WHPB=1.00000                                                    &')
  87         md_input.append('CUTOFF=7.00000 WCORR4=0.00000 WSCCOR=0.0')
  88     else:
  89         # Wagi pola E0LLY
  90         md_input.append('WLONG=1.00000 WSCP=1.23315 WELEC=0.84476 WBOND=1.00000 WANG=0.62954            &')
  91         md_input.append('WSCLOC=0.10554 WTOR=1.84316 WTORD=1.26571 WCORRH=0.19212 WCORR5=0.00000        &')
  92         md_input.append('WCORR6=0.00000 WEL_LOC=0.37357 WTURN3=1.40323 WTURN4=0.64673 WTURN6=0.00000    &')
  93         md_input.append('WVDWPP=0.23173 WHPB=1.00000 WSCCOR=0.0                                         &')
  94         md_input.append('CUTOFF=7.00000 WCORR4=0.00000')
  95     # Plik PDB
  96     md_input.append(params['pdb_file'].split('/')[-1])
  97     # Sekwencja aminokwasów
  98     md_input.append(len(params['sequence']))
  99     seq_str=params['sequence']
 100     while seq_str:
 101         md_input.append(seq_str[:80])
 102         seq_str=seq_str[80:]
 103     md_input.append(' 0')
 104     md_input.append(' 0')
 105
 106     return md_input
 107
 108
 109 def to_job_desc(params, proxy):
 110     QCG.start()
 111     desc = JobDescription()
 112
 113
 114     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 115                   'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
 116                   'preprocess', 'postprocess', 'persistent')
 117
 118     for name in direct_map:
 119         if params[name]:
 120             setattr(desc, name, params[name])
 121
 122
 123     if params['application']:
 124         desc.set_application(*params['application'])
 125         desc.stage_in += [params['master_file']]
 126         desc.arguments.insert(0, os.path.basename(params['master_file']))
 127     if params['script']:
 128         ftp = FTPOperation(proxy)
 129
 130         ftp.mkdir(constants.QCG_DATA_URL, parents=True)
 131         url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
 132         ftp.put(url)
 133
 134         for chunk in chunks(params['script'], 4096):
 135             ftp.stream.put(chunk)
 136         ftp.stream.put(None)
 137         ftp.wait()
 138
 139         desc.executable = url
 140     if params['nodes']:
 141         desc.set_nodes(*params['nodes'])
 142     if params['reservation']:
 143         desc.set_reservation(params['reservation'])
 144     if params['watch_output']:
 145         desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
 146
 147     #print "testowanie ",
 148     przemytnik = params['env_variables']
 149     kontrabanda =( (u'UNRESPORTAL_FORCEFIELD', 'force_field'), (u'UNRESPORTAL_SEQUENCE', 'sequence'),
 150                 (u'UNRESPORTAL_PDBFILE','pdb_file'), (u'UNRESPORTAL_NSTEP','nstep'),
 151                 (u'UNRESPORTAL_NTWE', 'ntwe'), (u'UNRESPORTAL_NTWX','ntwx'),
 152                 (u'UNRESPORTAL_DT', 'dt'), (u'UNRESPORTAL_DAMAX', 'damax'))
 153
 154     print "> Przed: ",przemytnik
 155     for p in przemytnik:
 156         print "-",p
 157         for k in kontrabanda:
 158             if p[0]==k[0]:
 159                 print "  usuwam",p
 160                 przemytnik.remove(p)
 161                 break
 162
 163     print "> Po: ",przemytnik
 164     for p in przemytnik:
 165         print "-",p
 166
 167     for k in kontrabanda:
 168         if params[k[1]]:
 169             przemytnik.append( [ k[0], params[k[1]] ] )
 170     '''
 171         if params[k[1]]:
 172             print params[k[1]]
 173             print k
 174             try:
 175                 setattr(przemytnik, k[0], params[k[1]] )
 176                 print "setattr"
 177             except AttributeError:
 178                 przemytnik.append( [ k[0], params[k[1]] ] )
 179                 print "append"
 180
 181
 182     if params['sequence']:
 183         przemytnik.append([u'UNRESPORTAL_SEQUENCE',params['sequence']])
 184     if params['pdb_file']:
 185         przemytnik.append([u'UNRESPORTAL_PDBFILE',params['pdb_file']])
 186     if params['nstep']:
 187         przemytnik.append([u'UNRESPORTAL_NSTEP',params['nstep']])
 188     if params['ntwe']:
 189         przemytnik.append([u'UNRESPORTAL_NTWE',params['ntwe']])
 190     if params['ntwx']:
 191         przemytnik.append([u'UNRESPORTAL_NTWX',params['ntwx']])
 192     if params['dt']:
 193         przemytnik.append([u'UNRESPORTAL_DT',params['dt']])
 194     if params['damax']:
 195         przemytnik.append([u'UNRESPORTAL_DAMAX',params['damax']])
 196     '''
 197     setattr(desc, 'env_variables', przemytnik)
 198
 199     # TODO monitoring
 200
 201     return desc
 202
 203
 204 def to_form_data(xml):
 205     # prevent circular import errors
 206     from forms import JobDescriptionForm
 207
 208     QCG.start()
 209     desc = JobDescription()
 210     desc.xml_description = xml
 211
 212     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 213                   'wall_time', 'modules', 'input', 'stage_in', 'native', 'persistent')
 214
 215     params = {}
 216     for name in direct_map:
 217         attr = getattr(desc, name)
 218         if isinstance(attr, bool) or attr:
 219             params[name] = attr
 220
 221     if desc.application is not None:
 222         app_name, app_ver = desc.application
 223         params['application'] = app_name if app_ver is None else app_name + '/' + app_name
 224         stage_in = params['stage_in']
 225         params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1]
 226         params['arguments'] = params['arguments'][1:]
 227     if desc.nodes is not None:
 228         params['nodes'] = ':'.join(map(str, desc.nodes))
 229     if desc.reservation is not None:
 230         res_id, res_type = desc.reservation
 231         params['reservation'] = res_id
 232     if desc.notify is not None:
 233         params['notify_type'], params['notify_address'] = desc.notify.split(':')
 234     if desc.watch_output is not None:
 235         watch_output, params['watch_output_pattern'] = desc.watch_output
 236         params['watch_output_type'], params['watch_output_address'] = watch_output.split(':')
 237     if desc.preprocess is not None:
 238         if desc.preprocess.startswith('gsiftp://'):
 239             params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT
 240             params['preprocess_script'] = desc.preprocess
 241         else:
 242             params['preprocess_type'] = JobDescriptionForm.Process.CMD
 243             params['preprocess_cmd'] = desc.preprocess
 244     if desc.postprocess is not None:
 245         if desc.postprocess.startswith('gsiftp://'):
 246             params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT
 247             params['postprocess_script'] = desc.postprocess
 248         else:
 249             params['postprocess_type'] = JobDescriptionForm.Process.CMD
 250             params['postprocess_cmd'] = desc.postprocess
 251     if desc.memory:
 252         params['memory'] = int(desc.memory)
 253     if desc.memory_per_slot:
 254         params['memory_per_slot'] = int(desc.memory_per_slot)
 255
 256     for item in desc.env_variables:
 257         if item[0] == u'UNRESPORTAL_FORCEFIELD':
 258             params['force_field'] = item[1]
 259         if item[0] == u'UNRESPORTAL_SEQUENCE':
 260             params['sequence'] = item[1]
 261         if item[0] == u'UNRESPORTAL_PDBFILE':
 262             params['pdb_file'] = item[1]
 263         if item[0] == u'UNRESPORTAL_NSTEP':
 264             params['nstep'] = item[1]
 265         if item[0] == u'UNRESPORTAL_NTWE':
 266             params['ntwe'] = item[1]
 267         if item[0] == u'UNRESPORTAL_NTWX':
 268             params['ntwx'] = item[1]
 269         if item[0] == u'UNRESPORTAL_DT':
 270             params['dt'] = item[1]
 271         if item[0] == u'UNRESPORTAL_DAMAX':
 272             params['damax'] = item[1]
 273     return params
 274
 275
 276 def restricted(view):
 277     return wraps(view)(cache_control(no_cache=True, must_revalidate=True, no_store=True)(login_required(view)))
 278
 279
 280 def cached_resources(proxy):
 281     hosts = resources_cache.get('hosts')
 282     if hosts is None:
 283         # prevent circular import errors
 284         from qcg.service import fetch_resources
 285
 286         hosts, _, applications, modules = map(make_choices, fetch_resources(proxy))
 287         resources_cache.set('hosts', hosts)
 288         resources_cache.set('applications', applications)
 289         resources_cache.set('modules', modules)
 290     else:
 291         applications = resources_cache.get('applications')
 292         modules = resources_cache.get('modules')
 293
 294     return hosts, applications, modules
 295
 296
 297 def make_choices(iterable):
 298     return ((None, ''),) + tuple((item, item) for item in sorted(iterable))