qcg/utils.py

   1 # coding=utf-8
   2
   3 from functools import wraps
   4
   5 import os
   6 import string
   7 import random
   8
   9 from django.contrib.auth.decorators import login_required
  10 from django.core.cache import caches
  11 from django.core.paginator import Paginator
  12 from django.utils.formats import date_format
  13 from django.utils.timezone import localtime
  14 from django.views.decorators.cache import cache_control
  15 from pyqcg import QCG
  16 from pyqcg.description import JobDescription
  17
  18 from filex.ftp import FTPOperation
  19 from qcg import constants
  20
  21
  22 from django.utils import encoding
  23
  24
  25 resources_cache = caches['resources']
  26
  27
  28 def get_attributes(obj, attrs):
  29     return {name: getattr(obj, name) for name in attrs if getattr(obj, name) is not None}
  30
  31
  32 def username_from_dn(dn):
  33     _, username = dn.rsplit('=', 1)
  34
  35     return username
  36
  37
  38 def try_parse_int(s, default=None, base=10):
  39     try:
  40         return int(s, base)
  41     except (TypeError, ValueError):
  42         return default
  43
  44
  45 def paginator_context(request, objects, per_page=constants.PER_PAGE):
  46     paginator = Paginator(objects, per_page)
  47
  48     page_num = try_parse_int(request.GET.get('page'), 1)
  49     if not (1 <= page_num <= paginator.num_pages):
  50         page_num = 1
  51
  52     pages_range = range(max(2, min(page_num - 2, paginator.num_pages - 4)),
  53                         min(max(page_num + 2, 5), paginator.num_pages) + 1)
  54
  55     return {'page': paginator.page(page_num), 'num_pages': paginator.num_pages, 'pages_range': pages_range}
  56
  57
  58 def localtime_str(datetime):
  59     return date_format(localtime(datetime), 'DATETIME_FORMAT')
  60
  61
  62 def random_id(size=8, chars=string.ascii_uppercase + string.digits):
  63     return ''.join(random.choice(chars) for _ in range(size))
  64
  65
  66 def chunks(seq, size):
  67     return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
  68
  69 def generate_md_inputfile(params):
  70     md_input = list()
  71     # Opis pliku wyjsciowego
  72     opis=params['note'][:80]
  73     md_input.append(encoding.smart_str(opis, encoding='ascii', errors='ignore'))
  74     # Dane kontrolne obliczen
  75     md_input.append('SEED=-3059743 PDBREF ONE_LETTER MD EXTCONF RESCALE_MODE=2')
  76     ctl_data='nstep='+str(params['nstep'])+' ntwe='+str(params['ntwe'])
  77     ctl_data+=' ntwx='+str(params['ntwx'])+' dt='+str(params['dt'])+' damax='+str(params['damax'])+'lang=1 scal_fric=0.01'
  78     md_input.append('{:<79}&'.format(ctl_data))
  79     md_input.append('tau_bath=1.0 t_bath=300 reset_vel=0 respa ntime_split=1 maxtime_split=512 mdpdb')
  80     # Paramatry pól siłowych
  81     if params['force_field'] == 'GAB':
  82         # Wagi pola GAB
  83         md_input.append('WLONG=1.35279 WSCP=1.59304 WELEC=0.71534 WBOND=1.00000 WANG=1.13873            &')
  84         md_input.append('WSCLOC=0.16258 WTOR=1.98599 WTORD=1.57069 WCORRH=0.42887 WCORR5=0.00000        &')
  85         md_input.append('WCORR6=0.00000 WEL_LOC=0.16036 WTURN3=1.68722 WTURN4=0.66230 WTURN6=0.00000    &')
  86         md_input.append('WVDWPP=0.11371 WHPB=1.00000                                                    &')
  87         md_input.append('CUTOFF=7.00000 WCORR4=0.00000 WSCCOR=0.0')
  88     else:
  89         # Wagi pola E0LLY
  90         md_input.append('WLONG=1.00000 WSCP=1.23315 WELEC=0.84476 WBOND=1.00000 WANG=0.62954            &')
  91         md_input.append('WSCLOC=0.10554 WTOR=1.84316 WTORD=1.26571 WCORRH=0.19212 WCORR5=0.00000        &')
  92         md_input.append('WCORR6=0.00000 WEL_LOC=0.37357 WTURN3=1.40323 WTURN4=0.64673 WTURN6=0.00000    &')
  93         md_input.append('WVDWPP=0.23173 WHPB=1.00000 WSCCOR=0.0                                         &')
  94         md_input.append('CUTOFF=7.00000 WCORR4=0.00000')
  95     # Plik PDB
  96     md_input.append(params['pdb_file'].split('/')[-1])
  97     # Sekwencja aminokwasów
  98     md_input.append(len(params['sequence']))
  99     seq_str=params['sequence']
 100     while seq_str:
 101         md_input.append(seq_str[:80])
 102         seq_str=seq_str[80:]
 103     md_input.append(' 0')
 104     md_input.append(' 0')
 105
 106     return md_input
 107
 108
 109 def to_job_desc(params, proxy):
 110     QCG.start()
 111     desc = JobDescription()
 112
 113
 114     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 115                   'wall_time', 'memory', 'memory_per_slot', 'modules', 'input', 'stage_in', 'native', 'notify',
 116                   'preprocess', 'postprocess', 'persistent')
 117
 118     for name in direct_map:
 119         if params[name]:
 120             setattr(desc, name, params[name])
 121
 122
 123     if params['application']:
 124         desc.set_application(*params['application'])
 125         desc.stage_in += [params['master_file']]
 126         desc.arguments.insert(0, os.path.basename(params['master_file']))
 127     if params['script']:
 128         ftp = FTPOperation(proxy)
 129
 130         ftp.mkdir(constants.QCG_DATA_URL, parents=True)
 131         url = os.path.join(constants.QCG_DATA_URL, 'script.{}.sh'.format(random_id()))
 132         ftp.put(url)
 133
 134         for chunk in chunks(params['script'], 4096):
 135             ftp.stream.put(chunk)
 136         ftp.stream.put(None)
 137         ftp.wait()
 138
 139         desc.executable = url
 140     if params['nodes']:
 141         desc.set_nodes(*params['nodes'])
 142     if params['reservation']:
 143         desc.set_reservation(params['reservation'])
 144     if params['watch_output']:
 145         desc.set_watch_output(params['watch_output'], params['watch_output_pattern'])
 146
 147     przemytnik = params['env_variables']
 148     if params['force_field']:
 149         przemytnik.append([u'UNRESPORTAL_FORCEFIELD',params['force_field']])
 150     if params['sequence']:
 151         przemytnik.append([u'UNRESPORTAL_SEQUENCE',params['sequence']])
 152     if params['pdb_file']:
 153         przemytnik.append([u'UNRESPORTAL_PDBFILE',params['pdb_file']])
 154     if params['nstep']:
 155         przemytnik.append([u'UNRESPORTAL_NSTEP',params['nstep']])
 156     if params['ntwe']:
 157         przemytnik.append([u'UNRESPORTAL_NTWE',params['ntwe']])
 158     if params['ntwx']:
 159         przemytnik.append([u'UNRESPORTAL_NTWX',params['ntwx']])
 160     if params['dt']:
 161         przemytnik.append([u'UNRESPORTAL_DT',params['dt']])
 162     if params['damax']:
 163         przemytnik.append([u'UNRESPORTAL_DAMAX',params['damax']])
 164
 165     setattr(desc, 'env_variables', przemytnik)
 166
 167     # TODO monitoring
 168
 169     return desc
 170
 171
 172 def to_form_data(xml):
 173     # prevent circular import errors
 174     from forms import JobDescriptionForm
 175
 176     QCG.start()
 177     desc = JobDescription()
 178     desc.xml_description = xml
 179
 180     direct_map = ('env_variables', 'executable', 'arguments', 'note', 'grant', 'hosts', 'properties', 'queue', 'procs',
 181                   'wall_time', 'modules', 'input', 'stage_in', 'native', 'persistent')
 182
 183     params = {}
 184     for name in direct_map:
 185         attr = getattr(desc, name)
 186         if isinstance(attr, bool) or attr:
 187             params[name] = attr
 188
 189     if desc.application is not None:
 190         app_name, app_ver = desc.application
 191         params['application'] = app_name if app_ver is None else app_name + '/' + app_name
 192         stage_in = params['stage_in']
 193         params['stage_in'], params['master_file'] = stage_in[:-1], stage_in[-1]
 194         params['arguments'] = params['arguments'][1:]
 195     if desc.nodes is not None:
 196         params['nodes'] = ':'.join(map(str, desc.nodes))
 197     if desc.reservation is not None:
 198         res_id, res_type = desc.reservation
 199         params['reservation'] = res_id
 200     if desc.notify is not None:
 201         params['notify_type'], params['notify_address'] = desc.notify.split(':')
 202     if desc.watch_output is not None:
 203         watch_output, params['watch_output_pattern'] = desc.watch_output
 204         params['watch_output_type'], params['watch_output_address'] = watch_output.split(':')
 205     if desc.preprocess is not None:
 206         if desc.preprocess.startswith('gsiftp://'):
 207             params['preprocess_type'] = JobDescriptionForm.Process.SCRIPT
 208             params['preprocess_script'] = desc.preprocess
 209         else:
 210             params['preprocess_type'] = JobDescriptionForm.Process.CMD
 211             params['preprocess_cmd'] = desc.preprocess
 212     if desc.postprocess is not None:
 213         if desc.postprocess.startswith('gsiftp://'):
 214             params['postprocess_type'] = JobDescriptionForm.Process.SCRIPT
 215             params['postprocess_script'] = desc.postprocess
 216         else:
 217             params['postprocess_type'] = JobDescriptionForm.Process.CMD
 218             params['postprocess_cmd'] = desc.postprocess
 219     if desc.memory:
 220         params['memory'] = int(desc.memory)
 221     if desc.memory_per_slot:
 222         params['memory_per_slot'] = int(desc.memory_per_slot)
 223
 224     for item in desc.env_variables:
 225         if item[0] == u'UNRESPORTAL_FORCEFIELD':
 226             params['force_field'] = item[1]
 227         if item[0] == u'UNRESPORTAL_SEQUENCE':
 228             params['sequence'] = item[1]
 229         if item[0] == u'UNRESPORTAL_PDBFILE':
 230             params['pdb_file'] = item[1]
 231         if item[0] == u'UNRESPORTAL_NSTEP':
 232             params['nstep'] = item[1]
 233         if item[0] == u'UNRESPORTAL_NTWE':
 234             params['ntwe'] = item[1]
 235         if item[0] == u'UNRESPORTAL_NTWX':
 236             params['ntwx'] = item[1]
 237         if item[0] == u'UNRESPORTAL_DT':
 238             params['dt'] = item[1]
 239         if item[0] == u'UNRESPORTAL_DAMAX':
 240             params['damax'] = item[1]
 241     return params
 242
 243
 244 def restricted(view):
 245     return wraps(view)(cache_control(no_cache=True, must_revalidate=True, no_store=True)(login_required(view)))
 246
 247
 248 def cached_resources(proxy):
 249     hosts = resources_cache.get('hosts')
 250     if hosts is None:
 251         # prevent circular import errors
 252         from qcg.service import fetch_resources
 253
 254         hosts, _, applications, modules = map(make_choices, fetch_resources(proxy))
 255         resources_cache.set('hosts', hosts)
 256         resources_cache.set('applications', applications)
 257         resources_cache.set('modules', modules)
 258     else:
 259         applications = resources_cache.get('applications')
 260         modules = resources_cache.get('modules')
 261
 262     return hosts, applications, modules
 263
 264
 265 def make_choices(iterable):
 266     return ((None, ''),) + tuple((item, item) for item in sorted(iterable))