X-Git-Url: http://mmka.chem.univ.gda.pl/gitweb/?a=blobdiff_plain;ds=sidebyside;f=filex%2Fftp.py;h=85161a4ba8f956e84ff88176f684d9e1d6a97c89;hb=HEAD;hp=c0b3e25d4d54419800d2433ef4f3a028d6cc0f53;hpb=be284bd6256a7a095d62786d9b58f815eefb769a;p=qcg-portal.git diff --git a/filex/ftp.py b/filex/ftp.py index c0b3e25..85161a4 100644 --- a/filex/ftp.py +++ b/filex/ftp.py @@ -1,18 +1,26 @@ from datetime import datetime from Queue import Queue, Empty +from itertools import chain +import os import re from threading import Event +from urlparse import urlparse, urlunparse +from django.utils.http import urlquote from django.utils.timezone import localtime, UTC -from gridftp import FTPClient, Buffer, HandleAttr, OperationAttr -class FTPException(Exception): - pass +class FTPError(Exception): + def __init__(self, message, verbose=None, *args, **kwargs): + super(FTPError, self).__init__(message, *args, **kwargs) + + self.verbose = verbose class FTPOperation: def __init__(self, proxy=None, buffer_size=4096): + from gridftp import FTPClient, Buffer, HandleAttr, OperationAttr + self._end = Event() self._error = None self._buffer = Buffer(buffer_size) @@ -68,7 +76,7 @@ class FTPOperation: msg = match.groups()[0] if match else "Unknown error" - raise FTPException(msg) + raise FTPError(msg, self._error) def listing(self, url): self.cli.verbose_list(url, self._done, None, self.op_attr) @@ -84,25 +92,26 @@ class FTPOperation: @staticmethod def _parse_mlst(listing): - data = [] - for item in listing.strip().splitlines(): # we may receive empty string when there are multiple consecutive newlines in listing if item: attrs, name = item.split(' ', 1) - attrs = dict((attr.split('=') for attr in attrs.split(';') if attr)) + attrs_dict = {} + for attr in attrs.split(';'): + try: + key, value = attr.split('=', 1) + except ValueError: + key, value = attr, '' - date = localtime(datetime.strptime(attrs['Modify'], "%Y%m%d%H%M%S").replace(tzinfo=UTC())) + attrs_dict[key] = value - data.append({ + yield { 'name': name, - 'type': 'file' if attrs['Type'] == 'file' else 'directory', - 'size': int(attrs['Size']), - 'date': date, - }) - - return data + 'type': 'directory' if attrs_dict['Type'].endswith('dir') else 'file', + 'size': int(attrs_dict['Size']), + 'date': localtime(datetime.strptime(attrs_dict['Modify'], "%Y%m%d%H%M%S").replace(tzinfo=UTC())), + } def get(self, url): self.cli.get(url, self._done, None, self.op_attr) @@ -125,3 +134,100 @@ class FTPOperation: self.cli.move(src, dst, self._done, None, self.op_attr) self.wait() + + def info(self, url): + data = self.listing(url).next() + + if data['name'] == '.': + data['name'] = os.path.basename(os.path.normpath(url)) + + return data + + def exists(self, url): + self.cli.exists(url, self._done, None, self.op_attr) + + try: + self.wait() + except FTPError as e: + if 'No such file or directory' in e.message: + return False + raise + else: + return True + + def delete(self, url): + self.cli.delete(url, self._done, None, self.op_attr) + + self.wait() + + def rmdir(self, url): + self.cli.rmdir(url, self._done, None, self.op_attr) + + self.wait() + + def mkdir(self, url, parents=False): + if parents: + if self.exists(url): + return + + u = urlparse(url) + parent_url = urlunparse((u.scheme, u.netloc, os.path.dirname(os.path.normpath(u.path)), '', '', '')) + + self.mkdir(parent_url, parents=True) + + self.cli.mkdir(url, self._done, None, self.op_attr) + + self.wait() + + @staticmethod + def match_ext(archive, *extensions): + for ext in extensions: + if archive.endswith(ext): + return True + return False + + def compress(self, server, path, files, archive): + self._check_disk_stack_args(*([path, archive] + files)) + + if self.match_ext(archive, '.tar.gz', '.tgz'): + cmd, args = 'tar', ['czf', archive, '-C', path] + files + elif self.match_ext(archive, '.tar.bz2', '.tbz'): + cmd, args = 'tar', ['cjf', archive, '-C', path] + files + elif self.match_ext(archive, '.zip'): + # zip doesn't support unicode file names + for arg in files: + try: + arg.encode('ascii') + except UnicodeEncodeError as e: + raise ValueError(u'Unsupported character `{}` in `{}`!'.format(arg[e.start:e.start + 1], arg)) + + cmd, args = 'jar', (['cMf', archive] + list(chain.from_iterable(('-C', path, f) for f in files))) + else: + raise ValueError('Unknown archive type: {}'.format(archive)) + + self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + map(urlquote, args))) + + return self.get(server) + + def extract(self, server, archive, dst): + self._check_disk_stack_args(*[archive, dst]) + + if self.match_ext(archive, '.tar.gz', '.tgz'): + cmd, args = 'tar', ('xzf', archive, '-C', dst) + elif self.match_ext(archive, '.tar.bz2', '.tbz'): + cmd, args = 'tar', ('xjf', archive, '-C', dst) + elif self.match_ext(archive, '.zip'): + cmd, args = 'unzip', ('-qo', archive, '-d', dst) + else: + raise ValueError('Unknown archive type: {}'.format(archive)) + + self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + map(urlquote, args))) + + return self.get(server) + + @staticmethod + def _check_disk_stack_args(*args): + for char in ['#', ';']: + for arg in args: + if char in arg: + raise ValueError(u'Unsupported character `{}` in `{}`!'.format(char, arg))