from datetime import datetime
from Queue import Queue, Empty
+from itertools import chain
import os
import re
from threading import Event
-from urlparse import urlparse
+from urlparse import urlparse, urlunparse
+from django.utils.http import urlquote
from django.utils.timezone import localtime, UTC
-from gridftp import FTPClient, Buffer, HandleAttr, OperationAttr
-class FTPException(Exception):
- pass
+class FTPError(Exception):
+ def __init__(self, message, verbose=None, *args, **kwargs):
+ super(FTPError, self).__init__(message, *args, **kwargs)
+
+ self.verbose = verbose
class FTPOperation:
def __init__(self, proxy=None, buffer_size=4096):
+ from gridftp import FTPClient, Buffer, HandleAttr, OperationAttr
+
self._end = Event()
self._error = None
self._buffer = Buffer(buffer_size)
self._end.clear()
if self._error is not None:
- # TODO logging
- print 'GridFTP ERROR:', self._error
-
match = re.search(r'A system call failed: (.*)$', self._error.replace('\r\n', '\n'), re.MULTILINE)
msg = match.groups()[0] if match else "Unknown error"
- raise FTPException(msg)
+ raise FTPError(msg, self._error)
def listing(self, url):
self.cli.verbose_list(url, self._done, None, self.op_attr)
for attr in attrs.split(';'):
try:
key, value = attr.split('=', 1)
- attrs_dict[key] = value
except ValueError:
- pass
+ key, value = attr, ''
+
+ attrs_dict[key] = value
yield {
'name': name,
- 'type': 'directory' if attrs_dict['Type'] == 'dir' else 'file',
+ 'type': 'directory' if attrs_dict['Type'].endswith('dir') else 'file',
'size': int(attrs_dict['Size']),
'date': localtime(datetime.strptime(attrs_dict['Modify'], "%Y%m%d%H%M%S").replace(tzinfo=UTC())),
}
data = self.listing(url).next()
if data['name'] == '.':
- data['name'] = os.path.basename(urlparse(url).path.rstrip('/')) or u'/'
+ data['name'] = os.path.basename(os.path.normpath(url))
return data
+ def exists(self, url):
+ self.cli.exists(url, self._done, None, self.op_attr)
+
+ try:
+ self.wait()
+ except FTPError as e:
+ if 'No such file or directory' in e.message:
+ return False
+ raise
+ else:
+ return True
+
def delete(self, url):
self.cli.delete(url, self._done, None, self.op_attr)
self.wait()
- def mkdir(self, url):
+ def mkdir(self, url, parents=False):
+ if parents:
+ if self.exists(url):
+ return
+
+ u = urlparse(url)
+ parent_url = urlunparse((u.scheme, u.netloc, os.path.dirname(os.path.normpath(u.path)), '', '', ''))
+
+ self.mkdir(parent_url, parents=True)
+
self.cli.mkdir(url, self._done, None, self.op_attr)
self.wait()
+ @staticmethod
+ def match_ext(archive, *extensions):
+ for ext in extensions:
+ if archive.endswith(ext):
+ return True
+ return False
+
def compress(self, server, path, files, archive):
- def match_ext(*extensions):
- for ext in extensions:
- if archive.endswith(ext):
- return True
- return False
-
- if match_ext('.tar.gz', '.tgz'):
- cmd, args = 'tar', ['cvzf', os.path.join(path, archive), '-C', path] + files
- elif match_ext('.tar.bz2', '.tbz'):
- cmd, args = 'tar', ['cvjf', os.path.join(path, archive), '-C', path] + files
- elif match_ext('.zip'):
- cmd, args = 'zip', ['-r', os.path.join(path, archive)] + [os.path.join(path, f) for f in files]
+ self._check_disk_stack_args(*([path, archive] + files))
+
+ if self.match_ext(archive, '.tar.gz', '.tgz'):
+ cmd, args = 'tar', ['czf', archive, '-C', path] + files
+ elif self.match_ext(archive, '.tar.bz2', '.tbz'):
+ cmd, args = 'tar', ['cjf', archive, '-C', path] + files
+ elif self.match_ext(archive, '.zip'):
+ # zip doesn't support unicode file names
+ for arg in files:
+ try:
+ arg.encode('ascii')
+ except UnicodeEncodeError as e:
+ raise ValueError(u'Unsupported character `{}` in `{}`!'.format(arg[e.start:e.start + 1], arg))
+
+ cmd, args = 'jar', (['cMf', archive] + list(chain.from_iterable(('-C', path, f) for f in files)))
+ else:
+ raise ValueError('Unknown archive type: {}'.format(archive))
+
+ self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + map(urlquote, args)))
+
+ return self.get(server)
+
+ def extract(self, server, archive, dst):
+ self._check_disk_stack_args(*[archive, dst])
+
+ if self.match_ext(archive, '.tar.gz', '.tgz'):
+ cmd, args = 'tar', ('xzf', archive, '-C', dst)
+ elif self.match_ext(archive, '.tar.bz2', '.tbz'):
+ cmd, args = 'tar', ('xjf', archive, '-C', dst)
+ elif self.match_ext(archive, '.zip'):
+ cmd, args = 'unzip', ('-qo', archive, '-d', dst)
else:
raise ValueError('Unknown archive type: {}'.format(archive))
- # FIXME handling filename with #
- self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + args))
+ self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + map(urlquote, args)))
return self.get(server)
+
+ @staticmethod
+ def _check_disk_stack_args(*args):
+ for char in ['#', ';']:
+ for arg in args:
+ if char in arg:
+ raise ValueError(u'Unsupported character `{}` in `{}`!'.format(char, arg))