1 from datetime import datetime
2 from Queue import Queue, Empty
3 from itertools import chain
6 from threading import Event
8 from django.utils.http import urlunquote
9 from django.utils.timezone import localtime, UTC
10 from gridftp import FTPClient, Buffer, HandleAttr, OperationAttr
13 class FTPError(Exception):
14 def __init__(self, message, verbose=None, *args, **kwargs):
15 super(FTPError, self).__init__(message, *args, **kwargs)
17 self.verbose = verbose
21 def __init__(self, proxy=None, buffer_size=4096):
24 self._buffer = Buffer(buffer_size)
26 self.attr = HandleAttr()
27 self.op_attr = OperationAttr()
30 self.op_attr.set_authorization(proxy)
32 self.cli = FTPClient(self.attr)
34 # limit size of a queue to 4 MB
35 self.stream = Queue((4 * 2**20) / buffer_size or 1)
39 self.op_attr.destroy()
42 def _read(self, arg, handle, error, buff, length, offset, eof):
44 self.stream.put(str(buff))
47 self.cli.register_read(self._buffer, self._read, None)
49 def _write(self, arg, handle, error, buff, length, offset, eof):
53 chunk = self.stream.get()
60 self._buffer.fill(chunk)
62 self.cli.register_write(self._buffer, size, offset, eof, self._write, None)
64 def _done(self, arg, handle, error):
72 if self._error is not None:
73 match = re.search(r'A system call failed: (.*)$', self._error.replace('\r\n', '\n'), re.MULTILINE)
75 msg = match.groups()[0] if match else "Unknown error"
77 raise FTPError(msg, self._error)
79 def listing(self, url):
80 self.cli.verbose_list(url, self._done, None, self.op_attr)
81 self.cli.register_read(self._buffer, self._read, None)
86 while not self.stream.empty():
87 result += self.stream.get()
89 return self._parse_mlst(result)
92 def _parse_mlst(listing):
93 for item in listing.strip().splitlines():
94 # we may receive empty string when there are multiple consecutive newlines in listing
96 attrs, name = item.split(' ', 1)
99 for attr in attrs.split(';'):
101 key, value = attr.split('=', 1)
103 key, value = attr, ''
105 attrs_dict[key] = value
109 'type': 'directory' if attrs_dict['Type'].endswith('dir') else 'file',
110 'size': int(attrs_dict['Size']),
111 'date': localtime(datetime.strptime(attrs_dict['Modify'], "%Y%m%d%H%M%S").replace(tzinfo=UTC())),
115 self.cli.get(url, self._done, None, self.op_attr)
116 self.cli.register_read(self._buffer, self._read, None)
120 yield self.stream.get(timeout=0.1)
122 if self._end.wait(0):
128 self.cli.put(url, self._done, None, self.op_attr)
129 self.cli.register_write(self._buffer, 0, 0, False, self._write, None)
131 def move(self, src, dst):
132 self.cli.move(src, dst, self._done, None, self.op_attr)
137 data = self.listing(url).next()
139 if data['name'] == '.':
140 data['name'] = os.path.basename(os.path.normpath(url))
144 def delete(self, url):
145 self.cli.delete(url, self._done, None, self.op_attr)
149 def rmdir(self, url):
150 self.cli.rmdir(url, self._done, None, self.op_attr)
154 def mkdir(self, url):
155 self.cli.mkdir(url, self._done, None, self.op_attr)
160 def match_ext(archive, *extensions):
161 for ext in extensions:
162 if archive.endswith(ext):
166 def compress(self, server, path, files, archive):
167 self._check_disk_stack_args(*([path, archive] + files))
169 if self.match_ext(archive, '.tar.gz', '.tgz'):
170 cmd, args = 'tar', ['cvzf', archive, '-C', path] + files
171 elif self.match_ext(archive, '.tar.bz2', '.tbz'):
172 cmd, args = 'tar', ['cvjf', archive, '-C', path] + files
173 elif self.match_ext(archive, '.zip'):
174 cmd, args = 'jar', (['cvMf', archive] + list(chain.from_iterable(('-C', path, f) for f in files)))
176 raise ValueError('Unknown archive type: {}'.format(archive))
178 self.op_attr.set_disk_stack('#'.join(["popen:argv=", cmd] + args))
180 return self.get(server)
182 def extract(self, server, archive, dst):
183 self._check_disk_stack_args(*[archive, dst])
185 if self.match_ext(archive, '.tar.gz', '.tgz'):
186 cmd, args = 'tar', ('xvzf', archive, '-C', dst)
187 elif self.match_ext(archive, '.tar.bz2', '.tbz'):
188 cmd, args = 'tar', ('xvjf', archive, '-C', dst)
189 elif self.match_ext(archive, '.zip'):
190 cmd, args = 'unzip', (archive, '-d', dst)
192 raise ValueError('Unknown archive type: {}'.format(archive))
194 self.op_attr.set_disk_stack('#'.join(("popen:argv=", cmd) + args))
196 return self.get(server)
199 def _check_disk_stack_args(*args):
200 for char in ['#', ',', ';', '%23', '%3B']:
203 raise ValueError('Unsupported character `{}` in `{}`!'.format(urlunquote(char), urlunquote(arg)))