diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b89c33c --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +Copyright (c) 2012, SmartFile + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.rst b/README.rst index 0f94018..c6c90d1 100644 --- a/README.rst +++ b/README.rst @@ -2,3 +2,17 @@ fs-dropbox ---------- File system for pyFilesystem which uses the dropbox API. + + + + + + +**A Note About Caching** + + +This library has built in caching. There are times when you will want to disable +caching. A user can change the file on the remote server and our code/cache will +not be notified of the change. One example of where this is an issue is before +you read/download a file from the remote side if you are using cached meta info +to specify the download size. diff --git a/dropboxfs.py b/dropboxfs.py index a318708..485ebad 100644 --- a/dropboxfs.py +++ b/dropboxfs.py @@ -7,13 +7,11 @@ """ import time -import stat import shutil import optparse import datetime import tempfile import calendar -import os.path from UserDict import UserDict from fs.base import * @@ -31,7 +29,7 @@ # The format Dropbox uses for times. TIME_FORMAT = '%a, %d %b %Y %H:%M:%S +0000' # Max size for spooling to memory before using disk (5M). -MAX_BUFFER = 1024**2*5 +MAX_BUFFER = 1024 ** 2 * 5 class ContextManagerStream(object): @@ -56,9 +54,9 @@ def __exit__(self, *args): self.close() -# TODO: these classes can probably be replaced with tempfile.SpooledTemporaryFile, however -# I am unsure at this moment if doing so would be bad since it is only available in Python -# 2.6+. +# TODO: these classes can probably be replaced with +# tempfile.SpooledTemporaryFile, however I am unsure at this moment if doing +# so would be bad since it is only available in Python 2.6+. class SpooledWriter(ContextManagerStream): """Spools bytes to a StringIO buffer until it reaches max_buffer. At that @@ -68,19 +66,23 @@ def __init__(self, client, name, max_buffer=MAX_BUFFER): self.max_buffer = max_buffer self.bytes = 0 super(SpooledWriter, self).__init__(StringIO(), name) - + def __len__(self): return self.bytes def write(self, data): if self.temp.tell() + len(data) >= self.max_buffer: - # We reached the max_buffer size that we want to keep in memory. Switch - # to an on-disk temp file. Copy what has been written so far to it. + # We reached the max_buffer size that we want to keep in memory. + # Switch to an on-disk temp file. Copy what has been written so + # far to it. temp = tempfile.TemporaryFile() self.temp.seek(0) shutil.copyfileobj(self.temp, temp) self.temp = temp - self.temp.write(data) + try: + self.temp.write(data) + except rest.ErrorResponse, e: + raise RemoteConnectionError(opname='write', path=name) self.bytes += len(data) def close(self): @@ -93,11 +95,17 @@ def close(self): class SpooledReader(ContextManagerStream): - """Reads the entire file from the remote server into a buffer or temporary file. - It can then satisfy read(), seek() and other calls using the local file.""" + """ + Reads the entire file from the remote server into a buffer or temporary + file. It can then satisfy read(), seek() and other calls using the local + file. + """ def __init__(self, client, name, max_buffer=MAX_BUFFER): self.client = client - r = self.client.get_file(name) + try: + r = self.client.get_file(name) + except rest.ErrorResponse, e: + raise RemoteConnectionError(opname='get_file', path=name) self.bytes = int(r.getheader('Content-Length')) if r > max_buffer: temp = tempfile.TemporaryFile() @@ -111,6 +119,110 @@ def __len__(self): return self.bytes +class ChunkedReader(ContextManagerStream): + """ A file-like that provides access to a file with dropbox API""" + """Reads the file from the remote server as requested. + It can then satisfy read().""" + def __init__(self, client, name): + self.client = client + try: + self.r = self.client.get_file(name) + except rest.ErrorResponse, e: + raise RemoteConnectionError(opname='get_file', path=name, + details=e) + self.bytes = int(self.r.getheader('Content-Length')) + self.name = name + self.closed = False + self.pos = 0 + self.seek_pos = 0 + + def __len__(self): + return self.bytes + + def __iter__(self): + return self + + def seek(self, offset, whence=0): + """ + Change the stream position to the given byte offset in the file-like + object. + """ + if (whence == 0): + self.seek_pos = offset + elif (whence == 1): + self.seek_pos += offset + elif (whence == 2): + self.seek_pos = self.size + offset + + def tell(self): + """ Return the current stream position. """ + return self.seek_pos + + def next(self): + """ + Read the data until all data is read. + data is empty string when there is no more data to read. + """ + data = self.read() + if data is None: + raise StopIteration() + return data + + def read(self, amt=None): + """ Read a piece of the file from dropbox. """ + if not self.r.isclosed(): + # Do some fake seeking + if self.seek_pos < self.pos: + self.r.close() + self.r = self.client.get_file(self.name) + self.r.read(self.seek_pos) + elif self.seek_pos > self.pos: + # Read ahead enough to reconcile pos and seek_pos + self.r.read(self.pos - self.seek_pos) + self.pos = self.seek_pos + + # Update position pointers + if amt: + self.pos += amt + self.seek_pos += amt + else: + self.pos = self.bytes + self.seek_pos = self.bytes + return self.r.read(amt) + else: + self.close() + + def readline(self, size=-1): + """ Not implemented. Read and return one line from the stream. """ + raise NotImplementedError() + + def readlines(self, hint=-1): + """ + Not implemented. Read and return a list of lines from the stream. + """ + raise NotImplementedError() + + def writable(self): + """ The stream does not support writing. """ + return False + + def writelines(self, lines): + """ Not implemented. Write a list of lines to the stream. """ + raise NotImplementedError() + + def close(self): + """ + Flush and close this stream. This method has no effect if the file + is already closed. As a convenience, it is allowed to call this method + more than once; only the first call, however, will have an effect. + """ + # It's a memory leak if self.r not closed. + if not self.r.isclosed(): + self.r.close() + if not self.closed: + self.closed = True + + class CacheItem(object): """Represents a path in the cache. There are two components to a path. It's individual metadata, and the children contained within it.""" @@ -173,18 +285,18 @@ def __init__(self, *args, **kwargs): # metadata() and children(). This allows for more fine-grained fetches # and caching. - def metadata(self, path): + def metadata(self, path, cache_read=True): "Gets metadata for a given path." - item = self.cache.get(path) + item = self.cache.get(path) if cache_read else None if not item or item.metadata is None or item.expired: try: - metadata = super(DropboxClient, self).metadata(path, - include_deleted=False, list=False) + metadata = super(DropboxClient, self).metadata( + path, include_deleted=False, list=False) except rest.ErrorResponse, e: if e.status == 404: raise ResourceNotFoundError(path) raise RemoteConnectionError(opname='metadata', path=path, - errno=e.status) + details=e) if metadata.get('is_deleted', False): raise ResourceNotFoundError(path) item = self.cache[path] = CacheItem(metadata) @@ -209,8 +321,8 @@ def children(self, path): update = True if update: try: - metadata = super(DropboxClient, self).metadata(path, hash=hash, - include_deleted=False, list=True) + metadata = super(DropboxClient, self).metadata( + path, hash=hash, include_deleted=False, list=True) children = [] contents = metadata.pop('contents') for child in contents: @@ -222,7 +334,7 @@ def children(self, path): except rest.ErrorResponse, e: if not item or e.status != 304: raise RemoteConnectionError(opname='metadata', path=path, - errno=e.status) + details=e) # We have an item from cache (perhaps expired), but it's # hash is still valid (as far as Dropbox is concerned), # so just renew it and keep using it. @@ -239,7 +351,7 @@ def file_create_folder(self, path): if e.status == 403: raise DestinationExistsError(path) raise RemoteConnectionError(opname='file_create_folder', path=path, - errno=e.status) + details=e) self.cache.set(path, metadata) def file_copy(self, src, dst): @@ -251,7 +363,7 @@ def file_copy(self, src, dst): if e.status == 403: raise DestinationExistsError(dst) raise RemoteConnectionError(opname='file_copy', path=path, - errno=e.status) + details=e) self.cache.set(dst, metadata) def file_move(self, src, dst): @@ -263,7 +375,7 @@ def file_move(self, src, dst): if e.status == 403: raise DestinationExistsError(dst) raise RemoteConnectionError(opname='file_move', path=path, - errno=e.status) + details=e) self.cache.pop(src, None) self.cache.set(dst, metadata) @@ -280,10 +392,10 @@ def file_delete(self, path): def put_file(self, path, f, overwrite=False): try: - metadata = super(DropboxClient, self).put_file(path, f, overwrite=overwrite) + super(DropboxClient, self).put_file(path, f, overwrite=overwrite) except rest.ErrorResponse, e: raise RemoteConnectionError(opname='put_file', path=path, - errno=e.status) + details=e) self.cache.pop(dirname(path), None) @@ -302,7 +414,10 @@ def metadata_to_info(metadata, localtime=False): 'isfile': not isdir, } try: - mtime = metadata.pop('modified', None) + if 'client_mtime' in metadata: + mtime = metadata.get('client_mtime') + else: + mtime = metadata.get('modified') if mtime: # Parse date/time from Dropbox as struct_time. mtime = time.strptime(mtime, TIME_FORMAT) @@ -321,17 +436,16 @@ def metadata_to_info(metadata, localtime=False): class DropboxFS(FS): """A FileSystem that stores data in Dropbox.""" - _meta = { 'thread_safe' : True, - 'virtual' : False, - 'read_only' : False, - 'unicode_paths' : True, - 'case_insensitive_paths' : True, - 'network' : True, - 'atomic.setcontents' : False, - 'atomic.makedir': True, - 'atomic.rename': True, - 'mime_type': 'virtual/dropbox', - } + _meta = {'thread_safe': True, + 'virtual': False, + 'read_only': False, + 'unicode_paths': True, + 'case_insensitive_paths': True, + 'network': True, + 'atomic.setcontents': False, + 'atomic.makedir': True, + 'atomic.rename': True, + 'mime_type': 'virtual/dropbox', } def __init__(self, app_key, app_secret, access_type, token_key, token_secret, localtime=False, thread_synchronize=True): @@ -345,7 +459,8 @@ def __init__(self, app_key, app_secret, access_type, token_key, :param thread_synchronize: set to True (default) to enable thread-safety """ super(DropboxFS, self).__init__(thread_synchronize=thread_synchronize) - self.client = create_client(app_key, app_secret, access_type, token_key, token_secret) + self.client = create_client(app_key, app_secret, access_type, + token_key, token_secret) self.localtime = localtime def __str__(self): @@ -362,7 +477,7 @@ def getmeta(self, meta_name, default=NoDefaultMeta): @synchronize def open(self, path, mode="rb", **kwargs): if 'r' in mode: - return SpooledReader(self.client, path) + return ChunkedReader(self.client, path) else: return SpooledWriter(self.client, path) @@ -405,15 +520,17 @@ def exists(self, path): except ResourceNotFoundError: return False - def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): + def listdir(self, path="/", wildcard=None, full=False, absolute=False, + dirs_only=False, files_only=False): path = abspath(normpath(path)) children = self.client.children(path) - return self._listdir_helper(path, children, wildcard, full, absolute, dirs_only, files_only) + return self._listdir_helper(path, children, wildcard, full, absolute, + dirs_only, files_only) @synchronize - def getinfo(self, path): + def getinfo(self, path, cache_read=True): path = abspath(normpath(path)) - metadata = self.client.metadata(path) + metadata = self.client.metadata(path, cache_read=cache_read) return metadata_to_info(metadata, localtime=self.localtime) def copy(self, src, dst, *args, **kwargs): @@ -460,12 +577,30 @@ def removedir(self, path, *args, **kwargs): def main(): - parser = optparse.OptionParser(prog="dropboxfs", description="CLI harness for DropboxFS.") - parser.add_option("-k", "--app-key", help="Your Dropbox app key.") - parser.add_option("-s", "--app-secret", help="Your Dropbox app secret.") - parser.add_option("-t", "--type", default='dropbox', choices=('dropbox', 'app_folder'), help="Your Dropbox app access type.") - parser.add_option("-a", "--token-key", help="Your access token key (if you previously obtained one.") - parser.add_option("-b", "--token-secret", help="Your access token secret (if you previously obtained one.") + parser = optparse.OptionParser(prog="dropboxfs", + description="CLI harness for DropboxFS.") + parser.add_option( + "-k", + "--app-key", + help="Your Dropbox app key.") + parser.add_option( + "-s", + "--app-secret", + help="Your Dropbox app secret.") + parser.add_option( + "-t", + "--type", + default='dropbox', + choices=('dropbox', 'app_folder'), + help="Your Dropbox app access type.") + parser.add_option( + "-a", + "--token-key", + help="Your access token key (if you previously obtained one.") + parser.add_option( + "-b", + "--token-secret", + help="Your access token secret (if you previously obtained one.") (options, args) = parser.parse_args() @@ -475,7 +610,8 @@ def main(): # Instantiate a client one way or another. if not options.token_key and not options.token_secret: - s = session.DropboxSession(options.app_key, options.app_secret, options.type) + s = session.DropboxSession(options.app_key, options.app_secret, + options.type) # Get a temporary token, so we can make oAuth calls. t = s.obtain_request_token() print "Please visit the following URL and authorize this application.\n" @@ -493,11 +629,13 @@ def main(): print "\nWhen you are done, please press ." raw_input() elif not options.token_key or not options.token_secret: - parser.error('You must provide both the access token and the access token secret.') + parser.error('You must provide both the access token and the ' + 'access token secret.') else: token_key, token_secret = options.token_key, options.token_secret - fs = DropboxFS(options.app_key, options.app_secret, options.type, token_key, token_secret) + fs = DropboxFS(options.app_key, options.app_secret, options.type, + token_key, token_secret) print fs.getinfo('/') print fs.getinfo('/Public') @@ -508,6 +646,17 @@ def main(): print fs.listdir('/') print fs.listdir('/Foo') + filelike = fs.open('/big-file.pdf') + print filelike.read(100) + filelike.seek(100) + chunk2 = filelike.read(100) + print chunk2 + filelike.seek(200) + print filelike.read(100) + filelike.seek(100) + chunk2a = filelike.read(100) + print chunk2a + assert chunk2 == chunk2a + if __name__ == '__main__': main() - diff --git a/setup.py b/setup.py index 8287503..cd73837 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,37 @@ -#!/bin/env python +#!/usr/bin/env python import os from distutils.core import setup name = 'dropboxfs' -version = '0.1' -release = '11' +version = '0.4.2' +release = '0' versrel = version + '-' + release readme = os.path.join(os.path.dirname(__file__), 'README.rst') long_description = file(readme).read() setup( - name = name, - version = versrel, - description = 'A pyFilesystem backend for the Dropbox API.', - long_description = long_description, - requires = [ + name=name, + version=versrel, + description='A PyFilesystem backend for the Dropbox API.', + long_description=long_description, + requires=[ 'fs', 'dropbox', ], - author = 'Ben Timby', - author_email = 'btimby@gmail.com', - maintainer = 'Ben Timby', - maintainer_email = 'btimby@gmail.com', - url = 'http://github.com/btimby/fs-dropbox/', - license = 'GPLv3', + author='SmartFile', + author_email='tcunningham@smartfile.com', + maintainer='Travis Cunningham', + maintainer_email='tcunningham@smartfile.com', + url='http://github.com/smartfile/fs-dropbox/', + license='GPLv3', py_modules=['dropboxfs'], package_data={'': ['README.rst']}, - classifiers = ( - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', + classifiers=( + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', ), )