diff --git a/dropboxfs.py b/dropboxfs.py index a2a2c41..2104165 100644 --- a/dropboxfs.py +++ b/dropboxfs.py @@ -29,7 +29,7 @@ # The format Dropbox uses for times. TIME_FORMAT = '%a, %d %b %Y %H:%M:%S +0000' # Max size for spooling to memory before using disk (5M). -MAX_BUFFER = 1024**2*5 +MAX_BUFFER = 1024 ** 2 * 5 class ContextManagerStream(object): @@ -54,9 +54,9 @@ def __exit__(self, *args): self.close() -# TODO: these classes can probably be replaced with tempfile.SpooledTemporaryFile, however -# I am unsure at this moment if doing so would be bad since it is only available in Python -# 2.6+. +# TODO: these classes can probably be replaced with +# tempfile.SpooledTemporaryFile, however I am unsure at this moment if doing +# so would be bad since it is only available in Python 2.6+. class SpooledWriter(ContextManagerStream): """Spools bytes to a StringIO buffer until it reaches max_buffer. At that @@ -66,14 +66,15 @@ def __init__(self, client, name, max_buffer=MAX_BUFFER): self.max_buffer = max_buffer self.bytes = 0 super(SpooledWriter, self).__init__(StringIO(), name) - + def __len__(self): return self.bytes def write(self, data): if self.temp.tell() + len(data) >= self.max_buffer: - # We reached the max_buffer size that we want to keep in memory. Switch - # to an on-disk temp file. Copy what has been written so far to it. + # We reached the max_buffer size that we want to keep in memory. + # Switch to an on-disk temp file. Copy what has been written so + # far to it. temp = tempfile.TemporaryFile() self.temp.seek(0) shutil.copyfileobj(self.temp, temp) @@ -91,8 +92,11 @@ def close(self): class SpooledReader(ContextManagerStream): - """Reads the entire file from the remote server into a buffer or temporary file. - It can then satisfy read(), seek() and other calls using the local file.""" + """ + Reads the entire file from the remote server into a buffer or temporary + file. It can then satisfy read(), seek() and other calls using the local + file. + """ def __init__(self, client, name, max_buffer=MAX_BUFFER): self.client = client r = self.client.get_file(name) @@ -123,6 +127,8 @@ def __init__(self, client, name): self.bytes = int(self.r.getheader('Content-Length')) self.name = name self.closed = False + self.pos = 0 + self.seek_pos = 0 def __len__(self): return self.bytes @@ -130,22 +136,83 @@ def __len__(self): def __iter__(self): return self + def seek(self, offset, whence=0): + """ + Change the stream position to the given byte offset in the file-like + object. + """ + if (whence == 0): + self.seek_pos = offset + elif (whence == 1): + self.seek_pos += offset + elif (whence == 2): + self.seek_pos = self.size + offset + + def tell(self): + """ Return the current stream position. """ + return self.seek_pos + def next(self): + """ + Read the data until all data is read. + data is empty string when there is no more data to read. + """ data = self.read() if data is None: raise StopIteration() return data - def read(self, size=16384): + def read(self, amt=None): + """ Read a piece of the file from dropbox. """ if not self.r.isclosed(): - return self.r.read(size) + # Do some fake seeking + if self.seek_pos < self.pos: + self.r.close() + self.r = self.client.get_file(self.name) + self.r.read(self.seek_pos) + elif self.seek_pos > self.pos: + # Read ahead enough to reconcile pos and seek_pos + self.r.read(self.pos - self.seek_pos) + self.pos = self.seek_pos + + # Update position pointers + if amt: + self.pos += amt + self.seek_pos += amt + else: + self.pos = self.bytes + self.seek_pos = self.bytes + return self.r.read(amt) else: self.close() - def readline(self): + def readline(self, size=-1): + """ Not implemented. Read and return one line from the stream. """ + raise NotImplementedError() + + def readlines(self, hint=-1): + """ + Not implemented. Read and return a list of lines from the stream. + """ + raise NotImplementedError() + + def writable(self): + """ The stream does not support writing. """ + return False + + def writelines(self, lines): + """ Not implemented. Write a list of lines to the stream. """ raise NotImplementedError() - + def close(self): + """ + Flush and close this stream. This method has no effect if the file + is already closed. As a convenience, it is allowed to call this method + more than once; only the first call, however, will have an effect. + """ + # It's a memory leak if self.r not closed. + if not self.r.isclosed(): + self.r.close() if not self.closed: self.closed = True @@ -217,8 +284,8 @@ def metadata(self, path, cache_read=True): item = self.cache.get(path) if cache_read else None if not item or item.metadata is None or item.expired: try: - metadata = super(DropboxClient, self).metadata(path, - include_deleted=False, list=False) + metadata = super(DropboxClient, self).metadata( + path, include_deleted=False, list=False) except rest.ErrorResponse, e: if e.status == 404: raise ResourceNotFoundError(path) @@ -248,8 +315,8 @@ def children(self, path): update = True if update: try: - metadata = super(DropboxClient, self).metadata(path, hash=hash, - include_deleted=False, list=True) + metadata = super(DropboxClient, self).metadata( + path, hash=hash, include_deleted=False, list=True) children = [] contents = metadata.pop('contents') for child in contents: @@ -343,7 +410,7 @@ def metadata_to_info(metadata, localtime=False): try: if 'client_mtime' in metadata: mtime = metadata.get('client_mtime') - else: + else: mtime = metadata.get('modified') if mtime: # Parse date/time from Dropbox as struct_time. @@ -363,17 +430,16 @@ def metadata_to_info(metadata, localtime=False): class DropboxFS(FS): """A FileSystem that stores data in Dropbox.""" - _meta = { 'thread_safe' : True, - 'virtual' : False, - 'read_only' : False, - 'unicode_paths' : True, - 'case_insensitive_paths' : True, - 'network' : True, - 'atomic.setcontents' : False, - 'atomic.makedir': True, - 'atomic.rename': True, - 'mime_type': 'virtual/dropbox', - } + _meta = {'thread_safe': True, + 'virtual': False, + 'read_only': False, + 'unicode_paths': True, + 'case_insensitive_paths': True, + 'network': True, + 'atomic.setcontents': False, + 'atomic.makedir': True, + 'atomic.rename': True, + 'mime_type': 'virtual/dropbox', } def __init__(self, app_key, app_secret, access_type, token_key, token_secret, localtime=False, thread_synchronize=True): @@ -387,7 +453,8 @@ def __init__(self, app_key, app_secret, access_type, token_key, :param thread_synchronize: set to True (default) to enable thread-safety """ super(DropboxFS, self).__init__(thread_synchronize=thread_synchronize) - self.client = create_client(app_key, app_secret, access_type, token_key, token_secret) + self.client = create_client(app_key, app_secret, access_type, + token_key, token_secret) self.localtime = localtime def __str__(self): @@ -447,10 +514,12 @@ def exists(self, path): except ResourceNotFoundError: return False - def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): + def listdir(self, path="/", wildcard=None, full=False, absolute=False, + dirs_only=False, files_only=False): path = abspath(normpath(path)) children = self.client.children(path) - return self._listdir_helper(path, children, wildcard, full, absolute, dirs_only, files_only) + return self._listdir_helper(path, children, wildcard, full, absolute, + dirs_only, files_only) @synchronize def getinfo(self, path, cache_read=True): @@ -502,12 +571,30 @@ def removedir(self, path, *args, **kwargs): def main(): - parser = optparse.OptionParser(prog="dropboxfs", description="CLI harness for DropboxFS.") - parser.add_option("-k", "--app-key", help="Your Dropbox app key.") - parser.add_option("-s", "--app-secret", help="Your Dropbox app secret.") - parser.add_option("-t", "--type", default='dropbox', choices=('dropbox', 'app_folder'), help="Your Dropbox app access type.") - parser.add_option("-a", "--token-key", help="Your access token key (if you previously obtained one.") - parser.add_option("-b", "--token-secret", help="Your access token secret (if you previously obtained one.") + parser = optparse.OptionParser(prog="dropboxfs", + description="CLI harness for DropboxFS.") + parser.add_option( + "-k", + "--app-key", + help="Your Dropbox app key.") + parser.add_option( + "-s", + "--app-secret", + help="Your Dropbox app secret.") + parser.add_option( + "-t", + "--type", + default='dropbox', + choices=('dropbox', 'app_folder'), + help="Your Dropbox app access type.") + parser.add_option( + "-a", + "--token-key", + help="Your access token key (if you previously obtained one.") + parser.add_option( + "-b", + "--token-secret", + help="Your access token secret (if you previously obtained one.") (options, args) = parser.parse_args() @@ -517,7 +604,8 @@ def main(): # Instantiate a client one way or another. if not options.token_key and not options.token_secret: - s = session.DropboxSession(options.app_key, options.app_secret, options.type) + s = session.DropboxSession(options.app_key, options.app_secret, + options.type) # Get a temporary token, so we can make oAuth calls. t = s.obtain_request_token() print "Please visit the following URL and authorize this application.\n" @@ -535,11 +623,13 @@ def main(): print "\nWhen you are done, please press ." raw_input() elif not options.token_key or not options.token_secret: - parser.error('You must provide both the access token and the access token secret.') + parser.error('You must provide both the access token and the ' + 'access token secret.') else: token_key, token_secret = options.token_key, options.token_secret - fs = DropboxFS(options.app_key, options.app_secret, options.type, token_key, token_secret) + fs = DropboxFS(options.app_key, options.app_secret, options.type, + token_key, token_secret) print fs.getinfo('/') print fs.getinfo('/Public') @@ -550,6 +640,17 @@ def main(): print fs.listdir('/') print fs.listdir('/Foo') + filelike = fs.open('/big-file.pdf') + print filelike.read(100) + filelike.seek(100) + chunk2 = filelike.read(100) + print chunk2 + filelike.seek(200) + print filelike.read(100) + filelike.seek(100) + chunk2a = filelike.read(100) + print chunk2a + assert chunk2 == chunk2a + if __name__ == '__main__': main() - diff --git a/setup.py b/setup.py index 4984c78..cd73837 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup name = 'dropboxfs' -version = '0.4.1' +version = '0.4.2' release = '0' versrel = version + '-' + release readme = os.path.join(os.path.dirname(__file__), 'README.rst')