Skip to content

Commit

Permalink
Merge pull request btimby#3 from smartfile/seek-read
Browse files Browse the repository at this point in the history 
Add seek to the chunkedReader. Closes btimby#2
  • Loading branch information
@freak3dot
freak3dot committed Jul 29, 2015
2 parents 0b2344b + 31de49c commit 35f7398
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 43 deletions.
185 changes: 143 additions & 42 deletions dropboxfs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
# The format Dropbox uses for times.
TIME_FORMAT = '%a, %d %b %Y %H:%M:%S +0000'
# Max size for spooling to memory before using disk (5M).
MAX_BUFFER = 1024**2*5
MAX_BUFFER = 1024 ** 2 * 5


class ContextManagerStream(object):
Expand All @@ -54,9 +54,9 @@ def __exit__(self, *args):
self.close()


# TODO: these classes can probably be replaced with tempfile.SpooledTemporaryFile, however
# I am unsure at this moment if doing so would be bad since it is only available in Python
# 2.6+.
# TODO: these classes can probably be replaced with
# tempfile.SpooledTemporaryFile, however I am unsure at this moment if doing
# so would be bad since it is only available in Python 2.6+.

class SpooledWriter(ContextManagerStream):
"""Spools bytes to a StringIO buffer until it reaches max_buffer. At that
Expand All @@ -66,14 +66,15 @@ def __init__(self, client, name, max_buffer=MAX_BUFFER):
self.max_buffer = max_buffer
self.bytes = 0
super(SpooledWriter, self).__init__(StringIO(), name)

def __len__(self):
return self.bytes

def write(self, data):
if self.temp.tell() + len(data) >= self.max_buffer:
# We reached the max_buffer size that we want to keep in memory. Switch
# to an on-disk temp file. Copy what has been written so far to it.
# We reached the max_buffer size that we want to keep in memory.
# Switch to an on-disk temp file. Copy what has been written so
# far to it.
temp = tempfile.TemporaryFile()
self.temp.seek(0)
shutil.copyfileobj(self.temp, temp)
Expand All @@ -91,8 +92,11 @@ def close(self):


class SpooledReader(ContextManagerStream):
"""Reads the entire file from the remote server into a buffer or temporary file.
It can then satisfy read(), seek() and other calls using the local file."""
"""
Reads the entire file from the remote server into a buffer or temporary
file. It can then satisfy read(), seek() and other calls using the local
file.
"""
def __init__(self, client, name, max_buffer=MAX_BUFFER):
self.client = client
r = self.client.get_file(name)
Expand Down Expand Up @@ -123,29 +127,92 @@ def __init__(self, client, name):
self.bytes = int(self.r.getheader('Content-Length'))
self.name = name
self.closed = False
self.pos = 0
self.seek_pos = 0

def __len__(self):
return self.bytes

def __iter__(self):
return self

def seek(self, offset, whence=0):
"""
Change the stream position to the given byte offset in the file-like
object.
"""
if (whence == 0):
self.seek_pos = offset
elif (whence == 1):
self.seek_pos += offset
elif (whence == 2):
self.seek_pos = self.size + offset

def tell(self):
""" Return the current stream position. """
return self.seek_pos

def next(self):
"""
Read the data until all data is read.
data is empty string when there is no more data to read.
"""
data = self.read()
if data is None:
raise StopIteration()
return data

def read(self, size=16384):
def read(self, amt=None):
""" Read a piece of the file from dropbox. """
if not self.r.isclosed():
return self.r.read(size)
# Do some fake seeking
if self.seek_pos < self.pos:
self.r.close()
self.r = self.client.get_file(self.name)
self.r.read(self.seek_pos)
elif self.seek_pos > self.pos:
# Read ahead enough to reconcile pos and seek_pos
self.r.read(self.pos - self.seek_pos)
self.pos = self.seek_pos

# Update position pointers
if amt:
self.pos += amt
self.seek_pos += amt
else:
self.pos = self.bytes
self.seek_pos = self.bytes
return self.r.read(amt)
else:
self.close()

def readline(self):
def readline(self, size=-1):
""" Not implemented. Read and return one line from the stream. """
raise NotImplementedError()

def readlines(self, hint=-1):
"""
Not implemented. Read and return a list of lines from the stream.
"""
raise NotImplementedError()

def writable(self):
""" The stream does not support writing. """
return False

def writelines(self, lines):
""" Not implemented. Write a list of lines to the stream. """
raise NotImplementedError()

def close(self):
"""
Flush and close this stream. This method has no effect if the file
is already closed. As a convenience, it is allowed to call this method
more than once; only the first call, however, will have an effect.
"""
# It's a memory leak if self.r not closed.
if not self.r.isclosed():
self.r.close()
if not self.closed:
self.closed = True

Expand Down Expand Up @@ -217,8 +284,8 @@ def metadata(self, path, cache_read=True):
item = self.cache.get(path) if cache_read else None
if not item or item.metadata is None or item.expired:
try:
metadata = super(DropboxClient, self).metadata(path,
include_deleted=False, list=False)
metadata = super(DropboxClient, self).metadata(
path, include_deleted=False, list=False)
except rest.ErrorResponse, e:
if e.status == 404:
raise ResourceNotFoundError(path)
Expand Down Expand Up @@ -248,8 +315,8 @@ def children(self, path):
update = True
if update:
try:
metadata = super(DropboxClient, self).metadata(path, hash=hash,
include_deleted=False, list=True)
metadata = super(DropboxClient, self).metadata(
path, hash=hash, include_deleted=False, list=True)
children = []
contents = metadata.pop('contents')
for child in contents:
Expand Down Expand Up @@ -343,7 +410,7 @@ def metadata_to_info(metadata, localtime=False):
try:
if 'client_mtime' in metadata:
mtime = metadata.get('client_mtime')
else:
else:
mtime = metadata.get('modified')
if mtime:
# Parse date/time from Dropbox as struct_time.
Expand All @@ -363,17 +430,16 @@ def metadata_to_info(metadata, localtime=False):
class DropboxFS(FS):
"""A FileSystem that stores data in Dropbox."""

_meta = { 'thread_safe' : True,
'virtual' : False,
'read_only' : False,
'unicode_paths' : True,
'case_insensitive_paths' : True,
'network' : True,
'atomic.setcontents' : False,
'atomic.makedir': True,
'atomic.rename': True,
'mime_type': 'virtual/dropbox',
}
_meta = {'thread_safe': True,
'virtual': False,
'read_only': False,
'unicode_paths': True,
'case_insensitive_paths': True,
'network': True,
'atomic.setcontents': False,
'atomic.makedir': True,
'atomic.rename': True,
'mime_type': 'virtual/dropbox', }

def __init__(self, app_key, app_secret, access_type, token_key,
token_secret, localtime=False, thread_synchronize=True):
Expand All @@ -387,7 +453,8 @@ def __init__(self, app_key, app_secret, access_type, token_key,
:param thread_synchronize: set to True (default) to enable thread-safety
"""
super(DropboxFS, self).__init__(thread_synchronize=thread_synchronize)
self.client = create_client(app_key, app_secret, access_type, token_key, token_secret)
self.client = create_client(app_key, app_secret, access_type,
token_key, token_secret)
self.localtime = localtime

def __str__(self):
Expand Down Expand Up @@ -447,10 +514,12 @@ def exists(self, path):
except ResourceNotFoundError:
return False

def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
def listdir(self, path="/", wildcard=None, full=False, absolute=False,
dirs_only=False, files_only=False):
path = abspath(normpath(path))
children = self.client.children(path)
return self._listdir_helper(path, children, wildcard, full, absolute, dirs_only, files_only)
return self._listdir_helper(path, children, wildcard, full, absolute,
dirs_only, files_only)

@synchronize
def getinfo(self, path, cache_read=True):
Expand Down Expand Up @@ -502,12 +571,30 @@ def removedir(self, path, *args, **kwargs):


def main():
parser = optparse.OptionParser(prog="dropboxfs", description="CLI harness for DropboxFS.")
parser.add_option("-k", "--app-key", help="Your Dropbox app key.")
parser.add_option("-s", "--app-secret", help="Your Dropbox app secret.")
parser.add_option("-t", "--type", default='dropbox', choices=('dropbox', 'app_folder'), help="Your Dropbox app access type.")
parser.add_option("-a", "--token-key", help="Your access token key (if you previously obtained one.")
parser.add_option("-b", "--token-secret", help="Your access token secret (if you previously obtained one.")
parser = optparse.OptionParser(prog="dropboxfs",
description="CLI harness for DropboxFS.")
parser.add_option(
"-k",
"--app-key",
help="Your Dropbox app key.")
parser.add_option(
"-s",
"--app-secret",
help="Your Dropbox app secret.")
parser.add_option(
"-t",
"--type",
default='dropbox',
choices=('dropbox', 'app_folder'),
help="Your Dropbox app access type.")
parser.add_option(
"-a",
"--token-key",
help="Your access token key (if you previously obtained one.")
parser.add_option(
"-b",
"--token-secret",
help="Your access token secret (if you previously obtained one.")

(options, args) = parser.parse_args()

Expand All @@ -517,7 +604,8 @@ def main():

# Instantiate a client one way or another.
if not options.token_key and not options.token_secret:
s = session.DropboxSession(options.app_key, options.app_secret, options.type)
s = session.DropboxSession(options.app_key, options.app_secret,
options.type)
# Get a temporary token, so we can make oAuth calls.
t = s.obtain_request_token()
print "Please visit the following URL and authorize this application.\n"
Expand All @@ -535,11 +623,13 @@ def main():
print "\nWhen you are done, please press <enter>."
raw_input()
elif not options.token_key or not options.token_secret:
parser.error('You must provide both the access token and the access token secret.')
parser.error('You must provide both the access token and the '
'access token secret.')
else:
token_key, token_secret = options.token_key, options.token_secret

fs = DropboxFS(options.app_key, options.app_secret, options.type, token_key, token_secret)
fs = DropboxFS(options.app_key, options.app_secret, options.type,
token_key, token_secret)

print fs.getinfo('/')
print fs.getinfo('/Public')
Expand All @@ -550,6 +640,17 @@ def main():
print fs.listdir('/')
print fs.listdir('/Foo')

filelike = fs.open('/big-file.pdf')
print filelike.read(100)
filelike.seek(100)
chunk2 = filelike.read(100)
print chunk2
filelike.seek(200)
print filelike.read(100)
filelike.seek(100)
chunk2a = filelike.read(100)
print chunk2a
assert chunk2 == chunk2a

if __name__ == '__main__':
main()

2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from distutils.core import setup

name = 'dropboxfs'
version = '0.4.1'
version = '0.4.2'
release = '0'
versrel = version + '-' + release
readme = os.path.join(os.path.dirname(__file__), 'README.rst')
Expand Down

0 comments on commit 35f7398

Please sign in to comment.