From 15751ea337e3d022df3a46e70ea505ec070c625a Mon Sep 17 00:00:00 2001 From: Nicolas Pieuchot Date: Sun, 7 Apr 2013 18:49:10 +0200 Subject: [PATCH] =?UTF-8?q?Speed=20up=20exists=20function=20on=20buckets?= =?UTF-8?q?=20with=20large=20number=20of=20files=20(>10=E2=81=B5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/backends/amazon-S3.rst | 4 ++++ storages/backends/s3boto.py | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst index 3773c02..718a8e0 100644 --- a/docs/backends/amazon-S3.rst +++ b/docs/backends/amazon-S3.rst @@ -58,6 +58,10 @@ To allow ``django-admin.py`` collectstatic to automatically put your static file STATICFILES_STORAGE = 'storages.backends.s3boto.S3BotoStorage' +``AWS_S3_NUMEROUS_FILES`` (optional) + +If you have numerous files in your bucket, some simple functions as `exists` might end up being very slow as it would first check across all the entries. This function can even take several minutes if you have hundred thousand files. This parameter set to `True` speeds up this function. + Fields ------ diff --git a/storages/backends/s3boto.py b/storages/backends/s3boto.py index 73d8cd2..0d7919a 100644 --- a/storages/backends/s3boto.py +++ b/storages/backends/s3boto.py @@ -224,6 +224,7 @@ class S3BotoStorage(Storage): custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') calling_format = setting('AWS_S3_CALLING_FORMAT', SubdomainCallingFormat()) secure_urls = setting('AWS_S3_SECURE_URLS', True) + numerous_files_bucket = setting('AWS_S3_NUMEROUS_FILES', False) file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8') gzip = setting('AWS_IS_GZIPPED', False) preload_metadata = setting('AWS_PRELOAD_METADATA', False) @@ -408,8 +409,9 @@ def delete(self, name): def exists(self, name): name = self._normalize_name(self._clean_name(name)) - if self.entries: - return name in self.entries + if not self.numerous_files_bucket: + if self.entries: + return name in self.entries k = self.bucket.new_key(self._encode_name(name)) return k.exists()