diff --git a/.hgignore b/.hgignore deleted file mode 100644 index 1a2d13161..000000000 --- a/.hgignore +++ /dev/null @@ -1,21 +0,0 @@ -syntax:glob - -*.DS_Store -*.egg -*.egg-info -*.elc -*.gz -*.log -*.orig -*.pyc -*.swp -*.tmp -*~ -.tox/ -_build/ -build/ -dist/* -django -local_settings.py -setuptools* -testdb.sqlite diff --git a/.travis.yml b/.travis.yml index 926d3fea2..0f03d69ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,32 @@ sudo: false language: python - -env: - - TOX_ENV=py27-django17 - - TOX_ENV=py33-django17 - - TOX_ENV=py34-django17 - - TOX_ENV=py27-django18 - - TOX_ENV=py33-django18 - - TOX_ENV=py34-django18 - - TOX_ENV=py35-django18 - - TOX_ENV=py27-django19 - - TOX_ENV=py34-django19 - - TOX_ENV=py35-django19 +cache: pip matrix: - # Python 3.5 not yet available on travis, watch this to see when it is. - allow_failures: - - env: TOX_ENV=py35-django18 - - env: TOX_ENV=py35-django19 + include: + - env: TOX_ENV=lint + - python: 2.7 + env: TOX_ENV=py27-django18 + - python: 3.3 + env: TOX_ENV=py33-django18 + - python: 3.4 + env: TOX_ENV=py34-django18 + - python: 3.5 + env: TOX_ENV=py35-django18 + - python: 2.7 + env: TOX_ENV=py27-django110 + - python: 3.4 + env: TOX_ENV=py34-django110 + - python: 3.5 + env: TOX_ENV=py35-django110 + - python: 2.7 + env: TOX_ENV=py27-django111 + - python: 3.4 + env: TOX_ENV=py34-django111 + - python: 3.5 + env: TOX_ENV=py35-django111 + - python: 3.6 + env: TOX_ENV=py36-django111 before_install: - pip install codecov diff --git a/AUTHORS b/AUTHORS index 8b34c4b33..25db7d015 100644 --- a/AUTHORS +++ b/AUTHORS @@ -24,6 +24,14 @@ By order of apparition, thanks: * Josh Schneier (Fork maintainer, Bugfixes, Py3K) * Anthony Monthe (Dropbox) * EunPyo (Andrew) Hong (Azure) + * Michael Barrientos (S3 with Boto3) + * piglei (patches) + * Matt Braymer-Hayes (S3 with Boto3) + * Eirik Martiniussen Sylliaas (Google Cloud Storage native support) + * Jody McIntyre (Google Cloud Storage native support) + * Stanislav Kaledin (Bug fixes in SFTPStorage) + * Filip Vavera (Google Cloud MIME types support) + * Max Malysh (Dropbox large file support) Extra thanks to Marty for adding this in Django, you can buy his very interesting book (Pro Django). diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a3ef4c311..195477e41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,22 +1,177 @@ django-storages change log ========================== -1.4.2 (XXXX-XX-XX) +1.6.5 (2017-08-01) ****************** -* Fix ``MANIFEST.in`` to not ship ``.pyc`` files. (`#145`_) thanks @fladi +* Fix Django 1.11 regression with gzipped content being saved twice + resulting in empty files (`#367`_, `#371`_, `#373`_) +* Fix the ``mtime`` when gzipping content on ``S3Boto3Storage`` (`#374`_) + +.. _#367: https://github.com/jschneier/django-storages/issues/367 +.. _#371: https://github.com/jschneier/django-storages/pull/371 +.. _#373: https://github.com/jschneier/django-storages/pull/373 +.. _#374: https://github.com/jschneier/django-storages/pull/374 + +1.6.4 (2017-07-27) +****************** + +* Files uploaded with ``GoogleCloudStorage`` will now set their appropriate mimetype (`#320`_) +* Fix ``DropBoxStorage.url`` to work. (`#357`_) +* Fix ``S3Boto3Storage`` when ``AWS_PRELOAD_METADATA = True`` (`#366`_) +* Fix ``S3Boto3Storage`` uploading file-like objects without names (`#195`_, `#368`_) +* ``S3Boto3Storage`` is now threadsafe - a separate session is created on a + per-thread basis (`#268`_, `#358`_) + +.. _#320: https://github.com/jschneier/django-storages/pull/320 +.. _#357: https://github.com/jschneier/django-storages/pull/357 +.. _#366: https://github.com/jschneier/django-storages/pull/366 +.. _#195: https://github.com/jschneier/django-storages/pull/195 +.. _#368: https://github.com/jschneier/django-storages/pull/368 +.. _#268: https://github.com/jschneier/django-storages/issues/268 +.. _#358: https://github.com/jschneier/django-storages/pull/358 + +1.6.3 (2017-06-23) +****************** + +* Revert default ``AWS_S3_SIGNATURE_VERSION`` to V2 to restore backwards + compatability in ``S3Boto3``. It's recommended that all new projects set + this to be ``'s3v4'``. (`#344`_) + +.. _#344: https://github.com/jschneier/django-storages/pull/344 + +1.6.2 (2017-06-22) +****************** + +* Fix regression in ``safe_join()`` to handle a trailing slash in an + intermediate path. (`#341`_) +* Fix regression in ``gs.GSBotoStorage`` getting an unexpected kwarg. + (`#342`_) + +.. _#341: https://github.com/jschneier/django-storages/pull/341 +.. _#342: https://github.com/jschneier/django-storages/pull/342 + +1.6.1 (2017-06-22) +****************** + +* Drop support for Django 1.9 (`e89db45`_) +* Fix regression in ``safe_join()`` to allow joining a base path with an empty + string. (`#336`_) + +.. _e89db45: https://github.com/jschneier/django-storages/commit/e89db451d7e617638b5991e31df4c8de196546a6 +.. _#336: https://github.com/jschneier/django-storages/pull/336 + +1.6 (2017-06-21) +****************** + +* **Breaking:** Remove backends deprecated in v1.5.1 (`#280`_) +* **Breaking:** ``DropBoxStorage`` has been upgrade to support v2 of the API, v1 will be shut off at the + end of the month - upgrading is recommended (`#273`_) +* **Breaking:** The ``SFTPStorage`` backend now checks for the existence of the fallback ``~/.ssh/known_hosts`` + before attempting to load it. If you had previously been passing in a path to a non-existent file it will no longer + attempt to load the fallback. (`#118`_, `#325`_) +* **Breaking:** The default version value for ``AWS_S3_SIGNATURE_VERSION`` is now ``'s3v4'``. No changes should + be required (`#335`_) +* **Deprecation:** The undocumented ``gs.GSBotoStorage`` backend. See the new ``gcloud.GoogleCloudStorage`` + or ``apache_libcloud.LibCloudStorage`` backends instead. (`#236`_) +* Add a new backend, ``gcloud.GoogleCloudStorage`` based on the ``google-cloud`` bindings. (`#236`_) +* Pass in the location constraint when auto creating a bucket in ``S3Boto3Storage`` (`#257`_, `#258`_) +* Add support for reading ``AWS_SESSION_TOKEN`` and ``AWS_SECURITY_TOKEN`` from the environment + to ``S3Boto3Storage`` and ``S3BotoStorage``. (`#283`_) +* Fix Boto3 non-ascii filenames on Python 2.7 (`#216`_, `#217`_) +* Fix ``collectstatic`` timezone handling in and add ``get_modified_time`` to ``S3BotoStorage`` (`#290`_) +* Add support for Django 1.11 (`#295`_) +* Add ``project`` keyword support to GCS in ``LibCloudStorage`` backend (`#269`_) +* Files that have a guessable encoding (e.g. gzip or compress) will be uploaded with that Content-Encoding in + the ``s3boto3`` backend (`#263`_, `#264`_) +* The Dropbox backend now properly translates backslashes in Windows paths into forward slashes (`e52a127`_) +* The S3 backends now permit colons in the keys (`#248`_, `#322`_) + +.. _#217: https://github.com/jschneier/django-storages/pull/217 +.. _#273: https://github.com/jschneier/django-storages/pull/273 +.. _#216: https://github.com/jschneier/django-storages/issues/216 +.. _#283: https://github.com/jschneier/django-storages/pull/283 +.. _#280: https://github.com/jschneier/django-storages/pull/280 +.. _#257: https://github.com/jschneier/django-storages/issues/257 +.. _#258: https://github.com/jschneier/django-storages/pull/258 +.. _#290: https://github.com/jschneier/django-storages/pull/290 +.. _#295: https://github.com/jschneier/django-storages/pull/295 +.. _#269: https://github.com/jschneier/django-storages/pull/269 +.. _#263: https://github.com/jschneier/django-storages/issues/263 +.. _#264: https://github.com/jschneier/django-storages/pull/264 +.. _e52a127: https://github.com/jschneier/django-storages/commit/e52a127523fdd5be50bb670ccad566c5d527f3d1 +.. _#236: https://github.com/jschneier/django-storages/pull/236 +.. _#118: https://github.com/jschneier/django-storages/issues/118 +.. _#325: https://github.com/jschneier/django-storages/pull/325 +.. _#248: https://github.com/jschneier/django-storages/issues/248 +.. _#322: https://github.com/jschneier/django-storages/pull/322 +.. _#335: https://github.com/jschneier/django-storages/pull/335 + +1.5.2 (2017-01-13) +****************** + +* Actually use ``SFTP_STORAGE_HOST`` in ``SFTPStorage`` backend (`#204`_) +* Fix ``S3Boto3Storage`` to avoid race conditions in a multi-threaded WSGI environment (`#238`_) +* Fix trying to localize a naive datetime when ``settings.USE_TZ`` is ``False`` in ``S3Boto3Storage.modified_time``. + (`#235`_, `#234`_) +* Fix automatic bucket creation in ``S3Boto3Storage`` when ``AWS_AUTO_CREATE_BUCKET`` is ``True`` (`#196`_) +* Improve the documentation for the S3 backends + +.. _#204: https://github.com/jschneier/django-storages/pull/204 +.. _#238: https://github.com/jschneier/django-storages/pull/238 +.. _#234: https://github.com/jschneier/django-storages/issues/234 +.. _#235: https://github.com/jschneier/django-storages/pull/235 +.. _#196: https://github.com/jschneier/django-storages/pull/196 + +1.5.1 (2016-09-13) +****************** + +* **Breaking:** Drop support for Django 1.7 (`#185`_) +* **Deprecation:** hashpath, image, overwrite, mogile, symlinkorcopy, database, mogile, couchdb. + See (`#202`_) to discuss maintenance going forward +* Use a fixed ``mtime`` argument for ``GzipFile`` in ``S3BotoStorage`` and ``S3Boto3Storage`` to ensure + a stable output for gzipped files +* Use ``.putfileobj`` instead of ``.put`` in ``S3Boto3Storage`` to use the transfer manager, + allowing files greater than 5GB to be put on S3 (`#194`_ , `#201`_) +* Update ``S3Boto3Storage`` for Django 1.10 (`#181`_) (``get_modified_time`` and ``get_accessed_time``) +* Fix bad kwarg name in ``S3Boto3Storage`` when `AWS_PRELOAD_METADATA` is `True` (`#189`_, `#190`_) + +.. _#202: https://github.com/jschneier/django-storages/issues/202 +.. _#201: https://github.com/jschneier/django-storages/pull/201 +.. _#194: https://github.com/jschneier/django-storages/issues/194 +.. _#190: https://github.com/jschneier/django-storages/pull/190 +.. _#189: https://github.com/jschneier/django-storages/issues/189 +.. _#185: https://github.com/jschneier/django-storages/pull/185 +.. _#181: https://github.com/jschneier/django-storages/pull/181 + +1.5.0 (2016-08-02) +****************** + +* Add new backend ``S3Boto3Storage`` (`#179`_) +* Add a `strict` option to `utils.setting` (`#176`_) +* Tests, documentation, fixing ``.close`` for ``SFTPStorage`` (`#177`_) +* Tests, documentation, add `.readlines` for ``FTPStorage`` (`#175`_) +* Tests and documentation for ``DropBoxStorage`` (`#174`_) +* Fix ``MANIFEST.in`` to not ship ``.pyc`` files. (`#145`_) +* Enable CI testing of Python 3.5 and fix test failure from api change (`#171`_) .. _#145: https://github.com/jschneier/django-storages/pull/145 +.. _#171: https://github.com/jschneier/django-storages/pull/171 +.. _#174: https://github.com/jschneier/django-storages/pull/174 +.. _#175: https://github.com/jschneier/django-storages/pull/175 +.. _#177: https://github.com/jschneier/django-storages/pull/177 +.. _#176: https://github.com/jschneier/django-storages/pull/176 +.. _#179: https://github.com/jschneier/django-storages/pull/179 1.4.1 (2016-04-07) ****************** * Files that have a guessable encoding (e.g. gzip or compress) will be uploaded with that Content-Encoding in the ``s3boto`` backend. Compressable types such as ``application/javascript`` will still be gzipped. - PR `#122`_ thanks @cambonf -* Fix ``DropBoxStorage.exists`` check and add ``DropBoxStorage.url`` (`#127`_) thanks @zuck + PR `#122`_ +* Fix ``DropBoxStorage.exists`` check and add ``DropBoxStorage.url`` (`#127`_) * Add ``GS_HOST`` setting (with a default of ``GSConnection.DefaultHost``) to fix ``GSBotoStorage``. - Issue `#124`_. Fixed in `#125`_. Thanks @patgmiller @dcgoss. + (`#124`_, `#125`_) .. _#122: https://github.com/jschneier/django-storages/pull/122 .. _#127: https://github.com/jschneier/django-storages/pull/127 @@ -32,10 +187,10 @@ django-storages change log 1.3.2 (2016-01-26) ****************** -* Fix memory leak from not closing underlying temp file in ``s3boto`` backend (`#106`_) thanks @kmmbvnr -* Allow easily specifying a custom expiry time when generating a url for ``S3BotoStorage`` (`#96`_) thanks @mattbriancon +* Fix memory leak from not closing underlying temp file in ``s3boto`` backend (`#106`_) +* Allow easily specifying a custom expiry time when generating a url for ``S3BotoStorage`` (`#96`_) * Check for bucket existence when the empty path ('') is passed to ``storage.exists`` in ``S3BotoStorage`` - - this prevents a crash when running ``collecstatic -c`` on Django 1.9.1 (`#112`_) fixed in `#116`_ thanks @xblitz + this prevents a crash when running ``collectstatic -c`` on Django 1.9.1 (`#112`_) fixed in `#116`_ .. _#106: https://github.com/jschneier/django-storages/pull/106 .. _#96: https://github.com/jschneier/django-storages/pull/96 @@ -46,12 +201,12 @@ django-storages change log 1.3.1 (2016-01-12) ****************** -* A few Azure Storage fixes [pass the content-type to Azure, handle chunked content, fix ``url``] (`#45`__) thanks @erlingbo -* Add support for a Dropbox (``dropbox``) storage backend, thanks @ZuluPro (`#76`_) +* A few Azure Storage fixes [pass the content-type to Azure, handle chunked content, fix ``url``] (`#45`__) +* Add support for a Dropbox (``dropbox``) storage backend * Various fixes to the ``apache_libcloud`` backend [return the number of bytes asked for by ``.read``, make ``.name`` non-private, don't - initialize to an empty ``BytesIO`` object] thanks @kaedroho (`#55`_) -* Fix multi-part uploads in ``s3boto`` backend not respecting ``AWS_S3_ENCRYPTION`` (`#94`_) thanks @andersontep -* Automatically gzip svg files thanks @comandrei (`#100`_) + initialize to an empty ``BytesIO`` object] (`#55`_) +* Fix multi-part uploads in ``s3boto`` backend not respecting ``AWS_S3_ENCRYPTION`` (`#94`_) +* Automatically gzip svg files (`#100`_) .. __: https://github.com/jschneier/django-storages/pull/45 .. _#76: https://github.com/jschneier/django-storages/pull/76 @@ -63,13 +218,13 @@ django-storages change log 1.3 (2015-08-14) **************** -* **Drop Support for Django 1.5 and Python2.6** -* Remove previously deprecated mongodb backend -* Remove previously deprecated ``parse_ts_extended`` from s3boto storage +* **Breaking:** Drop Support for Django 1.5 and Python 2.6 +* **Breaking:** Remove previously deprecated mongodb backend +* **Breaking:** Remove previously deprecated ``parse_ts_extended`` from s3boto storage * Add support for Django 1.8+ (`#36`__) * Add ``AWS_S3_PROXY_HOST`` and ``AWS_S3_PROXY_PORT`` settings for s3boto backend (`#41`_) * Fix Python3K compat issue in apache_libcloud (`#52`_) -* Fix Google Storage backend not respecting ``GS_IS_GZIPPED`` setting (`#51`__, `#60`_) thanks @stmos +* Fix Google Storage backend not respecting ``GS_IS_GZIPPED`` setting (`#51`__, `#60`_) * Rename FTP ``_name`` attribute to ``name`` which is what the Django ``File`` api is expecting (`#70`_) * Put ``StorageMixin`` first in inheritance to maintain backwards compat with older versions of Django (`#63`_) @@ -107,9 +262,9 @@ django-storages change log 1.2.1 (2014-12-31) ****************** +* **Deprecation:** Issue warning about ``parse_ts_extended`` +* **Deprecation:** mongodb backend - django-mongodb-engine now ships its own storage backend * Fix ``storage.modified_time`` crashing on new files when ``AWS_PRELOAD_METADATA=True`` (`#11`_, `#12`__, `#14`_) -* Deprecate and issue warning about ``parse_ts_extended`` -* Deprecate mongodb backend - django-mongodb-engine now ships its own storage backend .. _#11: https://github.com/jschneier/django-storages/pull/11 __ https://github.com/jschneier/django-storages/issues/12 @@ -119,11 +274,11 @@ __ https://github.com/jschneier/django-storages/issues/12 1.2 (2014-12-14) **************** +* **Breaking:** Remove legacy S3 storage (`#1`_) +* **Breaking:** Remove mosso files backend (`#2`_) * Add text/javascript mimetype to S3BotoStorage gzip allowed defaults * Add support for Django 1.7 migrations in S3BotoStorage and ApacheLibCloudStorage (`#5`_, `#8`_) * Python3K (3.3+) now available for S3Boto backend (`#4`_) -* Remove legacy S3 storage (`#1`_) -* Remove mosso files backend (`#2`_) .. _#8: https://github.com/jschneier/django-storages/pull/8 .. _#5: https://github.com/jschneier/django-storages/pull/5 @@ -268,4 +423,3 @@ since March 2013. .. _#89: https://bitbucket.org/david/django-storages/issue/89/112-broke-the-mosso-backend .. _pull request #5: https://bitbucket.org/david/django-storages/pull-request/5/fixed-path-bug-and-added-testcase-for - diff --git a/README.rst b/README.rst index da1fc5349..214ba0c97 100644 --- a/README.rst +++ b/README.rst @@ -2,14 +2,14 @@ django-storages =============== -.. image:: https://travis-ci.org/jschneier/django-storages.png?branch=master - :target: https://travis-ci.org/jschneier/django-storages - :alt: Build Status -.. image:: https://pypip.in/v/django-storages/badge.png +.. image:: https://img.shields.io/pypi/v/django-storages.svg :target: https://pypi.python.org/pypi/django-storages :alt: PyPI Version +.. image:: https://travis-ci.org/jschneier/django-storages.svg?branch=master + :target: https://travis-ci.org/jschneier/django-storages + :alt: Build Status Installation ============ @@ -23,9 +23,9 @@ hasn't been released yet) then the magic incantation you are looking for is:: pip install -e 'git+https://github.com/jschneier/django-storages.git#egg=django-storages' Once that is done add ``storages`` to your ``INSTALLED_APPS`` and set ``DEFAULT_FILE_STORAGE`` to the -backend of your choice. If, for example, you want to use the s3boto backend you would set:: +backend of your choice. If, for example, you want to use the boto3 backend you would set:: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' There are also a number of settings available to control how each storage backend functions, please consult the documentation for a comprehensive list. @@ -33,13 +33,14 @@ please consult the documentation for a comprehensive list. About ===== django-storages is a project to provide a variety of storage backends in a single library. -This library is compatible with Django >= 1.7. It should also works with 1.6.2+ but no guarantees are made. + +This library is usually compatible with the currently supported versions of +Django. Check the Trove classifiers in setup.py to be sure. History ======= This repo began as a fork of the original library under the package name of django-storages-redux and -became the official successor (releasing under django-storages on PyPI) in February of 2016. The initial -reasons for the fork are explained at the bottom of this document. +became the official successor (releasing under django-storages on PyPI) in February of 2016. Found a Bug? Something Unsupported? =================================== @@ -53,7 +54,7 @@ Issues are tracked via GitHub issues at the `project issue page Documentation ============= -The original documentation for django-storages is located at http://django-storages.readthedocs.org/. +The original documentation for django-storages is located at https://django-storages.readthedocs.org/. Stay tuned for forthcoming documentation updates. Contributing @@ -68,21 +69,3 @@ Contributing correctly. #. Bug me until I can merge your pull request. Also, don't forget to add yourself to ``AUTHORS``. - -Why Fork? -==================== -The BitBucket repo of the original django-storages has seen no commit applied -since March 2014 (it is currently December 2014) and no PyPi release since -March 2013 despite a wealth of bugfixes that were applied in that year-long -gap. There is plenty of community support for the django-storages project -(especially the S3BotoStorage piece) and I have a personal need for a Python3 -compatible version. - -All of the Python3 compatible forks that currently exist (and there are a few) -are lacking in some way. This can be anything from the fact that they don't -release to PyPi, have no ongoing testing, didn't apply many important bugfixes -that have occurred on the Bitbucket repo since forking or don't support older -versions of Python and Django (vital to finding bugs and keeping a large -community). For this fork I've done the small bit of work necessary to get a -tox + travis ci matrix going for all of the supported Python + Django versions. -In many cases the various forks are lacking in a few of the above ways. diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst index 4532aadd4..986030128 100644 --- a/docs/backends/amazon-S3.rst +++ b/docs/backends/amazon-S3.rst @@ -4,42 +4,73 @@ Amazon S3 Usage ***** -There is one backend for interacting with S3 based on the boto library. A legacy backend backed on the Amazon S3 Python library was removed in version 1.2. +There are two backends for interacting with Amazon's S3, one based +on boto3 and an older one based on boto. It is highly recommended that all +new projects (at least) use the boto3 backend since it has many bug fixes +and performance improvements over boto and is the future; boto is lightly +maintained if at all. The boto based backed will continue to be maintained +for the forseeable future. + +For historical completeness an extreme legacy backend was removed +in version 1.2 + +If using the boto backend on a new project (not recommended) it is recommended +that you configure it to also use `AWS Signature Version 4`_. This can be done +by adding ``S3_USE_SIGV4 = True`` to your settings and setting the ``AWS_S3_HOST`` +configuration option. For regions created after January 2014 this is your only +option if you insist on using the boto backend. Settings -------- -To use s3boto set:: +To use boto3 set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + +To use the boto version of the backend set:: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' -``AWS_ACCESS_KEY_ID`` +To allow ``django-admin.py`` collectstatic to automatically put your static files in your bucket set the following in your settings.py:: -Your Amazon Web Services access key, as a string. + STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -``AWS_SECRET_ACCESS_KEY`` +Available are numerous settings. It should be especially noted the following: + +``AWS_ACCESS_KEY_ID`` + Your Amazon Web Services access key, as a string. -Your Amazon Web Services secret access key, as a string. +``AWS_SECRET_ACCESS_KEY`` + Your Amazon Web Services secret access key, as a string. ``AWS_STORAGE_BUCKET_NAME`` + Your Amazon Web Services storage bucket name, as a string. -Your Amazon Web Services storage bucket name, as a string. +``AWS_DEFAULT_ACL`` (optional) + If set to ``private`` changes uploaded file's Access Control List from the default permission ``public-read`` to give owner full control and remove read access from everyone else. ``AWS_AUTO_CREATE_BUCKET`` (optional) + If set to ``True`` the bucket specified in ``AWS_STORAGE_BUCKET_NAME`` is automatically created. -If set to ``True`` the bucket specified in ``AWS_STORAGE_BUCKET_NAME`` is automatically created. +``AWS_HEADERS`` (optional - boto only, for boto3 see ``AWS_S3_OBJECT_PARAMETERS``) + If you'd like to set headers sent with each file of the storage:: + AWS_HEADERS = { + 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', + 'Cache-Control': 'max-age=86400', + } -``AWS_HEADERS`` (optional) +``AWS_S3_OBJECT_PARAMETERS`` (optional - boto3 only) + Use this to set object parameters on your object (such as CacheControl):: -If you'd like to set headers sent with each file of the storage:: - - # see http://developer.yahoo.com/performance/rules.html#expires - AWS_HEADERS = { - 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', - 'Cache-Control': 'max-age=86400', - } + AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', + } +``AWS_QUERYSTRING_AUTH`` (optional; default is ``True``) + Setting ``AWS_QUERYSTRING_AUTH`` to ``False`` to remove query parameter + authentication from generated URLs. This can be useful if your S3 buckets + are public. ``AWS_EXTRA_HEADERS`` (optional) @@ -51,50 +82,86 @@ AWS_EXTRA_HEADERS = [ ] To allow ``django-admin.py`` collectstatic to automatically put your static files in your bucket set the following in your settings.py:: +======= +``AWS_QUERYSTRING_EXPIRE`` (optional; default is 3600 seconds) + The number of seconds that a generated URL is valid for. - STATICFILES_STORAGE = 'storages.backends.s3boto.S3BotoStorage' +``AWS_S3_ENCRYPTION`` (optional; default is ``False``) + Enable server-side file encryption while at rest, by setting ``encrypt_key`` parameter to True. More info available here: http://boto.cloudhackers.com/en/latest/ref/s3.html -Fields ------- +``AWS_S3_FILE_OVERWRITE`` (optional: default is ``True``) + By default files with the same name will overwrite each other. Set this to ``False`` to have extra characters appended. -Once you're done, default_storage will be the S3 storage:: +``AWS_S3_HOST`` (optional - boto only, default is ``s3.amazonaws.com``) - >>> from django.core.files.storage import default_storage - >>> print default_storage.__class__ - + To ensure you use `AWS Signature Version 4`_ it is recommended to set this to the host of your bucket. See the + `S3 region list`_ to figure out the appropriate endpoint for your bucket. Also be sure to add + ``S3_USE_SIGV4 = True`` to settings.py -The above doesn't seem to be true for django 1.3+ instead look at:: + .. note:: - >>> from django.core.files.storage import default_storage - >>> print default_storage.connection - S3Connection:s3.amazonaws.com + The signature versions are not backwards compatible so be careful about url endpoints if making this change + for legacy projects. -This way, if you define a new FileField, it will use the S3 storage:: +``AWS_LOCATION`` (optional: default is `''`) + A path prefix that will be prepended to all uploads - >>> from django.db import models - >>> class Resume(models.Model): - ... pdf = models.FileField(upload_to='pdfs') - ... photos = models.ImageField(upload_to='photos') - ... - >>> resume = Resume() - >>> print resume.pdf.storage - +``AWS_IS_GZIPPED`` (optional: default is ``False``) + Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES`` -Tests -***** +``GZIP_CONTENT_TYPES`` (optional: default is ``text/css``, ``text/javascript``, ``application/javascript``, ``application/x-javascript``, ``image/svg+xml``) + When ``AWS_IS_GZIPPED`` is set to ``True`` the content types which will be gzipped -Initialization:: +``AWS_S3_REGION_NAME`` (optional: default is ``None``) + Name of the AWS S3 region to use (eg. eu-west-1) - >>> from django.core.files.storage import default_storage - >>> from django.core.files.base import ContentFile - >>> from django.core.cache import cache - >>> from models import MyStorage +``AWS_S3_USE_SSL`` (optional: default is ``True``) + Whether or not to use SSL when connecting to S3. + +``AWS_S3_ENDPOINT_URL`` (optional: default is ``None``) + Custom S3 URL to use when connecting to S3, including scheme. Overrides ``AWS_S3_REGION_NAME`` and ``AWS_S3_USE_SSL``. + +``AWS_S3_CALLING_FORMAT`` (optional: default is ``SubdomainCallingFormat()``) + Defines the S3 calling format to use to connect to the static bucket. + +``AWS_S3_SIGNATURE_VERSION`` (optional - boto3 only) + + All AWS regions support v4 of the signing protocol. To use it set this to ``'s3v4'``. It is recommended + to do this for all new projects and required for all regions launched after January 2014. To see + if your region is one of them you can view the `S3 region list`_. + + .. note:: + + The signature versions are not backwards compatible so be careful about url endpoints if making this change + for legacy projects. + +.. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +.. _S3 region list: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + +CloudFront +~~~~~~~~~~ + +If you're using S3 as a CDN (via CloudFront), you'll probably want this storage +to serve those files using that:: + + AWS_S3_CUSTOM_DOMAIN = 'cdn.mydomain.com' + +**NOTE:** Django's `STATIC_URL` `must end in a slash`_ and the `AWS_S3_CUSTOM_DOMAIN` *must not*. It is best to set this variable indepedently of `STATIC_URL`. + +.. _must end in a slash: https://docs.djangoproject.com/en/dev/ref/settings/#static-url + +Keep in mind you'll have to configure CloudFront to use the proper bucket as an +origin manually for this to work. + +If you need to use multiple storages that are served via CloudFront, pass the +`custom_domain` parameter to their constructors. Storage ------- Standard file access options are available, and work as expected:: + >>> from django.core.files.storage import default_storage >>> default_storage.exists('storage_test') False >>> file = default_storage.open('storage_test', 'w') diff --git a/docs/backends/apache_libcloud.rst b/docs/backends/apache_libcloud.rst index 0a1dc1d2c..7ea5ef0bd 100644 --- a/docs/backends/apache_libcloud.rst +++ b/docs/backends/apache_libcloud.rst @@ -6,6 +6,10 @@ It aims to provide a consistent API for dealing with cloud storage (and, more broadly, the many other services provided by cloud providers, such as device provisioning, load balancer configuration, and DNS configuration). +Use pip to install apache-libcloud from PyPI:: + + pip install apache-libcloud + As of v0.10.1, Libcloud supports the following cloud storage providers: * `Amazon S3`_ * `Google Cloud Storage`_ diff --git a/docs/backends/azure.rst b/docs/backends/azure.rst index 6433b17b1..b9fa2734b 100644 --- a/docs/backends/azure.rst +++ b/docs/backends/azure.rst @@ -1,32 +1,39 @@ Azure Storage -=========== +============= A custom storage system for Django using Windows Azure Storage backend. +Before you start configuration, you will need to install the Azure SDK for Python. -Settings -******* +Install the package:: + + pip install azure + +Add to your requirements file:: -``DEFAULT_FILE_STORAGE`` + pip freeze > requirements.txt -This setting sets the path to the Azure storage class:: + +Settings +******** + +To use `AzureStorage` set:: DEFAULT_FILE_STORAGE = 'storages.backends.azure_storage.AzureStorage' +The following settings are available: ``AZURE_ACCOUNT_NAME`` -This setting is the Windows Azure Storage Account name, which in many cases is also the first part of the url for instance: http://azure_account_name.blob.core.windows.net/ would mean:: - - AZURE_ACCOUNT_NAME = "azure_account_name" + This setting is the Windows Azure Storage Account name, which in many cases is also the first part of the url for instance: http://azure_account_name.blob.core.windows.net/ would mean:: + + AZURE_ACCOUNT_NAME = "azure_account_name" ``AZURE_ACCOUNT_KEY`` -This is the private key that gives your Django app access to your Windows Azure Account. + This is the private key that gives your Django app access to your Windows Azure Account. ``AZURE_CONTAINER`` -This is where the files uploaded through your Django app will be uploaded. -The container must be already created as the storage system will not attempt to create it. - - + This is where the files uploaded through your Django app will be uploaded. + The container must be already created as the storage system will not attempt to create it. diff --git a/docs/backends/couchdb.rst b/docs/backends/couchdb.rst deleted file mode 100644 index f93761b9d..000000000 --- a/docs/backends/couchdb.rst +++ /dev/null @@ -1,5 +0,0 @@ -CouchDB -======= - -A custom storage system for Django with CouchDB backend. - diff --git a/docs/backends/database.rst b/docs/backends/database.rst deleted file mode 100644 index bc86e1c31..000000000 --- a/docs/backends/database.rst +++ /dev/null @@ -1,59 +0,0 @@ -Database -======== - -Class DatabaseStorage can be used with either FileField or ImageField. It can be used to map filenames to database blobs: so you have to use it with a special additional table created manually. The table should contain a pk-column for filenames (better to use the same type that FileField uses: nvarchar(100)), blob field (image type for example) and size field (bigint). You can't just create blob column in the same table, where you defined FileField, since there is no way to find required row in the save() method. Also size field is required to obtain better perfomance (see size() method). - -So you can use it with different FileFields and even with different "upload_to" variables used. Thus it implements a kind of root filesystem, where you can define dirs using "upload_to" with FileField and store any files in these dirs. - -It uses either settings.DB_FILES_URL or constructor param 'base_url' (see __init__()) to create urls to files. Base url should be mapped to view that provides access to files. To store files in the same table, where FileField is defined you have to define your own field and provide extra argument (e.g. pk) to save(). - -Raw sql is used for all operations. In constructor or in DB_FILES of settings.py () you should specify a dictionary with db_table, fname_column, blob_column, size_column and 'base_url'. For example I just put to the settings.py the following line:: - - DB_FILES = { - 'db_table': 'FILES', - 'fname_column': 'FILE_NAME', - 'blob_column': 'BLOB', - 'size_column': 'SIZE', - 'base_url': 'http://localhost/dbfiles/' - } - -And use it with ImageField as following:: - - player_photo = models.ImageField(upload_to="player_photos", storage=DatabaseStorage() ) - -DatabaseStorage class uses your settings.py file to perform custom connection to your database. - -The reason to use custom connection: http://code.djangoproject.com/ticket/5135 Connection string looks like:: - - cnxn = pyodbc.connect('DRIVER={SQL Server};SERVER=localhost;DATABASE=testdb;UID=me;PWD=pass') - -It's based on pyodbc module, so can be used with any database supported by pyodbc. I've tested it with MS Sql Express 2005. - -Note: It returns special path, which should be mapped to special view, which returns requested file:: - - def image_view(request, filename): - import os - from django.http import HttpResponse - from django.conf import settings - from django.utils._os import safe_join - from filestorage import DatabaseStorage - from django.core.exceptions import ObjectDoesNotExist - - storage = DatabaseStorage() - - try: - image_file = storage.open(filename, 'rb') - file_content = image_file.read() - except: - filename = 'no_image.gif' - path = safe_join(os.path.abspath(settings.MEDIA_ROOT), filename) - if not os.path.exists(path): - raise ObjectDoesNotExist - no_image = open(path, 'rb') - file_content = no_image.read() - - response = HttpResponse(file_content, mimetype="image/jpeg") - response['Content-Disposition'] = 'inline; filename=%s'%filename - return response - -Note: If filename exist, blob will be overwritten, to change this remove get_available_name(self, name), so Storage.get_available_name(self, name) will be used to generate new filename. diff --git a/docs/backends/dropbox.rst b/docs/backends/dropbox.rst new file mode 100644 index 000000000..6b7aa1fbc --- /dev/null +++ b/docs/backends/dropbox.rst @@ -0,0 +1,27 @@ +DropBox +======= + +A custom storage system for Django using Dropbox Storage backend. + +Before you start configuration, you will need to install `Dropbox SDK for Python`_. + + +Install the package:: + + pip install dropbox + +Settings +-------- + +To use DropBoxStorage set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.dropbox.DropBoxStorage' + +``DROPBOX_OAUTH2_TOKEN`` + Your DropBox token, if you haven't follow this `guide step`_. + +``DROPBOX_ROOT_PATH`` + Allow to jail your storage to a defined directory. + +.. _`guide step`: https://www.dropbox.com/developers/documentation/python#tutorial +.. _`Dropbox SDK for Python`: https://www.dropbox.com/developers/documentation/python#tutorial diff --git a/docs/backends/ftp.rst b/docs/backends/ftp.rst index 6b894f0db..64320e54c 100644 --- a/docs/backends/ftp.rst +++ b/docs/backends/ftp.rst @@ -5,3 +5,11 @@ FTP This implementation was done preliminary for upload files in admin to remote FTP location and read them back on site by HTTP. It was tested mostly in this configuration, so read/write using FTPStorageFile class may break. +Settings +-------- + +``LOCATION`` + URL of the server that hold the files. Example ``'ftp://:@:'`` + +``BASE_URL`` + URL that serves the files stored at this location. Defaults to the value of your ``MEDIA_URL`` setting. diff --git a/docs/backends/gcloud.rst b/docs/backends/gcloud.rst new file mode 100644 index 000000000..4e126acf8 --- /dev/null +++ b/docs/backends/gcloud.rst @@ -0,0 +1,194 @@ +Google Cloud Storage +==================== + +Usage +***** + +This backend provides support for Google Cloud Storage using the +library provided by Google. + +It's possible to access Google Cloud Storage in S3 compatibility mode +using other libraries in django-storages, but this is the only library +offering native support. + +By default this library will use the credentials associated with the +current instance for authentication. To override this, see the +settings below. + + +Settings +-------- + +To use gcloud set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' + +``GS_BUCKET_NAME`` + +Your Google Storage bucket name, as a string. + +``GS_PROJECT_ID`` (optional) + +Your Google Cloud project ID. If unset, falls back to the default +inferred from the environment. + +``GS_CREDENTIALS`` (optional) + +The OAuth 2 credentials to use for the connection. If unset, falls +back to the default inferred from the environment. + +``GS_AUTO_CREATE_BUCKET`` (optional, default is ``False``) + +If True, attempt to create the bucket if it does not exist. + +``GS_AUTO_CREATE_ACL`` (optional, default is ``projectPrivate``) + +ACL used when creating a new bucket, from the +`list of predefined ACLs `_. +(A "JSON API" ACL is preferred but an "XML API/gsutil" ACL will be +translated.) + +Note that the ACL you select must still give the service account +running the gcloud backend to have OWNER permission on the bucket. If +you're using the default service account, this means you're restricted +to the ``projectPrivate`` ACL. + +``GS_FILE_CHARSET`` (optional) + +Allows overriding the character set used in filenames. + +``GS_FILE_OVERWRITE`` (optional: default is ``True``) + +By default files with the same name will overwrite each other. Set this to ``False`` to have extra characters appended. + +``GS_MAX_MEMORY_SIZE`` (optional) + +The maximum amount of memory a returned file can take up before being +rolled over into a temporary file on disk. Default is 0: Do not roll over. + +Fields +------ + +Once you're done, default_storage will be Google Cloud Storage:: + + >>> from django.core.files.storage import default_storage + >>> print default_storage.__class__ + + +This way, if you define a new FileField, it will use the Google Cloud Storage:: + + >>> from django.db import models + >>> class Resume(models.Model): + ... pdf = models.FileField(upload_to='pdfs') + ... photos = models.ImageField(upload_to='photos') + ... + >>> resume = Resume() + >>> print resume.pdf.storage + + +Storage +------- + +Standard file access options are available, and work as expected:: + + >>> default_storage.exists('storage_test') + False + >>> file = default_storage.open('storage_test', 'w') + >>> file.write('storage contents') + >>> file.close() + + >>> default_storage.exists('storage_test') + True + >>> file = default_storage.open('storage_test', 'r') + >>> file.read() + 'storage contents' + >>> file.close() + + >>> default_storage.delete('storage_test') + >>> default_storage.exists('storage_test') + False + +Model +----- + +An object without a file has limited functionality:: + + >>> obj1 = MyStorage() + >>> obj1.normal + + >>> obj1.normal.size + Traceback (most recent call last): + ... + ValueError: The 'normal' attribute has no file associated with it. + +Saving a file enables full functionality:: + + >>> obj1.normal.save('django_test.txt', ContentFile('content')) + >>> obj1.normal + + >>> obj1.normal.size + 7 + >>> obj1.normal.read() + 'content' + +Files can be read in a little at a time, if necessary:: + + >>> obj1.normal.open() + >>> obj1.normal.read(3) + 'con' + >>> obj1.normal.read() + 'tent' + >>> '-'.join(obj1.normal.chunks(chunk_size=2)) + 'co-nt-en-t' + +Save another file with the same name:: + + >>> obj2 = MyStorage() + >>> obj2.normal.save('django_test.txt', ContentFile('more content')) + >>> obj2.normal + + >>> obj2.normal.size + 12 + +Push the objects into the cache to make sure they pickle properly:: + + >>> cache.set('obj1', obj1) + >>> cache.set('obj2', obj2) + >>> cache.get('obj2').normal + + +Deleting an object deletes the file it uses, if there are no other objects still using that file:: + + >>> obj2.delete() + >>> obj2.normal.save('django_test.txt', ContentFile('more content')) + >>> obj2.normal + + +Default values allow an object to access a single file:: + + >>> obj3 = MyStorage.objects.create() + >>> obj3.default + + >>> obj3.default.read() + 'default content' + +But it shouldn't be deleted, even if there are no more objects using it:: + + >>> obj3.delete() + >>> obj3 = MyStorage() + >>> obj3.default.read() + 'default content' + +Verify the fix for #5655, making sure the directory is only determined once:: + + >>> obj4 = MyStorage() + >>> obj4.random.save('random_file', ContentFile('random content')) + >>> obj4.random + + +Clean up the temporary files:: + + >>> obj1.normal.delete() + >>> obj2.normal.delete() + >>> obj3.default.delete() + >>> obj4.random.delete() diff --git a/docs/backends/image.rst b/docs/backends/image.rst deleted file mode 100644 index b03e1d4ec..000000000 --- a/docs/backends/image.rst +++ /dev/null @@ -1,5 +0,0 @@ -Image -===== - -A custom FileSystemStorage made for normalizing extensions. It lets PIL look at the file to determine the format and append an always lower-case extension based on the results. - diff --git a/docs/backends/mogilefs.rst b/docs/backends/mogilefs.rst deleted file mode 100644 index a330fdda0..000000000 --- a/docs/backends/mogilefs.rst +++ /dev/null @@ -1,55 +0,0 @@ -MogileFS -======== - -This storage allows you to use MogileFS, it comes from this blog post. - -The MogileFS storage backend is fairly simple: it uses URLs (or, rather, parts of URLs) as keys into the mogile database. When the user requests a file stored by mogile (say, an avatar), the URL gets passed to a view which, using a client to the mogile tracker, retrieves the "correct" path (the path that points to the actual file data). The view will then either return the path(s) to perlbal to reproxy, or, if you're not using perlbal to reproxy (which you should), it serves the data of the file directly from django. - -* ``MOGILEFS_DOMAIN``: The mogile domain that files should read from/written to, e.g "production" -* ``MOGILEFS_TRACKERS``: A list of trackers to connect to, e.g. ["foo.sample.com:7001", "bar.sample.com:7001"] -* ``MOGILEFS_MEDIA_URL`` (optional): The prefix for URLs that point to mogile files. This is used in a similar way to ``MEDIA_URL``, e.g. "/mogilefs/" -* ``SERVE_WITH_PERLBAL``: Boolean that, when True, will pass the paths back in the response in the ``X-REPROXY-URL`` header. If False, django will serve all mogile media files itself (bad idea for production, but useful if you're testing on a setup that doesn't have perlbal running) -* ``DEFAULT_FILE_STORAGE``: This is the class that's used for the backend. You'll want to set this to ``project.app.storages.MogileFSStorage`` (or wherever you've installed the backend) - -Getting files into mogile -************************* - -The great thing about file backends is that we just need to specify the backend in the model file and everything is taken care for us - all the default save() methods work correctly. - -For Fluther, we have two main media types we use mogile for: avatars and thumbnails. Mogile defines "classes" that dictate how each type of file is replicated - so you can make sure you have 3 copies of the original avatar but only 1 of the thumbnail. - -In order for classes to behave nicely with the backend framework, we've had to do a little tomfoolery. (This is something that may change in future versions of the filestorage framework). - -Here's what the models.py file looks like for the avatars:: - - from django.core.filestorage import storage - - # TODO: Find a better way to deal with classes. Maybe a generator? - class AvatarStorage(storage.__class__): - mogile_class = 'avatar' - - class ThumbnailStorage(storage.__class__): - mogile_class = 'thumb' - - class Avatar(models.Model): - user = models.ForeignKey(User, null=True, blank=True) - image = models.ImageField(storage=AvatarStorage()) - thumb = models.ImageField(storage=ThumbnailStorage()) - -Each of the custom storage classes defines a class attribute which gets passed to the mogile backend behind the scenes. If you don't want to worry about mogile classes, don't need to define a custom storage engine or specify it in the field - the default should work just fine. - -Serving files from mogile -************************* - -Now, all we need to do is plug in the view that serves up mogile data. - -Here's what we use:: - - urlpatterns += patterns(", - (r'^%s(?P.*)' % settings.MOGILEFS_MEDIA_URL[1:], - 'MogileFSStorage.serve_mogilefs_file') - ) - -Any url beginning with the value of ``MOGILEFS_MEDIA_URL`` will get passed to our view. Since ``MOGILEFS_MEDIA_URL`` requires a leading slash (like ``MEDIA_URL``), we strip that off and pass the rest of the url over to the view. - -That's it! Happy mogiling! diff --git a/docs/backends/overwrite.rst b/docs/backends/overwrite.rst deleted file mode 100644 index 66aa87538..000000000 --- a/docs/backends/overwrite.rst +++ /dev/null @@ -1,5 +0,0 @@ -Overwrite -========= - -This is a simple implementation overwrite of the FileSystemStorage. It removes the addition of an '_' to the filename if the file already exists in the storage system. I needed a model in the admin area to act exactly like a file system (overwriting the file if it already exists). - diff --git a/docs/backends/sftp.rst b/docs/backends/sftp.rst index 6f19f9f56..9d18e074d 100644 --- a/docs/backends/sftp.rst +++ b/docs/backends/sftp.rst @@ -1,5 +1,59 @@ SFTP ==== -Take a look at the top of the backend's file for the documentation. +Settings +-------- +``SFTP_STORAGE_HOST`` + The hostname where you want the files to be saved. + +``SFTP_STORAGE_ROOT`` + The root directory on the remote host into which files should be placed. + Should work the same way that ``STATIC_ROOT`` works for local files. Must + include a trailing slash. + +``SFTP_STORAGE_PARAMS`` (optional) + A dictionary containing connection parameters to be passed as keyword + arguments to ``paramiko.SSHClient().connect()`` (do not include hostname here). + See `paramiko SSHClient.connect() documentation`_ for details + +``SFTP_STORAGE_INTERACTIVE`` (optional) + A boolean indicating whether to prompt for a password if the connection cannot + be made using keys, and there is not already a password in + ``SFTP_STORAGE_PARAMS``. You can set this to ``True`` to enable interactive + login when running ``manage.py collectstatic``, for example. + + .. warning:: + + DO NOT set SFTP_STORAGE_INTERACTIVE to True if you are using this storage + for files being uploaded to your site by users, because you'll have no way + to enter the password when they submit the form.. + +``SFTP_STORAGE_FILE_MODE`` (optional) + A bitmask for setting permissions on newly-created files. See + `Python os.chmod documentation`_ for acceptable values. + +``SFTP_STORAGE_DIR_MODE`` (optional) + A bitmask for setting permissions on newly-created directories. See + `Python os.chmod documentation`_ for acceptable values. + + .. note:: + + Hint: if you start the mode number with a 0 you can express it in octal + just like you would when doing "chmod 775 myfile" from bash. + +``SFTP_STORAGE_UID`` (optional) + UID of the account that should be set as owner of the files on the remote + host. You may have to be root to set this. + +``SFTP_STORAGE_GID`` (optional) + GID of the group that should be set on the files on the remote host. You have + to be a member of the group to set this. + +``SFTP_KNOWN_HOST_FILE`` (optional) + Absolute path of know host file, if it isn't set ``"~/.ssh/known_hosts"`` will be used. + + +.. _`paramiko SSHClient.connect() documentation`: http://docs.paramiko.org/en/latest/api/client.html#paramiko.client.SSHClient.connect + +.. _`Python os.chmod documentation`: http://docs.python.org/library/os.html#os.chmod diff --git a/docs/backends/symlinkcopy.rst b/docs/backends/symlinkcopy.rst deleted file mode 100644 index be4abe18e..000000000 --- a/docs/backends/symlinkcopy.rst +++ /dev/null @@ -1,6 +0,0 @@ -Symlink or copy -=============== - -Stores symlinks to files instead of actual files whenever possible - -When a file that's being saved is currently stored in the symlink_within directory, then symlink the file. Otherwise, copy the file. diff --git a/docs/conf.py b/docs/conf.py index c1052fa73..85cfc7942 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,7 +42,7 @@ # General information about the project. project = u'django-storages' -copyright = u'2011-2013, David Larlet, et. al.' +copyright = u'2011-2017, David Larlet, et. al.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -223,7 +223,7 @@ epub_title = u'django-storages' epub_author = u'David Larlet, et. al.' epub_publisher = u'David Larlet, et. al.' -epub_copyright = u'2011-2013, David Larlet, et. al.' +epub_copyright = u'2011-2017, David Larlet, et. al.' # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/requirements-tests.txt b/requirements-tests.txt index 307b874a3..229a8a288 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,9 @@ -Django>=1.7 -pytest-cov==2.2.1 +boto3>=1.2.3 boto>=2.32.0 -dropbox>=3.24 +dropbox>=8.0.0 +Django>=1.8 +flake8 +google-cloud-storage>=0.22.0 mock +paramiko +pytest-cov>=2.2.1 diff --git a/setup.cfg b/setup.cfg index 7c964b49e..1a0dfbc1b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,17 @@ -[wheel] +[bdist_wheel] universal=1 + +[flake8] +exclude = + .tox, + docs +max-line-length = 119 + +[isort] +combine_as_imports = true +default_section = THIRDPARTY +include_trailing_comma = true +known_first_party = storages +line_length = 79 +multi_line_output = 5 +not_skip = __init__.py diff --git a/setup.py b/setup.py index 6170482b5..876059f6b 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import setup + import storages @@ -11,6 +12,7 @@ def get_requirements_tests(): with open('requirements-tests.txt') as f: return f.readlines() + setup( name='django-storages', version=storages.__version__, @@ -18,13 +20,16 @@ def get_requirements_tests(): author='Josh Schneier', author_email='josh.schneier@gmail.com', license='BSD', - description='Support for many storages (S3, MogileFS, etc) in Django.', + description='Support for many storage backends in Django', long_description=read('README.rst') + '\n\n' + read('CHANGELOG.rst'), url='https://github.com/jschneier/django-storages', classifiers=[ - 'Framework :: Django', 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', + 'Framework :: Django', + 'Framework :: Django :: 1.8', + 'Framework :: Django :: 1.10', + 'Framework :: Django :: 1.11', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', @@ -35,6 +40,7 @@ def get_requirements_tests(): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', ], tests_require=get_requirements_tests(), test_suite='tests', diff --git a/storages/__init__.py b/storages/__init__.py index 8e3c933cd..f3df7f04b 100644 --- a/storages/__init__.py +++ b/storages/__init__.py @@ -1 +1 @@ -__version__ = '1.4.1' +__version__ = '1.6.5' diff --git a/storages/backends/apache_libcloud.py b/storages/backends/apache_libcloud.py index 4dc4b8b00..a2e5dc2e3 100644 --- a/storages/backends/apache_libcloud.py +++ b/storages/backends/apache_libcloud.py @@ -4,13 +4,13 @@ import os from django.conf import settings -from django.core.files.base import File from django.core.exceptions import ImproperlyConfigured -from django.utils.six import string_types +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible +from django.utils.six import BytesIO, string_types from django.utils.six.moves.urllib.parse import urljoin -from storages.compat import BytesIO, deconstructible, Storage - try: from libcloud.storage.providers import get_driver from libcloud.storage.types import ObjectDoesNotExistError, Provider @@ -33,6 +33,9 @@ def __init__(self, provider_name=None, option=None): extra_kwargs = {} if 'region' in self.provider: extra_kwargs['region'] = self.provider['region'] + # Used by the GoogleStorageDriver + if 'project' in self.provider: + extra_kwargs['project'] = self.provider['project'] try: provider_type = self.provider['type'] if isinstance(provider_type, string_types): diff --git a/storages/backends/azure_storage.py b/storages/backends/azure_storage.py index e1e4b5651..ea5d71c3f 100644 --- a/storages/backends/azure_storage.py +++ b/storages/backends/azure_storage.py @@ -1,12 +1,15 @@ -from datetime import datetime -import os.path import mimetypes +import os.path import time +from datetime import datetime from time import mktime -from django.core.files.base import ContentFile from django.core.exceptions import ImproperlyConfigured -from storages.compat import Storage +from django.core.files.base import ContentFile +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible + +from storages.utils import setting try: import azure # noqa @@ -23,13 +26,12 @@ from azure.storage import BlobService from azure import WindowsAzureMissingResourceError as AzureMissingResourceHttpError -from storages.utils import setting - def clean_name(name): return os.path.normpath(name).replace("\\", "/") +@deconstructible class AzureStorage(Storage): account_name = setting("AZURE_ACCOUNT_NAME") account_key = setting("AZURE_ACCOUNT_KEY") diff --git a/storages/backends/couchdb.py b/storages/backends/couchdb.py deleted file mode 100644 index 2bcecd8a2..000000000 --- a/storages/backends/couchdb.py +++ /dev/null @@ -1,133 +0,0 @@ -""" -This is a Custom Storage System for Django with CouchDB backend. -Created by Christian Klein. -(c) Copyright 2009 HUDORA GmbH. All Rights Reserved. -""" -import os - -from django.conf import settings -from django.core.files import File -from django.core.exceptions import ImproperlyConfigured - -from storages.compat import urlparse, BytesIO, Storage - -try: - import couchdb -except ImportError: - raise ImproperlyConfigured("Could not load couchdb dependency.\ - \nSee http://code.google.com/p/couchdb-python/") - -DEFAULT_SERVER= getattr(settings, 'COUCHDB_DEFAULT_SERVER', 'http://couchdb.local:5984') -STORAGE_OPTIONS= getattr(settings, 'COUCHDB_STORAGE_OPTIONS', {}) - - -class CouchDBStorage(Storage): - """ - CouchDBStorage - a Django Storage class for CouchDB. - - The CouchDBStorage can be configured in settings.py, e.g.:: - - COUCHDB_STORAGE_OPTIONS = { - 'server': "http://example.org", - 'database': 'database_name' - } - - Alternatively, the configuration can be passed as a dictionary. - """ - def __init__(self, **kwargs): - kwargs.update(STORAGE_OPTIONS) - self.base_url = kwargs.get('server', DEFAULT_SERVER) - server = couchdb.client.Server(self.base_url) - self.db = server[kwargs.get('database')] - - def _put_file(self, name, content): - self.db[name] = {'size': len(content)} - self.db.put_attachment(self.db[name], content, filename='content') - return name - - def get_document(self, name): - return self.db.get(name) - - def _open(self, name, mode='rb'): - couchdb_file = CouchDBFile(name, self, mode=mode) - return couchdb_file - - def _save(self, name, content): - content.open() - if hasattr(content, 'chunks'): - content_str = ''.join(chunk for chunk in content.chunks()) - else: - content_str = content.read() - name = name.replace('/', '-') - return self._put_file(name, content_str) - - def exists(self, name): - return name in self.db - - def size(self, name): - doc = self.get_document(name) - if doc: - return doc['size'] - return 0 - - def url(self, name): - return urlparse.urljoin(self.base_url, - os.path.join(urlparse.quote_plus(self.db.name), - urlparse.quote_plus(name), - 'content')) - - def delete(self, name): - try: - del self.db[name] - except couchdb.client.ResourceNotFound: - raise IOError("File not found: %s" % name) - - #def listdir(self, name): - # _all_docs? - # pass - - -class CouchDBFile(File): - """ - CouchDBFile - a Django File-like class for CouchDB documents. - """ - - def __init__(self, name, storage, mode): - self._name = name - self._storage = storage - self._mode = mode - self._is_dirty = False - - try: - self._doc = self._storage.get_document(name) - - tmp, ext = os.path.split(name) - if ext: - filename = "content." + ext - else: - filename = "content" - attachment = self._storage.db.get_attachment(self._doc, filename=filename) - self.file = BytesIO(attachment) - except couchdb.client.ResourceNotFound: - if 'r' in self._mode: - raise ValueError("The file cannot be reopened.") - else: - self.file = BytesIO() - self._is_dirty = True - - @property - def size(self): - return self._doc['size'] - - def write(self, content): - if 'w' not in self._mode: - raise AttributeError("File was opened for read-only access.") - self.file = BytesIO(content) - self._is_dirty = True - - def close(self): - if self._is_dirty: - self._storage._put_file(self._name, self.file.getvalue()) - self.file.close() - - diff --git a/storages/backends/database.py b/storages/backends/database.py deleted file mode 100644 index e0057ab16..000000000 --- a/storages/backends/database.py +++ /dev/null @@ -1,132 +0,0 @@ -# DatabaseStorage for django. -# 2009 (c) GameKeeper Gambling Ltd, Ivanov E. - -from django.conf import settings -from django.core.files import File -from django.core.exceptions import ImproperlyConfigured - -from storages.compat import urlparse, BytesIO, Storage - -try: - import pyodbc -except ImportError: - raise ImproperlyConfigured("Could not load pyodbc dependency.\ - \nSee https://github.com/mkleehammer/pyodbc") - -REQUIRED_FIELDS = ('db_table', 'fname_column', 'blob_column', 'size_column', 'base_url') - - -class DatabaseStorage(Storage): - """ - Class DatabaseStorage provides storing files in the database. - """ - - def __init__(self, option=settings.DB_FILES): - """Constructor. - - Constructs object using dictionary either specified in contucotr or -in settings.DB_FILES. - - @param option dictionary with 'db_table', 'fname_column', -'blob_column', 'size_column', 'base_url' keys. - - option['db_table'] - Table to work with. - option['fname_column'] - Column in the 'db_table' containing filenames (filenames can -contain pathes). Values should be the same as where FileField keeps -filenames. - It is used to map filename to blob_column. In sql it's simply -used in where clause. - option['blob_column'] - Blob column (for example 'image' type), created manually in the -'db_table', used to store image. - option['size_column'] - Column to store file size. Used for optimization of size() -method (another way is to open file and get size) - option['base_url'] - Url prefix used with filenames. Should be mapped to the view, -that returns an image as result. - """ - - if not option or not all([field in option for field in REQUIRED_FIELDS]): - raise ValueError("You didn't specify required options") - - self.db_table = option['db_table'] - self.fname_column = option['fname_column'] - self.blob_column = option['blob_column'] - self.size_column = option['size_column'] - self.base_url = option['base_url'] - - #get database settings - self.DATABASE_ODBC_DRIVER = settings.DATABASE_ODBC_DRIVER - self.DATABASE_NAME = settings.DATABASE_NAME - self.DATABASE_USER = settings.DATABASE_USER - self.DATABASE_PASSWORD = settings.DATABASE_PASSWORD - self.DATABASE_HOST = settings.DATABASE_HOST - - self.connection = pyodbc.connect('DRIVER=%s;SERVER=%s;DATABASE=%s;UID=%s;PWD=%s'%(self.DATABASE_ODBC_DRIVER,self.DATABASE_HOST,self.DATABASE_NAME, - self.DATABASE_USER, self.DATABASE_PASSWORD) ) - self.cursor = self.connection.cursor() - - def _open(self, name, mode='rb'): - """Open a file from database. - - @param name filename or relative path to file based on base_url. path should contain only "/", but not "\". Apache sends pathes with "/". - If there is no such file in the db, returs None - """ - - assert mode == 'rb', "You've tried to open binary file without specifying binary mode! You specified: %s"%mode - - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.blob_column,self.db_table,self.fname_column,name) ).fetchone() - if row is None: - return None - inMemFile = BytesIO(row[0]) - inMemFile.name = name - inMemFile.mode = mode - - retFile = File(inMemFile) - return retFile - - def _save(self, name, content): - """Save 'content' as file named 'name'. - - @note '\' in path will be converted to '/'. - """ - - name = name.replace('\\', '/') - binary = pyodbc.Binary(content.read()) - size = len(binary) - - #todo: check result and do something (exception?) if failed. - if self.exists(name): - self.cursor.execute("UPDATE %s SET %s = ?, %s = ? WHERE %s = '%s'"%(self.db_table,self.blob_column,self.size_column,self.fname_column,name), - (binary, size) ) - else: - self.cursor.execute("INSERT INTO %s VALUES(?, ?, ?)"%(self.db_table), (name, binary, size) ) - self.connection.commit() - return name - - def exists(self, name): - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.fname_column,self.db_table,self.fname_column,name)).fetchone() - return row is not None - - def get_available_name(self, name, max_length=None): - return name - - def delete(self, name): - if self.exists(name): - self.cursor.execute("DELETE FROM %s WHERE %s = '%s'"%(self.db_table,self.fname_column,name)) - self.connection.commit() - - def url(self, name): - if self.base_url is None: - raise ValueError("This file is not accessible via a URL.") - return urlparse.urljoin(self.base_url, name).replace('\\', '/') - - def size(self, name): - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.size_column,self.db_table,self.fname_column,name)).fetchone() - if row is None: - return 0 - else: - return int(row[0]) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index dc1958757..bae6deadb 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -6,20 +6,25 @@ # # Add below to settings.py: # DROPBOX_OAUTH2_TOKEN = 'YourOauthToken' +# DROPBOX_ROOT_PATH = '/dir/' from __future__ import absolute_import from datetime import datetime +from shutil import copyfileobj +from tempfile import SpooledTemporaryFile -from django.core.files.base import File from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils._os import safe_join +from django.utils.deconstruct import deconstructible +from dropbox import Dropbox +from dropbox.exceptions import ApiError +from dropbox.files import CommitInfo, UploadSessionCursor -from storages.compat import BytesIO, Storage from storages.utils import setting -from dropbox.client import DropboxClient -from dropbox.rest import ErrorResponse - DATE_FORMAT = '%a, %d %b %Y %X +0000' @@ -28,39 +33,55 @@ class DropBoxStorageException(Exception): class DropBoxFile(File): - def __init__(self, name, storage, mode='rb'): + def __init__(self, name, storage): self.name = name self._storage = storage - def read(self, num_bytes=None): - return self._storage._read(self.name, num_bytes=num_bytes) - - def write(self, content): - self._storage._save(self.name, content) + @property + def file(self): + if not hasattr(self, '_file'): + response = self._storage.client.files_download(self.name) + self._file = SpooledTemporaryFile() + copyfileobj(response, self._file) + self._file.seek(0) + return self._file +@deconstructible class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" - def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')): + CHUNK_SIZE = 4 * 1024 * 1024 + + def __init__(self, oauth2_access_token=None, root_path=None): + oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') + self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") - self.client = DropboxClient(oauth2_access_token) + self.client = Dropbox(oauth2_access_token) + + def _full_path(self, name): + if name == '/': + name = '' + return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): - self.client.file_delete(name) + self.client.files_delete(self._full_path(name)) def exists(self, name): try: - return bool(self.client.metadata(name)) - except ErrorResponse: + return bool(self.client.files_get_metadata(self._full_path(name))) + except ApiError: return False def listdir(self, path): directories, files = [], [] - metadata = self.client.metadata(path) + full_path = self._full_path(path) + metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: + entry['path'] = entry['path'].replace(full_path, '', 1) + entry['path'] = entry['path'].replace('/', '', 1) if entry['is_dir']: directories.append(entry['path']) else: @@ -68,31 +89,53 @@ def listdir(self, path): return directories, files def size(self, name): - metadata = self.client.metadata(name) + metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): - metadata = self.client.metadata(name) + metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): - metadata = self.client.metadata(name) + metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): - media = self.client.media(name) - return media['url'] + media = self.client.files_get_temporary_link(self._full_path(name)) + return media.link def _open(self, name, mode='rb'): - remote_file = DropBoxFile(name, self) + remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): - self.client.put_file(name, content) + content.open() + if content.size <= self.CHUNK_SIZE: + self.client.files_upload(content.read(), self._full_path(name)) + else: + self._chunked_upload(content, self._full_path(name)) + content.close() return name - def _read(self, name, num_bytes=None): - data = self.client.get_file(name) - return data.read(num_bytes) + def _chunked_upload(self, content, dest_path): + upload_session = self.client.files_upload_session_start( + content.read(self.CHUNK_SIZE) + ) + cursor = UploadSessionCursor( + session_id=upload_session.session_id, + offset=content.tell() + ) + commit = CommitInfo(path=dest_path) + + while content.tell() < content.size: + if (content.size - content.tell()) <= self.CHUNK_SIZE: + self.client.files_upload_session_finish( + content.read(self.CHUNK_SIZE), cursor, commit + ) + else: + self.client.files_upload_session_append_v2( + content.read(self.CHUNK_SIZE), cursor + ) + cursor.offset = content.tell() diff --git a/storages/backends/ftp.py b/storages/backends/ftp.py index fd7ae4bb0..0b28280ac 100644 --- a/storages/backends/ftp.py +++ b/storages/backends/ftp.py @@ -14,26 +14,37 @@ # class FTPTest(models.Model): # file = models.FileField(upload_to='a/b/c/', storage=fs) +import ftplib import os from datetime import datetime -import ftplib from django.conf import settings -from django.core.files.base import File from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible +from django.utils.six import BytesIO +from django.utils.six.moves.urllib import parse as urlparse -from storages.compat import urlparse, BytesIO, Storage +from storages.utils import setting class FTPStorageException(Exception): pass +@deconstructible class FTPStorage(Storage): """FTP Storage class for Django pluggable storage system.""" - def __init__(self, location=settings.FTP_STORAGE_LOCATION, - base_url=settings.MEDIA_URL): + def __init__(self, location=None, base_url=None): + location = location or setting('FTP_STORAGE_LOCATION') + if location is None: + raise ImproperlyConfigured("You must set a location at " + "instanciation or at " + " settings.FTP_STORAGE_LOCATION'.") + self.location = location + base_url = base_url or settings.MEDIA_URL self._config = self._decode_location(location) self._base_url = base_url self._connection = None @@ -134,6 +145,7 @@ def _read(self, name): self._connection.retrbinary('RETR ' + os.path.basename(name), memory_file.write) self._connection.cwd(pwd) + memory_file.seek(0) return memory_file except ftplib.all_errors: raise FTPStorageException('Error reading file %s' % name) @@ -184,7 +196,7 @@ def listdir(self, path): self._start_connection() try: dirs, files = self._get_dir_details(path) - return dirs.keys(), files.keys() + return list(dirs.keys()), list(files.keys()) except FTPStorageException: raise @@ -248,12 +260,18 @@ def size(self): self._size = self._storage.size(self.name) return self._size - def read(self, num_bytes=None): + def readlines(self): if not self._is_read: self._storage._start_connection() self.file = self._storage._read(self.name) self._is_read = True + return self.file.readlines() + def read(self, num_bytes=None): + if not self._is_read: + self._storage._start_connection() + self.file = self._storage._read(self.name) + self._is_read = True return self.file.read(num_bytes) def write(self, content): diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py new file mode 100644 index 000000000..6b433c602 --- /dev/null +++ b/storages/backends/gcloud.py @@ -0,0 +1,236 @@ +import mimetypes +from tempfile import SpooledTemporaryFile + +from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils import timezone +from django.utils.deconstruct import deconstructible +from django.utils.encoding import force_bytes, smart_str + +from storages.utils import clean_name, safe_join, setting + +try: + from google.cloud.storage.client import Client + from google.cloud.storage.blob import Blob + from google.cloud.exceptions import NotFound +except ImportError: + raise ImproperlyConfigured("Could not load Google Cloud Storage bindings.\n" + "See https://github.com/GoogleCloudPlatform/gcloud-python") + + +class GoogleCloudFile(File): + def __init__(self, name, mode, storage): + self.name = name + self.mime_type = mimetypes.guess_type(name)[0] + self._mode = mode + self._storage = storage + self.blob = storage.bucket.get_blob(name) + if not self.blob and 'w' in mode: + self.blob = Blob(self.name, storage.bucket) + self._file = None + self._is_dirty = False + + @property + def size(self): + return self.blob.size + + def _get_file(self): + if self._file is None: + self._file = SpooledTemporaryFile( + max_size=self._storage.max_memory_size, + suffix=".GSStorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR", None) + ) + if 'r' in self._mode: + self._is_dirty = False + self.blob.download_to_file(self._file) + self._file.seek(0) + return self._file + + def _set_file(self, value): + self._file = value + + file = property(_get_file, _set_file) + + def read(self, num_bytes=None): + if 'r' not in self._mode: + raise AttributeError("File was not opened in read mode.") + + if num_bytes is None: + num_bytes = -1 + + return super(GoogleCloudFile, self).read(num_bytes) + + def write(self, content): + if 'w' not in self._mode: + raise AttributeError("File was not opened in write mode.") + self._is_dirty = True + return super(GoogleCloudFile, self).write(force_bytes(content)) + + def close(self): + if self._file is not None: + if self._is_dirty: + self.file.seek(0) + self.blob.upload_from_file(self.file, content_type=self.mime_type) + self._file.close() + self._file = None + + +@deconstructible +class GoogleCloudStorage(Storage): + project_id = setting('GS_PROJECT_ID', None) + credentials = setting('GS_CREDENTIALS', None) + bucket_name = setting('GS_BUCKET_NAME', None) + auto_create_bucket = setting('GS_AUTO_CREATE_BUCKET', False) + auto_create_acl = setting('GS_AUTO_CREATE_ACL', 'projectPrivate') + file_name_charset = setting('GS_FILE_NAME_CHARSET', 'utf-8') + file_overwrite = setting('GS_FILE_OVERWRITE', True) + # The max amount of memory a returned file can take up before being + # rolled over into a temporary file on disk. Default is 0: Do not roll over. + max_memory_size = setting('GS_MAX_MEMORY_SIZE', 0) + + def __init__(self, **settings): + # check if some of the settings we've provided as class attributes + # need to be overwritten with values passed in here + for name, value in settings.items(): + if hasattr(self, name): + setattr(self, name, value) + + self._bucket = None + self._client = None + + @property + def client(self): + if self._client is None: + self._client = Client( + project=self.project_id, + credentials=self.credentials + ) + return self._client + + @property + def bucket(self): + if self._bucket is None: + self._bucket = self._get_or_create_bucket(self.bucket_name) + return self._bucket + + def _get_or_create_bucket(self, name): + """ + Retrieves a bucket if it exists, otherwise creates it. + """ + try: + return self.client.get_bucket(name) + except NotFound: + if self.auto_create_bucket: + bucket = self.client.create_bucket(name) + bucket.acl.save_predefined(self.auto_create_acl) + return bucket + raise ImproperlyConfigured("Bucket %s does not exist. Buckets " + "can be automatically created by " + "setting GS_AUTO_CREATE_BUCKET to " + "``True``." % name) + + def _normalize_name(self, name): + """ + Normalizes the name so that paths like /path/to/ignored/../something.txt + and ./file.txt work. Note that clean_name adds ./ to some paths so + they need to be fixed here. + """ + return safe_join('', name) + + def _encode_name(self, name): + return smart_str(name, encoding=self.file_name_charset) + + def _open(self, name, mode='rb'): + name = self._normalize_name(clean_name(name)) + file_object = GoogleCloudFile(name, mode, self) + if not file_object.blob: + raise IOError(u'File does not exist: %s' % name) + return file_object + + def _save(self, name, content): + cleaned_name = clean_name(name) + name = self._normalize_name(cleaned_name) + + content.name = cleaned_name + encoded_name = self._encode_name(name) + file = GoogleCloudFile(encoded_name, 'rw', self) + file.blob.upload_from_file(content, size=content.size, + content_type=file.mime_type) + return cleaned_name + + def delete(self, name): + name = self._normalize_name(clean_name(name)) + self.bucket.delete_blob(self._encode_name(name)) + + def exists(self, name): + if not name: # root element aka the bucket + try: + self.bucket + return True + except ImproperlyConfigured: + return False + + name = self._normalize_name(clean_name(name)) + return bool(self.bucket.get_blob(self._encode_name(name))) + + def listdir(self, name): + name = self._normalize_name(clean_name(name)) + # for the bucket.list and logic below name needs to end in / + # But for the root path "" we leave it as an empty string + if name and not name.endswith('/'): + name += '/' + + files_list = list(self.bucket.list_blobs(prefix=self._encode_name(name))) + files = [] + dirs = set() + + base_parts = name.split("/")[:-1] + for item in files_list: + parts = item.name.split("/") + parts = parts[len(base_parts):] + if len(parts) == 1 and parts[0]: + # File + files.append(parts[0]) + elif len(parts) > 1 and parts[0]: + # Directory + dirs.add(parts[0]) + return list(dirs), files + + def _get_blob(self, name): + # Wrap google.cloud.storage's blob to raise if the file doesn't exist + blob = self.bucket.get_blob(name) + + if blob is None: + raise NotFound(u'File does not exist: {}'.format(name)) + + return blob + + def size(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return blob.size + + def modified_time(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return timezone.make_naive(blob.updated) + + def get_modified_time(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + updated = blob.updated + return updated if setting('USE_TZ') else timezone.make_naive(updated) + + def url(self, name): + # Preserve the trailing slash after normalizing the path. + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return blob.public_url + + def get_available_name(self, name, max_length=None): + if self.file_overwrite: + name = clean_name(name) + return name + return super(GoogleCloudStorage, self).get_available_name(name, max_length) diff --git a/storages/backends/gs.py b/storages/backends/gs.py index 2ee2ccf79..38256d002 100644 --- a/storages/backends/gs.py +++ b/storages/backends/gs.py @@ -1,8 +1,11 @@ +import warnings + from django.core.exceptions import ImproperlyConfigured +from django.utils.deconstruct import deconstructible +from django.utils.six import BytesIO from storages.backends.s3boto import S3BotoStorage, S3BotoStorageFile from storages.utils import setting -from storages.compat import BytesIO try: from boto.gs.connection import GSConnection, SubdomainCallingFormat @@ -13,6 +16,17 @@ "See https://github.com/boto/boto") +warnings.warn("DEPRECATION NOTICE: This backend is deprecated in favour of the " + "\"gcloud\" backend. This backend uses Google Cloud Storage's XML " + "Interoperable API which uses keyed-hash message authentication code " + "(a.k.a. developer keys) that are linked to your Google account. The " + "interoperable API is really meant for migration to Google Cloud " + "Storage. The biggest problem with the developer keys is security and " + "privacy. Developer keys should not be shared with anyone as they can " + "be used to gain access to other Google Cloud Storage buckets linked " + "to your Google account.", DeprecationWarning) + + class GSBotoStorageFile(S3BotoStorageFile): def write(self, content): @@ -30,6 +44,7 @@ def close(self): self.key.close() +@deconstructible class GSBotoStorage(S3BotoStorage): connection_class = GSConnection connection_response_error = GSResponseError @@ -65,6 +80,11 @@ class GSBotoStorage(S3BotoStorage): url_protocol = setting('GS_URL_PROTOCOL', 'http:') host = setting('GS_HOST', GSConnection.DefaultHost) + def _get_connection_kwargs(self): + kwargs = super(GSBotoStorage, self)._get_connection_kwargs() + del kwargs['security_token'] + return kwargs + def _save_content(self, key, content, headers): # only pass backwards incompatible arguments if they vary from the default options = {} @@ -84,7 +104,7 @@ def _get_or_create_bucket(self, name): storage_class = 'STANDARD' try: return self.connection.get_bucket(name, - validate=self.auto_create_bucket) + validate=self.auto_create_bucket) except self.connection_response_error: if self.auto_create_bucket: bucket = self.connection.create_bucket(name, storage_class=storage_class) diff --git a/storages/backends/hashpath.py b/storages/backends/hashpath.py deleted file mode 100644 index 44343c9ef..000000000 --- a/storages/backends/hashpath.py +++ /dev/null @@ -1,42 +0,0 @@ -import os, hashlib, errno - -from django.utils.encoding import force_text, force_bytes -from storages.compat import FileSystemStorage - - -class HashPathStorage(FileSystemStorage): - """ - Creates a hash from the uploaded file to build the path. - """ - - def save(self, name, content, max_length=None): - # Get the content name if name is not given - if name is None: - name = content.name - - # Get the SHA1 hash of the uploaded file - sha1 = hashlib.sha1() - for chunk in content.chunks(): - sha1.update(force_bytes(chunk)) - sha1sum = sha1.hexdigest() - - # Build the new path and split it into directory and filename - name = os.path.join(os.path.split(name)[0], sha1sum[:1], sha1sum[1:2], sha1sum) - dir_name, file_name = os.path.split(name) - - # Return the name if the file is already there - if self.exists(name): - return name - - # Try to create the directory relative to location specified in __init__ - try: - os.makedirs(os.path.join(self.location, dir_name)) - except OSError as e: - if e.errno is not errno.EEXIST: - raise e - - # Save the file - name = self._save(name, content) - - # Store filenames with forward slashes, even on Windows - return force_text(name.replace('\\', '/')) diff --git a/storages/backends/image.py b/storages/backends/image.py deleted file mode 100644 index 637ae8b6b..000000000 --- a/storages/backends/image.py +++ /dev/null @@ -1,55 +0,0 @@ - -import os - -from django.core.exceptions import ImproperlyConfigured -from storages.compat import FileSystemStorage - -try: - from PIL import ImageFile as PILImageFile -except ImportError: - raise ImproperlyConfigured("Could not load PIL dependency.\ - \nSee http://www.pythonware.com/products/pil/") - - -class ImageStorage(FileSystemStorage): - """ - A FileSystemStorage which normalizes extensions for images. - - Comes from http://www.djangosnippets.org/snippets/965/ - """ - - def find_extension(self, format): - """Normalizes PIL-returned format into a standard, lowercase extension.""" - format = format.lower() - - if format == 'jpeg': - format = 'jpg' - - return format - - def save(self, name, content, max_length=None): - dirname = os.path.dirname(name) - basename = os.path.basename(name) - - # Use PIL to determine filetype - - p = PILImageFile.Parser() - while 1: - data = content.read(1024) - if not data: - break - p.feed(data) - if p.image: - im = p.image - break - - extension = self.find_extension(im.format) - - # Does the basename already have an extension? If so, replace it. - # bare as in without extension - bare_basename, _ = os.path.splitext(basename) - basename = bare_basename + '.' + extension - - name = os.path.join(dirname, basename) - return super(ImageStorage, self).save(name, content) - diff --git a/storages/backends/mogile.py b/storages/backends/mogile.py deleted file mode 100644 index 5a31f663a..000000000 --- a/storages/backends/mogile.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import print_function - -import mimetypes - -from django.conf import settings -from django.core.cache import cache -from django.utils.text import force_text -from django.http import HttpResponse, HttpResponseNotFound -from django.core.exceptions import ImproperlyConfigured - -from storages.compat import urlparse, BytesIO, Storage - -try: - import mogilefs -except ImportError: - raise ImproperlyConfigured("Could not load mogilefs dependency.\ - \nSee http://mogilefs.pbworks.com/Client-Libraries") - - -class MogileFSStorage(Storage): - """MogileFS filesystem storage""" - def __init__(self, base_url=settings.MEDIA_URL): - - # the MOGILEFS_MEDIA_URL overrides MEDIA_URL - if hasattr(settings, 'MOGILEFS_MEDIA_URL'): - self.base_url = settings.MOGILEFS_MEDIA_URL - else: - self.base_url = base_url - - for var in ('MOGILEFS_TRACKERS', 'MOGILEFS_DOMAIN',): - if not hasattr(settings, var): - raise ImproperlyConfigured("You must define %s to use the MogileFS backend." % var) - - self.trackers = settings.MOGILEFS_TRACKERS - self.domain = settings.MOGILEFS_DOMAIN - self.client = mogilefs.Client(self.domain, self.trackers) - - def get_mogile_paths(self, filename): - return self.client.get_paths(filename) - - # The following methods define the Backend API - - def filesize(self, filename): - raise NotImplemented - #return os.path.getsize(self._get_absolute_path(filename)) - - def path(self, filename): - paths = self.get_mogile_paths(filename) - if paths: - return self.get_mogile_paths(filename)[0] - else: - return None - - def url(self, filename): - return urlparse.urljoin(self.base_url, filename).replace('\\', '/') - - def open(self, filename, mode='rb'): - raise NotImplemented - #return open(self._get_absolute_path(filename), mode) - - def exists(self, filename): - return filename in self.client - - def save(self, filename, raw_contents, max_length=None): - filename = self.get_available_name(filename, max_length) - - if not hasattr(self, 'mogile_class'): - self.mogile_class = None - - # Write the file to mogile - success = self.client.send_file(filename, BytesIO(raw_contents), self.mogile_class) - if success: - print("Wrote file to key %s, %s@%s" % (filename, self.domain, self.trackers[0])) - else: - print("FAILURE writing file %s" % (filename)) - - return force_text(filename.replace('\\', '/')) - - def delete(self, filename): - self.client.delete(filename) - - -def serve_mogilefs_file(request, key=None): - """ - Called when a user requests an image. - Either reproxy the path to perlbal, or serve the image outright - """ - # not the best way to do this, since we create a client each time - mimetype = mimetypes.guess_type(key)[0] or "application/x-octet-stream" - client = mogilefs.Client(settings.MOGILEFS_DOMAIN, settings.MOGILEFS_TRACKERS) - if hasattr(settings, "SERVE_WITH_PERLBAL") and settings.SERVE_WITH_PERLBAL: - # we're reproxying with perlbal - - # check the path cache - - path = cache.get(key) - - if not path: - path = client.get_paths(key) - cache.set(key, path, 60) - - if path: - response = HttpResponse(content_type=mimetype) - response['X-REPROXY-URL'] = path[0] - else: - response = HttpResponseNotFound() - - else: - # we don't have perlbal, let's just serve the image via django - file_data = client[key] - if file_data: - response = HttpResponse(file_data, mimetype=mimetype) - else: - response = HttpResponseNotFound() - - return response diff --git a/storages/backends/overwrite.py b/storages/backends/overwrite.py deleted file mode 100644 index 84969bddd..000000000 --- a/storages/backends/overwrite.py +++ /dev/null @@ -1,19 +0,0 @@ -from storages.compat import FileSystemStorage - - -class OverwriteStorage(FileSystemStorage): - """ - Comes from http://www.djangosnippets.org/snippets/976/ - (even if it already exists in S3Storage for ages) - - See also Django #4339, which might add this functionality to core. - """ - - def get_available_name(self, name, max_length=None): - """ - Returns a filename that's free on the target storage system, and - available for new content to be written to. - """ - if self.exists(name): - self.delete(name) - return name diff --git a/storages/backends/s3boto.py b/storages/backends/s3boto.py index 432397e74..1015dd957 100644 --- a/storages/backends/s3boto.py +++ b/storages/backends/s3boto.py @@ -1,18 +1,26 @@ +import mimetypes import os -import posixpath import re -import mimetypes +import urlparse from datetime import datetime from gzip import GzipFile from tempfile import SpooledTemporaryFile -from django.core.files.base import File from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation -from django.utils.encoding import force_text, smart_str, filepath_to_uri, force_bytes +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils import timezone as tz +from django.utils.deconstruct import deconstructible +from django.utils.encoding import ( + filepath_to_uri, force_bytes, force_text, smart_str, +) +from django.utils.six import BytesIO + +from storages.utils import clean_name, safe_join, setting try: from boto import __version__ as boto_version - from boto.s3.connection import S3Connection, SubdomainCallingFormat + from boto.s3.connection import S3Connection, SubdomainCallingFormat, Location from boto.exception import S3ResponseError from boto.s3.key import Key as S3Key from boto.utils import parse_ts, ISO8601 @@ -20,8 +28,6 @@ raise ImproperlyConfigured("Could not load Boto's S3 bindings.\n" "See https://github.com/boto/boto") -from storages.utils import setting -from storages.compat import urlparse, BytesIO, deconstructible, Storage boto_version_info = tuple([int(i) for i in boto_version.split('-')[0].split('.')]) @@ -112,8 +118,8 @@ def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, - suffix=".S3BotoStorageFile", - dir=setting("FILE_UPLOAD_TEMP_DIR", None) + suffix='.S3BotoStorageFile', + dir=setting('FILE_UPLOAD_TEMP_DIR', None) ) if 'r' in self._mode: self._is_dirty = False @@ -130,12 +136,12 @@ def _set_file(self, value): def read(self, *args, **kwargs): if 'r' not in self._mode: - raise AttributeError("File was not opened in read mode.") + raise AttributeError('File was not opened in read mode.') return super(S3BotoStorageFile, self).read(*args, **kwargs) def write(self, content, *args, **kwargs): if 'w' not in self._mode: - raise AttributeError("File was not opened in write mode.") + raise AttributeError('File was not opened in write mode.') self._is_dirty = True if self._multipart is None: provider = self.key.bucket.connection.provider @@ -164,9 +170,6 @@ def _buffer_file_size(self): return length def _flush_write_buffer(self): - """ - Flushes the write buffer. - """ if self._buffer_file_size: self._write_counter += 1 self.file.seek(0) @@ -179,7 +182,7 @@ def close(self): self._flush_write_buffer() self._multipart.complete_upload() else: - if not self._multipart is None: + if self._multipart is not None: self._multipart.cancel_upload() self.key.close() if self._file is not None: @@ -204,6 +207,7 @@ class S3BotoStorage(Storage): # used for looking up the access and secret key from env vars access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] + security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) @@ -218,6 +222,7 @@ class S3BotoStorage(Storage): querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600) reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False) location = setting('AWS_LOCATION', '') + origin = setting('AWS_ORIGIN', Location.DEFAULT) encryption = setting('AWS_S3_ENCRYPTION', False) custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') calling_format = setting('AWS_S3_CALLING_FORMAT', SubdomainCallingFormat()) @@ -266,25 +271,36 @@ def __init__(self, acl=None, bucket=None, **settings): self._entries = {} self._bucket = None self._connection = None + self._loaded_meta = False + self.security_token = None if not self.access_key and not self.secret_key: self.access_key, self.secret_key = self._get_access_keys() + self.security_token = self._get_security_token() @property def connection(self): if self._connection is None: + kwargs = self._get_connection_kwargs() + self._connection = self.connection_class( self.access_key, self.secret_key, - is_secure=self.use_ssl, - calling_format=self.calling_format, - host=self.host, - port=self.port, - proxy=self.proxy, - proxy_port=self.proxy_port + **kwargs ) return self._connection + def _get_connection_kwargs(self): + return dict( + security_token=self.security_token, + is_secure=self.use_ssl, + calling_format=self.calling_format, + host=self.host, + port=self.port, + proxy=self.proxy, + proxy_port=self.proxy_port + ) + @property def bucket(self): """ @@ -300,28 +316,34 @@ def entries(self): """ Get the locally cached files for the bucket. """ - if self.preload_metadata and not self._entries: - self._entries = dict((self._decode_name(entry.key), entry) - for entry in self.bucket.list(prefix=self.location)) + if self.preload_metadata and not self._loaded_meta: + self._entries.update({ + self._decode_name(entry.key): entry + for entry in self.bucket.list(prefix=self.location) + }) + self._loaded_meta = True return self._entries + def _lookup_env(self, names): + for name in names: + value = os.environ.get(name) + if value: + return value + def _get_access_keys(self): """ Gets the access keys to use when accessing S3. If none are provided to the class in the constructor or in the settings then get them from the environment variables. """ - - def lookup_env(names): - for name in names: - value = os.environ.get(name) - if value: - return value - - access_key = self.access_key or lookup_env(self.access_key_names) - secret_key = self.secret_key or lookup_env(self.secret_key_names) + access_key = self.access_key or self._lookup_env(self.access_key_names) + secret_key = self.secret_key or self._lookup_env(self.secret_key_names) return access_key, secret_key + def _get_security_token(self): + security_token = self._lookup_env(self.security_token_names) + return security_token + def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. @@ -330,13 +352,13 @@ def _get_or_create_bucket(self, name): return self.connection.get_bucket(name, validate=self.auto_create_bucket) except self.connection_response_error: if self.auto_create_bucket: - bucket = self.connection.create_bucket(name) + bucket = self.connection.create_bucket(name, location=self.origin) bucket.set_acl(self.bucket_acl) return bucket - raise ImproperlyConfigured("Bucket %s does not exist. Buckets " - "can be automatically created by " - "setting AWS_AUTO_CREATE_BUCKET to " - "``True``." % name) + raise ImproperlyConfigured('Bucket %s does not exist. Buckets ' + 'can be automatically created by ' + 'setting AWS_AUTO_CREATE_BUCKET to ' + '``True``.' % name) def _get_headers(self): return self.headers.copy() @@ -345,16 +367,7 @@ def _clean_name(self, name): """ Cleans the name so that Windows style paths work """ - # Normalize Windows style paths - clean_name = posixpath.normpath(name).replace('\\', '/') - - # os.path.normpath() can strip trailing slashes so we implement - # a workaround here. - if name.endswith('/') and not clean_name.endswith('/'): - # Add a trailing slash as it was stripped. - return clean_name + '/' - else: - return clean_name + return clean_name(name) def _normalize_name(self, name): """ @@ -377,7 +390,11 @@ def _decode_name(self, name): def _compress_content(self, content): """Gzip a given string content.""" zbuf = BytesIO() - zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) + # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html) + # This means each time a file is compressed it changes even if the other contents don't change + # For S3 this defeats detection of changes using MD5 sums on gzipped files + # Fixing the mtime at 0.0 at compression time avoids this problem + zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf, mtime=0.0) try: zfile.write(force_bytes(content.read())) finally: @@ -435,6 +452,12 @@ def _save_content(self, key, content, headers): reduced_redundancy=self.reduced_redundancy, rewind=True, **kwargs) + def _get_key(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + return self.entries.get(name) + return self.bucket.get_key(self._encode_name(name)) + def delete(self, name): name = self._normalize_name(self._clean_name(name)) self.bucket.delete_key(self._encode_name(name)) @@ -447,11 +470,7 @@ def exists(self, name): except ImproperlyConfigured: return False - name = self._normalize_name(self._clean_name(name)) - if self.entries: - return name in self.entries - k = self.bucket.new_key(self._encode_name(name)) - return k.exists() + return self._get_key(name) is not None def listdir(self, name): name = self._normalize_name(self._clean_name(name)) @@ -463,9 +482,9 @@ def listdir(self, name): dirlist = self.bucket.list(self._encode_name(name)) files = [] dirs = set() - base_parts = name.split("/")[:-1] + base_parts = name.split('/')[:-1] for item in dirlist: - parts = item.name.split("/") + parts = item.name.split('/') parts = parts[len(base_parts):] if len(parts) == 1: # File @@ -476,29 +495,21 @@ def listdir(self, name): return list(dirs), files def size(self, name): - name = self._normalize_name(self._clean_name(name)) - if self.entries: - entry = self.entries.get(name) - if entry: - return entry.size - return 0 - return self.bucket.get_key(self._encode_name(name)).size + return self._get_key(name).size + + def get_modified_time(self, name): + dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) + return dt if setting('USE_TZ') else tz.make_naive(dt) def modified_time(self, name): - name = self._normalize_name(self._clean_name(name)) - entry = self.entries.get(name) - # only call self.bucket.get_key() if the key is not found - # in the preloaded metadata. - if entry is None: - entry = self.bucket.get_key(self._encode_name(name)) - # Parse the last_modified string to a local datetime object. - return parse_ts(entry.last_modified) + dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) + return tz.make_naive(dt) def url(self, name, headers=None, response_headers=None, expire=None): # Preserve the trailing slash after normalizing the path. name = self._normalize_name(self._clean_name(name)) if self.custom_domain: - return "%s//%s/%s" % (self.url_protocol, + return '%s//%s/%s' % (self.url_protocol, self.custom_domain, filepath_to_uri(name)) if expire is None: diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py new file mode 100644 index 000000000..9caae4d0a --- /dev/null +++ b/storages/backends/s3boto3.py @@ -0,0 +1,582 @@ +import mimetypes +import os +import posixpath +import threading +from gzip import GzipFile +from tempfile import SpooledTemporaryFile + +from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible +from django.utils.encoding import ( + filepath_to_uri, force_bytes, force_text, smart_text, +) +from django.utils.six import BytesIO +from django.utils.six.moves.urllib import parse as urlparse +from django.utils.timezone import is_naive, localtime + +from storages.utils import safe_join, setting + +try: + import boto3.session + from boto3 import __version__ as boto3_version + from botocore.client import Config + from botocore.exceptions import ClientError +except ImportError: + raise ImproperlyConfigured("Could not load Boto3's S3 bindings.\n" + "See https://github.com/boto/boto3") + + +boto3_version_info = tuple([int(i) for i in boto3_version.split('.')]) + +if boto3_version_info[:2] < (1, 2): + raise ImproperlyConfigured("The installed Boto3 library must be 1.2.0 or " + "higher.\nSee https://github.com/boto/boto3") + + +@deconstructible +class S3Boto3StorageFile(File): + + """ + The default file object used by the S3Boto3Storage backend. + + This file implements file streaming using boto's multipart + uploading functionality. The file can be opened in read or + write mode. + + This class extends Django's File class. However, the contained + data is only the data contained in the current buffer. So you + should not access the contained file object directly. You should + access the data via this class. + + Warning: This file *must* be closed using the close() method in + order to properly write the file to S3. Be sure to close the file + in your application. + """ + # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing. + # TODO: When Django drops support for Python 2.5, rewrite to use the + # BufferedIO streams in the Python 2.6 io module. + buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) + + def __init__(self, name, mode, storage, buffer_size=None): + self._storage = storage + self.name = name[len(self._storage.location):].lstrip('/') + self._mode = mode + self.obj = storage.bucket.Object(storage._encode_name(name)) + if 'w' not in mode: + # Force early RAII-style exception if object does not exist + self.obj.load() + self._is_dirty = False + self._file = None + self._multipart = None + # 5 MB is the minimum part size (if there is more than one part). + # Amazon allows up to 10,000 parts. The default supports uploads + # up to roughly 50 GB. Increase the part size to accommodate + # for files larger than this. + if buffer_size is not None: + self.buffer_size = buffer_size + self._write_counter = 0 + + @property + def size(self): + return self.obj.content_length + + def _get_file(self): + if self._file is None: + self._file = SpooledTemporaryFile( + max_size=self._storage.max_memory_size, + suffix=".S3Boto3StorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR", None) + ) + if 'r' in self._mode: + self._is_dirty = False + self._file.write(self.obj.get()['Body'].read()) + self._file.seek(0) + if self._storage.gzip and self.obj.content_encoding == 'gzip': + self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0) + return self._file + + def _set_file(self, value): + self._file = value + + file = property(_get_file, _set_file) + + def read(self, *args, **kwargs): + if 'r' not in self._mode: + raise AttributeError("File was not opened in read mode.") + return super(S3Boto3StorageFile, self).read(*args, **kwargs) + + def write(self, content): + if 'w' not in self._mode: + raise AttributeError("File was not opened in write mode.") + self._is_dirty = True + if self._multipart is None: + parameters = self._storage.object_parameters.copy() + parameters['ACL'] = self._storage.default_acl + parameters['ContentType'] = (mimetypes.guess_type(self.obj.key)[0] or + self._storage.default_content_type) + if self._storage.reduced_redundancy: + parameters['StorageClass'] = 'REDUCED_REDUNDANCY' + if self._storage.encryption: + parameters['ServerSideEncryption'] = 'AES256' + self._multipart = self.obj.initiate_multipart_upload(**parameters) + if self.buffer_size <= self._buffer_file_size: + self._flush_write_buffer() + return super(S3Boto3StorageFile, self).write(force_bytes(content)) + + @property + def _buffer_file_size(self): + pos = self.file.tell() + self.file.seek(0, os.SEEK_END) + length = self.file.tell() + self.file.seek(pos) + return length + + def _flush_write_buffer(self): + """ + Flushes the write buffer. + """ + if self._buffer_file_size: + self._write_counter += 1 + self.file.seek(0) + part = self._multipart.Part(self._write_counter) + part.upload(Body=self.file.read()) + + def close(self): + if self._is_dirty: + self._flush_write_buffer() + # TODO: Possibly cache the part ids as they're being uploaded + # instead of requesting parts from server. For now, emulating + # s3boto's behavior. + parts = [{'ETag': part.e_tag, 'PartNumber': part.part_number} + for part in self._multipart.parts.all()] + self._multipart.complete( + MultipartUpload={'Parts': parts}) + else: + if self._multipart is not None: + self._multipart.abort() + if self._file is not None: + self._file.close() + self._file = None + + +@deconstructible +class S3Boto3Storage(Storage): + """ + Amazon Simple Storage Service using Boto3 + + This storage backend supports opening files in read or write + mode and supports streaming(buffering) data in chunks to S3 + when writing. + """ + default_content_type = 'application/octet-stream' + # If config provided in init, signature_version and addressing_style settings/args are ignored. + config = None + + # used for looking up the access and secret key from env vars + access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] + secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] + security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] + + access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) + secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) + file_overwrite = setting('AWS_S3_FILE_OVERWRITE', True) + object_parameters = setting('AWS_S3_OBJECT_PARAMETERS', {}) + bucket_name = setting('AWS_STORAGE_BUCKET_NAME') + auto_create_bucket = setting('AWS_AUTO_CREATE_BUCKET', False) + default_acl = setting('AWS_DEFAULT_ACL', 'public-read') + bucket_acl = setting('AWS_BUCKET_ACL', default_acl) + querystring_auth = setting('AWS_QUERYSTRING_AUTH', True) + querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600) + signature_version = setting('AWS_S3_SIGNATURE_VERSION') + reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False) + location = setting('AWS_LOCATION', '') + encryption = setting('AWS_S3_ENCRYPTION', False) + custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') + addressing_style = setting('AWS_S3_ADDRESSING_STYLE') + secure_urls = setting('AWS_S3_SECURE_URLS', True) + file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8') + gzip = setting('AWS_IS_GZIPPED', False) + preload_metadata = setting('AWS_PRELOAD_METADATA', False) + gzip_content_types = setting('GZIP_CONTENT_TYPES', ( + 'text/css', + 'text/javascript', + 'application/javascript', + 'application/x-javascript', + 'image/svg+xml', + )) + url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:') + endpoint_url = setting('AWS_S3_ENDPOINT_URL', None) + region_name = setting('AWS_S3_REGION_NAME', None) + use_ssl = setting('AWS_S3_USE_SSL', True) + + # The max amount of memory a returned file can take up before being + # rolled over into a temporary file on disk. Default is 0: Do not roll over. + max_memory_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0) + + def __init__(self, acl=None, bucket=None, **settings): + # check if some of the settings we've provided as class attributes + # need to be overwritten with values passed in here + for name, value in settings.items(): + if hasattr(self, name): + setattr(self, name, value) + + # For backward-compatibility of old differing parameter names + if acl is not None: + self.default_acl = acl + if bucket is not None: + self.bucket_name = bucket + + self.location = (self.location or '').lstrip('/') + # Backward-compatibility: given the anteriority of the SECURE_URL setting + # we fall back to https if specified in order to avoid the construction + # of unsecure urls. + if self.secure_urls: + self.url_protocol = 'https:' + + self._entries = {} + self._bucket = None + self._connections = threading.local() + + self.security_token = None + if not self.access_key and not self.secret_key: + self.access_key, self.secret_key = self._get_access_keys() + self.security_token = self._get_security_token() + + if not self.config: + self.config = Config(s3={'addressing_style': self.addressing_style}, + signature_version=self.signature_version) + + @property + def connection(self): + # TODO: Support host, port like in s3boto + # Note that proxies are handled by environment variables that the underlying + # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338 + # and http://docs.python-requests.org/en/latest/user/advanced/#proxies + connection = getattr(self._connections, 'connection', None) + if connection is None: + session = boto3.session.Session() + self._connections.connection = session.resource( + 's3', + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.security_token, + region_name=self.region_name, + use_ssl=self.use_ssl, + endpoint_url=self.endpoint_url, + config=self.config + ) + return self._connections.connection + + @property + def bucket(self): + """ + Get the current bucket. If there is no current bucket object + create it. + """ + if self._bucket is None: + self._bucket = self._get_or_create_bucket(self.bucket_name) + return self._bucket + + @property + def entries(self): + """ + Get the locally cached files for the bucket. + """ + if self.preload_metadata and not self._entries: + self._entries = { + self._decode_name(entry.key): entry + for entry in self.bucket.objects.filter(Prefix=self.location) + } + return self._entries + + def _lookup_env(self, names): + for name in names: + value = os.environ.get(name) + if value: + return value + + def _get_access_keys(self): + """ + Gets the access keys to use when accessing S3. If none + are provided to the class in the constructor or in the + settings then get them from the environment variables. + """ + access_key = self.access_key or self._lookup_env(self.access_key_names) + secret_key = self.secret_key or self._lookup_env(self.secret_key_names) + return access_key, secret_key + + def _get_security_token(self): + security_token = self._lookup_env(self.security_token_names) + return security_token + + def _get_or_create_bucket(self, name): + """ + Retrieves a bucket if it exists, otherwise creates it. + """ + bucket = self.connection.Bucket(name) + if self.auto_create_bucket: + try: + # Directly call head_bucket instead of bucket.load() because head_bucket() + # fails on wrong region, while bucket.load() does not. + bucket.meta.client.head_bucket(Bucket=name) + except ClientError as err: + if err.response['ResponseMetadata']['HTTPStatusCode'] == 301: + raise ImproperlyConfigured("Bucket %s exists, but in a different " + "region than we are connecting to. Set " + "the region to connect to by setting " + "AWS_S3_REGION_NAME to the correct region." % name) + + elif err.response['ResponseMetadata']['HTTPStatusCode'] == 404: + # Notes: When using the us-east-1 Standard endpoint, you can create + # buckets in other regions. The same is not true when hitting region specific + # endpoints. However, when you create the bucket not in the same region, the + # connection will fail all future requests to the Bucket after the creation + # (301 Moved Permanently). + # + # For simplicity, we enforce in S3Boto3Storage that any auto-created + # bucket must match the region that the connection is for. + # + # Also note that Amazon specifically disallows "us-east-1" when passing bucket + # region names; LocationConstraint *must* be blank to create in US Standard. + bucket_params = {'ACL': self.bucket_acl} + region_name = self.connection.meta.client.meta.region_name + if region_name != 'us-east-1': + bucket_params['CreateBucketConfiguration'] = { + 'LocationConstraint': region_name} + bucket.create(**bucket_params) + else: + raise ImproperlyConfigured("Bucket %s does not exist. Buckets " + "can be automatically created by " + "setting AWS_AUTO_CREATE_BUCKET to " + "``True``." % name) + return bucket + + def _clean_name(self, name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name += '/' + return clean_name + + def _normalize_name(self, name): + """ + Normalizes the name so that paths like /path/to/ignored/../something.txt + work. We check to make sure that the path pointed to is not outside + the directory specified by the LOCATION setting. + """ + try: + return safe_join(self.location, name) + except ValueError: + raise SuspiciousOperation("Attempted access to '%s' denied." % + name) + + def _encode_name(self, name): + return smart_text(name, encoding=self.file_name_charset) + + def _decode_name(self, name): + return force_text(name, encoding=self.file_name_charset) + + def _compress_content(self, content): + """Gzip a given string content.""" + content.seek(0) + zbuf = BytesIO() + # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html) + # This means each time a file is compressed it changes even if the other contents don't change + # For S3 this defeats detection of changes using MD5 sums on gzipped files + # Fixing the mtime at 0.0 at compression time avoids this problem + zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf, mtime=0.0) + try: + zfile.write(force_bytes(content.read())) + finally: + zfile.close() + zbuf.seek(0) + # Boto 2 returned the InMemoryUploadedFile with the file pointer replaced, + # but Boto 3 seems to have issues with that. No need for fp.name in Boto3 + # so just returning the BytesIO directly + return zbuf + + def _open(self, name, mode='rb'): + name = self._normalize_name(self._clean_name(name)) + try: + f = S3Boto3StorageFile(name, mode, self) + except ClientError as err: + if err.response['ResponseMetadata']['HTTPStatusCode'] == 404: + raise IOError('File does not exist: %s' % name) + raise # Let it bubble up if it was some other error + return f + + def _save(self, name, content): + cleaned_name = self._clean_name(name) + name = self._normalize_name(cleaned_name) + parameters = self.object_parameters.copy() + _type, encoding = mimetypes.guess_type(name) + content_type = getattr(content, 'content_type', + _type or self.default_content_type) + + # setting the content_type in the key object is not enough. + parameters.update({'ContentType': content_type}) + + if self.gzip and content_type in self.gzip_content_types: + content = self._compress_content(content) + parameters.update({'ContentEncoding': 'gzip'}) + elif encoding: + # If the content already has a particular encoding, set it + parameters.update({'ContentEncoding': encoding}) + + encoded_name = self._encode_name(name) + obj = self.bucket.Object(encoded_name) + if self.preload_metadata: + self._entries[encoded_name] = obj + + # If both `name` and `content.name` are empty or None, your request + # can be rejected with `XAmzContentSHA256Mismatch` error, because in + # `django.core.files.storage.Storage.save` method your file-like object + # will be wrapped in `django.core.files.File` if no `chunks` method + # provided. `File.__bool__` method is Django-specific and depends on + # file name, for this reason`botocore.handlers.calculate_md5` can fail + # even if wrapped file-like object exists. To avoid Django-specific + # logic, pass internal file-like object if `content` is `File` + # class instance. + if isinstance(content, File): + content = content.file + + self._save_content(obj, content, parameters=parameters) + # Note: In boto3, after a put, last_modified is automatically reloaded + # the next time it is accessed; no need to specifically reload it. + return cleaned_name + + def _save_content(self, obj, content, parameters): + # only pass backwards incompatible arguments if they vary from the default + put_parameters = parameters.copy() if parameters else {} + if self.encryption: + put_parameters['ServerSideEncryption'] = 'AES256' + if self.reduced_redundancy: + put_parameters['StorageClass'] = 'REDUCED_REDUNDANCY' + if self.default_acl: + put_parameters['ACL'] = self.default_acl + content.seek(0, os.SEEK_SET) + obj.upload_fileobj(content, ExtraArgs=put_parameters) + + def delete(self, name): + name = self._normalize_name(self._clean_name(name)) + self.bucket.Object(self._encode_name(name)).delete() + + def exists(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + return name in self.entries + try: + self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name) + return True + except ClientError: + return False + + def listdir(self, name): + name = self._normalize_name(self._clean_name(name)) + # for the bucket.objects.filter and logic below name needs to end in / + # But for the root path "" we leave it as an empty string + if name and not name.endswith('/'): + name += '/' + + files = [] + dirs = set() + base_parts = name.split("/")[:-1] + for item in self.bucket.objects.filter(Prefix=self._encode_name(name)): + parts = item.key.split("/") + parts = parts[len(base_parts):] + if len(parts) == 1: + # File + files.append(parts[0]) + elif len(parts) > 1: + # Directory + dirs.add(parts[0]) + return list(dirs), files + + def size(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + entry = self.entries.get(name) + if entry: + return entry.size if hasattr(entry, 'size') else entry.content_length + return 0 + return self.bucket.Object(self._encode_name(name)).content_length + + def get_modified_time(self, name): + """ + Returns an (aware) datetime object containing the last modified time if + USE_TZ is True, otherwise returns a naive datetime in the local timezone. + """ + name = self._normalize_name(self._clean_name(name)) + entry = self.entries.get(name) + # only call self.bucket.Object() if the key is not found + # in the preloaded metadata. + if entry is None: + entry = self.bucket.Object(self._encode_name(name)) + if setting('USE_TZ'): + # boto3 returns TZ aware timestamps + return entry.last_modified + else: + return localtime(entry.last_modified).replace(tzinfo=None) + + def modified_time(self, name): + """Returns a naive datetime object containing the last modified time.""" + # If USE_TZ=False then get_modified_time will return a naive datetime + # so we just return that, else we have to localize and strip the tz + mtime = self.get_modified_time(name) + return mtime if is_naive(mtime) else localtime(mtime).replace(tzinfo=None) + + def _strip_signing_parameters(self, url): + # Boto3 does not currently support generating URLs that are unsigned. Instead we + # take the signed URLs and strip any querystring params related to signing and expiration. + # Note that this may end up with URLs that are still invalid, especially if params are + # passed in that only work with signed URLs, e.g. response header params. + # The code attempts to strip all query parameters that match names of known parameters + # from v2 and v4 signatures, regardless of the actual signature version used. + split_url = urlparse.urlsplit(url) + qs = urlparse.parse_qsl(split_url.query, keep_blank_values=True) + blacklist = { + 'x-amz-algorithm', 'x-amz-credential', 'x-amz-date', + 'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature', + 'x-amz-security-token', 'awsaccesskeyid', 'expires', 'signature', + } + filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist) + # Note: Parameters that did not have a value in the original query string will have + # an '=' sign appended to it, e.g ?foo&bar becomes ?foo=&bar= + joined_qs = ('='.join(keyval) for keyval in filtered_qs) + split_url = split_url._replace(query="&".join(joined_qs)) + return split_url.geturl() + + def url(self, name, parameters=None, expire=None): + # Preserve the trailing slash after normalizing the path. + # TODO: Handle force_http=not self.secure_urls like in s3boto + name = self._normalize_name(self._clean_name(name)) + if self.custom_domain: + return "%s//%s/%s" % (self.url_protocol, + self.custom_domain, filepath_to_uri(name)) + if expire is None: + expire = self.querystring_expire + + params = parameters.copy() if parameters else {} + params['Bucket'] = self.bucket.name + params['Key'] = self._encode_name(name) + url = self.bucket.meta.client.generate_presigned_url('get_object', Params=params, + ExpiresIn=expire) + if self.querystring_auth: + return url + return self._strip_signing_parameters(url) + + def get_available_name(self, name, max_length=None): + """Overwrite existing file with the same name.""" + if self.file_overwrite: + name = self._clean_name(name) + return name + return super(S3Boto3Storage, self).get_available_name(name, max_length) diff --git a/storages/backends/sftpstorage.py b/storages/backends/sftpstorage.py index ce4c144e7..f07e8cf5d 100644 --- a/storages/backends/sftpstorage.py +++ b/storages/backends/sftpstorage.py @@ -1,87 +1,50 @@ -from __future__ import print_function # SFTP storage backend for Django. # Author: Brent Tubbs # License: MIT # # Modeled on the FTP storage by Rafal Jonca -# -# Settings: -# -# SFTP_STORAGE_HOST - The hostname where you want the files to be saved. -# -# SFTP_STORAGE_ROOT - The root directory on the remote host into which files -# should be placed. Should work the same way that STATIC_ROOT works for local -# files. Must include a trailing slash. -# -# SFTP_STORAGE_PARAMS (Optional) - A dictionary containing connection -# parameters to be passed as keyword arguments to -# paramiko.SSHClient().connect() (do not include hostname here). See -# http://www.lag.net/paramiko/docs/paramiko.SSHClient-class.html#connect for -# details -# -# SFTP_STORAGE_INTERACTIVE (Optional) - A boolean indicating whether to prompt -# for a password if the connection cannot be made using keys, and there is not -# already a password in SFTP_STORAGE_PARAMS. You can set this to True to -# enable interactive login when running 'manage.py collectstatic', for example. -# -# DO NOT set SFTP_STORAGE_INTERACTIVE to True if you are using this storage -# for files being uploaded to your site by users, because you'll have no way -# to enter the password when they submit the form.. -# -# SFTP_STORAGE_FILE_MODE (Optional) - A bitmask for setting permissions on -# newly-created files. See http://docs.python.org/library/os.html#os.chmod for -# acceptable values. -# -# SFTP_STORAGE_DIR_MODE (Optional) - A bitmask for setting permissions on -# newly-created directories. See -# http://docs.python.org/library/os.html#os.chmod for acceptable values. -# -# Hint: if you start the mode number with a 0 you can express it in octal -# just like you would when doing "chmod 775 myfile" from bash. -# -# SFTP_STORAGE_UID (Optional) - uid of the account that should be set as owner -# of the files on the remote host. You have to be root to set this. -# -# SFTP_STORAGE_GID (Optional) - gid of the group that should be set on the -# files on the remote host. You have to be a member of the group to set this. -# SFTP_KNOWN_HOST_FILE (Optional) - absolute path of know host file, if it isn't -# set "~/.ssh/known_hosts" will be used - +from __future__ import print_function import getpass import os -import paramiko import posixpath import stat from datetime import datetime -from django.conf import settings +import paramiko from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils.deconstruct import deconstructible +from django.utils.six import BytesIO +from django.utils.six.moves.urllib import parse as urlparse -from storages.compat import urlparse, BytesIO, Storage +from storages.utils import setting +@deconstructible class SFTPStorage(Storage): - def __init__(self): - self._host = settings.SFTP_STORAGE_HOST + def __init__(self, host=None, params=None, interactive=None, file_mode=None, + dir_mode=None, uid=None, gid=None, known_host_file=None, + root_path=None, base_url=None): + self._host = host or setting('SFTP_STORAGE_HOST') - # if present, settings.SFTP_STORAGE_PARAMS should be a dict with params - # matching the keyword arguments to paramiko.SSHClient().connect(). So - # you can put username/password there. Or you can omit all that if - # you're using keys. - self._params = getattr(settings, 'SFTP_STORAGE_PARAMS', {}) - self._interactive = getattr(settings, 'SFTP_STORAGE_INTERACTIVE', - False) - self._file_mode = getattr(settings, 'SFTP_STORAGE_FILE_MODE', None) - self._dir_mode = getattr(settings, 'SFTP_STORAGE_DIR_MODE', None) + self._params = params or setting('SFTP_STORAGE_PARAMS', {}) + self._interactive = setting('SFTP_STORAGE_INTERACTIVE', False) \ + if interactive is None else interactive + self._file_mode = setting('SFTP_STORAGE_FILE_MODE') \ + if file_mode is None else file_mode + self._dir_mode = setting('SFTP_STORAGE_DIR_MODE') if \ + dir_mode is None else dir_mode - self._uid = getattr(settings, 'SFTP_STORAGE_UID', None) - self._gid = getattr(settings, 'SFTP_STORAGE_GID', None) - self._known_host_file = getattr(settings, 'SFTP_KNOWN_HOST_FILE', None) + self._uid = setting('SFTP_STORAGE_UID') if uid is None else uid + self._gid = setting('SFTP_STORAGE_GID') if gid is None else gid + self._known_host_file = setting('SFTP_KNOWN_HOST_FILE') \ + if known_host_file is None else known_host_file - self._root_path = settings.SFTP_STORAGE_ROOT - self._base_url = settings.MEDIA_URL + self._root_path = setting('SFTP_STORAGE_ROOT', '') \ + if root_path is None else root_path + self._base_url = setting('MEDIA_URL') if base_url is None else base_url # for now it's all posix paths. Maybe someday we'll support figuring # out if the remote host is windows. @@ -90,11 +53,12 @@ def __init__(self): def _connect(self): self._ssh = paramiko.SSHClient() - if self._known_host_file is not None: - self._ssh.load_host_keys(self._known_host_file) - else: - # automatically add host keys from current user. - self._ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts"))) + known_host_file = self._known_host_file or os.path.expanduser( + os.path.join("~", ".ssh", "known_hosts") + ) + + if os.path.exists(known_host_file): + self._ssh.load_host_keys(known_host_file) # and automatically add new host keys for hosts we haven't seen before. self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -189,6 +153,7 @@ def delete(self, name): def exists(self, name): # Try to retrieve file info. Return true on success, false on failure. remote_path = self._remote_path(name) + try: self.sftp.stat(remote_path) return True @@ -263,5 +228,5 @@ def write(self, content): def close(self): if self._is_dirty: - self._storage._save(self._name, self.file.getvalue()) + self._storage._save(self._name, self) self.file.close() diff --git a/storages/backends/symlinkorcopy.py b/storages/backends/symlinkorcopy.py deleted file mode 100644 index 881042358..000000000 --- a/storages/backends/symlinkorcopy.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -from django.conf import settings -from storages.compat import FileSystemStorage - -__doc__ = """ -I needed to efficiently create a mirror of a directory tree (so that -"origin pull" CDNs can automatically pull files). The trick was that -some files could be modified, and some could be identical to the original. -Of course it doesn't make sense to store the exact same data twice on the -file system. So I created SymlinkOrCopyStorage. - -SymlinkOrCopyStorage allows you to symlink a file when it's identical to -the original file and to copy the file if it's modified. -Of course, it's impossible to know if a file is modified just by looking -at the file, without knowing what the original file was. -That's what the symlinkWithin parameter is for. It accepts one or more paths -(if multiple, they should be concatenated using a colon (:)). -Files that will be saved using SymlinkOrCopyStorage are then checked on their -location: if they are within one of the symlink_within directories, -they will be symlinked, otherwise they will be copied. - -The rationale is that unmodified files will exist in their original location, -e.g. /htdocs/example.com/image.jpg and modified files will be stored in -a temporary directory, e.g. /tmp/image.jpg. -""" - - -class SymlinkOrCopyStorage(FileSystemStorage): - """Stores symlinks to files instead of actual files whenever possible - - When a file that's being saved is currently stored in the symlink_within - directory, then symlink the file. Otherwise, copy the file. - """ - def __init__(self, location=settings.MEDIA_ROOT, base_url=settings.MEDIA_URL, - symlink_within=None): - super(SymlinkOrCopyStorage, self).__init__(location, base_url) - self.symlink_within = symlink_within.split(":") - - def _save(self, name, content): - full_path_dst = self.path(name) - - directory = os.path.dirname(full_path_dst) - if not os.path.exists(directory): - os.makedirs(directory) - elif not os.path.isdir(directory): - raise IOError("%s exists and is not a directory." % directory) - - full_path_src = os.path.abspath(content.name) - - symlinked = False - # Only symlink if the current platform supports it. - if getattr(os, "symlink", False): - for path in self.symlink_within: - if full_path_src.startswith(path): - os.symlink(full_path_src, full_path_dst) - symlinked = True - break - - if not symlinked: - super(SymlinkOrCopyStorage, self)._save(name, content) - - return name diff --git a/storages/compat.py b/storages/compat.py deleted file mode 100644 index 1ac3e1d0a..000000000 --- a/storages/compat.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.utils.six.moves.urllib import parse as urlparse -from django.utils.six import BytesIO -import django - -try: - from django.utils.deconstruct import deconstructible -except ImportError: # Django 1.7+ migrations - deconstructible = lambda klass, *args, **kwargs: klass - -# Storage only accepts `max_length` in 1.8+ -if django.VERSION >= (1, 8): - from django.core.files.storage import Storage, FileSystemStorage -else: - from django.core.files.storage import Storage as DjangoStorage - from django.core.files.storage import FileSystemStorage as DjangoFileSystemStorage - - class StorageMixin(object): - def save(self, name, content, max_length=None): - return super(StorageMixin, self).save(name, content) - - def get_available_name(self, name, max_length=None): - return super(StorageMixin, self).get_available_name(name) - - class Storage(StorageMixin, DjangoStorage): - pass - - class FileSystemStorage(StorageMixin, DjangoFileSystemStorage): - pass diff --git a/storages/utils.py b/storages/utils.py index 810e0c596..566aa5127 100644 --- a/storages/utils.py +++ b/storages/utils.py @@ -1,9 +1,82 @@ +import posixpath + from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.utils.encoding import force_text -def setting(name, default=None): +def setting(name, default=None, strict=False): """ - Helper function to get a Django setting by name or (optionally) return - a default (or else ``None``). + Helper function to get a Django setting by name. If setting doesn't exists + it can return a default or raise an error if in strict mode. + + :param name: Name of setting + :type name: str + :param default: Value if setting is unfound + :param strict: Define if return default value or raise an error + :type strict: bool + :returns: Setting's value + :raises: django.core.exceptions.ImproperlyConfigured if setting is unfound + and strict mode """ + if strict and not hasattr(settings, name): + msg = "You must provide settings.%s" % name + raise ImproperlyConfigured(msg) return getattr(settings, name, default) + + +def clean_name(name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name = clean_name + '/' + + # Given an empty string, os.path.normpath() will return ., which we don't want + if clean_name == '.': + clean_name = '' + + return clean_name + + +def safe_join(base, *paths): + """ + A version of django.utils._os.safe_join for S3 paths. + + Joins one or more path components to the base path component + intelligently. Returns a normalized version of the final path. + + The final path must be located inside of the base path component + (otherwise a ValueError is raised). + + Paths outside the base path indicate a possible security + sensitive operation. + """ + base_path = force_text(base) + base_path = base_path.rstrip('/') + paths = [force_text(p) for p in paths] + + final_path = base_path + '/' + for path in paths: + _final_path = posixpath.normpath(posixpath.join(final_path, path)) + # posixpath.normpath() strips the trailing /. Add it back. + if path.endswith('/') or _final_path + '/' == final_path: + _final_path += '/' + final_path = _final_path + if final_path == base_path: + final_path += '/' + + # Ensure final_path starts with base_path and that the next character after + # the base path is /. + base_path_len = len(base_path) + if (not final_path.startswith(base_path) or final_path[base_path_len] != '/'): + raise ValueError('the joined path is located outside of the base path' + ' component') + + return final_path.lstrip('/') diff --git a/tests/settings.py b/tests/settings.py index d84433b1d..43047b6cc 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -28,3 +28,6 @@ AWS_IS_GZIPPED = True GS_IS_GZIPPED = True SECRET_KEY = 'hailthesunshine' + +USE_TZ = True +TIME_ZONE = 'America/Chicago' diff --git a/tests/test_dropbox.py b/tests/test_dropbox.py index 41be28a13..58d503628 100644 --- a/tests/test_dropbox.py +++ b/tests/test_dropbox.py @@ -1,14 +1,23 @@ -import re from datetime import datetime + +from django.core.exceptions import ( + ImproperlyConfigured, SuspiciousFileOperation, +) +from django.core.files.base import ContentFile, File +from django.test import TestCase +from django.utils.six import BytesIO + +from storages.backends import dropbox + try: from unittest import mock except ImportError: # Python 3.2 and below import mock -from django.test import TestCase -from django.core.files.base import File, ContentFile -from storages.backends import dropbox +class F(object): + pass + FILE_DATE = datetime(2015, 8, 24, 15, 6, 41) FILE_FIXTURE = { @@ -48,63 +57,57 @@ 'size': '0 bytes', 'thumb_exists': False } -FILE_MEDIA_FIXTURE = { - 'url': 'https://dl.dropboxusercontent.com/1/view/foo', - 'expires': 'Fri, 16 Sep 2011 01:01:25 +0000', -} - -__all__ = [ - 'DropBoxTest', - 'DropBoxFileTest' -] +FILE_MEDIA_FIXTURE = F() +FILE_MEDIA_FIXTURE.link = 'https://dl.dropboxusercontent.com/1/view/foo' class DropBoxTest(TestCase): - @mock.patch('dropbox.client._OAUTH2_ACCESS_TOKEN_PATTERN', - re.compile(r'.*')) - @mock.patch('dropbox.client.DropboxOAuth2Session') def setUp(self, *args): - self.storage = dropbox.DropBoxStorage('') + self.storage = dropbox.DropBoxStorage('foo') - @mock.patch('dropbox.client.DropboxClient.file_delete', + def test_no_access_token(self, *args): + with self.assertRaises(ImproperlyConfigured): + dropbox.DropBoxStorage(None) + + @mock.patch('dropbox.Dropbox.files_delete', return_value=FILE_FIXTURE) def test_delete(self, *args): self.storage.delete('foo') - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=[FILE_FIXTURE]) def test_exists(self, *args): exists = self.storage.exists('foo') self.assertTrue(exists) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=[]) def test_not_exists(self, *args): exists = self.storage.exists('bar') self.assertFalse(exists) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILES_FIXTURE) def test_listdir(self, *args): dirs, files = self.storage.listdir('/') self.assertGreater(len(dirs), 0) self.assertGreater(len(files), 0) - self.assertEqual(dirs[0], '/bar') - self.assertEqual(files[0], '/foo.txt') + self.assertEqual(dirs[0], 'bar') + self.assertEqual(files[0], 'foo.txt') - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_size(self, *args): size = self.storage.size('foo') self.assertEqual(size, FILE_FIXTURE['bytes']) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_modified_time(self, *args): mtime = self.storage.modified_time('foo') self.assertEqual(mtime, FILE_DATE) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_accessed_time(self, *args): mtime = self.storage.accessed_time('foo') @@ -114,39 +117,68 @@ def test_open(self, *args): obj = self.storage._open('foo') self.assertIsInstance(obj, File) - @mock.patch('dropbox.client.DropboxClient.put_file', + @mock.patch('dropbox.Dropbox.files_upload', return_value='foo') - def test_save(self, *args): - self.storage._save('foo', b'bar') - - @mock.patch('dropbox.client.DropboxClient.get_file', - return_value=ContentFile('bar')) - def test_read(self, *args): - content = self.storage._read('foo') - self.assertEqual(content, 'bar') - - @mock.patch('dropbox.client.DropboxClient.media', + def test_save(self, files_upload, *args): + self.storage._save('foo', File(BytesIO(b'bar'), 'foo')) + self.assertTrue(files_upload.called) + + @mock.patch('dropbox.Dropbox.files_upload') + @mock.patch('dropbox.Dropbox.files_upload_session_finish') + @mock.patch('dropbox.Dropbox.files_upload_session_append_v2') + @mock.patch('dropbox.Dropbox.files_upload_session_start', + return_value=mock.MagicMock(session_id='foo')) + def test_chunked_upload(self, start, append, finish, upload): + large_file = File(BytesIO(b'bar' * self.storage.CHUNK_SIZE), 'foo') + self.storage._save('foo', large_file) + self.assertTrue(start.called) + self.assertTrue(append.called) + self.assertTrue(finish.called) + self.assertFalse(upload.called) + + @mock.patch('dropbox.Dropbox.files_get_temporary_link', return_value=FILE_MEDIA_FIXTURE) def test_url(self, *args): url = self.storage.url('foo') - self.assertEqual(url, FILE_MEDIA_FIXTURE['url']) + self.assertEqual(url, FILE_MEDIA_FIXTURE.link) + + def test_formats(self, *args): + self.storage = dropbox.DropBoxStorage('foo') + files = self.storage._full_path('') + self.assertEqual(files, self.storage._full_path('/')) + self.assertEqual(files, self.storage._full_path('.')) + self.assertEqual(files, self.storage._full_path('..')) + self.assertEqual(files, self.storage._full_path('../..')) class DropBoxFileTest(TestCase): - @mock.patch('dropbox.client._OAUTH2_ACCESS_TOKEN_PATTERN', - re.compile(r'.*')) - @mock.patch('dropbox.client.DropboxOAuth2Session') def setUp(self, *args): - self.storage = dropbox.DropBoxStorage('') + self.storage = dropbox.DropBoxStorage('foo') self.file = dropbox.DropBoxFile('/foo.txt', self.storage) - @mock.patch('dropbox.client.DropboxClient.put_file', - return_value='foo') - def test_write(self, *args): - self.storage._save('foo', b'bar') - - @mock.patch('dropbox.client.DropboxClient.get_file', - return_value=ContentFile('bar')) + @mock.patch('dropbox.Dropbox.files_download', + return_value=ContentFile(b'bar')) def test_read(self, *args): - content = self.storage._read('foo') - self.assertEqual(content, 'bar') + file = self.storage._open(b'foo') + self.assertEqual(file.read(), b'bar') + + +@mock.patch('dropbox.Dropbox.files_get_metadata', + return_value={'contents': []}) +class DropBoxRootPathTest(TestCase): + def test_jailed(self, *args): + self.storage = dropbox.DropBoxStorage('foo', '/bar') + dirs, files = self.storage.listdir('/') + self.assertFalse(dirs) + self.assertFalse(files) + + def test_suspicious(self, *args): + self.storage = dropbox.DropBoxStorage('foo', '/bar') + with self.assertRaises((SuspiciousFileOperation, ValueError)): + self.storage._full_path('..') + + def test_formats(self, *args): + self.storage = dropbox.DropBoxStorage('foo', '/bar') + files = self.storage._full_path('') + self.assertEqual(files, self.storage._full_path('/')) + self.assertEqual(files, self.storage._full_path('.')) diff --git a/tests/test_ftp.py b/tests/test_ftp.py new file mode 100644 index 000000000..34ae7140b --- /dev/null +++ b/tests/test_ftp.py @@ -0,0 +1,239 @@ +try: + from unittest.mock import patch +except ImportError: + from mock import patch +from datetime import datetime + +from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import File +from django.test import TestCase +from django.utils.six import BytesIO + +from storages.backends import ftp + +USER = 'foo' +PASSWORD = 'b@r' +HOST = 'localhost' +PORT = 2121 +URL = "ftp://{user}:{passwd}@{host}:{port}/".format(user=USER, passwd=PASSWORD, + host=HOST, port=PORT) + +LIST_FIXTURE = """drwxr-xr-x 2 ftp nogroup 4096 Jul 27 09:46 dir +-rw-r--r-- 1 ftp nogroup 1024 Jul 27 09:45 fi +-rw-r--r-- 1 ftp nogroup 2048 Jul 27 09:50 fi2""" + + +def list_retrlines(cmd, func): + for line in LIST_FIXTURE.splitlines(): + func(line) + + +class FTPTest(TestCase): + def setUp(self): + self.storage = ftp.FTPStorage(location=URL) + + def test_init_no_location(self): + with self.assertRaises(ImproperlyConfigured): + ftp.FTPStorage() + + @patch('storages.backends.ftp.setting', return_value=URL) + def test_init_location_from_setting(self, mock_setting): + storage = ftp.FTPStorage() + self.assertTrue(mock_setting.called) + self.assertEqual(storage.location, URL) + + def test_decode_location(self): + config = self.storage._decode_location(URL) + wanted_config = { + 'passwd': 'b@r', + 'host': 'localhost', + 'user': 'foo', + 'active': False, + 'path': '/', + 'port': 2121, + } + self.assertEqual(config, wanted_config) + # Test active FTP + config = self.storage._decode_location('a'+URL) + wanted_config = { + 'passwd': 'b@r', + 'host': 'localhost', + 'user': 'foo', + 'active': True, + 'path': '/', + 'port': 2121, + } + self.assertEqual(config, wanted_config) + + def test_decode_location_error(self): + with self.assertRaises(ImproperlyConfigured): + self.storage._decode_location('foo') + with self.assertRaises(ImproperlyConfigured): + self.storage._decode_location('http://foo.pt') + # TODO: Cannot not provide a port + # with self.assertRaises(ImproperlyConfigured): + # self.storage._decode_location('ftp://') + + @patch('ftplib.FTP') + def test_start_connection(self, mock_ftp): + self.storage._start_connection() + self.assertIsNotNone(self.storage._connection) + # Start active + storage = ftp.FTPStorage(location='a'+URL) + storage._start_connection() + + @patch('ftplib.FTP', **{'return_value.pwd.side_effect': IOError()}) + def test_start_connection_timeout(self, mock_ftp): + self.storage._start_connection() + self.assertIsNotNone(self.storage._connection) + + @patch('ftplib.FTP', **{'return_value.connect.side_effect': IOError()}) + def test_start_connection_error(self, mock_ftp): + with self.assertRaises(ftp.FTPStorageException): + self.storage._start_connection() + + @patch('ftplib.FTP', **{'return_value.quit.return_value': None}) + def test_disconnect(self, mock_ftp_quit): + self.storage._start_connection() + self.storage.disconnect() + self.assertIsNone(self.storage._connection) + + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) + def test_mkremdirs(self, mock_ftp): + self.storage._start_connection() + self.storage._mkremdirs('foo/bar') + + @patch('ftplib.FTP', **{ + 'return_value.pwd.return_value': 'foo', + 'return_value.storbinary.return_value': None + }) + def test_put_file(self, mock_ftp): + self.storage._start_connection() + self.storage._put_file('foo', File(BytesIO(b'foo'), 'foo')) + + @patch('ftplib.FTP', **{ + 'return_value.pwd.return_value': 'foo', + 'return_value.storbinary.side_effect': IOError() + }) + def test_put_file_error(self, mock_ftp): + self.storage._start_connection() + with self.assertRaises(ftp.FTPStorageException): + self.storage._put_file('foo', File(BytesIO(b'foo'), 'foo')) + + def test_open(self): + remote_file = self.storage._open('foo') + self.assertIsInstance(remote_file, ftp.FTPStorageFile) + + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) + def test_read(self, mock_ftp): + self.storage._start_connection() + self.storage._read('foo') + + @patch('ftplib.FTP', **{'return_value.pwd.side_effect': IOError()}) + def test_read2(self, mock_ftp): + self.storage._start_connection() + with self.assertRaises(ftp.FTPStorageException): + self.storage._read('foo') + + @patch('ftplib.FTP', **{ + 'return_value.pwd.return_value': 'foo', + 'return_value.storbinary.return_value': None + }) + def test_save(self, mock_ftp): + self.storage._save('foo', File(BytesIO(b'foo'), 'foo')) + + @patch('ftplib.FTP', **{'return_value.sendcmd.return_value': '213 20160727094506'}) + def test_modified_time(self, mock_ftp): + self.storage._start_connection() + modif_date = self.storage.modified_time('foo') + self.assertEqual(modif_date, datetime(2016, 7, 27, 9, 45, 6)) + + @patch('ftplib.FTP', **{'return_value.sendcmd.return_value': '500'}) + def test_modified_time_error(self, mock_ftp): + self.storage._start_connection() + with self.assertRaises(ftp.FTPStorageException): + self.storage.modified_time('foo') + + @patch('ftplib.FTP', **{'return_value.retrlines': list_retrlines}) + def test_listdir(self, mock_retrlines): + dirs, files = self.storage.listdir('/') + self.assertEqual(len(dirs), 1) + self.assertEqual(dirs, ['dir']) + self.assertEqual(len(files), 2) + self.assertEqual(sorted(files), sorted(['fi', 'fi2'])) + + @patch('ftplib.FTP', **{'return_value.retrlines.side_effect': IOError()}) + def test_listdir_error(self, mock_ftp): + with self.assertRaises(ftp.FTPStorageException): + self.storage.listdir('/') + + @patch('ftplib.FTP', **{'return_value.nlst.return_value': ['foo', 'foo2']}) + def test_exists(self, mock_ftp): + self.assertTrue(self.storage.exists('foo')) + self.assertFalse(self.storage.exists('bar')) + + @patch('ftplib.FTP', **{'return_value.nlst.side_effect': IOError()}) + def test_exists_error(self, mock_ftp): + with self.assertRaises(ftp.FTPStorageException): + self.storage.exists('foo') + + @patch('ftplib.FTP', **{ + 'return_value.delete.return_value': None, + 'return_value.nlst.return_value': ['foo', 'foo2'] + }) + def test_delete(self, mock_ftp): + self.storage.delete('foo') + self.assertTrue(mock_ftp.return_value.delete.called) + + @patch('ftplib.FTP', **{'return_value.retrlines': list_retrlines}) + def test_size(self, mock_ftp): + self.assertEqual(1024, self.storage.size('fi')) + self.assertEqual(2048, self.storage.size('fi2')) + self.assertEqual(0, self.storage.size('bar')) + + @patch('ftplib.FTP', **{'return_value.retrlines.side_effect': IOError()}) + def test_size_error(self, mock_ftp): + self.assertEqual(0, self.storage.size('foo')) + + def test_url(self): + with self.assertRaises(ValueError): + self.storage._base_url = None + self.storage.url('foo') + self.storage = ftp.FTPStorage(location=URL, base_url='http://foo.bar/') + self.assertEqual('http://foo.bar/foo', self.storage.url('foo')) + + +class FTPStorageFileTest(TestCase): + def setUp(self): + self.storage = ftp.FTPStorage(location=URL) + + @patch('ftplib.FTP', **{'return_value.retrlines': list_retrlines}) + def test_size(self, mock_ftp): + file_ = ftp.FTPStorageFile('fi', self.storage, 'wb') + self.assertEqual(file_.size, 1024) + + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) + @patch('storages.backends.ftp.FTPStorage._read', return_value=BytesIO(b'foo')) + def test_readlines(self, mock_ftp, mock_storage): + file_ = ftp.FTPStorageFile('fi', self.storage, 'wb') + self.assertEqual([b'foo'], file_.readlines()) + + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) + @patch('storages.backends.ftp.FTPStorage._read', return_value=BytesIO(b'foo')) + def test_read(self, mock_ftp, mock_storage): + file_ = ftp.FTPStorageFile('fi', self.storage, 'wb') + self.assertEqual(b'foo', file_.read()) + + def test_write(self): + file_ = ftp.FTPStorageFile('fi', self.storage, 'wb') + file_.write(b'foo') + file_.seek(0) + self.assertEqual(file_.file.read(), b'foo') + + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) + @patch('storages.backends.ftp.FTPStorage._read', return_value=BytesIO(b'foo')) + def test_close(self, mock_ftp, mock_storage): + file_ = ftp.FTPStorageFile('fi', self.storage, 'wb') + file_.is_dirty = True + file_.read() + file_.close() diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py new file mode 100644 index 000000000..e1c4cb603 --- /dev/null +++ b/tests/test_gcloud.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +try: + from unittest import mock +except ImportError: # Python 3.2 and below + import mock + +import datetime +import mimetypes + +from django.core.files.base import ContentFile +from django.test import TestCase +from django.utils import timezone +from google.cloud.exceptions import NotFound +from google.cloud.storage.blob import Blob + +from storages.backends import gcloud + + +class GCloudTestCase(TestCase): + def setUp(self): + self.bucket_name = 'test_bucket' + self.filename = 'test_file.txt' + + self.storage = gcloud.GoogleCloudStorage(bucket_name=self.bucket_name) + + self.client_patcher = mock.patch('storages.backends.gcloud.Client') + self.client_patcher.start() + + def tearDown(self): + self.client_patcher.stop() + + +class GCloudStorageTests(GCloudTestCase): + + def test_open_read(self): + """ + Test opening a file and reading from it + """ + data = b'This is some test read data.' + + f = self.storage.open(self.filename) + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + f.blob.download_to_file = lambda tmpfile: tmpfile.write(data) + self.assertEqual(f.read(), data) + + def test_open_read_num_bytes(self): + data = b'This is some test read data.' + num_bytes = 10 + + f = self.storage.open(self.filename) + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + f.blob.download_to_file = lambda tmpfile: tmpfile.write(data) + self.assertEqual(f.read(num_bytes), data[0:num_bytes]) + + def test_open_read_nonexistent(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(IOError, self.storage.open, self.filename) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_open_read_nonexistent_unicode(self): + filename = 'ủⓝï℅ⅆℇ.txt' + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(IOError, self.storage.open, filename) + + @mock.patch('storages.backends.gcloud.Blob') + def test_open_write(self, MockBlob): + """ + Test opening a file and writing to it + """ + data = 'This is some test write data.' + + # Simulate the file not existing before the write + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + f = self.storage.open(self.filename, 'wb') + MockBlob.assert_called_with(self.filename, self.storage._bucket) + + f.write(data) + tmpfile = f._file + # File data is not actually written until close(), so do that. + f.close() + + MockBlob().upload_from_file.assert_called_with( + tmpfile, content_type=mimetypes.guess_type(self.filename)[0]) + + def test_save(self): + data = 'This is some test content.' + content = ContentFile(data) + + self.storage.save(self.filename, content) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob().upload_from_file.assert_called_with( + content, size=len(data), content_type=mimetypes.guess_type(self.filename)[0]) + + def test_save2(self): + data = 'This is some test ủⓝï℅ⅆℇ content.' + filename = 'ủⓝï℅ⅆℇ.txt' + content = ContentFile(data) + + self.storage.save(filename, content) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob().upload_from_file.assert_called_with( + content, size=len(data), content_type=mimetypes.guess_type(filename)[0]) + + def test_delete(self): + self.storage.delete(self.filename) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.delete_blob.assert_called_with(self.filename) + + def test_exists(self): + self.storage._bucket = mock.MagicMock() + self.assertTrue(self.storage.exists(self.filename)) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + self.storage._bucket.reset_mock() + self.storage._bucket.get_blob.return_value = None + self.assertFalse(self.storage.exists(self.filename)) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_exists_no_bucket(self): + # exists('') should return False if the bucket doesn't exist + self.storage._client = mock.MagicMock() + self.storage._client.get_bucket.side_effect = NotFound('dang') + self.assertFalse(self.storage.exists('')) + + def test_exists_bucket(self): + # exists('') should return True if the bucket exists + self.assertTrue(self.storage.exists('')) + + def test_exists_bucket_auto_create(self): + # exists('') should automatically create the bucket if + # auto_create_bucket is configured + self.storage.auto_create_bucket = True + self.storage._client = mock.MagicMock() + self.storage._client.get_bucket.side_effect = NotFound('dang') + + self.assertTrue(self.storage.exists('')) + self.storage._client.create_bucket.assert_called_with(self.bucket_name) + + def test_listdir(self): + file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"] + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.list_blobs.return_value = [] + for name in file_names: + blob = mock.MagicMock(spec=Blob) + blob.name = name + self.storage._bucket.list_blobs.return_value.append(blob) + + dirs, files = self.storage.listdir('') + + self.assertEqual(len(dirs), 2) + for directory in ["some", "other"]: + self.assertTrue(directory in dirs, + """ "%s" not in directory list "%s".""" % ( + directory, dirs)) + + self.assertEqual(len(files), 2) + for filename in ["2.txt", "4.txt"]: + self.assertTrue(filename in files, + """ "%s" not in file list "%s".""" % ( + filename, files)) + + def test_listdir_subdir(self): + file_names = ["some/path/1.txt", "some/2.txt"] + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.list_blobs.return_value = [] + for name in file_names: + blob = mock.MagicMock(spec=Blob) + blob.name = name + self.storage._bucket.list_blobs.return_value.append(blob) + + dirs, files = self.storage.listdir('some/') + + self.assertEqual(len(dirs), 1) + self.assertTrue('path' in dirs, + """ "path" not in directory list "%s".""" % (dirs,)) + + self.assertEqual(len(files), 1) + self.assertTrue('2.txt' in files, + """ "2.txt" not in files list "%s".""" % (files,)) + + def test_size(self): + size = 1234 + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.size = size + self.storage._bucket.get_blob.return_value = blob + + self.assertEqual(self.storage.size(self.filename), size) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_size_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.size, self.filename) + + def test_modified_time(self): + naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678) + aware_date = timezone.make_aware(naive_date, timezone.utc) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.updated = aware_date + self.storage._bucket.get_blob.return_value = blob + + with self.settings(TIME_ZONE='UTC'): + mt = self.storage.modified_time(self.filename) + self.assertTrue(timezone.is_naive(mt)) + self.assertEqual(mt, naive_date) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_get_modified_time(self): + naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678) + aware_date = timezone.make_aware(naive_date, timezone.utc) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.updated = aware_date + self.storage._bucket.get_blob.return_value = blob + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=False): + mt = self.storage.get_modified_time(self.filename) + self.assertTrue(timezone.is_naive(mt)) + naive_date_montreal = timezone.make_naive(aware_date) + self.assertEqual(mt, naive_date_montreal) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=True): + mt = self.storage.get_modified_time(self.filename) + self.assertTrue(timezone.is_aware(mt)) + self.assertEqual(mt, aware_date) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_modified_time_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.modified_time, self.filename) + + def test_url(self): + url = 'https://example.com/mah-bukkit/{}'.format(self.filename) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.public_url = url + self.storage._bucket.get_blob.return_value = blob + + self.assertEqual(self.storage.url(self.filename), url) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_url_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.url, self.filename) + + def test_get_available_name(self): + self.storage.file_overwrite = True + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + self.storage.file_overwrite = False + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_get_available_name_unicode(self): + filename = 'ủⓝï℅ⅆℇ.txt' + self.assertEqual(self.storage.get_available_name(filename), filename) diff --git a/tests/test_gs.py b/tests/test_gs.py index 814fc3391..48ad71e78 100644 --- a/tests/test_gs.py +++ b/tests/test_gs.py @@ -1,5 +1,5 @@ -from django.test import TestCase from django.core.files.base import ContentFile +from django.test import TestCase from storages.backends import gs, s3boto diff --git a/tests/test_hashpath.py b/tests/test_hashpath.py deleted file mode 100644 index 5cc4d6571..000000000 --- a/tests/test_hashpath.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil - -from django.test import TestCase -from django.core.files.base import ContentFile -from django.conf import settings - -from storages.backends.hashpath import HashPathStorage - -TEST_PATH_PREFIX = 'django-storages-test' - - -class HashPathStorageTest(TestCase): - - def setUp(self): - self.test_path = os.path.join(settings.MEDIA_ROOT, TEST_PATH_PREFIX) - self.storage = HashPathStorage(location=self.test_path) - - # make sure the profile upload folder exists - if not os.path.exists(self.test_path): - os.makedirs(self.test_path) - - def tearDown(self): - # remove uploaded profile picture - if os.path.exists(self.test_path): - shutil.rmtree(self.test_path) - - def test_save_same_file(self): - """ - saves a file twice, the file should only be stored once, because the - content/hash is the same - """ - path_1 = self.storage.save('test', ContentFile('new content')) - path_2 = self.storage.save('test', ContentFile('new content')) - self.assertEqual(path_1, path_2) diff --git a/tests/test_s3boto.py b/tests/test_s3boto.py index f0739df19..b6aae6373 100644 --- a/tests/test_s3boto.py +++ b/tests/test_s3boto.py @@ -1,4 +1,7 @@ -import unittest +import urlparse + +import django + try: from unittest import mock except ImportError: # Python 3.2 and below @@ -6,22 +9,15 @@ import datetime -from django.test import TestCase -from django.core.files.base import ContentFile -import django - from boto.exception import S3ResponseError from boto.s3.key import Key -from boto.utils import parse_ts, ISO8601 +from boto.utils import ISO8601, parse_ts +from django.core.files.base import ContentFile +from django.test import TestCase +from django.utils import timezone as tz, unittest -from storages.compat import urlparse from storages.backends import s3boto -__all__ = ( - 'SafeJoinTest', - 'S3BotoStorageTests', -) - class S3BotoTestCase(TestCase): @mock.patch('storages.backends.s3boto.S3Connection') @@ -30,73 +26,16 @@ def setUp(self, S3Connection): self.storage._connection = mock.MagicMock() -class SafeJoinTest(TestCase): - def test_normal(self): - path = s3boto.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") - self.assertEquals(path, "path/to/somewhere/other/path/to/somewhere") - - def test_with_dot(self): - path = s3boto.safe_join("", "path/./somewhere/../other", "..", - ".", "to/./somewhere") - self.assertEquals(path, "path/to/somewhere") - - def test_base_url(self): - path = s3boto.safe_join("base_url", "path/to/somewhere") - self.assertEquals(path, "base_url/path/to/somewhere") - - def test_base_url_with_slash(self): - path = s3boto.safe_join("base_url/", "path/to/somewhere") - self.assertEquals(path, "base_url/path/to/somewhere") - - def test_suspicious_operation(self): - self.assertRaises(ValueError, - s3boto.safe_join, "base", "../../../../../../../etc/passwd") - - def test_trailing_slash(self): - """ - Test safe_join with paths that end with a trailing slash. - """ - path = s3boto.safe_join("base_url/", "path/to/somewhere/") - self.assertEquals(path, "base_url/path/to/somewhere/") - - def test_trailing_slash_multi(self): - """ - Test safe_join with multiple paths that end with a trailing slash. - """ - path = s3boto.safe_join("base_url/", "path/to/" "somewhere/") - self.assertEquals(path, "base_url/path/to/somewhere/") - - class S3BotoStorageTests(S3BotoTestCase): def test_clean_name(self): """ - Test the base case of _clean_name + Test the base case of _clean_name - more tests are performed in + test_utils """ path = self.storage._clean_name("path/to/somewhere") self.assertEqual(path, "path/to/somewhere") - def test_clean_name_normalize(self): - """ - Test the normalization of _clean_name - """ - path = self.storage._clean_name("path/to/../somewhere") - self.assertEqual(path, "path/somewhere") - - def test_clean_name_trailing_slash(self): - """ - Test the _clean_name when the path has a trailing slash - """ - path = self.storage._clean_name("path/to/somewhere/") - self.assertEqual(path, "path/to/somewhere/") - - def test_clean_name_windows(self): - """ - Test the _clean_name when the path has a trailing slash - """ - path = self.storage._clean_name("path\\to\\somewhere") - self.assertEqual(path, "path/to/somewhere") - def test_storage_url_slashes(self): """ Test URL generation. @@ -153,8 +92,7 @@ def test_storage_save_gzip(self): """ Test saving a file with gzip enabled. """ - if not s3boto.S3BotoStorage.gzip: # Gzip not available. - return + self.storage.gzip = True name = 'test_storage_save.css' content = ContentFile("I should be gzip'd") self.storage.save(name, content) @@ -172,8 +110,6 @@ def test_compress_content_len(self): """ Test that file returned by _compress_content() is readable. """ - if not s3boto.S3BotoStorage.gzip: # Gzip not available. - return content = ContentFile("I should be gzip'd") content = self.storage._compress_content(content) self.assertTrue(len(content.read()) > 0) @@ -214,7 +150,7 @@ def test_storage_open_write(self): file._multipart.upload_part_from_file.assert_called_with( _file, 1, headers=self.storage.headers, ) - file._multipart.complete_upload.assert_called_once() + file._multipart.complete_upload.assert_called_once_with() def test_storage_exists_bucket(self): self.storage._connection.get_bucket.side_effect = S3ResponseError(404, 'No bucket') @@ -224,13 +160,11 @@ def test_storage_exists_bucket(self): self.assertTrue(self.storage.exists('')) def test_storage_exists(self): - key = self.storage.bucket.new_key.return_value - key.exists.return_value = True + self.storage.bucket.get_key.return_value = mock.MagicMock(spec=Key) self.assertTrue(self.storage.exists("file.txt")) def test_storage_exists_false(self): - key = self.storage.bucket.new_key.return_value - key.exists.return_value = False + self.storage.bucket.get_key.return_value = None self.assertFalse(self.storage.exists("file.txt")) def test_storage_delete(self): @@ -290,17 +224,17 @@ def test_storage_url(self): url = 'http://aws.amazon.com/%s' % name self.storage.connection.generate_url.return_value = url - kwargs = dict( - method='GET', - bucket=self.storage.bucket.name, - key=name, - query_auth=self.storage.querystring_auth, - force_http=not self.storage.secure_urls, - headers=None, - response_headers=None, - ) - - self.assertEquals(self.storage.url(name), url) + kwargs = { + 'method': 'GET', + 'bucket': self.storage.bucket.name, + 'key': name, + 'query_auth': self.storage.querystring_auth, + 'force_http': not self.storage.secure_urls, + 'headers': None, + 'response_headers': None, + } + + self.assertEqual(self.storage.url(name), url) self.storage.connection.generate_url.assert_called_with( self.storage.querystring_expire, **kwargs @@ -308,7 +242,7 @@ def test_storage_url(self): custom_expire = 123 - self.assertEquals(self.storage.url(name, expire=custom_expire), url) + self.assertEqual(self.storage.url(name, expire=custom_expire), url) self.storage.connection.generate_url.assert_called_with( custom_expire, **kwargs @@ -327,7 +261,7 @@ def test_new_file_modified_time(self): name = 'test_storage_save.txt' content = ContentFile('new content') utcnow = datetime.datetime.utcnow() - with mock.patch('storages.backends.s3boto.datetime') as mock_datetime: + with mock.patch('storages.backends.s3boto.datetime') as mock_datetime, self.settings(TIME_ZONE='UTC'): mock_datetime.utcnow.return_value = utcnow self.storage.save(name, content) self.assertEqual(self.storage.modified_time(name), @@ -352,3 +286,26 @@ def test_get_extra_headers(self): self.assertDictEqual(result, extra_headers[0][1]) result = self.storage.get_extra_headers("storage/not/matching/file") self.assertDictEqual(result, {}) + + @mock.patch('storages.backends.s3boto.S3BotoStorage._get_key') + def test_get_modified_time(self, getkey): + utcnow = datetime.datetime.utcnow().strftime(ISO8601) + + with self.settings(USE_TZ=True, TIME_ZONE='America/New_York'): + key = mock.MagicMock(spec=Key) + key.last_modified = utcnow + getkey.return_value = key + modtime = self.storage.get_modified_time('foo') + self.assertFalse(tz.is_naive(modtime)) + self.assertEqual(modtime, + tz.make_aware(datetime.datetime.strptime(utcnow, ISO8601), tz.utc)) + + with self.settings(USE_TZ=False, TIME_ZONE='America/New_York'): + key = mock.MagicMock(spec=Key) + key.last_modified = utcnow + getkey.return_value = key + modtime = self.storage.get_modified_time('foo') + self.assertTrue(tz.is_naive(modtime)) + self.assertEqual(modtime, + tz.make_naive(tz.make_aware( + datetime.datetime.strptime(utcnow, ISO8601), tz.utc))) diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py new file mode 100644 index 000000000..ef1a263e3 --- /dev/null +++ b/tests/test_s3boto3.py @@ -0,0 +1,389 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import gzip +import threading +from datetime import datetime +from unittest import skipIf + +from botocore.exceptions import ClientError +from django.conf import settings +from django.core.files.base import ContentFile +from django.test import TestCase +from django.utils.six.moves.urllib import parse as urlparse +from django.utils.timezone import is_aware, utc + +from storages.backends import s3boto3 + +try: + from unittest import mock +except ImportError: # Python 3.2 and below + import mock + + +class S3Boto3TestCase(TestCase): + def setUp(self): + self.storage = s3boto3.S3Boto3Storage() + self.storage._connections.connection = mock.MagicMock() + + +class S3Boto3StorageTests(S3Boto3TestCase): + + def test_clean_name(self): + """ + Test the base case of _clean_name + """ + path = self.storage._clean_name("path/to/somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_clean_name_normalize(self): + """ + Test the normalization of _clean_name + """ + path = self.storage._clean_name("path/to/../somewhere") + self.assertEqual(path, "path/somewhere") + + def test_clean_name_trailing_slash(self): + """ + Test the _clean_name when the path has a trailing slash + """ + path = self.storage._clean_name("path/to/somewhere/") + self.assertEqual(path, "path/to/somewhere/") + + def test_clean_name_windows(self): + """ + Test the _clean_name when the path has a trailing slash + """ + path = self.storage._clean_name("path\\to\\somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_storage_url_slashes(self): + """ + Test URL generation. + """ + self.storage.custom_domain = 'example.com' + + # We expect no leading slashes in the path, + # and trailing slashes should be preserved. + self.assertEqual(self.storage.url(''), 'https://example.com/') + self.assertEqual(self.storage.url('path'), 'https://example.com/path') + self.assertEqual(self.storage.url('path/'), 'https://example.com/path/') + self.assertEqual(self.storage.url('path/1'), 'https://example.com/path/1') + self.assertEqual(self.storage.url('path/1/'), 'https://example.com/path/1/') + + def test_storage_save(self): + """ + Test saving a file + """ + name = 'test_storage_save.txt' + content = ContentFile('new content') + self.storage.save(name, content) + self.storage.bucket.Object.assert_called_once_with(name) + + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + content.file, + ExtraArgs={ + 'ContentType': 'text/plain', + 'ACL': self.storage.default_acl, + } + ) + + def test_storage_save_gzipped(self): + """ + Test saving a gzipped file + """ + name = 'test_storage_save.gz' + content = ContentFile("I am gzip'd") + self.storage.save(name, content) + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + content.file, + ExtraArgs={ + 'ContentType': 'application/octet-stream', + 'ContentEncoding': 'gzip', + 'ACL': self.storage.default_acl, + } + ) + + def test_storage_save_gzip(self): + """ + Test saving a file with gzip enabled. + """ + self.storage.gzip = True + name = 'test_storage_save.css' + content = ContentFile("I should be gzip'd") + self.storage.save(name, content) + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + mock.ANY, + ExtraArgs={ + 'ContentType': 'text/css', + 'ContentEncoding': 'gzip', + 'ACL': self.storage.default_acl, + } + ) + args, kwargs = obj.upload_fileobj.call_args + content = args[0] + zfile = gzip.GzipFile(mode='rb', fileobj=content) + self.assertEqual(zfile.read(), b"I should be gzip'd") + + def test_storage_save_gzip_twice(self): + """ + Test saving the same file content twice with gzip enabled. + """ + # Given + self.storage.gzip = True + name = 'test_storage_save.css' + content = ContentFile("I should be gzip'd") + + # When + self.storage.save(name, content) + self.storage.save('test_storage_save_2.css', content) + + # Then + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + mock.ANY, + ExtraArgs={ + 'ContentType': 'text/css', + 'ContentEncoding': 'gzip', + 'ACL': self.storage.default_acl, + } + ) + args, kwargs = obj.upload_fileobj.call_args + content = args[0] + zfile = gzip.GzipFile(mode='rb', fileobj=content) + self.assertEqual(zfile.read(), b"I should be gzip'd") + + def test_compress_content_len(self): + """ + Test that file returned by _compress_content() is readable. + """ + self.storage.gzip = True + content = ContentFile("I should be gzip'd") + content = self.storage._compress_content(content) + self.assertTrue(len(content.read()) > 0) + + def test_storage_open_write(self): + """ + Test opening a file in write mode + """ + name = 'test_open_for_writïng.txt' + content = 'new content' + + # Set the encryption flag used for multipart uploads + self.storage.encryption = True + self.storage.reduced_redundancy = True + self.storage.default_acl = 'public-read' + + file = self.storage.open(name, 'w') + self.storage.bucket.Object.assert_called_with(name) + obj = self.storage.bucket.Object.return_value + # Set the name of the mock object + obj.key = name + + file.write(content) + obj.initiate_multipart_upload.assert_called_with( + ACL='public-read', + ContentType='text/plain', + ServerSideEncryption='AES256', + StorageClass='REDUCED_REDUNDANCY' + ) + + # Save the internal file before closing + multipart = obj.initiate_multipart_upload.return_value + multipart.parts.all.return_value = [mock.MagicMock(e_tag='123', part_number=1)] + file.close() + multipart.Part.assert_called_with(1) + part = multipart.Part.return_value + part.upload.assert_called_with(Body=content.encode('utf-8')) + multipart.complete.assert_called_once_with( + MultipartUpload={'Parts': [{'ETag': '123', 'PartNumber': 1}]}) + + def test_auto_creating_bucket(self): + self.storage.auto_create_bucket = True + Bucket = mock.MagicMock() + self.storage._connections.connection.Bucket.return_value = Bucket + self.storage._connections.connection.meta.client.meta.region_name = 'sa-east-1' + + Bucket.meta.client.head_bucket.side_effect = ClientError({'Error': {}, + 'ResponseMetadata': {'HTTPStatusCode': 404}}, + 'head_bucket') + self.storage._get_or_create_bucket('testbucketname') + Bucket.create.assert_called_once_with( + ACL='public-read', + CreateBucketConfiguration={ + 'LocationConstraint': 'sa-east-1', + } + ) + + def test_storage_exists(self): + self.assertTrue(self.storage.exists("file.txt")) + self.storage.connection.meta.client.head_object.assert_called_with( + Bucket=self.storage.bucket_name, + Key="file.txt", + ) + + def test_storage_exists_false(self): + self.storage.connection.meta.client.head_object.side_effect = ClientError( + {'Error': {'Code': '404', 'Message': 'Not Found'}}, + 'HeadObject', + ) + self.assertFalse(self.storage.exists("file.txt")) + self.storage.connection.meta.client.head_object.assert_called_with( + Bucket=self.storage.bucket_name, + Key='file.txt', + ) + + def test_storage_exists_doesnt_create_bucket(self): + with mock.patch.object(self.storage, '_get_or_create_bucket') as method: + self.storage.exists('file.txt') + method.assert_not_called() + + def test_storage_delete(self): + self.storage.delete("path/to/file.txt") + self.storage.bucket.Object.assert_called_with('path/to/file.txt') + self.storage.bucket.Object.return_value.delete.assert_called_with() + + def test_storage_listdir_base(self): + file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"] + + result = [] + for p in file_names: + obj = mock.MagicMock() + obj.key = p + result.append(obj) + self.storage.bucket.objects.filter.return_value = iter(result) + + dirs, files = self.storage.listdir("") + self.storage.bucket.objects.filter.assert_called_with(Prefix="") + + self.assertEqual(len(dirs), 2) + for directory in ["some", "other"]: + self.assertTrue(directory in dirs, + """ "%s" not in directory list "%s".""" % ( + directory, dirs)) + + self.assertEqual(len(files), 2) + for filename in ["2.txt", "4.txt"]: + self.assertTrue(filename in files, + """ "%s" not in file list "%s".""" % ( + filename, files)) + + def test_storage_listdir_subdir(self): + file_names = ["some/path/1.txt", "some/2.txt"] + + result = [] + for p in file_names: + obj = mock.MagicMock() + obj.key = p + result.append(obj) + self.storage.bucket.objects.filter.return_value = iter(result) + + dirs, files = self.storage.listdir("some/") + self.storage.bucket.objects.filter.assert_called_with(Prefix="some/") + + self.assertEqual(len(dirs), 1) + self.assertTrue('path' in dirs, + """ "path" not in directory list "%s".""" % (dirs,)) + + self.assertEqual(len(files), 1) + self.assertTrue('2.txt' in files, + """ "2.txt" not in files list "%s".""" % (files,)) + + def test_storage_size(self): + obj = self.storage.bucket.Object.return_value + obj.content_length = 4098 + + name = 'file.txt' + self.assertEqual(self.storage.size(name), obj.content_length) + + def test_storage_mtime(self): + # Test both USE_TZ cases + for use_tz in (True, False): + with self.settings(USE_TZ=use_tz): + self._test_storage_mtime(use_tz) + + def _test_storage_mtime(self, use_tz): + obj = self.storage.bucket.Object.return_value + obj.last_modified = datetime.now(utc) + + name = 'file.txt' + self.assertFalse( + is_aware(self.storage.modified_time(name)), + 'Naive datetime object expected from modified_time()' + ) + + self.assertIs( + settings.USE_TZ, + is_aware(self.storage.get_modified_time(name)), + '%s datetime object expected from get_modified_time() when USE_TZ=%s' % ( + ('Naive', 'Aware')[settings.USE_TZ], + settings.USE_TZ + ) + ) + + def test_storage_url(self): + name = 'test_storage_size.txt' + url = 'http://aws.amazon.com/%s' % name + self.storage.bucket.meta.client.generate_presigned_url.return_value = url + self.storage.bucket.name = 'bucket' + self.assertEqual(self.storage.url(name), url) + self.storage.bucket.meta.client.generate_presigned_url.assert_called_with( + 'get_object', + Params={'Bucket': self.storage.bucket.name, 'Key': name}, + ExpiresIn=self.storage.querystring_expire + ) + + custom_expire = 123 + + self.assertEqual(self.storage.url(name, expire=custom_expire), url) + self.storage.bucket.meta.client.generate_presigned_url.assert_called_with( + 'get_object', + Params={'Bucket': self.storage.bucket.name, 'Key': name}, + ExpiresIn=custom_expire + ) + + def test_generated_url_is_encoded(self): + self.storage.custom_domain = "mock.cloudfront.net" + filename = "whacky & filename.mp4" + url = self.storage.url(filename) + parsed_url = urlparse.urlparse(url) + self.assertEqual(parsed_url.path, + "/whacky%20%26%20filename.mp4") + self.assertFalse(self.storage.bucket.meta.client.generate_presigned_url.called) + + def test_special_characters(self): + self.storage.custom_domain = "mock.cloudfront.net" + + name = "ãlöhâ.jpg" + content = ContentFile('new content') + self.storage.save(name, content) + self.storage.bucket.Object.assert_called_once_with(name) + + url = self.storage.url(name) + parsed_url = urlparse.urlparse(url) + self.assertEqual(parsed_url.path, "/%C3%A3l%C3%B6h%C3%A2.jpg") + + def test_strip_signing_parameters(self): + expected = 'http://bucket.s3-aws-region.amazonaws.com/foo/bar' + self.assertEqual(self.storage._strip_signing_parameters( + '%s?X-Amz-Date=12345678&X-Amz-Signature=Signature' % expected), expected) + self.assertEqual(self.storage._strip_signing_parameters( + '%s?expires=12345678&signature=Signature' % expected), expected) + + @skipIf(threading is None, 'Test requires threading') + def test_connection_threading(self): + connections = [] + + def thread_storage_connection(): + connections.append(self.storage.connection) + + for x in range(2): + t = threading.Thread(target=thread_storage_connection) + t.start() + t.join() + + # Connection for each thread needs to be unique + self.assertIsNot(connections[0], connections[1]) diff --git a/tests/test_sftp.py b/tests/test_sftp.py new file mode 100644 index 000000000..754e98703 --- /dev/null +++ b/tests/test_sftp.py @@ -0,0 +1,163 @@ +import os +import stat +from datetime import datetime + +from django.core.files.base import File +from django.test import TestCase +from django.utils.six import BytesIO + +from storages.backends import sftpstorage + +try: + from unittest.mock import patch, MagicMock +except ImportError: # Python 3.2 and below + from mock import patch, MagicMock + + +class SFTPStorageTest(TestCase): + def setUp(self): + self.storage = sftpstorage.SFTPStorage('foo') + + def test_init(self): + pass + + @patch('paramiko.SSHClient') + def test_no_known_hosts_file(self, mock_ssh): + self.storage._known_host_file = "not_existed_file" + self.storage._connect() + self.assertEqual('foo', mock_ssh.return_value.connect.call_args[0][0]) + + @patch.object(os.path, "expanduser", return_value="/path/to/known_hosts") + @patch.object(os.path, "exists", return_value=True) + @patch('paramiko.SSHClient') + def test_error_when_known_hosts_file_not_defined(self, mock_ssh, *a): + self.storage._connect() + self.storage._ssh.load_host_keys.assert_called_once_with("/path/to/known_hosts") + + @patch('paramiko.SSHClient') + def test_connect(self, mock_ssh): + self.storage._connect() + self.assertEqual('foo', mock_ssh.return_value.connect.call_args[0][0]) + + def test_open(self): + file_ = self.storage._open('foo') + self.assertIsInstance(file_, sftpstorage.SFTPStorageFile) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_read(self, mock_sftp): + self.storage._read('foo') + self.assertTrue(mock_sftp.open.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_chown(self, mock_sftp): + self.storage._chown('foo', 1, 1) + self.assertEqual(mock_sftp.chown.call_args[0], ('foo', 1, 1)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_mkdir(self, mock_sftp): + self.storage._mkdir('foo') + self.assertEqual(mock_sftp.mkdir.call_args[0], ('foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': (IOError(), True) + }) + def test_mkdir_parent(self, mock_sftp): + self.storage._mkdir('bar/foo') + self.assertEqual(mock_sftp.mkdir.call_args_list[0][0], ('bar',)) + self.assertEqual(mock_sftp.mkdir.call_args_list[1][0], ('bar/foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_save(self, mock_sftp): + self.storage._save('foo', File(BytesIO(b'foo'), 'foo')) + self.assertTrue(mock_sftp.open.return_value.write.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': (IOError(), True) + }) + def test_save_in_subdir(self, mock_sftp): + self.storage._save('bar/foo', File(BytesIO(b'foo'), 'foo')) + self.assertEqual(mock_sftp.mkdir.call_args_list[0][0], ('bar',)) + self.assertTrue(mock_sftp.open.return_value.write.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_delete(self, mock_sftp): + self.storage.delete('foo') + self.assertEqual(mock_sftp.remove.call_args_list[0][0], ('foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_exists(self, mock_sftp): + self.assertTrue(self.storage.exists('foo')) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': IOError() + }) + def test_not_exists(self, mock_sftp): + self.assertFalse(self.storage.exists('foo')) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'listdir_attr.return_value': + [MagicMock(filename='foo', st_mode=stat.S_IFDIR), + MagicMock(filename='bar', st_mode=None)]}) + def test_listdir(self, mock_sftp): + dirs, files = self.storage.listdir('/') + self.assertTrue(dirs) + self.assertTrue(files) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_size': 42, + }) + def test_size(self, mock_sftp): + self.assertEqual(self.storage.size('foo'), 42) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_atime': 1469674684.000000, + }) + def test_accessed_time(self, mock_sftp): + self.assertEqual(self.storage.accessed_time('foo'), + datetime(2016, 7, 27, 21, 58, 4)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_mtime': 1469674684.000000, + }) + def test_modified_time(self, mock_sftp): + self.assertEqual(self.storage.modified_time('foo'), + datetime(2016, 7, 27, 21, 58, 4)) + + def test_url(self): + self.assertEqual(self.storage.url('foo'), '/media/foo') + # Test custom + self.storage._base_url = 'http://bar.pt/' + self.assertEqual(self.storage.url('foo'), 'http://bar.pt/foo') + # Test error + with self.assertRaises(ValueError): + self.storage._base_url = None + self.storage.url('foo') + + +class SFTPStorageFileTest(TestCase): + def setUp(self): + self.storage = sftpstorage.SFTPStorage('foo') + self.file = sftpstorage.SFTPStorageFile('bar', self.storage, 'wb') + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_size': 42, + }) + def test_size(self, mock_sftp): + self.assertEqual(self.file.size, 42) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'open.return_value.read.return_value': b'foo', + }) + def test_read(self, mock_sftp): + self.assertEqual(self.file.read(), b'foo') + self.assertTrue(mock_sftp.open.called) + + def test_write(self): + self.file.write(b'foo') + self.assertEqual(self.file.file.read(), b'foo') + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_close(self, mock_sftp): + self.file.write(b'foo') + self.file.close() + self.assertTrue(mock_sftp.open.return_value.write.called) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..eb309acd9 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,117 @@ +import datetime + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.test import TestCase + +from storages import utils + + +class SettingTest(TestCase): + def test_get_setting(self): + value = utils.setting('SECRET_KEY') + self.assertEqual(settings.SECRET_KEY, value) + + def test_setting_unfound(self): + self.assertIsNone(utils.setting('FOO')) + self.assertEqual(utils.setting('FOO', 'bar'), 'bar') + with self.assertRaises(ImproperlyConfigured): + utils.setting('FOO', strict=True) + + +class CleanNameTests(TestCase): + def test_clean_name(self): + """ + Test the base case of clean_name + """ + path = utils.clean_name("path/to/somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_clean_name_normalize(self): + """ + Test the normalization of clean_name + """ + path = utils.clean_name("path/to/../somewhere") + self.assertEqual(path, "path/somewhere") + + def test_clean_name_trailing_slash(self): + """ + Test the clean_name when the path has a trailing slash + """ + path = utils.clean_name("path/to/somewhere/") + self.assertEqual(path, "path/to/somewhere/") + + def test_clean_name_windows(self): + """ + Test the clean_name when the path has a trailing slash + """ + path = utils.clean_name("path\\to\\somewhere") + self.assertEqual(path, "path/to/somewhere") + + +class SafeJoinTest(TestCase): + def test_normal(self): + path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") + self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere") + + def test_with_dot(self): + path = utils.safe_join("", "path/./somewhere/../other", "..", + ".", "to/./somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_with_only_dot(self): + path = utils.safe_join("", ".") + self.assertEqual(path, "") + + def test_base_url(self): + path = utils.safe_join("base_url", "path/to/somewhere") + self.assertEqual(path, "base_url/path/to/somewhere") + + def test_base_url_with_slash(self): + path = utils.safe_join("base_url/", "path/to/somewhere") + self.assertEqual(path, "base_url/path/to/somewhere") + + def test_suspicious_operation(self): + with self.assertRaises(ValueError): + utils.safe_join("base", "../../../../../../../etc/passwd") + with self.assertRaises(ValueError): + utils.safe_join("base", "/etc/passwd") + + def test_trailing_slash(self): + """ + Test safe_join with paths that end with a trailing slash. + """ + path = utils.safe_join("base_url/", "path/to/somewhere/") + self.assertEqual(path, "base_url/path/to/somewhere/") + + def test_trailing_slash_multi(self): + """ + Test safe_join with multiple paths that end with a trailing slash. + """ + path = utils.safe_join("base_url/", "path/to/", "somewhere/") + self.assertEqual(path, "base_url/path/to/somewhere/") + + def test_datetime_isoformat(self): + dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456) + path = utils.safe_join('base_url', dt.isoformat()) + self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456') + + def test_join_empty_string(self): + path = utils.safe_join('base_url', '') + self.assertEqual(path, 'base_url/') + + def test_with_base_url_and_dot(self): + path = utils.safe_join('base_url', '.') + self.assertEqual(path, 'base_url/') + + def test_with_base_url_and_dot_and_path_and_slash(self): + path = utils.safe_join('base_url', '.', 'path/to/', '.') + self.assertEqual(path, 'base_url/path/to/') + + def test_join_nothing(self): + path = utils.safe_join('') + self.assertEqual(path, '') + + def test_with_base_url_join_nothing(self): + path = utils.safe_join('base_url') + self.assertEqual(path, 'base_url/') diff --git a/tox.ini b/tox.ini index 9cadef9a4..05dc3d145 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] envlist = - {py27,py33,py34}-django17, - {py27,py33,py34,py35}-django18, - {py27,py34,py35}-django19 + lint + {py27,py33,py34,py35}-django18 + {py27,py34,py35}-django110 + {py27,py34,py35,py36}-django111 [testenv] @@ -11,10 +12,22 @@ setenv = PYTHONDONTWRITEBYTECODE=1 DJANGO_SETTINGS_MODULE=tests.settings deps = - django17: Django>=1.7, <1.8 django18: Django>=1.8, <1.9 - django19: Django>=1.9, <1.10 - py27: mock==1.0.1 + django110: Django>=1.10, <1.11 + django111: Django>=1.11, <2.0 + py27: mock + boto3>=1.2.3 boto>=2.32.0 - pytest-cov==2.2.1 - dropbox>=3.24 + dropbox>=8.0.0 + google-cloud-storage>=0.22.0 + paramiko + pytest-cov>=2.2.1 + + +[testenv:lint] +deps = + flake8 + isort +commands = + flake8 + isort --recursive --check-only --diff storages/ tests/