diff --git a/cachalot/monkey_patch.py b/cachalot/monkey_patch.py index 37b293255..a87e30cf7 100644 --- a/cachalot/monkey_patch.py +++ b/cachalot/monkey_patch.py @@ -1,4 +1,5 @@ import re +import types from collections.abc import Iterable from functools import wraps from time import time @@ -62,6 +63,10 @@ def _get_result_or_execute_query(execute_query_func, cache, pass result = execute_query_func() + + if result.__class__ == types.GeneratorType and not cachalot_settings.CACHALOT_CACHE_ITERATORS: + return result + if result.__class__ not in ITERABLES and isinstance(result, Iterable): result = list(result) diff --git a/cachalot/settings.py b/cachalot/settings.py index 2bf5c817a..05000b992 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -49,6 +49,7 @@ class Settings(object): CACHALOT_DATABASES = 'supported_only' CACHALOT_TIMEOUT = None CACHALOT_CACHE_RANDOM = False + CACHALOT_CACHE_ITERATORS = False CACHALOT_INVALIDATE_RAW = True CACHALOT_ONLY_CACHABLE_TABLES = () CACHALOT_ONLY_CACHABLE_APPS = () diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index aee07fbab..09ce34e2f 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -244,13 +244,20 @@ def test_distinct(self): self.assert_query_cached(qs, [self.t1]) def test_iterator(self): - with self.assertNumQueries(1): + with self.assertNumQueries(2): data1 = list(Test.objects.iterator()) - with self.assertNumQueries(0): data2 = list(Test.objects.iterator()) self.assertListEqual(data2, data1) self.assertListEqual(data2, [self.t1, self.t2]) + with self.settings(CACHALOT_CACHE_ITERATORS=True): + with self.assertNumQueries(1): + data1 = list(Test.objects.iterator()) + with self.assertNumQueries(0): + data2 = list(Test.objects.iterator()) + self.assertListEqual(data2, data1) + self.assertListEqual(data2, [self.t1, self.t2]) + def test_in_bulk(self): with self.assertNumQueries(1): data1 = Test.objects.in_bulk((5432, self.t2.pk, 9200)) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a86198c03..ac5585d71 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -106,6 +106,20 @@ Settings :Description: If set to ``True``, caches random queries (those with ``order_by('?')``). +``CACHALOT_CACHE_ITERATORS`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Default: ``False`` +:Description: + If set to ``True``, cache results from QuerySets that return + generators. This is useful for caching the result sets of QuerySets that + use ``.iterator()``. + + .. warnings:: + ``.iterator()`` is often used for large result sets. Caching these can use large + amounts of local memory because django-cachalot has to first convert them to a list to + store them in the cache. + .. _CACHALOT_INVALIDATE_RAW: ``CACHALOT_INVALIDATE_RAW``