Accept string based primary keys

install_xapian.sh

@@ -3,7 +3,7 @@
 VERSION=$1
 
 # prepare
-mkdir $VIRTUAL_ENV/packages && cd $VIRTUAL_ENV/packages
+mkdir -p $VIRTUAL_ENV/packages && cd $VIRTUAL_ENV/packages
 
 CORE=xapian-core-$VERSION
 BINDINGS=xapian-bindings-$VERSION
@@ -15,8 +15,19 @@ curl -O https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz
 
 # extract
 echo "Extracting source..."
-tar xf ${CORE}.tar.xz
-tar xf ${BINDINGS}.tar.xz
+if [ $OSTYPE = 'darwin*' ]; then
+    xz -d ${CORE}.tar.xz
+    xz -d ${BINDINGS}.tar.xz
+
+    tar xf ${CORE}.tar
+    tar xf ${BINDINGS}.tar
+else
+    tar xf ${CORE}.tar.xz
+    tar xf ${BINDINGS}.tar.xz
+fi
+
+test -e $VIRTUAL_ENV/packages/${BINDINGS} || exit 1
+test -e $VIRTUAL_ENV/packages/${CORE} || exit 1
 
 # install
 echo "Installing Xapian-core..."

tests/xapian_tests/models.py

@@ -15,7 +15,7 @@ class Document(models.Model):
     text = models.TextField()
 
 
-class BlogEntry(models.Model):
+class AbstractBlogEntry(models.Model):
     """
     Same as tests.core.MockModel with a few extra fields for testing various
     sorting and ordering criteria.
@@ -36,6 +36,17 @@ class BlogEntry(models.Model):
     float_number = models.FloatField()
     decimal_number = models.DecimalField(max_digits=4, decimal_places=2)
 
+    class Meta:
+        abstract = True
+
+
+class BlogEntry(AbstractBlogEntry):
+    pass
+
+
+class UUIDModel(models.Model):
+    uuid = models.CharField(primary_key=True, max_length=20)
+
 
 class DjangoContentType(models.Model):
     content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)

tests/xapian_tests/search_indexes.py

@@ -82,6 +82,13 @@ def prepare_empty(self, obj):
         return ''
 
 
+class UUIDModelSearchIndex(indexes.SearchIndex):
+    text = indexes.CharField(document=True)
+
+    def get_model(self):
+        return models.UUIDModel
+
+
 class CompleteBlogEntryIndex(indexes.SearchIndex):
     text = indexes.CharField(model_attr='text', document=True)
     author = indexes.CharField(model_attr='author')

tests/xapian_tests/tests/test_backend.py

@@ -18,8 +18,8 @@
 from haystack.utils.loading import UnifiedIndex
 
 from ..search_indexes import XapianNGramIndex, XapianEdgeNGramIndex, \
-    CompleteBlogEntryIndex, BlogSearchIndex, DjangoContentTypeIndex
-from ..models import BlogEntry, AnotherMockModel, MockTag, DjangoContentType
+    CompleteBlogEntryIndex, BlogSearchIndex, DjangoContentTypeIndex, UUIDModelSearchIndex
+from ..models import BlogEntry, AnotherMockModel, MockTag, DjangoContentType, UUIDModel
 
 
 XAPIAN_VERSION = [int(x) for x in xapian.__version__.split('.')]
@@ -682,6 +682,54 @@ def test_more_like_this_with_unindexed_model(self):
         self.assertRaises(InvalidIndexError, self.backend.more_like_this, mock)
 
 
+class StringBasedPKModelBackendFeaturesTestCase(HaystackBackendTestCase, TestCase):
+    """
+    Covers #138, Must not assume django_id is an int
+    """
+
+    def get_index(self):
+        return UUIDModelSearchIndex()
+
+    @staticmethod
+    def get_entry(i):
+        entry = UUIDModel(uuid='uuid-%s' % i)
+        return entry
+
+    def setUp(self):
+        super(StringBasedPKModelBackendFeaturesTestCase, self).setUp()
+
+        for i in range(1, 4):
+            entry = self.get_entry(i)
+            entry.save()
+
+        self.backend.update(self.index, UUIDModel.objects.all())
+
+    def test_update(self):
+        self.assertEqual(pks(self.backend.search(xapian.Query(''))['results']),
+                         ['uuid-1', 'uuid-2', 'uuid-3'])
+
+    def test_order_by_django_id(self):
+        """
+        We need this test because ordering on more than
+        10 entries was not correct at some point.
+        """
+        sample_objs = []
+        pk_list = []
+        for i in range(101, 200):
+            entry = self.get_entry(i)
+            sample_objs.append(entry)
+            pk_list.append('uuid-%03d' % i)
+
+        for obj in sample_objs:
+            obj.save()
+
+        self.backend.clear()
+        self.backend.update(self.index, sample_objs)
+
+        results = self.backend.search(xapian.Query(''), sort_by=['-django_id'])
+        self.assertEqual(pks(results['results']), list(reversed(pk_list)))
+
+
 class IndexationNGramTestCase(HaystackBackendTestCase, TestCase):
     def get_index(self):
         return XapianNGramIndex()

tests/xapian_tests/tests/test_interface.py

@@ -9,8 +9,8 @@
 from haystack.inputs import AutoQuery
 from haystack.query import SearchQuerySet
 
-from ..models import Document
-from ..search_indexes import DocumentIndex
+from ..models import Document, UUIDModel
+from ..search_indexes import DocumentIndex, UUIDModelSearchIndex
 from ..tests.test_backend import pks
 
 
@@ -203,3 +203,29 @@ def test_multi_values_exact_search(self):
         self.assertEqual(len(self.queryset.filter(tags__exact='tag')), 12)
         self.assertEqual(len(self.queryset.filter(tags__exact='tag-test')), 8)
         self.assertEqual(len(self.queryset.filter(tags__exact='tag-test-test')), 4)
+
+
+class UUIDInterfaceTestCase(TestCase):
+    def setUp(self):
+        super(UUIDInterfaceTestCase, self).setUp()
+
+        for i in range(1, 3):
+            doc = UUIDModel(uuid='uuid-%d' % i)
+            doc.save()
+
+        self.index = UUIDModelSearchIndex()
+        self.ui = connections['default'].get_unified_index()
+        self.ui.build(indexes=[self.index])
+
+        self.backend = connections['default'].get_backend()
+        self.backend.update(self.index, UUIDModel.objects.all())
+
+        self.queryset = SearchQuerySet()
+
+    def tearDown(self):
+        UUIDModel.objects.all().delete()
+        super(UUIDInterfaceTestCase, self).tearDown()
+
+    def test_search(self):
+        self.assertEqual(pks(self.queryset.filter(django_id='uuid-2')),
+                         pks(UUIDModel.objects.filter(uuid='uuid-2')))

tests/xapian_tests/tests/test_query.py

@@ -335,6 +335,10 @@ def test_multiple_filter_types(self):
                                  'zzzzzzzzzzzzzzzzzzzzzzzzz AND'
                                  ' (QQ000000000001 OR QQ000000000002 OR QQ000000000003))')
 
+    def test_filter_string_based_django_pk(self):
+        self.sq.add_filter(SQ(django_id='uuid-1'))
+        self.assertExpectedQuery(self.sq.build_query(), 'QQuuid-1')
+
     def test_log_query(self):
         reset_search_queries()
         self.assertEqual(len(connections['default'].queries), 0)

xapian_backend.py

@@ -285,7 +285,7 @@ def update(self, index, iterable):
             term_generator.set_stemmer(xapian.Stem(self.language))
             try:
                 term_generator.set_stemming_strategy(self.stemming_strategy)
-            except AttributeError:  
+            except AttributeError:
                 # Versions before Xapian 1.2.11 do not support stemming strategies for TermGenerator
                 pass
             if self.include_spelling is True:
@@ -433,7 +433,11 @@ def add_datetime_to_document(termpos, prefix, term, weight):
                         # `django_id` is an int and `django_ct` is text;
                         # besides, they are indexed by their (unstemmed) value.
                         if field['field_name'] == DJANGO_ID:
-                            value = int(value)
+                            try:
+                                value = int(value)
+                            except ValueError:
+                                # Django_id is a string
+                                field['type'] = 'text'
                         value = _term_to_xapian_value(value, field['type'])
 
                         document.add_term(TERM_PREFIXES[field['field_name']] + value, weight)
@@ -1423,7 +1427,7 @@ def _filter_exact(self, term, field_name, field_type, is_not):
 
         Assumes term is not a list.
         """
-        if field_type == 'text' and field_name not in (DJANGO_CT,):
+        if field_type == 'text' and field_name not in (DJANGO_CT, DJANGO_ID) :
             term = '^ %s $' % term
             query = self._phrase_query(term.split(), field_name, field_type)
         else:
@@ -1494,7 +1498,12 @@ def _term_query(self, term, field_name, field_type, stemmed=True):
         if field_name in (ID, DJANGO_ID, DJANGO_CT):
             # to ensure the value is serialized correctly.
             if field_name == DJANGO_ID:
-                term = int(term)
+                try:
+                    term = int(term)
+                except ValueError:
+                    # Django_id is a string
+                    field_type = 'text'
+
             term = _term_to_xapian_value(term, field_type)
             return xapian.Query('%s%s' % (TERM_PREFIXES[field_name], term))