Add index to history_date field to improve performance.

Opt-out of the index with `SETTINGS_HISTORY_DATE_INDEX=False`.

Allow history_date indexing to be disabled or composite with model pk

Co-authored-by: jeking3 <jim.king@cloudtruth.com>

Author: nex2hex

CHANGES.rst

@@ -4,6 +4,13 @@ Changes
 Unreleased
 ----------
 
+Upgrade Implications:
+
+- Run `makemigrations` after upgrading to realize the benefit of indexing changes.
+
+Full list of changes:
+
+- Added index on `history_date` column; opt-out with setting `SIMPLE_HISTORY_DATE_INDEX` (gh-565)
 - Fixed ``prev_record`` and ``next_record`` performance when using ``excluded_fields`` (gh-791)
 - Fixed `update_change_reason` in pk (gh-806)
 - Fixed bug where serializer of djangorestframework crashed if used with ``OrderingFilter`` (gh-821)

docs/historical_model.rst

@@ -86,6 +86,26 @@ model, will work too.
     my_poll.save()
 
 
+Indexed ``history_date``
+------------------------
+
+Many queries use ``history_date`` as a filter.  The as_of queries combine this with the
+original model's promary key to extract point-in-time snapshots of history.  By default
+the ``history_date`` field is indexed.  You can control this behavior using settings.py.
+
+.. code-block:: python
+
+    # disable indexing on history_date
+    SIMPLE_HISTORY_DATE_INDEX = False
+
+    # enable indexing on history_date (default setting)
+    SIMPLE_HISTORY_DATE_INDEX = True
+
+    # enable composite indexing on history_date and model pk (to improve as_of queries)
+    # the string is case-insensitive
+    SIMPLE_HISTORY_DATE_INDEX = "Composite"
+
+
 Custom history table name
 -------------------------
 

simple_history/models.py

@@ -7,7 +7,7 @@
 from django.conf import settings
 from django.contrib import admin
 from django.contrib.auth import get_user_model
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
 from django.db import models
 from django.db.models import ManyToManyField
 from django.db.models.fields.proxy import OrderWrt
@@ -426,7 +426,7 @@ def get_default_history_user(instance):
 
         extra_fields = {
             "history_id": self._get_history_id_field(),
-            "history_date": models.DateTimeField(),
+            "history_date": models.DateTimeField(db_index=self._date_indexing is True),
             "history_change_reason": self._get_history_change_reason_field(),
             "history_type": models.CharField(
                 max_length=1,
@@ -451,6 +451,23 @@ def get_default_history_user(instance):
 
         return extra_fields
 
+    @property
+    def _date_indexing(self):
+        """False, True, or 'composite'; default is True"""
+        result = getattr(settings, "SIMPLE_HISTORY_DATE_INDEX", True)
+        valid = True
+        if isinstance(result, str):
+            result = result.lower()
+            if result not in ("composite",):
+                valid = False
+        elif not isinstance(result, bool):
+            valid = False
+        if not valid:
+            raise ImproperlyConfigured(
+                "SIMPLE_HISTORY_DATE_INDEX must be one of (False, True, 'Composite')"
+            )
+        return result
+
     def get_meta_options(self, model):
         """
         Returns a dictionary of fields that will be added to
@@ -467,6 +484,8 @@ def get_meta_options(self, model):
         meta_fields["verbose_name"] = name
         if self.app:
             meta_fields["app_label"] = self.app
+        if self._date_indexing == "composite":
+            meta_fields["index_together"] = (("history_date", model._meta.pk.attname),)
         return meta_fields
 
     def post_save(self, instance, created, using=None, **kwargs):

simple_history/registry_tests/migration_test_app/migrations/0004_history_date_indexing.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.0.dev20210811195242 on 2021-08-13 10:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "migration_test_app",
+            "0003_alter_historicalmodelwithcustomattrforeignkey_options_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="historicalmodelwithcustomattrforeignkey",
+            name="history_date",
+            field=models.DateTimeField(db_index=True),
+        ),
+        migrations.AlterField(
+            model_name="historicalyar",
+            name="history_date",
+            field=models.DateTimeField(db_index=True),
+        ),
+    ]

simple_history/tests/tests/test_index.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+from django.db import models
+from django.test import TestCase, override_settings
+
+from simple_history.models import HistoricalRecords
+
+
+@override_settings(SIMPLE_HISTORY_DATE_INDEX="Composite")
+class HistoricalIndexTest(TestCase):
+    def test_has_composite_index(self):
+        self.assertEqual(settings.SIMPLE_HISTORY_DATE_INDEX, "Composite")
+
+        class Foo(models.Model):
+            history = HistoricalRecords()
+
+        self.assertEqual(
+            ("history_date", "id"), Foo.history.model._meta.index_together[0]
+        )

simple_history/tests/tests/test_models.py

@@ -5,8 +5,9 @@
 
 import django
 from django.apps import apps
+from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
 from django.core.files.base import ContentFile
 from django.db import IntegrityError, models
 from django.db.models.fields.proxy import OrderWrt
@@ -1038,6 +1039,25 @@ def test_most_recent_nonexistant(self):
         poll.delete()
         self.assertRaises(Poll.DoesNotExist, poll.history.most_recent)
 
+    def test_date_indexing_options(self):
+        records = HistoricalRecords()
+        delattr(settings, "SIMPLE_HISTORY_DATE_INDEX")
+        self.assertTrue(records._date_indexing)
+        settings.SIMPLE_HISTORY_DATE_INDEX = False
+        self.assertFalse(records._date_indexing)
+        settings.SIMPLE_HISTORY_DATE_INDEX = "Composite"
+        self.assertEqual(records._date_indexing, "composite")
+        settings.SIMPLE_HISTORY_DATE_INDEX = "foo"
+        with self.assertRaises(ImproperlyConfigured):
+            records._date_indexing
+        settings.SIMPLE_HISTORY_DATE_INDEX = 42
+        with self.assertRaises(ImproperlyConfigured):
+            records._date_indexing
+        settings.SIMPLE_HISTORY_DATE_INDEX = None
+        with self.assertRaises(ImproperlyConfigured):
+            records._date_indexing
+        delattr(settings, "SIMPLE_HISTORY_DATE_INDEX")
+
     def test_as_of(self):
         poll = Poll.objects.create(question="what's up?", pub_date=today)
         poll.question = "how's it going?"