From 6b54dca3ae3d6c623266dcad677389ca5ca0a3b6 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 3 Apr 2025 12:27:23 -0400 Subject: [PATCH 01/10] feat: add DatabaseStore for persistent debug data storage - Introduced `DatabaseStore` to store debug toolbar data in the database. - Added `DebugToolbarEntry` model and migrations for persistent storage. - Updated documentation to include configuration for `DatabaseStore`. - Added tests for `DatabaseStore` functionality, including CRUD operations and cache size enforcement. Fixes #2073 --- debug_toolbar/migrations/0001_initial.py | 24 +++++ debug_toolbar/migrations/__init__.py | 0 debug_toolbar/models.py | 15 +++ debug_toolbar/store.py | 97 ++++++++++++++++++ docs/changes.rst | 2 + docs/configuration.rst | 29 +++++- tests/test_models.py | 28 ++++++ tests/test_store.py | 123 +++++++++++++++++++++++ 8 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 debug_toolbar/migrations/0001_initial.py create mode 100644 debug_toolbar/migrations/__init__.py create mode 100644 debug_toolbar/models.py create mode 100644 tests/test_models.py diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py new file mode 100644 index 000000000..1acb1f3fc --- /dev/null +++ b/debug_toolbar/migrations/0001_initial.py @@ -0,0 +1,24 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + operations = [ + migrations.CreateModel( + name="DebugToolbarEntry", + fields=[ + ( + "request_id", + models.UUIDField(primary_key=True, serialize=False), + ), + ("data", models.JSONField(default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "verbose_name": "Debug Toolbar Entry", + "verbose_name_plural": "Debug Toolbar Entries", + "ordering": ["-created_at"], + }, + ), + ] diff --git a/debug_toolbar/migrations/__init__.py b/debug_toolbar/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py new file mode 100644 index 000000000..fcd4f3a96 --- /dev/null +++ b/debug_toolbar/models.py @@ -0,0 +1,15 @@ +from django.db import models + + +class DebugToolbarEntry(models.Model): + request_id = models.UUIDField(primary_key=True) + data = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + verbose_name = "Debug Toolbar Entry" + verbose_name_plural = "Debug Toolbar Entries" + ordering = ["-created_at"] + + def __str__(self): + return str(self.request_id) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 122c2dfef..a54a911c5 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -6,10 +6,12 @@ from typing import Any from django.core.serializers.json import DjangoJSONEncoder +from django.utils import timezone from django.utils.encoding import force_str from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings +from debug_toolbar.models import DebugToolbarEntry logger = logging.getLogger(__name__) @@ -140,5 +142,100 @@ def panels(cls, request_id: str) -> Any: yield panel, deserialize(data) +class DatabaseStore(BaseStore): + @classmethod + def _cleanup_old_entries(cls): + """ + Enforce the cache size limit - keeping only the most recently used entries + up to RESULTS_CACHE_SIZE. + """ + # Get the cache size limit from settings + cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] + + # Determine which entries to keep (the most recent ones up to cache_size) + keep_ids = list( + DebugToolbarEntry.objects.order_by("-created_at")[:cache_size].values_list( + "request_id", flat=True + ) + ) + + # Delete all entries not in the keep list + if keep_ids: + DebugToolbarEntry.objects.exclude(request_id__in=keep_ids).delete() + + @classmethod + def request_ids(cls): + """Return all stored request ids within the cache size limit""" + cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] + return list( + DebugToolbarEntry.objects.order_by("-created_at")[:cache_size].values_list( + "request_id", flat=True + ) + ) + + @classmethod + def exists(cls, request_id: str) -> bool: + """Check if the given request_id exists in the store""" + return DebugToolbarEntry.objects.filter(request_id=request_id).exists() + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store and clean up old entries""" + # Create or update the entry + obj, created = DebugToolbarEntry.objects.get_or_create(request_id=request_id) + if not created: + # Update timestamp to mark as recently used + obj.created_at = timezone.now() + obj.save(update_fields=["created_at"]) + + # Enforce the cache size limit to clean up old entries + cls._cleanup_old_entries() + + @classmethod + def clear(cls): + """Remove all requests from the store""" + DebugToolbarEntry.objects.all().delete() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + DebugToolbarEntry.objects.filter(request_id=request_id).delete() + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + # First ensure older entries are cleared if we exceed cache size + cls.set(request_id) + + # Ensure the request exists + obj, _ = DebugToolbarEntry.objects.get_or_create(request_id=request_id) + store_data = obj.data + store_data[panel_id] = serialize(data) + obj.data = store_data + obj.save() + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = DebugToolbarEntry.objects.get(request_id=request_id).data + panel_data = data.get(panel_id) + if panel_data is None: + return {} + return deserialize(panel_data) + except DebugToolbarEntry.DoesNotExist: + return {} + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all panel data for the given request_id""" + try: + data = DebugToolbarEntry.objects.get(request_id=request_id).data + for panel_id, panel_data in data.items(): + yield panel_id, deserialize(panel_data) + except DebugToolbarEntry.DoesNotExist: + return {} + + def get_store() -> BaseStore: return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/docs/changes.rst b/docs/changes.rst index d6ca3ec37..138aa238d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -24,6 +24,8 @@ Serializable (don't include in main) * Update all panels to utilize data from ``Panel.get_stats()`` to load content to render. Specifically for ``Panel.title`` and ``Panel.nav_title``. * Extend example app to contain an async version. +* Added ``debug_toolbar.store.DatabaseStore`` for persistent debug data + storage. Pending ------- diff --git a/docs/configuration.rst b/docs/configuration.rst index a1c5e1406..c8ac1501b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -109,7 +109,8 @@ Toolbar options Default: ``25`` - The toolbar keeps up to this many results in memory. + The toolbar keeps up to this many results in memory or persistent storage. + .. _ROOT_TAG_EXTRA_ATTRS: @@ -186,6 +187,24 @@ Toolbar options The path to the class to be used for storing the toolbar's data per request. + Available store classes: + + * ``debug_toolbar.store.MemoryStore`` - Stores data in memory + * ``debug_toolbar.store.DatabaseStore`` - Stores data in the database + + The DatabaseStore provides persistence and automatically cleans up old + entries based on the ``RESULTS_CACHE_SIZE`` setting. + + Note: For full functionality, DatabaseStore requires migrations for + the debug_toolbar app: + + .. code-block:: bash + + python manage.py migrate debug_toolbar + + For the DatabaseStore to work properly, you need to run migrations for the + debug_toolbar app. The migrations create the necessary database table to store + toolbar data. .. _TOOLBAR_LANGUAGE: @@ -394,6 +413,14 @@ Here's what a slightly customized toolbar configuration might look like:: 'SQL_WARNING_THRESHOLD': 100, # milliseconds } +Here's an example of using a persistent store to keep debug data between server +restarts:: + + DEBUG_TOOLBAR_CONFIG = { + 'TOOLBAR_STORE_CLASS': 'debug_toolbar.store.DatabaseStore', + 'RESULTS_CACHE_SIZE': 100, # Store up to 100 requests + } + Theming support --------------- The debug toolbar uses CSS variables to define fonts and colors. This allows diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 000000000..9abd55558 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,28 @@ +import uuid + +from django.test import TestCase + +from debug_toolbar.models import DebugToolbarEntry + + +class DebugToolbarEntryTestCase(TestCase): + def test_str_method(self): + test_uuid = uuid.uuid4() + entry = DebugToolbarEntry(request_id=test_uuid) + self.assertEqual(str(entry), str(test_uuid)) + + def test_data_field_default(self): + """Test that the data field defaults to an empty dict""" + entry = DebugToolbarEntry(request_id=uuid.uuid4()) + self.assertEqual(entry.data, {}) + + def test_model_persistence(self): + """Test saving and retrieving a model instance""" + test_uuid = uuid.uuid4() + entry = DebugToolbarEntry(request_id=test_uuid, data={"test": True}) + entry.save() + + # Retrieve from database and verify + saved_entry = DebugToolbarEntry.objects.get(request_id=test_uuid) + self.assertEqual(saved_entry.data, {"test": True}) + self.assertEqual(str(saved_entry), str(test_uuid)) diff --git a/tests/test_store.py b/tests/test_store.py index 41be4b1a7..7f47e4c39 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,3 +1,5 @@ +import uuid + from django.test import TestCase from django.test.utils import override_settings @@ -109,3 +111,124 @@ def test_get_store(self): ) def test_get_store_with_setting(self): self.assertIs(store.get_store(), StubStore) + + +class DatabaseStoreTestCase(TestCase): + @classmethod + def setUpTestData(cls) -> None: + cls.store = store.DatabaseStore + + def tearDown(self) -> None: + self.store.clear() + + def test_ids(self): + id1 = str(uuid.uuid4()) + id2 = str(uuid.uuid4()) + self.store.set(id1) + self.store.set(id2) + # Convert the UUIDs to strings for comparison + request_ids = {str(id) for id in self.store.request_ids()} + self.assertEqual(request_ids, {id1, id2}) + + def test_exists(self): + missing_id = str(uuid.uuid4()) + self.assertFalse(self.store.exists(missing_id)) + id1 = str(uuid.uuid4()) + self.store.set(id1) + self.assertTrue(self.store.exists(id1)) + + def test_set(self): + id1 = str(uuid.uuid4()) + self.store.set(id1) + self.assertTrue(self.store.exists(id1)) + + def test_set_max_size(self): + with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1}): + # Clear any existing entries first + self.store.clear() + + # Add first entry + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "foo.panel", "foo.value") + + # Verify it exists + self.assertTrue(self.store.exists(id1)) + self.assertEqual(self.store.panel(id1, "foo.panel"), "foo.value") + + # Add second entry, which should push out the first one due to size limit=1 + id2 = str(uuid.uuid4()) + self.store.save_panel(id2, "bar.panel", {"a": 1}) + + # Verify only the bar entry exists now + # Convert the UUIDs to strings for comparison + request_ids = {str(id) for id in self.store.request_ids()} + self.assertEqual(request_ids, {id2}) + self.assertFalse(self.store.exists(id1)) + self.assertEqual(self.store.panel(id1, "foo.panel"), {}) + self.assertEqual(self.store.panel(id2, "bar.panel"), {"a": 1}) + + def test_clear(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.store.clear() + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel(id1, "bar.panel"), {}) + + def test_delete(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.store.delete(id1) + self.assertEqual(list(self.store.request_ids()), []) + self.assertEqual(self.store.panel(id1, "bar.panel"), {}) + # Make sure it doesn't error + self.store.delete(id1) + + def test_save_panel(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.assertTrue(self.store.exists(id1)) + self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) + + def test_update_panel(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "test.panel", {"original": True}) + self.assertEqual(self.store.panel(id1, "test.panel"), {"original": True}) + + # Update the panel + self.store.save_panel(id1, "test.panel", {"updated": True}) + self.assertEqual(self.store.panel(id1, "test.panel"), {"updated": True}) + + def test_panels_nonexistent_request(self): + missing_id = str(uuid.uuid4()) + panels = dict(self.store.panels(missing_id)) + self.assertEqual(panels, {}) + + def test_panel(self): + id1 = str(uuid.uuid4()) + missing_id = str(uuid.uuid4()) + self.assertEqual(self.store.panel(missing_id, "missing"), {}) + self.store.save_panel(id1, "bar.panel", {"a": 1}) + self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) + + def test_panels(self): + id1 = str(uuid.uuid4()) + self.store.save_panel(id1, "panel1", {"a": 1}) + self.store.save_panel(id1, "panel2", {"b": 2}) + panels = dict(self.store.panels(id1)) + self.assertEqual(len(panels), 2) + self.assertEqual(panels["panel1"], {"a": 1}) + self.assertEqual(panels["panel2"], {"b": 2}) + + def test_cleanup_old_entries(self): + # Create multiple entries + ids = [str(uuid.uuid4()) for _ in range(5)] + for id in ids: + self.store.save_panel(id, "test.panel", {"test": True}) + + # Set a small cache size + with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 2}): + # Trigger cleanup + self.store._cleanup_old_entries() + + # Check that only the most recent 2 entries remain + self.assertEqual(len(list(self.store.request_ids())), 2) From 17b27ceb7f5468a3704a25df3f0cbd53856e2e78 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 3 Apr 2025 15:43:35 -0400 Subject: [PATCH 02/10] refactor: rename DebugToolbarEntry to HistoryEntry and more - Updated model name from `DebugToolbarEntry` to `HistoryEntry` to make string representations of the app_model less redundant. - Adjusted verbose names to use translations with `gettext_lazy`. - Updated all references in `store.py` to use the new model name. - Modified tests to reflect the model name change. - Added a test to check the default ordering of the model and make it the defaul ordering in methods reliable. --- debug_toolbar/migrations/0001_initial.py | 2 +- debug_toolbar/models.py | 7 ++-- debug_toolbar/store.py | 43 +++++++++--------------- tests/test_models.py | 16 +++++---- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py index 1acb1f3fc..8c2766cc8 100644 --- a/debug_toolbar/migrations/0001_initial.py +++ b/debug_toolbar/migrations/0001_initial.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name="DebugToolbarEntry", + name="HistoryEntry", fields=[ ( "request_id", diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py index fcd4f3a96..8b3866c8c 100644 --- a/debug_toolbar/models.py +++ b/debug_toolbar/models.py @@ -1,14 +1,15 @@ from django.db import models +from django.utils.translation import gettext_lazy as _ -class DebugToolbarEntry(models.Model): +class HistoryEntry(models.Model): request_id = models.UUIDField(primary_key=True) data = models.JSONField(default=dict) created_at = models.DateTimeField(auto_now_add=True) class Meta: - verbose_name = "Debug Toolbar Entry" - verbose_name_plural = "Debug Toolbar Entries" + verbose_name = _("debug toolbar entry") + verbose_name_plural = _("debug toolbar entries") ordering = ["-created_at"] def __str__(self): diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index a54a911c5..2a63160a7 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -6,12 +6,11 @@ from typing import Any from django.core.serializers.json import DjangoJSONEncoder -from django.utils import timezone from django.utils.encoding import force_str from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings -from debug_toolbar.models import DebugToolbarEntry +from debug_toolbar.models import HistoryEntry logger = logging.getLogger(__name__) @@ -152,41 +151,32 @@ def _cleanup_old_entries(cls): # Get the cache size limit from settings cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] - # Determine which entries to keep (the most recent ones up to cache_size) - keep_ids = list( - DebugToolbarEntry.objects.order_by("-created_at")[:cache_size].values_list( - "request_id", flat=True - ) + # Get the IDs to keep as a subquery + recent_ids = HistoryEntry.objects.all()[:cache_size].values_list( + "request_id", flat=True ) # Delete all entries not in the keep list - if keep_ids: - DebugToolbarEntry.objects.exclude(request_id__in=keep_ids).delete() + HistoryEntry.objects.exclude(request_id__in=recent_ids).delete() @classmethod def request_ids(cls): """Return all stored request ids within the cache size limit""" cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] return list( - DebugToolbarEntry.objects.order_by("-created_at")[:cache_size].values_list( - "request_id", flat=True - ) + HistoryEntry.objects.all()[:cache_size].values_list("request_id", flat=True) ) @classmethod def exists(cls, request_id: str) -> bool: """Check if the given request_id exists in the store""" - return DebugToolbarEntry.objects.filter(request_id=request_id).exists() + return HistoryEntry.objects.filter(request_id=request_id).exists() @classmethod def set(cls, request_id: str): """Set a request_id in the store and clean up old entries""" - # Create or update the entry - obj, created = DebugToolbarEntry.objects.get_or_create(request_id=request_id) - if not created: - # Update timestamp to mark as recently used - obj.created_at = timezone.now() - obj.save(update_fields=["created_at"]) + # Create the entry if it doesn't exist (ignore otherwise) + HistoryEntry.objects.get_or_create(request_id=request_id) # Enforce the cache size limit to clean up old entries cls._cleanup_old_entries() @@ -194,12 +184,12 @@ def set(cls, request_id: str): @classmethod def clear(cls): """Remove all requests from the store""" - DebugToolbarEntry.objects.all().delete() + HistoryEntry.objects.all().delete() @classmethod def delete(cls, request_id: str): """Delete the stored request for the given request_id""" - DebugToolbarEntry.objects.filter(request_id=request_id).delete() + HistoryEntry.objects.filter(request_id=request_id).delete() @classmethod def save_panel(cls, request_id: str, panel_id: str, data: Any = None): @@ -207,8 +197,7 @@ def save_panel(cls, request_id: str, panel_id: str, data: Any = None): # First ensure older entries are cleared if we exceed cache size cls.set(request_id) - # Ensure the request exists - obj, _ = DebugToolbarEntry.objects.get_or_create(request_id=request_id) + obj, _ = HistoryEntry.objects.get_or_create(request_id=request_id) store_data = obj.data store_data[panel_id] = serialize(data) obj.data = store_data @@ -218,22 +207,22 @@ def save_panel(cls, request_id: str, panel_id: str, data: Any = None): def panel(cls, request_id: str, panel_id: str) -> Any: """Fetch the panel data for the given request_id""" try: - data = DebugToolbarEntry.objects.get(request_id=request_id).data + data = HistoryEntry.objects.get(request_id=request_id).data panel_data = data.get(panel_id) if panel_data is None: return {} return deserialize(panel_data) - except DebugToolbarEntry.DoesNotExist: + except HistoryEntry.DoesNotExist: return {} @classmethod def panels(cls, request_id: str) -> Any: """Fetch all panel data for the given request_id""" try: - data = DebugToolbarEntry.objects.get(request_id=request_id).data + data = HistoryEntry.objects.get(request_id=request_id).data for panel_id, panel_data in data.items(): yield panel_id, deserialize(panel_data) - except DebugToolbarEntry.DoesNotExist: + except HistoryEntry.DoesNotExist: return {} diff --git a/tests/test_models.py b/tests/test_models.py index 9abd55558..7ee2c621a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,27 +2,31 @@ from django.test import TestCase -from debug_toolbar.models import DebugToolbarEntry +from debug_toolbar.models import HistoryEntry -class DebugToolbarEntryTestCase(TestCase): +class HistoryEntryTestCase(TestCase): def test_str_method(self): test_uuid = uuid.uuid4() - entry = DebugToolbarEntry(request_id=test_uuid) + entry = HistoryEntry(request_id=test_uuid) self.assertEqual(str(entry), str(test_uuid)) def test_data_field_default(self): """Test that the data field defaults to an empty dict""" - entry = DebugToolbarEntry(request_id=uuid.uuid4()) + entry = HistoryEntry(request_id=uuid.uuid4()) self.assertEqual(entry.data, {}) def test_model_persistence(self): """Test saving and retrieving a model instance""" test_uuid = uuid.uuid4() - entry = DebugToolbarEntry(request_id=test_uuid, data={"test": True}) + entry = HistoryEntry(request_id=test_uuid, data={"test": True}) entry.save() # Retrieve from database and verify - saved_entry = DebugToolbarEntry.objects.get(request_id=test_uuid) + saved_entry = HistoryEntry.objects.get(request_id=test_uuid) self.assertEqual(saved_entry.data, {"test": True}) self.assertEqual(str(saved_entry), str(test_uuid)) + + def test_default_ordering(self): + """Test that the default ordering is by created_at in descending order""" + self.assertEqual(HistoryEntry._meta.ordering, ["-created_at"]) From a020331a9fc3f64d69bdb5d55da58a1f515ea7a3 Mon Sep 17 00:00:00 2001 From: Felipe Villegas Date: Fri, 4 Apr 2025 00:32:49 -0400 Subject: [PATCH 03/10] Update 0001_initial.py Fix verbose names and capitalization. --- debug_toolbar/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py index 8c2766cc8..e4d30fede 100644 --- a/debug_toolbar/migrations/0001_initial.py +++ b/debug_toolbar/migrations/0001_initial.py @@ -16,8 +16,8 @@ class Migration(migrations.Migration): ("created_at", models.DateTimeField(auto_now_add=True)), ], options={ - "verbose_name": "Debug Toolbar Entry", - "verbose_name_plural": "Debug Toolbar Entries", + "verbose_name": "history entry", + "verbose_name_plural": "history entries", "ordering": ["-created_at"], }, ), From 4e71a4a3033e1be38417fe9b68f609e1747e6673 Mon Sep 17 00:00:00 2001 From: Felipe Villegas Date: Fri, 4 Apr 2025 00:33:50 -0400 Subject: [PATCH 04/10] Update models.py Fix verbose name --- debug_toolbar/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py index 8b3866c8c..686ac4cfa 100644 --- a/debug_toolbar/models.py +++ b/debug_toolbar/models.py @@ -8,8 +8,8 @@ class HistoryEntry(models.Model): created_at = models.DateTimeField(auto_now_add=True) class Meta: - verbose_name = _("debug toolbar entry") - verbose_name_plural = _("debug toolbar entries") + verbose_name = _("history entry") + verbose_name_plural = _("history entries") ordering = ["-created_at"] def __str__(self): From d37468f3ae97c0a26c261d7bd18ce68c1162813f Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Fri, 4 Apr 2025 00:48:12 -0400 Subject: [PATCH 05/10] Pin django-csp<4.0 in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c8f4a6815..3a84468bb 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = pygments selenium>=4.8.0 sqlparse - django-csp + django-csp<4.0 passenv= CI COVERAGE_ARGS From 677a80a607618ccff1aae8d10cb3024e2435d7da Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Fri, 4 Apr 2025 09:59:06 -0400 Subject: [PATCH 06/10] Change _clean_up_old_entries to list --- debug_toolbar/store.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 2a63160a7..69adfcb20 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -151,13 +151,16 @@ def _cleanup_old_entries(cls): # Get the cache size limit from settings cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] - # Get the IDs to keep as a subquery - recent_ids = HistoryEntry.objects.all()[:cache_size].values_list( - "request_id", flat=True + # Determine which entries to keep (the most recent ones up to cache_size) + keep_ids = list( + HistoryEntry.objects.all()[:cache_size].values_list( + "request_id", flat=True + ) ) # Delete all entries not in the keep list - HistoryEntry.objects.exclude(request_id__in=recent_ids).delete() + if keep_ids: + HistoryEntry.objects.exclude(request_id__in=keep_ids).delete() @classmethod def request_ids(cls): From 9d25201dd72df48cfea44edeed7d7f3d3a19f4e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Apr 2025 13:59:24 +0000 Subject: [PATCH 07/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- debug_toolbar/store.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 69adfcb20..b18c62df4 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -153,9 +153,7 @@ def _cleanup_old_entries(cls): # Determine which entries to keep (the most recent ones up to cache_size) keep_ids = list( - HistoryEntry.objects.all()[:cache_size].values_list( - "request_id", flat=True - ) + HistoryEntry.objects.all()[:cache_size].values_list("request_id", flat=True) ) # Delete all entries not in the keep list From 4aadebe11a198c0f9328cff4f22f41e19ad47907 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Wed, 9 Apr 2025 12:45:44 -0400 Subject: [PATCH 08/10] Refactor _cleanup_old_entries to eliminate code duplication --- debug_toolbar/store.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index b18c62df4..91c0384ce 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -148,13 +148,8 @@ def _cleanup_old_entries(cls): Enforce the cache size limit - keeping only the most recently used entries up to RESULTS_CACHE_SIZE. """ - # Get the cache size limit from settings - cache_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] - - # Determine which entries to keep (the most recent ones up to cache_size) - keep_ids = list( - HistoryEntry.objects.all()[:cache_size].values_list("request_id", flat=True) - ) + # Determine which entries to keep + keep_ids = cls.request_ids() # Delete all entries not in the keep list if keep_ids: From fff3fae69ccc54d182a437ef06003df353c806fe Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Wed, 9 Apr 2025 13:02:47 -0400 Subject: [PATCH 09/10] Optimize entry creation logic to clean up old entries only when new entries are added --- debug_toolbar/store.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 91c0384ce..231102b49 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -172,10 +172,11 @@ def exists(cls, request_id: str) -> bool: def set(cls, request_id: str): """Set a request_id in the store and clean up old entries""" # Create the entry if it doesn't exist (ignore otherwise) - HistoryEntry.objects.get_or_create(request_id=request_id) + _, created = HistoryEntry.objects.get_or_create(request_id=request_id) - # Enforce the cache size limit to clean up old entries - cls._cleanup_old_entries() + # Only enforce cache size limit when new entries are created + if created: + cls._cleanup_old_entries() @classmethod def clear(cls): From d3329d1dbd88a5b8cec2851469fec64f7fe298a4 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 10 Apr 2025 12:20:32 -0400 Subject: [PATCH 10/10] Wrap creation and update methods in atomic transactions --- debug_toolbar/store.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 231102b49..7e220d7a2 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -6,6 +6,7 @@ from typing import Any from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction from django.utils.encoding import force_str from django.utils.module_loading import import_string @@ -171,12 +172,13 @@ def exists(cls, request_id: str) -> bool: @classmethod def set(cls, request_id: str): """Set a request_id in the store and clean up old entries""" - # Create the entry if it doesn't exist (ignore otherwise) - _, created = HistoryEntry.objects.get_or_create(request_id=request_id) + with transaction.atomic(): + # Create the entry if it doesn't exist (ignore otherwise) + _, created = HistoryEntry.objects.get_or_create(request_id=request_id) - # Only enforce cache size limit when new entries are created - if created: - cls._cleanup_old_entries() + # Only enforce cache size limit when new entries are created + if created: + cls._cleanup_old_entries() @classmethod def clear(cls): @@ -194,11 +196,12 @@ def save_panel(cls, request_id: str, panel_id: str, data: Any = None): # First ensure older entries are cleared if we exceed cache size cls.set(request_id) - obj, _ = HistoryEntry.objects.get_or_create(request_id=request_id) - store_data = obj.data - store_data[panel_id] = serialize(data) - obj.data = store_data - obj.save() + with transaction.atomic(): + obj, _ = HistoryEntry.objects.get_or_create(request_id=request_id) + store_data = obj.data + store_data[panel_id] = serialize(data) + obj.data = store_data + obj.save() @classmethod def panel(cls, request_id: str, panel_id: str) -> Any: