diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index ba64c8273..ffe68270b 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -18,6 +18,8 @@ def _is_running_tests(): CONFIG_DEFAULTS = { # Toolbar options + "CACHE_BACKEND": "default", + "CACHE_KEY_PREFIX": "djdt:", "DISABLE_PANELS": { "debug_toolbar.panels.profiling.ProfilingPanel", "debug_toolbar.panels.redirects.RedirectsPanel", diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index b4e32788d..e22b44938 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -1,9 +1,11 @@ import contextlib +import functools import json from collections import defaultdict, deque from collections.abc import Iterable from typing import Any +from django.core.cache import caches from django.core.serializers.json import DjangoJSONEncoder from django.db import transaction from django.utils.module_loading import import_string @@ -219,5 +221,175 @@ def panels(cls, request_id: str) -> Any: return {} +class _UntrackedCache: + """ + Wrapper around a Django cache backend that suppresses debug toolbar tracking. + + The cache panel's monkey-patched methods check ``cache._djdt_panel`` and skip + recording when it is ``None``. This proxy temporarily sets that attribute to + ``None`` around every call so the toolbar's own cache operations are invisible. + """ + + def __init__(self, cache): + self._cache = cache + + def __getattr__(self, name): + attr = getattr(self._cache, name) + if not callable(attr): + return attr + + @functools.wraps(attr) + def untracked(*args, **kwargs): + panel = getattr(self._cache, "_djdt_panel", None) + self._cache._djdt_panel = None + try: + return attr(*args, **kwargs) + finally: + self._cache._djdt_panel = panel + + return untracked + + +class CacheStore(BaseStore): + """ + Store that uses Django's cache framework to persist debug toolbar data. + """ + + _cache_table_registered = False + + @classmethod + def _get_cache(cls): + """Get the Django cache backend, wrapped to bypass toolbar tracking.""" + cache = _UntrackedCache(caches[dt_settings.get_config()["CACHE_BACKEND"]]) + + # Register the cache table with DDT_MODELS to filter SQL queries + if not cls._cache_table_registered: + cls._register_cache_table_for_sql_filtering(cache._cache) + cls._cache_table_registered = True + + return cache + + @classmethod + def _register_cache_table_for_sql_filtering(cls, cache): + """ + Add the cache table to DDT_MODELS. + + This ensures that when using DatabaseCache, the cache table's SQL queries + don't appear in the SQLPanel. + """ + # Only proceed if this is a DatabaseCache backend + if cache.__class__.__name__ != "DatabaseCache": + return + + # Get the cache table name + cache_table = getattr(cache, "_table", None) + if cache_table: + # Import here to avoid circular dependency: + # store.py -> panels/sql/tracking.py -> panels/sql/forms.py -> toolbar.py -> store.py + from debug_toolbar.panels.sql import tracking + + tracking.DDT_MODELS.add(cache_table) + + @classmethod + def _key_prefix(cls) -> str: + """Get the cache key prefix from settings.""" + return dt_settings.get_config()["CACHE_KEY_PREFIX"] + + @classmethod + def _request_ids_key(cls) -> str: + """Return the cache key for the request IDs list.""" + return f"{cls._key_prefix()}request_ids" + + @classmethod + def _request_key(cls, request_id: str) -> str: + """Return the cache key for a specific request's data.""" + return f"{cls._key_prefix()}req:{request_id}" + + @classmethod + def request_ids(cls) -> Iterable: + """The stored request ids.""" + return cls._get_cache().get(cls._request_ids_key(), []) + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the store.""" + return request_id in cls.request_ids() + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store.""" + cache = cls._get_cache() + ids_key = cls._request_ids_key() + request_ids = deque(cache.get(ids_key, [])) + + if request_id not in request_ids: + request_ids.append(request_id) + + # Enforce RESULTS_CACHE_SIZE limit + max_size = dt_settings.get_config()["RESULTS_CACHE_SIZE"] + while len(request_ids) > max_size: + removed_id = request_ids.popleft() + cache.delete(cls._request_key(removed_id)) + + cache.set(ids_key, list(request_ids), None) + + @classmethod + def clear(cls): + """Remove all requests from the request store.""" + cache = cls._get_cache() + ids_key = cls._request_ids_key() + request_ids = cache.get(ids_key, []) + + # Delete all request data + if request_ids: + cache.delete_many([cls._request_key(_id) for _id in request_ids]) + + # Clear the request IDs list + cache.delete(ids_key) + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id.""" + cache = cls._get_cache() + ids_key = cls._request_ids_key() + request_ids = list(cache.get(ids_key, [])) + + # Remove from the list if present + if request_id in request_ids: + request_ids.remove(request_id) + cache.set(ids_key, request_ids, None) + + # Delete the request data + cache.delete(cls._request_key(request_id)) + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id.""" + cls.set(request_id) + cache = cls._get_cache() + request_key = cls._request_key(request_id) + request_data = cache.get(request_key, {}) + request_data[panel_id] = serialize(data) + cache.set(request_key, request_data, None) + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id.""" + cache = cls._get_cache() + request_data = cache.get(cls._request_key(request_id), {}) + panel_data = request_data.get(panel_id) + if panel_data is None: + return {} + return deserialize(panel_data) + + @classmethod + def panels(cls, request_id: str) -> Any: + """Fetch all the panel data for the given request_id.""" + cache = cls._get_cache() + request_data = cache.get(cls._request_key(request_id), {}) + for panel_id, panel_data in request_data.items(): + yield panel_id, deserialize(panel_data) + + def get_store() -> BaseStore: return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/tests/test_store.py b/tests/test_store.py index 51e614826..3cbe15d33 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,10 +1,14 @@ import uuid -from django.test import TestCase -from django.test.utils import override_settings +from django.core.management import call_command +from django.db import connection +from django.http import HttpResponse +from django.test import RequestFactory, TestCase +from django.test.utils import CaptureQueriesContext, override_settings from django.utils.safestring import SafeData, mark_safe from debug_toolbar import store +from debug_toolbar.toolbar import DebugToolbar class SerializationTestCase(TestCase): @@ -50,59 +54,102 @@ def test_methods_are_not_implemented(self): store.BaseStore.panel("", "") -class MemoryStoreTestCase(TestCase): - @classmethod - def setUpTestData(cls) -> None: - cls.store = store.MemoryStore +class CommonStoreTestsMixin: + """ + Mixin class with common tests that apply to all store implementations. + Subclasses must set self.store to the appropriate store class. + Subclasses can override _get_request_id() to provide appropriate ID types. + """ - def tearDown(self) -> None: - self.store.clear() + def _get_request_id(self, name: str) -> str: + """ + Generate a request ID for testing. + """ + return name def test_ids(self): - self.store.set("foo") - self.store.set("bar") - self.assertEqual(list(self.store.request_ids()), ["foo", "bar"]) + foo_id = self._get_request_id("foo") + bar_id = self._get_request_id("bar") + self.store.set(foo_id) + self.store.set(bar_id) + request_ids = {str(id) for id in self.store.request_ids()} + self.assertEqual(request_ids, {str(foo_id), str(bar_id)}) def test_exists(self): - self.assertFalse(self.store.exists("missing")) - self.store.set("exists") - self.assertTrue(self.store.exists("exists")) + missing_id = self._get_request_id("missing") + exists_id = self._get_request_id("exists") + self.assertFalse(self.store.exists(missing_id)) + self.store.set(exists_id) + self.assertTrue(self.store.exists(exists_id)) def test_set(self): - self.store.set("foo") - self.assertEqual(list(self.store.request_ids()), ["foo"]) + foo_id = self._get_request_id("foo") + self.store.set(foo_id) + self.assertTrue(self.store.exists(foo_id)) def test_set_max_size(self): + foo_id = self._get_request_id("foo") + bar_id = self._get_request_id("bar") with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1}): - self.store.save_panel("foo", "foo.panel", "foo.value") - self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.assertEqual(list(self.store.request_ids()), ["bar"]) - self.assertEqual(self.store.panel("foo", "foo.panel"), {}) - self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + self.store.save_panel(foo_id, "foo.panel", "foo.value") + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) + request_ids = [str(id) for id in self.store.request_ids()] + self.assertEqual(len(request_ids), 1) + self.assertIn(str(bar_id), request_ids) + self.assertEqual(self.store.panel(foo_id, "foo.panel"), {}) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {"a": 1}) def test_clear(self): - self.store.save_panel("bar", "bar.panel", {"a": 1}) + bar_id = self._get_request_id("bar") + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) self.store.clear() self.assertEqual(list(self.store.request_ids()), []) - self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {}) def test_delete(self): - self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.store.delete("bar") + bar_id = self._get_request_id("bar") + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) + self.store.delete(bar_id) self.assertEqual(list(self.store.request_ids()), []) - self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {}) # Make sure it doesn't error - self.store.delete("bar") + self.store.delete(bar_id) def test_save_panel(self): - self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.assertEqual(list(self.store.request_ids()), ["bar"]) - self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + bar_id = self._get_request_id("bar") + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) + self.assertTrue(self.store.exists(bar_id)) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {"a": 1}) def test_panel(self): - self.assertEqual(self.store.panel("missing", "missing"), {}) - self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + missing_id = self._get_request_id("missing") + bar_id = self._get_request_id("bar") + self.assertEqual(self.store.panel(missing_id, "missing"), {}) + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {"a": 1}) + + def test_panels(self): + bar_id = self._get_request_id("bar") + self.store.save_panel(bar_id, "panel1", {"a": 1}) + self.store.save_panel(bar_id, "panel2", {"b": 2}) + panels = dict(self.store.panels(bar_id)) + self.assertEqual(len(panels), 2) + self.assertEqual(panels["panel1"], {"a": 1}) + self.assertEqual(panels["panel2"], {"b": 2}) + + def test_panels_nonexistent_request(self): + missing_id = self._get_request_id("missing") + panels = dict(self.store.panels(missing_id)) + self.assertEqual(panels, {}) + + +class MemoryStoreTestCase(CommonStoreTestsMixin, TestCase): + @classmethod + def setUpTestData(cls) -> None: + cls.store = store.MemoryStore + + def tearDown(self) -> None: + self.store.clear() def test_serialize_safestring(self): before = {"string": mark_safe("safe")} @@ -135,36 +182,29 @@ def test_get_store_with_setting(self): @override_settings( DEBUG_TOOLBAR_CONFIG={"TOOLBAR_STORE_CLASS": "debug_toolbar.store.DatabaseStore"} ) -class DatabaseStoreTestCase(TestCase): +class DatabaseStoreTestCase(CommonStoreTestsMixin, TestCase): @classmethod def setUpTestData(cls) -> None: cls.store = store.DatabaseStore + def setUp(self) -> None: + # Cache UUIDs so the same name returns the same UUID within a test + self._uuid_cache = {} + def tearDown(self) -> None: self.store.clear() - def test_ids(self): - id1 = str(uuid.uuid4()) - id2 = str(uuid.uuid4()) - self.store.set(id1) - self.store.set(id2) - # Convert the UUIDs to strings for comparison - request_ids = {str(id) for id in self.store.request_ids()} - self.assertEqual(request_ids, {id1, id2}) - - def test_exists(self): - missing_id = str(uuid.uuid4()) - self.assertFalse(self.store.exists(missing_id)) - id1 = str(uuid.uuid4()) - self.store.set(id1) - self.assertTrue(self.store.exists(id1)) - - def test_set(self): - id1 = str(uuid.uuid4()) - self.store.set(id1) - self.assertTrue(self.store.exists(id1)) + def _get_request_id(self, name: str) -> str: + """Generate a UUID for DatabaseStore tests, cached by name.""" + if name not in self._uuid_cache: + self._uuid_cache[name] = str(uuid.uuid4()) + return self._uuid_cache[name] def test_set_max_size(self): + """ + DatabaseStore test for max size using set() instead of save_panel(). + The cleanup logic is triggered by set(), not save_panel(). + """ with self.settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 1}): # Clear any existing entries first self.store.clear() @@ -180,34 +220,11 @@ def test_set_max_size(self): id2 = str(uuid.uuid4()) self.store.set(id2) - # Verify only the bar entry exists now - # Convert the UUIDs to strings for comparison + # Verify only the second entry exists now request_ids = {str(id) for id in self.store.request_ids()} self.assertEqual(request_ids, {id2}) self.assertFalse(self.store.exists(id1)) - def test_clear(self): - id1 = str(uuid.uuid4()) - self.store.save_panel(id1, "bar.panel", {"a": 1}) - self.store.clear() - self.assertEqual(list(self.store.request_ids()), []) - self.assertEqual(self.store.panel(id1, "bar.panel"), {}) - - def test_delete(self): - id1 = str(uuid.uuid4()) - self.store.save_panel(id1, "bar.panel", {"a": 1}) - self.store.delete(id1) - self.assertEqual(list(self.store.request_ids()), []) - self.assertEqual(self.store.panel(id1, "bar.panel"), {}) - # Make sure it doesn't error - self.store.delete(id1) - - def test_save_panel(self): - id1 = str(uuid.uuid4()) - self.store.save_panel(id1, "bar.panel", {"a": 1}) - self.assertTrue(self.store.exists(id1)) - self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) - def test_update_panel(self): id1 = str(uuid.uuid4()) self.store.save_panel(id1, "test.panel", {"original": True}) @@ -217,27 +234,6 @@ def test_update_panel(self): self.store.save_panel(id1, "test.panel", {"updated": True}) self.assertEqual(self.store.panel(id1, "test.panel"), {"updated": True}) - def test_panels_nonexistent_request(self): - missing_id = str(uuid.uuid4()) - panels = dict(self.store.panels(missing_id)) - self.assertEqual(panels, {}) - - def test_panel(self): - id1 = str(uuid.uuid4()) - missing_id = str(uuid.uuid4()) - self.assertEqual(self.store.panel(missing_id, "missing"), {}) - self.store.save_panel(id1, "bar.panel", {"a": 1}) - self.assertEqual(self.store.panel(id1, "bar.panel"), {"a": 1}) - - def test_panels(self): - id1 = str(uuid.uuid4()) - self.store.save_panel(id1, "panel1", {"a": 1}) - self.store.save_panel(id1, "panel2", {"b": 2}) - panels = dict(self.store.panels(id1)) - self.assertEqual(len(panels), 2) - self.assertEqual(panels["panel1"], {"a": 1}) - self.assertEqual(panels["panel2"], {"b": 2}) - def test_cleanup_old_entries(self): # Create multiple entries ids = [str(uuid.uuid4()) for _ in range(5)] @@ -251,3 +247,197 @@ def test_cleanup_old_entries(self): # Check that only the most recent 2 entries remain self.assertEqual(len(list(self.store.request_ids())), 2) + + def test_database_queries_are_efficient(self): + """Verify that DatabaseStore uses efficient database queries.""" + id1 = str(uuid.uuid4()) + + # Test that panel retrieval uses a single query + self.store.save_panel(id1, "test.panel", {"data": "value"}) + with CaptureQueriesContext(connection) as context: + self.store.panel(id1, "test.panel") + self.assertEqual(len(context.captured_queries), 1) + + # Test that panels() uses a single query + self.store.save_panel(id1, "panel2", {"data": "value2"}) + with CaptureQueriesContext(connection) as context: + list(self.store.panels(id1)) + self.assertEqual(len(context.captured_queries), 1) + + # Test that exists() uses a single query + with CaptureQueriesContext(connection) as context: + self.store.exists(id1) + self.assertEqual(len(context.captured_queries), 1) + + +@override_settings( + DEBUG_TOOLBAR_CONFIG={ + "CACHES": { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + }, + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.CacheStore", + } +) +class CacheStoreWithMemoryBackendTestCase(CommonStoreTestsMixin, TestCase): + """ + Test CacheStore with LocMemCache backend (in-memory caching). + """ + + @classmethod + def setUpTestData(cls) -> None: + cls.store = store.CacheStore + + def tearDown(self) -> None: + self.store.clear() + + def test_custom_cache_backend(self): + with self.settings( + DEBUG_TOOLBAR_CONFIG={ + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.CacheStore", + "CACHE_BACKEND": "default", + } + ): + self.store.save_panel("test", "test.panel", {"value": 123}) + self.assertEqual(self.store.panel("test", "test.panel"), {"value": 123}) + + def test_custom_key_prefix(self): + with self.settings( + DEBUG_TOOLBAR_CONFIG={ + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.CacheStore", + "CACHE_KEY_PREFIX": "custom:", + } + ): + # Verify the key prefix is used + self.assertEqual(self.store._key_prefix(), "custom:") + self.assertEqual(self.store._request_ids_key(), "custom:request_ids") + self.assertEqual(self.store._request_key("test"), "custom:req:test") + + def test_cache_store_operations_not_tracked_by_cache_panel(self): + """Verify that CacheStore operations don't appear in CachePanel data.""" + # Set up a toolbar with CachePanel + request = RequestFactory().get("/") + toolbar = DebugToolbar(request, lambda req: HttpResponse()) + panel = toolbar.get_panel_by_id("CachePanel") + panel.enable_instrumentation() + + try: + # Record the initial number of cache calls + initial_call_count = len(panel.calls) + + # Perform various CacheStore operations + self.store.set("test_req") + self.store.save_panel("test_req", "test.panel", {"data": "value"}) + self.store.exists("test_req") + self.store.panel("test_req", "test.panel") + self.store.panels("test_req") + self.store.delete("test_req") + + # Verify that no cache operations were recorded + # All CacheStore operations should be invisible to the CachePanel + self.assertEqual( + len(panel.calls), + initial_call_count, + "CacheStore operations should not be tracked by CachePanel", + ) + finally: + panel.disable_instrumentation() + + +@override_settings( + DEBUG_TOOLBAR_CONFIG={ + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.CacheStore", + "CACHE_BACKEND": "ddt_db_cache", + }, + CACHES={ + "ddt_db_cache": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "test_cache_store_table", + } + }, +) +class CacheStoreWithDatabaseBackendTestCase(CommonStoreTestsMixin, TestCase): + """ + Test CacheStore with DatabaseCache backend. + This ensures CacheStore works correctly when using database-backed caching. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Create the database cache table + call_command("createcachetable", "test_cache_store_table", verbosity=0) + cls.store = store.CacheStore + + @classmethod + def tearDownClass(cls): + # Drop the cache table + with connection.cursor() as cursor: + cursor.execute("DROP TABLE IF EXISTS test_cache_store_table") + super().tearDownClass() + + def tearDown(self) -> None: + self.store.clear() + + def test_set_max_size(self): + """Override to preserve cache backend settings.""" + foo_id = self._get_request_id("foo") + bar_id = self._get_request_id("bar") + with self.settings( + DEBUG_TOOLBAR_CONFIG={ + "RESULTS_CACHE_SIZE": 1, + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.CacheStore", + "CACHE_BACKEND": "ddt_db_cache", + } + ): + self.store.save_panel(foo_id, "foo.panel", "foo.value") + self.store.save_panel(bar_id, "bar.panel", {"a": 1}) + request_ids = [str(id) for id in self.store.request_ids()] + self.assertEqual(len(request_ids), 1) + self.assertIn(str(bar_id), request_ids) + self.assertEqual(self.store.panel(foo_id, "foo.panel"), {}) + self.assertEqual(self.store.panel(bar_id, "bar.panel"), {"a": 1}) + + def test_database_backend_not_tracked_by_sql_panel(self): + """ + Verify that CacheStore operations using DatabaseCache backend + don't appear in SQLPanel data. + + The _UntrackedCache wrapper prevents CachePanel tracking by setting + cache._djdt_panel = None. Additionally, SQL queries to the cache table + are filtered out because the cache table is dynamically added to + DDT_MODELS when CacheStore is configured with DatabaseCache. + """ + # Set up a toolbar with SQLPanel + request = RequestFactory().get("/") + toolbar = DebugToolbar(request, lambda req: HttpResponse()) + sql_panel = toolbar.get_panel_by_id("SQLPanel") + sql_panel.enable_instrumentation() + + try: + # Record the initial number of SQL queries + initial_query_count = len(sql_panel._queries) + + # Perform various CacheStore operations that will trigger DatabaseCache SQL queries + self.store.set("test_req") + self.store.save_panel("test_req", "test.panel", {"data": "value"}) + self.store.exists("test_req") + self.store.panel("test_req", "test.panel") + self.store.panels("test_req") + self.store.delete("test_req") + + # Verify that no SQL queries to the cache table were recorded + # All CacheStore DatabaseCache operations should be invisible to the SQLPanel + cache_queries = [ + q + for q in sql_panel._queries[initial_query_count:] + if "test_cache_store_table" in q.get("sql", "").lower() + ] + + self.assertEqual( + len(cache_queries), + 0, + f"CacheStore DatabaseCache operations should not be tracked by SQLPanel, " + f"but found {len(cache_queries)} queries to 'test_cache_store_table' table", + ) + finally: + sql_panel.disable_instrumentation()