From 202c301c7a59c0f106fc5c1112b4bb7f24e2a91a Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 16 May 2023 16:44:41 +0200 Subject: [PATCH 01/37] Fixed #1780 -- Adjusted system check to allow for nested template loaders. Django's cached loader wraps a list of child loaders, which may correctly contain the required app directories loader. --- debug_toolbar/apps.py | 18 ++++++++++++++++++ docs/changes.rst | 2 ++ tests/test_checks.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/debug_toolbar/apps.py b/debug_toolbar/apps.py index c55b75392..f84bcca06 100644 --- a/debug_toolbar/apps.py +++ b/debug_toolbar/apps.py @@ -32,8 +32,26 @@ def check_template_config(config): included in the loaders. If custom loaders are specified, then APP_DIRS must be True. """ + + def flat_loaders(loaders): + """ + Recursively flatten the settings list of template loaders. + + Check for (loader, [child_loaders]) tuples. + Django's default cached loader uses this pattern. + """ + for loader in loaders: + if isinstance(loader, tuple): + yield loader[0] + yield from flat_loaders(loader[1]) + else: + yield loader + app_dirs = config.get("APP_DIRS", False) loaders = config.get("OPTIONS", {}).get("loaders", None) + if loaders: + loaders = list(flat_loaders(loaders)) + # By default the app loader is included. has_app_loaders = ( loaders is None or "django.template.loaders.app_directories.Loader" in loaders diff --git a/docs/changes.rst b/docs/changes.rst index ad6607e34..06a0cc744 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,8 @@ Change log Pending ------- +* Adjusted app directories system check to allow for nested template loaders. + 4.1.0 (2023-05-15) ------------------ diff --git a/tests/test_checks.py b/tests/test_checks.py index 1e24688da..3f17922c0 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -198,3 +198,32 @@ def test_check_w006_invalid(self): ) def test_check_w006_valid(self): self.assertEqual(run_checks(), []) + + @override_settings( + TEMPLATES=[ + { + "NAME": "use_loaders", + "BACKEND": "django.template.backends.django.DjangoTemplates", + "APP_DIRS": False, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + "loaders": [ + ( + "django.template.loaders.cached.Loader", + [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", + ], + ), + ], + }, + }, + ] + ) + def test_check_w006_valid_nested_loaders(self): + self.assertEqual(run_checks(), []) From 4a696123ff296df0e6e7a65df045fcec42dc201d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 17 May 2023 05:54:00 +0200 Subject: [PATCH 02/37] Include all files in sdist archives Remove the restriction on files inclduded in sdist archives in order to include documentation sources and tests there. This is the default hatchling behavior and it is helpful to packagers (such as Linux distributions or Conda) as it permits using sdist archives to do packaging (instead of GitHub archives that are not guaranteed to be reproducible). Most of the ordinary users will not be affected since starlette is a pure Python package and therefore pip will prefer wheels to install it everywhere. --- pyproject.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8729cb911..d012cbaa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,12 +43,6 @@ dependencies = [ Download = "https://pypi.org/project/django-debug-toolbar/" Homepage = "https://github.com/jazzband/django-debug-toolbar" -[tool.hatch.build.targets.sdist] -include = [ - "/debug_toolbar", - "/CONTRIBUTING.md", -] - [tool.hatch.build.targets.wheel] packages = ["debug_toolbar"] From 5ec1ad2ec4c4852d82ad53e978ff882682ad28c2 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Mon, 18 Apr 2022 17:13:15 +0300 Subject: [PATCH 03/37] Fix confusing variable shadowing --- debug_toolbar/panels/sql/tracking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index a85ac51ad..e844d9a7b 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -182,7 +182,7 @@ def _record(self, method, sql, params): else: sql = str(sql) - params = { + kwargs = { "vendor": vendor, "alias": alias, "sql": self._last_executed_query(sql, params), @@ -228,7 +228,7 @@ def _record(self, method, sql, params): else: trans_id = None - params.update( + kwargs.update( { "trans_id": trans_id, "trans_status": conn.info.transaction_status, @@ -237,7 +237,7 @@ def _record(self, method, sql, params): ) # We keep `sql` to maintain backwards compatibility - self.logger.record(**params) + self.logger.record(**kwargs) def callproc(self, procname, params=None): return self._record(super().callproc, procname, params) From c387acf19038f5eae0c97280e959769df15f9c9b Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Mon, 18 Apr 2022 17:15:27 +0300 Subject: [PATCH 04/37] Remove unused query attributes --- debug_toolbar/panels/sql/tracking.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index e844d9a7b..6ac88c091 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -191,8 +191,6 @@ def _record(self, method, sql, params): "params": _params, "raw_params": params, "stacktrace": get_stack_trace(skip=2), - "start_time": start_time, - "stop_time": stop_time, "is_slow": ( duration > dt_settings.get_config()["SQL_WARNING_THRESHOLD"] ), From 7093210b3882c0f680552d3e3f5119778351f6c9 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Tue, 19 Apr 2022 14:18:47 +0300 Subject: [PATCH 05/37] Post-process two query attributes There is no reason for them to be computed when recording, so move them to SQLPanel.generate_stats(). --- debug_toolbar/panels/sql/panel.py | 9 +++++++++ debug_toolbar/panels/sql/tracking.py | 5 ----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index c8576e16f..9cdb1003a 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -6,6 +6,7 @@ from django.urls import path from django.utils.translation import gettext_lazy as _, ngettext +from debug_toolbar import settings as dt_settings from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels import Panel from debug_toolbar.panels.sql import views @@ -204,6 +205,8 @@ def generate_stats(self, request, response): duplicate_query_groups = defaultdict(list) if self._queries: + sql_warning_threshold = dt_settings.get_config()["SQL_WARNING_THRESHOLD"] + width_ratio_tally = 0 factor = int(256.0 / (len(self._databases) * 2.5)) for n, db in enumerate(self._databases.values()): @@ -261,6 +264,12 @@ def generate_stats(self, request, response): if query["sql"]: query["sql"] = reformat_sql(query["sql"], with_toggle=True) + + query["is_slow"] = query["duration"] > sql_warning_threshold + query["is_select"] = ( + query["raw_sql"].lower().lstrip().startswith("select") + ) + query["rgb_color"] = self._databases[alias]["rgb_color"] try: query["width_ratio"] = (query["duration"] / self._sql_time) * 100 diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 6ac88c091..35f0806f1 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -7,7 +7,6 @@ from django.db.backends.utils import CursorWrapper from django.utils.encoding import force_str -from debug_toolbar import settings as dt_settings from debug_toolbar.utils import get_stack_trace, get_template_info try: @@ -191,10 +190,6 @@ def _record(self, method, sql, params): "params": _params, "raw_params": params, "stacktrace": get_stack_trace(skip=2), - "is_slow": ( - duration > dt_settings.get_config()["SQL_WARNING_THRESHOLD"] - ), - "is_select": sql.lower().strip().startswith("select"), "template_info": template_info, } From 1490ba1dd854f1913677646143dae923b324f68c Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Tue, 31 May 2022 17:22:58 +0300 Subject: [PATCH 06/37] Remove unused SQLPanel._offset attribute Not used since 5aff6ee75f8af3dd46254953b0b0de7c8e19c8e2 (March 2011). --- debug_toolbar/panels/sql/panel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 9cdb1003a..5b450a358 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -111,7 +111,6 @@ class SQLPanel(Panel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._offset = {k: len(connections[k].queries) for k in connections} self._sql_time = 0 self._num_queries = 0 self._queries = [] From 5d420444c4cbe7d079e7ae31a2a287113cd51043 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Tue, 31 May 2022 17:01:35 +0300 Subject: [PATCH 07/37] Drop unneeded SQLPanel._num_queries attribute The same information can be retrieved from len(self._queries). --- debug_toolbar/panels/sql/panel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 5b450a358..984a2074a 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -112,7 +112,6 @@ class SQLPanel(Panel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._sql_time = 0 - self._num_queries = 0 self._queries = [] self._databases = {} # synthetic transaction IDs, keyed by DB alias @@ -151,7 +150,6 @@ def record(self, **kwargs): self._databases[alias]["time_spent"] += kwargs["duration"] self._databases[alias]["num_queries"] += 1 self._sql_time += kwargs["duration"] - self._num_queries += 1 # Implement the Panel API @@ -159,12 +157,13 @@ def record(self, **kwargs): @property def nav_subtitle(self): + query_count = len(self._queries) return ngettext( "%(query_count)d query in %(sql_time).2fms", "%(query_count)d queries in %(sql_time).2fms", - self._num_queries, + query_count, ) % { - "query_count": self._num_queries, + "query_count": query_count, "sql_time": self._sql_time, } From b7af52dce7052494456b55f42d734699b5a83ba1 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Mon, 15 May 2023 14:35:03 +0300 Subject: [PATCH 08/37] Improve comment in wrap_cursor() More fully explain the reasoning for skipping monkey patching when running under a Django SimpleTestCase. --- debug_toolbar/panels/sql/tracking.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 35f0806f1..ee75f8e06 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -33,8 +33,14 @@ class SQLQueryTriggered(Exception): def wrap_cursor(connection): - # If running a Django SimpleTestCase, which isn't allowed to access the database, - # don't perform any monkey patching. + # When running a SimpleTestCase, Django monkey patches some DatabaseWrapper + # methods, including .cursor() and .chunked_cursor(), to raise an exception + # if the test code tries to access the database, and then undoes the monkey + # patching when the test case is finished. If we monkey patch those methods + # also, Django's process of undoing those monkey patches will fail. To + # avoid this failure, and because database access is not allowed during a + # SimpleTextCase anyway, skip applying our instrumentation monkey patches if + # we detect that Django has already monkey patched DatabaseWrapper.cursor(). if isinstance(connection.cursor, django.test.testcases._DatabaseFailure): return if not hasattr(connection, "_djdt_cursor"): From 39e5135e219d26edc98334a40d9638b88afa4cd9 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 16 May 2023 12:09:53 +0200 Subject: [PATCH 09/37] Use ruff for linting Use ruff instead of flake8+isort+pyupgrade+pygrep-hooks. This change configures ruff to apply all the same rules that these tools apply. I've omitted E501 for now since there are a few violations. --- .flake8 | 4 ---- .pre-commit-config.yaml | 22 +++++----------------- debug_toolbar/panels/sql/utils.py | 2 +- docs/contributing.rst | 2 +- pyproject.toml | 26 ++++++++++++++++++++++---- requirements_dev.txt | 2 -- 6 files changed, 29 insertions(+), 29 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 7d44b7eca..000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -# TODO: move this to pyproject.toml when supported, see https://github.com/PyCQA/flake8/issues/234 - -[flake8] -extend-ignore = E203, E501 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 124892d78..f28cdd625 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,35 +7,18 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: mixed-line-ending -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - repo: https://github.com/pycqa/doc8 rev: v1.1.1 hooks: - id: doc8 -- repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 - hooks: - - id: pyupgrade - args: [--py38-plus] - repo: https://github.com/adamchainz/django-upgrade rev: 1.13.0 hooks: - id: django-upgrade args: [--target-version, "3.2"] -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - - id: python-check-blanket-noqa - - id: python-check-mock-methods - - id: python-no-eval - - id: python-no-log-warn - id: rst-backticks - id: rst-directive-colons - repo: https://github.com/pre-commit/mirrors-prettier @@ -53,6 +36,11 @@ repos: types: [file] args: - --fix +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.267' + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black rev: 23.3.0 hooks: diff --git a/debug_toolbar/panels/sql/utils.py b/debug_toolbar/panels/sql/utils.py index efd7c1637..120674c96 100644 --- a/debug_toolbar/panels/sql/utils.py +++ b/debug_toolbar/panels/sql/utils.py @@ -132,7 +132,7 @@ def contrasting_color_generator(): """ def rgb_to_hex(rgb): - return "#%02x%02x%02x" % tuple(rgb) + return "#{:02x}{:02x}{:02x}".format(*tuple(rgb)) triples = [ (1, 0, 0), diff --git a/docs/contributing.rst b/docs/contributing.rst index 079a8b195..5e11ee603 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -101,7 +101,7 @@ Style ----- The Django Debug Toolbar uses `black `__ to -format code and additionally uses flake8 and isort. The toolbar uses +format code and additionally uses ruff. The toolbar uses `pre-commit `__ to automatically apply our style guidelines when a commit is made. Set up pre-commit before committing with:: diff --git a/pyproject.toml b/pyproject.toml index d012cbaa3..ff8c22874 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,10 +49,6 @@ packages = ["debug_toolbar"] [tool.hatch.version] path = "debug_toolbar/__init__.py" -[tool.isort] -combine_as_imports = true -profile = "black" - [tool.coverage.html] skip_covered = true skip_empty = true @@ -69,3 +65,25 @@ source = ["src", ".tox/*/site-packages"] # Update coverage badge link in README.rst when fail_under changes fail_under = 94 show_missing = true + +[tool.ruff.isort] +combine-as-imports = true + +[tool.ruff] +select = [ + # flake8/Pyflakes + "F", + # flake8/pycodestyle + "E", + "W", + # isort + "I", + # pyupgrade + "UP", + # pygrep-hooks + "PGH", +] +ignore = [ + "E501", +] +target-version = "py38" diff --git a/requirements_dev.txt b/requirements_dev.txt index ade334aba..4b90beb08 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -7,9 +7,7 @@ Jinja2 # Testing coverage[toml] -flake8 html5lib -isort selenium tox black From 52ff5eb9c5a0b20c9dd77cb0ceed8a746a23fffc Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 22 May 2023 11:52:14 +0200 Subject: [PATCH 10/37] Mention ruff in the changelog --- .pre-commit-config.yaml | 2 ++ docs/changes.rst | 2 ++ docs/spelling_wordlist.txt | 25 +++++++++++++------------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f28cdd625..559e3d696 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,8 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: mixed-line-ending + - id: file-contents-sorter + files: docs/spelling_wordlist.txt - repo: https://github.com/pycqa/doc8 rev: v1.1.1 hooks: diff --git a/docs/changes.rst b/docs/changes.rst index 06a0cc744..3ba8b7155 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,6 +5,8 @@ Pending ------- * Adjusted app directories system check to allow for nested template loaders. +* Switched from flake8, isort and pyupgrade to `ruff + `__. 4.1.0 (2023-05-15) ------------------ diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d5aa73afe..2ab01758c 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1,3 +1,12 @@ +Hatchling +Hotwire +Jazzband +Makefile +Pympler +Roboto +Transifex +Werkzeug +async backend backends backported @@ -9,17 +18,13 @@ fallbacks flamegraph flatpages frontend -Hatchling -Hotwire htmx inlining isort -Jazzband -jinja jQuery +jinja jrestclient js -Makefile margins memcache memcached @@ -36,22 +41,18 @@ psycopg py pyflame pylibmc -Pympler +pyupgrade querysets refactoring resizing -Roboto spellchecking spooler stacktrace stacktraces startup -timeline theming +timeline tox -Transifex -unhashable uWSGI +unhashable validator -Werkzeug -async From 2929bb404fdf8526b5487b16b99efac563f0f790 Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Tue, 23 May 2023 08:49:12 +0200 Subject: [PATCH 11/37] Fixed references to django.core.cache in the docs See https://github.com/django/django/commit/c3862735cd8c268e99fb8d54c3955aacc4f2dc25 --- docs/changes.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 3ba8b7155..4847290fd 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -154,7 +154,7 @@ Deprecated features 3.3.0 (2022-04-28) ------------------ -* Track calls to :py:meth:`django.core.caches.cache.get_or_set`. +* Track calls to :py:meth:`django.core.cache.cache.get_or_set`. * Removed support for Django < 3.2. * Updated check ``W006`` to look for ``django.template.loaders.app_directories.Loader``. @@ -176,7 +176,7 @@ Deprecated features * Changed cache monkey-patching for Django 3.2+ to iterate over existing caches and patch them individually rather than attempting to patch - ``django.core.caches`` as a whole. The ``middleware.cache`` is still + ``django.core.cache`` as a whole. The ``middleware.cache`` is still being patched as a whole in order to attempt to catch any cache usages before ``enable_instrumentation`` is called. * Add check ``W006`` to warn that the toolbar is incompatible with @@ -298,7 +298,7 @@ Deprecated features ``localStorage``. * Updated the code to avoid a few deprecation warnings and resource warnings. * Started loading JavaScript as ES6 modules. -* Added support for :meth:`cache.touch() ` when +* Added support for :meth:`cache.touch() ` when using django-debug-toolbar. * Eliminated more inline CSS. * Updated ``tox.ini`` and ``Makefile`` to use isort>=5. From 2ac592d73c280f74b95d53383bec2e118dde578a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 08:55:01 +0200 Subject: [PATCH 12/37] [pre-commit.ci] pre-commit autoupdate (#1788) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-eslint: v8.40.0 → v8.41.0](https://github.com/pre-commit/mirrors-eslint/compare/v8.40.0...v8.41.0) - [github.com/charliermarsh/ruff-pre-commit: v0.0.267 → v0.0.269](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.267...v0.0.269) - [github.com/abravalheri/validate-pyproject: v0.12.2 → v0.13](https://github.com/abravalheri/validate-pyproject/compare/v0.12.2...v0.13) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 559e3d696..dfa139b1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.40.0 + rev: v8.41.0 hooks: - id: eslint files: \.js?$ @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.267' + rev: 'v0.0.269' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -54,6 +54,6 @@ repos: hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.12.2 + rev: v0.13 hooks: - id: validate-pyproject From 857e329a24a2e41f5cc30998523be799f12c2700 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Mon, 15 May 2023 15:48:56 +0300 Subject: [PATCH 13/37] Minor updates to tox.ini * Simplify envlist specification (no functional changes) * Tweak dj42 django dependency specification * Enable selenium tests for Python 3.11/Django 4.2 instead of for Python 3.10/Django 4.1. * Minor file formatting tweaks. --- tox.ini | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tox.ini b/tox.ini index d751d5325..b1c194c61 100644 --- a/tox.ini +++ b/tox.ini @@ -3,23 +3,22 @@ isolated_build = true envlist = docs packaging - py{38,39,310}-dj{32}-{sqlite,postgresql,postgis,mysql} - py{310}-dj{40}-{sqlite} - py{310,311}-dj{41}-{sqlite,postgresql,postgis,mysql} - py{310,311}-dj{42,main}-{sqlite,postgresql,postgis,mysql} - py{310,311}-dj{42,main}-psycopg3 + py{38,39,310}-dj32-{sqlite,postgresql,postgis,mysql} + py310-dj40-sqlite + py{310,311}-dj41-{sqlite,postgresql,postgis,mysql} + py{310,311}-dj{42,main}-{sqlite,postgresql,psycopg3,postgis,mysql} [testenv] deps = dj32: django~=3.2.9 dj40: django~=4.0.0 dj41: django~=4.1.3 - dj42: django>=4.2a1,<5 + dj42: django~=4.2.1 + djmain: https://github.com/django/django/archive/main.tar.gz postgresql: psycopg2-binary psycopg3: psycopg[binary] postgis: psycopg2-binary mysql: mysqlclient - djmain: https://github.com/django/django/archive/main.tar.gz coverage[toml] Jinja2 html5lib @@ -40,7 +39,7 @@ setenv = PYTHONPATH = {toxinidir} PYTHONWARNINGS = d py39-dj32-postgresql: DJANGO_SELENIUM_TESTS = true - py310-dj41-postgresql: DJANGO_SELENIUM_TESTS = true + py311-dj42-postgresql: DJANGO_SELENIUM_TESTS = true DB_NAME = {env:DB_NAME:debug_toolbar} DB_USER = {env:DB_USER:debug_toolbar} DB_HOST = {env:DB_HOST:localhost} @@ -56,7 +55,6 @@ setenv = DB_BACKEND = postgresql DB_PORT = {env:DB_PORT:5432} - [testenv:py{38,39,310,311}-dj{32,40,41,42,main}-postgis] setenv = {[testenv]setenv} From c335601eddb2914d07f6c43e094ba6197f730bcf Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Mon, 15 May 2023 15:59:56 +0300 Subject: [PATCH 14/37] Avoid selenium deprecation warning when testing Switch from setting the options.headless attribute to calling options.add_argument("-headless"). See [1] for more info. [1] https://www.selenium.dev/blog/2023/headless-is-going-away/ --- tests/test_integration.py | 3 ++- tox.ini | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 71340709a..b77b7cede 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -533,7 +533,8 @@ class DebugToolbarLiveTestCase(StaticLiveServerTestCase): def setUpClass(cls): super().setUpClass() options = Options() - options.headless = bool(os.environ.get("CI")) + if os.environ.get("CI"): + options.add_argument("-headless") cls.selenium = webdriver.Firefox(options=options) @classmethod diff --git a/tox.ini b/tox.ini index b1c194c61..87c8ff175 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ deps = Jinja2 html5lib pygments - selenium + selenium>=4.8.0 sqlparse passenv= CI From 476b5ce2a9d8261886b6f4a60fcbe7c62693d7de Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Tue, 16 May 2023 14:33:05 +0300 Subject: [PATCH 15/37] Fix BytesWarnings in tests Don't pass bytes objects as filter arguments for CharField fields. They get passed to str() internally within Django which results in a BytesWaring. --- tests/panels/test_sql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/panels/test_sql.py b/tests/panels/test_sql.py index 7b3452935..f4dcfaa2a 100644 --- a/tests/panels/test_sql.py +++ b/tests/panels/test_sql.py @@ -21,7 +21,7 @@ psycopg = None from ..base import BaseMultiDBTestCase, BaseTestCase -from ..models import PostgresJSON +from ..models import Binary, PostgresJSON def sql_call(use_iterator=False): @@ -149,7 +149,7 @@ def test_non_ascii_query(self): self.assertEqual(len(self.panel._queries), 2) # non-ASCII bytes parameters - list(User.objects.filter(username="café".encode())) + list(Binary.objects.filter(field__in=["café".encode()])) self.assertEqual(len(self.panel._queries), 3) response = self.panel.process_request(self.request) @@ -335,7 +335,7 @@ def test_insert_content(self): Test that the panel only inserts content after generate_stats and not the process_request. """ - list(User.objects.filter(username="café".encode())) + list(User.objects.filter(username="café")) response = self.panel.process_request(self.request) # ensure the panel does not have content yet. self.assertNotIn("café", self.panel.content) @@ -351,7 +351,7 @@ def test_insert_locals(self): Test that the panel inserts locals() content. """ local_var = "" # noqa: F841 - list(User.objects.filter(username="café".encode())) + list(User.objects.filter(username="café")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) self.assertIn("local_var", self.panel.content) @@ -365,7 +365,7 @@ def test_not_insert_locals(self): """ Test that the panel does not insert locals() content. """ - list(User.objects.filter(username="café".encode())) + list(User.objects.filter(username="café")) response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) self.assertNotIn("djdt-locals", self.panel.content) From 8e99db486fcdae741bebc6223f10ea10633e689c Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Wed, 24 May 2023 10:56:34 +0200 Subject: [PATCH 16/37] Removed Safari/Chrome workaround for system fonts See https://code.djangoproject.com/ticket/34592 --- debug_toolbar/static/debug_toolbar/css/toolbar.css | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debug_toolbar/static/debug_toolbar/css/toolbar.css b/debug_toolbar/static/debug_toolbar/css/toolbar.css index 4adb0abb5..a35286a1f 100644 --- a/debug_toolbar/static/debug_toolbar/css/toolbar.css +++ b/debug_toolbar/static/debug_toolbar/css/toolbar.css @@ -1,10 +1,9 @@ /* Variable definitions */ :root { /* Font families are the same as in Django admin/css/base.css */ - --djdt-font-family-primary: -apple-system, BlinkMacSystemFont, "Segoe UI", - system-ui, Roboto, "Helvetica Neue", Arial, sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", - "Noto Color Emoji"; + --djdt-font-family-primary: "Segoe UI", system-ui, Roboto, "Helvetica Neue", + Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", + "Segoe UI Symbol", "Noto Color Emoji"; --djdt-font-family-monospace: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono", "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro", "Fira Mono", "Droid Sans Mono", "Courier New", From 3e6e1df74f70000b4932b03962496f20f7ff89de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 18:09:03 +0200 Subject: [PATCH 17/37] [pre-commit.ci] pre-commit autoupdate (#1790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.269 → v0.0.270](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.269...v0.0.270) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dfa139b1e..46d8f5329 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.269' + rev: 'v0.0.270' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 3b87b93a40d298d59010f36005d429391657e13e Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 5 Jun 2023 12:55:42 +0200 Subject: [PATCH 18/37] Fix #1792: Lowercase all cookie keys, actually allow overriding the samesite value --- debug_toolbar/static/debug_toolbar/js/toolbar.js | 6 +++--- docs/changes.rst | 2 ++ docs/panels.rst | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/debug_toolbar/static/debug_toolbar/js/toolbar.js b/debug_toolbar/static/debug_toolbar/js/toolbar.js index ef2e617f9..9546ef27e 100644 --- a/debug_toolbar/static/debug_toolbar/js/toolbar.js +++ b/debug_toolbar/static/debug_toolbar/js/toolbar.js @@ -341,9 +341,9 @@ const djdt = { options.path ? "; path=" + options.path : "", options.domain ? "; domain=" + options.domain : "", options.secure ? "; secure" : "", - "sameSite" in options - ? "; sameSite=" + options.samesite - : "; sameSite=Lax", + "samesite" in options + ? "; samesite=" + options.samesite + : "; samesite=lax", ].join(""); return value; diff --git a/docs/changes.rst b/docs/changes.rst index 4847290fd..ab69ef99f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -7,6 +7,8 @@ Pending * Adjusted app directories system check to allow for nested template loaders. * Switched from flake8, isort and pyupgrade to `ruff `__. +* Converted cookie keys to lowercase. Fixed the ``samesite`` argument to + ``djdt.cookie.set``. 4.1.0 (2023-05-15) ------------------ diff --git a/docs/panels.rst b/docs/panels.rst index 519571574..61a23ce61 100644 --- a/docs/panels.rst +++ b/docs/panels.rst @@ -421,7 +421,9 @@ common methods available. :param value: The value to be set. :param options: The options for the value to be set. It should contain the - properties ``expires`` and ``path``. + properties ``expires`` and ``path``. The properties ``domain``, + ``secure`` and ``samesite`` are also supported. ``samesite`` defaults + to ``lax`` if not provided. .. js:function:: djdt.hide_toolbar From 904f2eb17d637b7b147bb5d9fb2f153f948c4302 Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 5 Jun 2023 14:16:13 +0200 Subject: [PATCH 19/37] Activate more ruff rules --- debug_toolbar/forms.py | 4 +- debug_toolbar/panels/headers.py | 2 +- debug_toolbar/panels/history/panel.py | 2 +- debug_toolbar/panels/profiling.py | 4 +- debug_toolbar/panels/sql/forms.py | 4 +- debug_toolbar/panels/sql/panel.py | 2 +- debug_toolbar/panels/templates/panel.py | 4 +- debug_toolbar/settings.py | 1 + debug_toolbar/toolbar.py | 2 +- pyproject.toml | 75 +++++++++++++++++++------ tests/context_processors.py | 2 +- tests/models.py | 9 +++ 12 files changed, 80 insertions(+), 31 deletions(-) diff --git a/debug_toolbar/forms.py b/debug_toolbar/forms.py index 1263c3aff..61444b43c 100644 --- a/debug_toolbar/forms.py +++ b/debug_toolbar/forms.py @@ -38,8 +38,8 @@ def clean_signed(self): signing.Signer(salt=self.salt).unsign(self.cleaned_data["signed"]) ) return verified - except signing.BadSignature: - raise ValidationError("Bad signature") + except signing.BadSignature as exc: + raise ValidationError("Bad signature") from exc def verified_data(self): return self.is_valid() and self.cleaned_data["signed"] diff --git a/debug_toolbar/panels/headers.py b/debug_toolbar/panels/headers.py index ed20d6178..e1ea6da1e 100644 --- a/debug_toolbar/panels/headers.py +++ b/debug_toolbar/panels/headers.py @@ -33,7 +33,7 @@ class HeadersPanel(Panel): template = "debug_toolbar/panels/headers.html" def process_request(self, request): - wsgi_env = list(sorted(request.META.items())) + wsgi_env = sorted(request.META.items()) self.request_headers = { unmangle(k): v for (k, v) in wsgi_env if is_http_header(k) } diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 8bd0e8f65..503cade31 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -22,7 +22,7 @@ class HistoryPanel(Panel): def get_headers(self, request): headers = super().get_headers(request) observe_request = self.toolbar.get_observe_request() - store_id = getattr(self.toolbar, "store_id") + store_id = self.toolbar.store_id if store_id and observe_request(request): headers["djdt-store-id"] = store_id return headers diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index ca32b98c2..9947b17dc 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -13,7 +13,7 @@ class FunctionCall: def __init__( - self, statobj, func, depth=0, stats=None, id=0, parent_ids=[], hsv=(0, 0.5, 1) + self, statobj, func, depth=0, stats=None, id=0, parent_ids=None, hsv=(0, 0.5, 1) ): self.statobj = statobj self.func = func @@ -23,7 +23,7 @@ def __init__( self.stats = statobj.stats[func][:4] self.depth = depth self.id = id - self.parent_ids = parent_ids + self.parent_ids = parent_ids or [] self.hsv = hsv def parent_classes(self): diff --git a/debug_toolbar/panels/sql/forms.py b/debug_toolbar/panels/sql/forms.py index 8d2709777..0515c5c8e 100644 --- a/debug_toolbar/panels/sql/forms.py +++ b/debug_toolbar/panels/sql/forms.py @@ -37,8 +37,8 @@ def clean_params(self): try: return json.loads(value) - except ValueError: - raise ValidationError("Is not valid JSON") + except ValueError as exc: + raise ValidationError("Is not valid JSON") from exc def clean_alias(self): value = self.cleaned_data["alias"] diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 984a2074a..58c1c2738 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -90,7 +90,7 @@ def _duplicate_query_key(query): def _process_query_groups(query_groups, databases, colors, name): counts = defaultdict(int) - for (alias, key), query_group in query_groups.items(): + for (alias, _key), query_group in query_groups.items(): count = len(query_group) # Queries are similar / duplicates only if there are at least 2 of them. if count > 1: diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index 35d5b5191..a1e45e83f 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -192,9 +192,7 @@ def generate_stats(self, request, response): template = self.templates[0]["template"] # django templates have the 'engine' attribute, while jinja # templates use 'backend' - engine_backend = getattr(template, "engine", None) or getattr( - template, "backend" - ) + engine_backend = getattr(template, "engine", None) or template.backend template_dirs = engine_backend.dirs else: context_processors = None diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index bf534a7da..eb6b59209 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -83,6 +83,7 @@ def get_panels(): warnings.warn( f"Please remove {logging_panel} from your DEBUG_TOOLBAR_PANELS setting.", DeprecationWarning, + stacklevel=1, ) return PANELS diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index 31010f47f..11f8a1daa 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -86,7 +86,7 @@ def render_toolbar(self): "The debug toolbar requires the staticfiles contrib app. " "Add 'django.contrib.staticfiles' to INSTALLED_APPS and " "define STATIC_URL in your settings." - ) + ) from None else: raise diff --git a/pyproject.toml b/pyproject.toml index ff8c22874..e04e3a42d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,24 +66,65 @@ source = ["src", ".tox/*/site-packages"] fail_under = 94 show_missing = true -[tool.ruff.isort] -combine-as-imports = true - [tool.ruff] -select = [ - # flake8/Pyflakes - "F", - # flake8/pycodestyle - "E", - "W", - # isort - "I", - # pyupgrade - "UP", - # pygrep-hooks - "PGH", +extend-select = [ + # pyflakes, pycodestyle + "F", "E", "W", + # mmcabe + # "C90", + # isort + "I", + # pep8-naming + # "N", + # pyupgrade + "UP", + # flake8-2020 + # "YTT", + # flake8-boolean-trap + # "FBT", + # flake8-bugbear + "B", + # flake8-builtins + # "A", + # flake8-comprehensions + "C4", + # flake8-django + "DJ", + # flake8-logging-format + "G", + # flake8-pie + # "PIE", + # flake8-simplify + # "SIM", + # flake8-gettext + "INT", + # pygrep-hooks + "PGH", + # pylint + # "PL", + # unused noqa + "RUF100", ] -ignore = [ - "E501", +extend-ignore = [ + # Allow zip() without strict= + "B905", + # No line length errors + "E501", ] +fix = true +show-fixes = true target-version = "py38" + +[tool.ruff.isort] +combine-as-imports = true + +[tool.ruff.mccabe] +max-complexity = 15 + +[tool.ruff.per-file-ignores] +"*/migrat*/*" = [ + # Allow using PascalCase model names in migrations + "N806", + # Ignore the fact that migration files are invalid module names + "N999", +] diff --git a/tests/context_processors.py b/tests/context_processors.py index 6fe220dba..69e112a39 100644 --- a/tests/context_processors.py +++ b/tests/context_processors.py @@ -1,2 +1,2 @@ def broken(request): - request.non_existing_attribute + _read = request.non_existing_attribute diff --git a/tests/models.py b/tests/models.py index 95696020a..e19bfe59d 100644 --- a/tests/models.py +++ b/tests/models.py @@ -11,13 +11,22 @@ def __repr__(self): class Binary(models.Model): field = models.BinaryField() + def __str__(self): + return "" + class PostgresJSON(models.Model): field = JSONField() + def __str__(self): + return "" + if settings.USE_GIS: from django.contrib.gis.db import models as gismodels class Location(gismodels.Model): point = gismodels.PointField() + + def __str__(self): + return "" From 52dedc4d87d90872ee859e88cb918f82f54f8a5f Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 5 Jun 2023 14:17:58 +0200 Subject: [PATCH 20/37] Use flake8-boolean-trap --- debug_toolbar/panels/sql/utils.py | 2 +- debug_toolbar/panels/templates/panel.py | 2 +- pyproject.toml | 2 +- tests/panels/test_sql.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug_toolbar/panels/sql/utils.py b/debug_toolbar/panels/sql/utils.py index 120674c96..cb4eda348 100644 --- a/debug_toolbar/panels/sql/utils.py +++ b/debug_toolbar/panels/sql/utils.py @@ -86,7 +86,7 @@ def process(stmt): return "".join(escaped_value(token) for token in stmt.flatten()) -def reformat_sql(sql, with_toggle=False): +def reformat_sql(sql, *, with_toggle=False): formatted = parse_sql(sql) if not with_toggle: return formatted diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index a1e45e83f..72565f016 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -117,7 +117,7 @@ def _store_template_info(self, sender, **kwargs): value.model._meta.label, ) else: - token = allow_sql.set(False) + token = allow_sql.set(False) # noqa: FBT003 try: saferepr(value) # this MAY trigger a db query except SQLQueryTriggered: diff --git a/pyproject.toml b/pyproject.toml index e04e3a42d..307574a5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ extend-select = [ # flake8-2020 # "YTT", # flake8-boolean-trap - # "FBT", + "FBT", # flake8-bugbear "B", # flake8-builtins diff --git a/tests/panels/test_sql.py b/tests/panels/test_sql.py index f4dcfaa2a..fd248663f 100644 --- a/tests/panels/test_sql.py +++ b/tests/panels/test_sql.py @@ -24,7 +24,7 @@ from ..models import Binary, PostgresJSON -def sql_call(use_iterator=False): +def sql_call(*, use_iterator=False): qs = User.objects.all() if use_iterator: qs = qs.iterator() @@ -105,7 +105,7 @@ async def test_cursor_wrapper_asyncio_ctx(self, mock_wrapper): await sync_to_async(sql_call)() async def task(): - sql_tracking.allow_sql.set(False) + sql_tracking.allow_sql.set(False) # noqa: FBT003 # By disabling sql_tracking.allow_sql, we are indicating that any # future SQL queries should be stopped. If SQL query occurs, # it raises an exception. From fcfdc7c6f4c2252a46f2da1a785eb479d1b790cb Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 5 Jun 2023 14:18:30 +0200 Subject: [PATCH 21/37] Activate flake8-pie --- debug_toolbar/panels/__init__.py | 1 - pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index 6d9491b1f..57f385a5e 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -124,7 +124,6 @@ def ready(cls): be done unconditionally for the panel regardless of whether it is enabled for a particular request. It should be idempotent. """ - pass # URLs for panel-specific views diff --git a/pyproject.toml b/pyproject.toml index 307574a5c..ce6bcf740 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ extend-select = [ # flake8-logging-format "G", # flake8-pie - # "PIE", + "PIE", # flake8-simplify # "SIM", # flake8-gettext From ebd6ea3bc2adcdbf1190dfbc3540b12793ba28d9 Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Mon, 5 Jun 2023 14:19:58 +0200 Subject: [PATCH 22/37] Activate flake8-simplify --- debug_toolbar/panels/cache.py | 5 +---- debug_toolbar/panels/history/panel.py | 6 +++--- debug_toolbar/panels/profiling.py | 5 +---- debug_toolbar/panels/request.py | 4 +--- debug_toolbar/panels/sql/tracking.py | 11 ++++------- debug_toolbar/utils.py | 5 +---- pyproject.toml | 2 +- tests/panels/test_profiling.py | 5 ++--- 8 files changed, 14 insertions(+), 29 deletions(-) diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 31ce70988..4c7bf5af7 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -117,10 +117,7 @@ def _store_call_info( else: self.hits += 1 elif name == "get_many": - if "keys" in kwargs: - keys = kwargs["keys"] - else: - keys = args[0] + keys = kwargs["keys"] if "keys" in kwargs else args[0] self.hits += len(return_value) self.misses += len(keys) - len(return_value) time_taken *= 1000 diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 503cade31..508a60577 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -1,3 +1,4 @@ +import contextlib import json from django.http.request import RawPostDataException @@ -62,10 +63,9 @@ def generate_stats(self, request, response): and request.body and request.headers.get("content-type") == "application/json" ): - try: + with contextlib.suppress(ValueError): data = json.loads(request.body) - except ValueError: - pass + except RawPostDataException: # It is not guaranteed that we may read the request data (again). data = None diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 9947b17dc..9d10229ad 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -88,10 +88,7 @@ def subfuncs(self): for func, stats in self.statobj.all_callees[self.func].items(): i += 1 h1 = h + (i / count) / (self.depth + 1) - if stats[3] == 0: - s1 = 0 - else: - s1 = s * (stats[3] / self.stats[3]) + s1 = 0 if stats[3] == 0 else s * (stats[3] / self.stats[3]) yield FunctionCall( self.statobj, func, diff --git a/debug_toolbar/panels/request.py b/debug_toolbar/panels/request.py index bfb485ae7..a936eba6b 100644 --- a/debug_toolbar/panels/request.py +++ b/debug_toolbar/panels/request.py @@ -64,7 +64,5 @@ def generate_stats(self, request, response): (k, request.session.get(k)) for k in sorted(request.session.keys()) ] except TypeError: - session_list = [ - (k, request.session.get(k)) for k in request.session.keys() - ] + session_list = [(k, request.session.get(k)) for k in request.session] self.record_stats({"session": {"list": session_list}}) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index ee75f8e06..9fab89b8b 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -1,3 +1,4 @@ +import contextlib import contextvars import datetime import json @@ -59,10 +60,7 @@ def cursor(*args, **kwargs): cursor = connection._djdt_cursor(*args, **kwargs) if logger is None: return cursor - if allow_sql.get(): - wrapper = NormalCursorWrapper - else: - wrapper = ExceptionCursorWrapper + wrapper = NormalCursorWrapper if allow_sql.get() else ExceptionCursorWrapper return wrapper(cursor.cursor, connection, logger) def chunked_cursor(*args, **kwargs): @@ -174,10 +172,9 @@ def _record(self, method, sql, params): stop_time = perf_counter() duration = (stop_time - start_time) * 1000 _params = "" - try: + with contextlib.suppress(TypeError): + # object JSON serializable? _params = json.dumps(self._decode(params)) - except TypeError: - pass # object not JSON serializable template_info = get_template_info() # Sql might be an object (such as psycopg Composed). diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index 4a7e9b2c3..968160078 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -168,10 +168,7 @@ def get_template_source_from_exception_info( def get_name_from_obj(obj: Any) -> str: - if hasattr(obj, "__name__"): - name = obj.__name__ - else: - name = obj.__class__.__name__ + name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ if hasattr(obj, "__module__"): module = obj.__module__ diff --git a/pyproject.toml b/pyproject.toml index ce6bcf740..1ce4f3787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ extend-select = [ # flake8-pie "PIE", # flake8-simplify - # "SIM", + "SIM", # flake8-gettext "INT", # pygrep-hooks diff --git a/tests/panels/test_profiling.py b/tests/panels/test_profiling.py index 2169932b2..ff613dfe1 100644 --- a/tests/panels/test_profiling.py +++ b/tests/panels/test_profiling.py @@ -88,7 +88,6 @@ def test_view_executed_once(self): self.assertContains(response, "Profiling") self.assertEqual(User.objects.count(), 1) - with self.assertRaises(IntegrityError): - with transaction.atomic(): - response = self.client.get("/new_user/") + with self.assertRaises(IntegrityError), transaction.atomic(): + response = self.client.get("/new_user/") self.assertEqual(User.objects.count(), 1) From 99a61c97b7534bb226e7351f32973621bec91f69 Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Tue, 6 Jun 2023 18:32:18 +0200 Subject: [PATCH 23/37] Let's see if this works Refs #1779. --- docs/conf.py | 2 +- requirements_dev.txt | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 2e4886c9c..219a2f440 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "default" +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/requirements_dev.txt b/requirements_dev.txt index 4b90beb08..8b24a8fbb 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -16,6 +16,7 @@ black Sphinx sphinxcontrib-spelling +sphinx-rtd-theme>1 # Other tools diff --git a/tox.ini b/tox.ini index 87c8ff175..5154d4907 100644 --- a/tox.ini +++ b/tox.ini @@ -78,6 +78,7 @@ commands = make -C {toxinidir}/docs {posargs:spelling} deps = Sphinx sphinxcontrib-spelling + sphinx-rtd-theme [testenv:packaging] commands = From e535c9d01520e2fd7ee5e05c6efded0071ab7f30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:32:12 +0000 Subject: [PATCH 24/37] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-eslint: v8.41.0 → v8.42.0](https://github.com/pre-commit/mirrors-eslint/compare/v8.41.0...v8.42.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46d8f5329..851b82459 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.41.0 + rev: v8.42.0 hooks: - id: eslint files: \.js?$ From e15c53860c74d9f074e7efd8e3750d5e3bac6664 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sat, 17 Jun 2023 10:16:24 -0500 Subject: [PATCH 25/37] Fix CI tests with MariaDB. (#1797) * Fix CI tests with MariaDB. The docker image now uses MARIADB_ROOT_PASSWORD for the environment variable for the password rather than MYSQL_ROOT_PASSWORD. * It's mariadb-admin now not mysqladmin As per https://github.com/MariaDB/mariadb-docker/issues/497 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc4b9a456..9dece68ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,9 +20,9 @@ jobs: mariadb: image: mariadb env: - MYSQL_ROOT_PASSWORD: debug_toolbar + MARIADB_ROOT_PASSWORD: debug_toolbar options: >- - --health-cmd "mysqladmin ping" + --health-cmd "mariadb-admin ping" --health-interval 10s --health-timeout 5s --health-retries 5 From 797a861abdab64e321897a3d52dd7b374e195520 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Jun 2023 10:07:18 +0200 Subject: [PATCH 26/37] [pre-commit.ci] pre-commit autoupdate (#1796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/adamchainz/django-upgrade: 1.13.0 → 1.14.0](https://github.com/adamchainz/django-upgrade/compare/1.13.0...1.14.0) - [github.com/pre-commit/mirrors-eslint: v8.42.0 → v8.43.0](https://github.com/pre-commit/mirrors-eslint/compare/v8.42.0...v8.43.0) - [github.com/charliermarsh/ruff-pre-commit: v0.0.270 → v0.0.272](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.270...v0.0.272) - [github.com/tox-dev/pyproject-fmt: 0.11.2 → 0.12.0](https://github.com/tox-dev/pyproject-fmt/compare/0.11.2...0.12.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 34 +++++++++++++++++----------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 851b82459..f9231dd6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: hooks: - id: doc8 - repo: https://github.com/adamchainz/django-upgrade - rev: 1.13.0 + rev: 1.14.0 hooks: - id: django-upgrade args: [--target-version, "3.2"] @@ -31,7 +31,7 @@ repos: args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.42.0 + rev: v8.43.0 hooks: - id: eslint files: \.js?$ @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.270' + rev: 'v0.0.272' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -50,7 +50,7 @@ repos: language_version: python3 entry: black --target-version=py38 - repo: https://github.com/tox-dev/pyproject-fmt - rev: 0.11.2 + rev: 0.12.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject diff --git a/pyproject.toml b/pyproject.toml index 1ce4f3787..685fe42c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,23 +49,6 @@ packages = ["debug_toolbar"] [tool.hatch.version] path = "debug_toolbar/__init__.py" -[tool.coverage.html] -skip_covered = true -skip_empty = true - -[tool.coverage.run] -branch = true -parallel = true -source = ["debug_toolbar"] - -[tool.coverage.paths] -source = ["src", ".tox/*/site-packages"] - -[tool.coverage.report] -# Update coverage badge link in README.rst when fail_under changes -fail_under = 94 -show_missing = true - [tool.ruff] extend-select = [ # pyflakes, pycodestyle @@ -128,3 +111,20 @@ max-complexity = 15 # Ignore the fact that migration files are invalid module names "N999", ] + +[tool.coverage.html] +skip_covered = true +skip_empty = true + +[tool.coverage.run] +branch = true +parallel = true +source = ["debug_toolbar"] + +[tool.coverage.paths] +source = ["src", ".tox/*/site-packages"] + +[tool.coverage.report] +# Update coverage badge link in README.rst when fail_under changes +fail_under = 94 +show_missing = true From 58f5ae30a2a559b05c67dc31e0c93f0165c9ebe3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:42:32 +0000 Subject: [PATCH 27/37] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.272 → v0.0.275](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.272...v0.0.275) - [github.com/tox-dev/pyproject-fmt: 0.12.0 → 0.12.1](https://github.com/tox-dev/pyproject-fmt/compare/0.12.0...0.12.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9231dd6e..8eee18339 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.272' + rev: 'v0.0.275' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -50,7 +50,7 @@ repos: language_version: python3 entry: black --target-version=py38 - repo: https://github.com/tox-dev/pyproject-fmt - rev: 0.12.0 + rev: 0.12.1 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject From 47d4eedbce345434b57369bb170499c03f556cc7 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Mon, 3 Jul 2023 02:13:13 -0500 Subject: [PATCH 28/37] Switch StaticFilesPanel to use ContextVar. (#1801) The StaticFilesPanel thread collector was not closing connections to the database. This approach allows those connections to be closed while still collecting context across threads. The test case is to hit an admin login screen with ab -n 200 http://localhost:8000/admin/login/ As far as I can tell, all the static files are collected properly and connections don't stay open. --- debug_toolbar/panels/staticfiles.py | 52 +++++++++++++---------------- debug_toolbar/utils.py | 36 -------------------- docs/changes.rst | 2 ++ 3 files changed, 25 insertions(+), 65 deletions(-) diff --git a/debug_toolbar/panels/staticfiles.py b/debug_toolbar/panels/staticfiles.py index bee336249..5f9efb5c3 100644 --- a/debug_toolbar/panels/staticfiles.py +++ b/debug_toolbar/panels/staticfiles.py @@ -1,3 +1,5 @@ +import contextlib +from contextvars import ContextVar from os.path import join, normpath from django.conf import settings @@ -7,12 +9,6 @@ from django.utils.translation import gettext_lazy as _, ngettext from debug_toolbar import panels -from debug_toolbar.utils import ThreadCollector - -try: - import threading -except ImportError: - threading = None class StaticFile: @@ -33,15 +29,8 @@ def url(self): return storage.staticfiles_storage.url(self.path) -class FileCollector(ThreadCollector): - def collect(self, path, thread=None): - # handle the case of {% static "admin/" %} - if path.endswith("/"): - return - super().collect(StaticFile(path), thread) - - -collector = FileCollector() +# This will collect the StaticFile instances across threads. +used_static_files = ContextVar("djdt_static_used_static_files") class DebugConfiguredStorage(LazyObject): @@ -65,15 +54,16 @@ def _setup(self): configured_storage_cls = get_storage_class(settings.STATICFILES_STORAGE) class DebugStaticFilesStorage(configured_storage_cls): - def __init__(self, collector, *args, **kwargs): - super().__init__(*args, **kwargs) - self.collector = collector - def url(self, path): - self.collector.collect(path) + with contextlib.suppress(LookupError): + # For LookupError: + # The ContextVar wasn't set yet. Since the toolbar wasn't properly + # configured to handle this request, we don't need to capture + # the static file. + used_static_files.get().append(StaticFile(path)) return super().url(path) - self._wrapped = DebugStaticFilesStorage(collector) + self._wrapped = DebugStaticFilesStorage() _original_storage = storage.staticfiles_storage @@ -97,7 +87,7 @@ def title(self): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.num_found = 0 - self._paths = {} + self.used_paths = [] def enable_instrumentation(self): storage.staticfiles_storage = DebugConfiguredStorage() @@ -120,18 +110,22 @@ def nav_subtitle(self): ) % {"num_used": num_used} def process_request(self, request): - collector.clear_collection() - return super().process_request(request) + reset_token = used_static_files.set([]) + response = super().process_request(request) + # Make a copy of the used paths so that when the + # ContextVar is reset, our panel still has the data. + self.used_paths = used_static_files.get().copy() + # Reset the ContextVar to be empty again, removing the reference + # to the list of used files. + used_static_files.reset(reset_token) + return response def generate_stats(self, request, response): - used_paths = collector.get_collection() - self._paths[threading.current_thread()] = used_paths - self.record_stats( { "num_found": self.num_found, - "num_used": len(used_paths), - "staticfiles": used_paths, + "num_used": len(self.used_paths), + "staticfiles": self.used_paths, "staticfiles_apps": self.get_staticfiles_apps(), "staticfiles_dirs": self.get_staticfiles_dirs(), "staticfiles_finders": self.get_staticfiles_finders(), diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index 968160078..7234f1f77 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -14,12 +14,6 @@ from debug_toolbar import _stubs as stubs, settings as dt_settings -try: - import threading -except ImportError: - threading = None - - _local_data = Local() @@ -357,33 +351,3 @@ def get_stack_trace(*, skip=0): def clear_stack_trace_caches(): if hasattr(_local_data, "stack_trace_recorder"): del _local_data.stack_trace_recorder - - -class ThreadCollector: - def __init__(self): - if threading is None: - raise NotImplementedError( - "threading module is not available, " - "this panel cannot be used without it" - ) - self.collections = {} # a dictionary that maps threads to collections - - def get_collection(self, thread=None): - """ - Returns a list of collected items for the provided thread, of if none - is provided, returns a list for the current thread. - """ - if thread is None: - thread = threading.current_thread() - if thread not in self.collections: - self.collections[thread] = [] - return self.collections[thread] - - def clear_collection(self, thread=None): - if thread is None: - thread = threading.current_thread() - if thread in self.collections: - del self.collections[thread] - - def collect(self, item, thread=None): - self.get_collection(thread).append(item) diff --git a/docs/changes.rst b/docs/changes.rst index ab69ef99f..cfd950229 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -9,6 +9,8 @@ Pending `__. * Converted cookie keys to lowercase. Fixed the ``samesite`` argument to ``djdt.cookie.set``. +* Converted ``StaticFilesPanel`` to no longer use a thread collector. Instead, + it collects the used static files in a ``ContextVar``. 4.1.0 (2023-05-15) ------------------ From 4a641ec7e079b2dbcb207bcc2a73a73fd7885741 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Tue, 4 Jul 2023 00:53:07 -0500 Subject: [PATCH 29/37] Check JavaScript files content type. (#1802) A common problem with windows set ups is that the registry is misconfigured when resolving the content type for a JavaScript file. This check captures this sooner to explain why the toolbar doesn't show and attempts to tell the user what to change to make it work. --- debug_toolbar/apps.py | 32 ++++++++++++++++++++++++++++++++ docs/changes.rst | 2 ++ docs/checks.rst | 3 +++ tests/test_checks.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/debug_toolbar/apps.py b/debug_toolbar/apps.py index f84bcca06..0a10a4b08 100644 --- a/debug_toolbar/apps.py +++ b/debug_toolbar/apps.py @@ -1,4 +1,5 @@ import inspect +import mimetypes from django.apps import AppConfig from django.conf import settings @@ -174,3 +175,34 @@ def check_panels(app_configs, **kwargs): ) ) return errors + + +@register() +def js_mimetype_check(app_configs, **kwargs): + """ + Check that JavaScript files are resolving to the correct content type. + """ + # Ideally application/javascript is returned, but text/javascript is + # acceptable. + javascript_types = {"application/javascript", "text/javascript"} + check_failed = not set(mimetypes.guess_type("toolbar.js")).intersection( + javascript_types + ) + if check_failed: + return [ + Warning( + "JavaScript files are resolving to the wrong content type.", + hint="The Django Debug Toolbar may not load properly while mimetypes are misconfigured. " + "See the Django documentation for an explanation of why this occurs.\n" + "https://docs.djangoproject.com/en/stable/ref/contrib/staticfiles/#static-file-development-view\n" + "\n" + "This typically occurs on Windows machines. The suggested solution is to modify " + "HKEY_CLASSES_ROOT in the registry to specify the content type for JavaScript " + "files.\n" + "\n" + "[HKEY_CLASSES_ROOT\\.js]\n" + '"Content Type"="application/javascript"', + id="debug_toolbar.W007", + ) + ] + return [] diff --git a/docs/changes.rst b/docs/changes.rst index cfd950229..2e9f6f29d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -11,6 +11,8 @@ Pending ``djdt.cookie.set``. * Converted ``StaticFilesPanel`` to no longer use a thread collector. Instead, it collects the used static files in a ``ContextVar``. +* Added check ``debug_toolbar.W007`` to warn when JavaScript files are + resolving to the wrong content type. 4.1.0 (2023-05-15) ------------------ diff --git a/docs/checks.rst b/docs/checks.rst index b76f761a0..6ed1e88f4 100644 --- a/docs/checks.rst +++ b/docs/checks.rst @@ -18,3 +18,6 @@ Django Debug Toolbar setup and configuration: configuration needs to have ``django.template.loaders.app_directories.Loader`` included in ``["OPTIONS"]["loaders"]`` or ``APP_DIRS`` set to ``True``. +* **debug_toolbar.W007**: JavaScript files are resolving to the wrong content + type. Refer to :external:ref:`Django's explanation of + mimetypes on Windows `. diff --git a/tests/test_checks.py b/tests/test_checks.py index 3f17922c0..8e4f8e62f 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -1,5 +1,6 @@ import os import unittest +from unittest.mock import patch import django from django.conf import settings @@ -227,3 +228,33 @@ def test_check_w006_valid(self): ) def test_check_w006_valid_nested_loaders(self): self.assertEqual(run_checks(), []) + + @patch("debug_toolbar.apps.mimetypes.guess_type") + def test_check_w007_valid(self, mocked_guess_type): + mocked_guess_type.return_value = ("text/javascript", None) + self.assertEqual(run_checks(), []) + mocked_guess_type.return_value = ("application/javascript", None) + self.assertEqual(run_checks(), []) + + @patch("debug_toolbar.apps.mimetypes.guess_type") + def test_check_w007_invalid(self, mocked_guess_type): + mocked_guess_type.return_value = ("text/plain", None) + self.assertEqual( + run_checks(), + [ + Warning( + "JavaScript files are resolving to the wrong content type.", + hint="The Django Debug Toolbar may not load properly while mimetypes are misconfigured. " + "See the Django documentation for an explanation of why this occurs.\n" + "https://docs.djangoproject.com/en/stable/ref/contrib/staticfiles/#static-file-development-view\n" + "\n" + "This typically occurs on Windows machines. The suggested solution is to modify " + "HKEY_CLASSES_ROOT in the registry to specify the content type for JavaScript " + "files.\n" + "\n" + "[HKEY_CLASSES_ROOT\\.js]\n" + '"Content Type"="application/javascript"', + id="debug_toolbar.W007", + ) + ], + ) From 43c076a64646df0b82de30ab17d2fd0526e0f25f Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Thu, 6 Jul 2023 17:03:59 +0200 Subject: [PATCH 30/37] pre-commit updates; disable two of the more annoying ruff rulesets --- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8eee18339..0504a99ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,14 +24,14 @@ repos: - id: rst-backticks - id: rst-directive-colons - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode + rev: v3.0.0 hooks: - id: prettier types_or: [javascript, css] args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.43.0 + rev: v8.44.0 hooks: - id: eslint files: \.js?$ @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.275' + rev: 'v0.0.277' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -50,7 +50,7 @@ repos: language_version: python3 entry: black --target-version=py38 - repo: https://github.com/tox-dev/pyproject-fmt - rev: 0.12.1 + rev: 0.13.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject diff --git a/pyproject.toml b/pyproject.toml index 685fe42c3..637dada5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,14 +67,10 @@ extend-select = [ "FBT", # flake8-bugbear "B", - # flake8-builtins - # "A", # flake8-comprehensions "C4", # flake8-django "DJ", - # flake8-logging-format - "G", # flake8-pie "PIE", # flake8-simplify From acd69dfe4349c62003c06fad56328ef76c3946d0 Mon Sep 17 00:00:00 2001 From: Lucidiot Date: Fri, 7 Jul 2023 10:07:46 +0200 Subject: [PATCH 31/37] Handle logging queries encoded as bytes under PostgreSQL (#1812) --- debug_toolbar/panels/sql/tracking.py | 5 ++++- docs/changes.rst | 2 ++ tests/panels/test_sql.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 9fab89b8b..83be91378 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -180,7 +180,10 @@ def _record(self, method, sql, params): # Sql might be an object (such as psycopg Composed). # For logging purposes, make sure it's str. if vendor == "postgresql" and not isinstance(sql, str): - sql = sql.as_string(conn) + if isinstance(sql, bytes): + sql = sql.decode("utf-8") + else: + sql = sql.as_string(conn) else: sql = str(sql) diff --git a/docs/changes.rst b/docs/changes.rst index 2e9f6f29d..e06d4c615 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -13,6 +13,8 @@ Pending it collects the used static files in a ``ContextVar``. * Added check ``debug_toolbar.W007`` to warn when JavaScript files are resolving to the wrong content type. +* Fixed SQL statement recording under PostgreSQL for queries encoded as byte + strings. 4.1.0 (2023-05-15) ------------------ diff --git a/tests/panels/test_sql.py b/tests/panels/test_sql.py index fd248663f..d6b31ca2b 100644 --- a/tests/panels/test_sql.py +++ b/tests/panels/test_sql.py @@ -158,6 +158,17 @@ def test_non_ascii_query(self): # ensure the panel renders correctly self.assertIn("café", self.panel.content) + @unittest.skipUnless( + connection.vendor == "postgresql", "Test valid only on PostgreSQL" + ) + def test_bytes_query(self): + self.assertEqual(len(self.panel._queries), 0) + + with connection.cursor() as cursor: + cursor.execute(b"SELECT 1") + + self.assertEqual(len(self.panel._queries), 1) + def test_param_conversion(self): self.assertEqual(len(self.panel._queries), 0) From 53747ef8cb9302478ce6cc5f50485b3c92f61b58 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:22:07 +0200 Subject: [PATCH 32/37] [pre-commit.ci] pre-commit autoupdate (#1815) --- .pre-commit-config.yaml | 8 ++++---- debug_toolbar/panels/redirects.py | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0504a99ac..36e3206e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,20 +31,20 @@ repos: args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.44.0 + rev: v8.45.0 hooks: - id: eslint files: \.js?$ types: [file] args: - --fix -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.277' +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.0.278' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black language_version: python3 diff --git a/debug_toolbar/panels/redirects.py b/debug_toolbar/panels/redirects.py index f6a00b574..195d0cf11 100644 --- a/debug_toolbar/panels/redirects.py +++ b/debug_toolbar/panels/redirects.py @@ -18,9 +18,7 @@ def process_request(self, request): if 300 <= response.status_code < 400: redirect_to = response.get("Location") if redirect_to: - status_line = "{} {}".format( - response.status_code, response.reason_phrase - ) + status_line = f"{response.status_code} {response.reason_phrase}" cookies = response.cookies context = {"redirect_to": redirect_to, "status_line": status_line} # Using SimpleTemplateResponse avoids running global context processors. From 66eb88d79669ee1c4b3a6d73a4d349f66a9db697 Mon Sep 17 00:00:00 2001 From: Matthias Kestenholz Date: Tue, 1 Aug 2023 14:51:38 +0200 Subject: [PATCH 33/37] Fix a typo --- debug_toolbar/panels/sql/tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 83be91378..14b2cb7ab 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -40,7 +40,7 @@ def wrap_cursor(connection): # patching when the test case is finished. If we monkey patch those methods # also, Django's process of undoing those monkey patches will fail. To # avoid this failure, and because database access is not allowed during a - # SimpleTextCase anyway, skip applying our instrumentation monkey patches if + # SimpleTestCase anyway, skip applying our instrumentation monkey patches if # we detect that Django has already monkey patched DatabaseWrapper.cursor(). if isinstance(connection.cursor, django.test.testcases._DatabaseFailure): return From 6e55663ab396fc1dfa6f2c64000dfa535f14df3a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:53:51 +0200 Subject: [PATCH 34/37] [pre-commit.ci] pre-commit autoupdate (#1817) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36e3206e3..a81ecfbdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: args: - --trailing-comma=es5 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.45.0 + rev: v8.46.0 hooks: - id: eslint files: \.js?$ @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.278' + rev: 'v0.0.281' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 7677183eb31130b0ea2e8408930b8622e1778171 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Thu, 10 Aug 2023 20:20:54 -0500 Subject: [PATCH 35/37] Patch CursorWrapper dynamically to allow multiple base classes. (#1820) * Patch CursorWrapper dynamically to allow multiple base classes. When the debug SQL logs are enabled, the wrapper class is CursorDebugWrapper not CursorWrapper. Since we have inspections based on that specific class they are removing the CursorDebugWrapper causing the SQL logs to not appear. This attempts to dynamically patch the CursorWrapper or CursorDebugWrapper with the functionality we need. This doesn't do a full regression test, but it may be possible to get it to work with: TEST_ARGS='--debug-sql' make test Which causes the current set of tests to fail since they are keyed to CursorWrapper. * Allow mixin as a valid word in our docs. * Support tests with --debug-sql --- debug_toolbar/panels/sql/tracking.py | 31 +++++++----- docs/changes.rst | 2 + docs/spelling_wordlist.txt | 1 + tests/panels/test_sql.py | 71 +++++++++++++++++++++------- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 14b2cb7ab..0c53dc2c5 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -5,7 +5,6 @@ from time import perf_counter import django.test.testcases -from django.db.backends.utils import CursorWrapper from django.utils.encoding import force_str from debug_toolbar.utils import get_stack_trace, get_template_info @@ -60,34 +59,42 @@ def cursor(*args, **kwargs): cursor = connection._djdt_cursor(*args, **kwargs) if logger is None: return cursor - wrapper = NormalCursorWrapper if allow_sql.get() else ExceptionCursorWrapper - return wrapper(cursor.cursor, connection, logger) + mixin = NormalCursorMixin if allow_sql.get() else ExceptionCursorMixin + return patch_cursor_wrapper_with_mixin(cursor.__class__, mixin)( + cursor.cursor, connection, logger + ) def chunked_cursor(*args, **kwargs): # prevent double wrapping # solves https://github.com/jazzband/django-debug-toolbar/issues/1239 logger = connection._djdt_logger cursor = connection._djdt_chunked_cursor(*args, **kwargs) - if logger is not None and not isinstance(cursor, DjDTCursorWrapper): - if allow_sql.get(): - wrapper = NormalCursorWrapper - else: - wrapper = ExceptionCursorWrapper - return wrapper(cursor.cursor, connection, logger) + if logger is not None and not isinstance(cursor, DjDTCursorWrapperMixin): + mixin = NormalCursorMixin if allow_sql.get() else ExceptionCursorMixin + return patch_cursor_wrapper_with_mixin(cursor.__class__, mixin)( + cursor.cursor, connection, logger + ) return cursor connection.cursor = cursor connection.chunked_cursor = chunked_cursor -class DjDTCursorWrapper(CursorWrapper): +def patch_cursor_wrapper_with_mixin(base_wrapper, mixin): + class DjDTCursorWrapper(mixin, base_wrapper): + pass + + return DjDTCursorWrapper + + +class DjDTCursorWrapperMixin: def __init__(self, cursor, db, logger): super().__init__(cursor, db) # logger must implement a ``record`` method self.logger = logger -class ExceptionCursorWrapper(DjDTCursorWrapper): +class ExceptionCursorMixin(DjDTCursorWrapperMixin): """ Wraps a cursor and raises an exception on any operation. Used in Templates panel. @@ -97,7 +104,7 @@ def __getattr__(self, attr): raise SQLQueryTriggered() -class NormalCursorWrapper(DjDTCursorWrapper): +class NormalCursorMixin(DjDTCursorWrapperMixin): """ Wraps a cursor and logs queries. """ diff --git a/docs/changes.rst b/docs/changes.rst index e06d4c615..5b7aaf1c1 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -15,6 +15,8 @@ Pending resolving to the wrong content type. * Fixed SQL statement recording under PostgreSQL for queries encoded as byte strings. +* Patch the ``CursorWrapper`` class with a mixin class to support multiple + base wrapper classes. 4.1.0 (2023-05-15) ------------------ diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2ab01758c..7a15d9aeb 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -30,6 +30,7 @@ memcache memcached middleware middlewares +mixin mousedown mouseup multi diff --git a/tests/panels/test_sql.py b/tests/panels/test_sql.py index d6b31ca2b..932a0dd92 100644 --- a/tests/panels/test_sql.py +++ b/tests/panels/test_sql.py @@ -2,12 +2,13 @@ import datetime import os import unittest -from unittest.mock import patch +from unittest.mock import call, patch import django from asgiref.sync import sync_to_async from django.contrib.auth.models import User from django.db import connection, transaction +from django.db.backends.utils import CursorDebugWrapper, CursorWrapper from django.db.models import Count from django.db.utils import DatabaseError from django.shortcuts import render @@ -68,39 +69,59 @@ def test_recording_chunked_cursor(self): self.assertEqual(len(self.panel._queries), 1) @patch( - "debug_toolbar.panels.sql.tracking.NormalCursorWrapper", - wraps=sql_tracking.NormalCursorWrapper, + "debug_toolbar.panels.sql.tracking.patch_cursor_wrapper_with_mixin", + wraps=sql_tracking.patch_cursor_wrapper_with_mixin, ) - def test_cursor_wrapper_singleton(self, mock_wrapper): + def test_cursor_wrapper_singleton(self, mock_patch_cursor_wrapper): sql_call() - # ensure that cursor wrapping is applied only once - self.assertEqual(mock_wrapper.call_count, 1) + self.assertIn( + mock_patch_cursor_wrapper.mock_calls, + [ + [call(CursorWrapper, sql_tracking.NormalCursorMixin)], + # CursorDebugWrapper is used if the test is called with `--debug-sql` + [call(CursorDebugWrapper, sql_tracking.NormalCursorMixin)], + ], + ) @patch( - "debug_toolbar.panels.sql.tracking.NormalCursorWrapper", - wraps=sql_tracking.NormalCursorWrapper, + "debug_toolbar.panels.sql.tracking.patch_cursor_wrapper_with_mixin", + wraps=sql_tracking.patch_cursor_wrapper_with_mixin, ) - def test_chunked_cursor_wrapper_singleton(self, mock_wrapper): + def test_chunked_cursor_wrapper_singleton(self, mock_patch_cursor_wrapper): sql_call(use_iterator=True) # ensure that cursor wrapping is applied only once - self.assertEqual(mock_wrapper.call_count, 1) + self.assertIn( + mock_patch_cursor_wrapper.mock_calls, + [ + [call(CursorWrapper, sql_tracking.NormalCursorMixin)], + # CursorDebugWrapper is used if the test is called with `--debug-sql` + [call(CursorDebugWrapper, sql_tracking.NormalCursorMixin)], + ], + ) @patch( - "debug_toolbar.panels.sql.tracking.NormalCursorWrapper", - wraps=sql_tracking.NormalCursorWrapper, + "debug_toolbar.panels.sql.tracking.patch_cursor_wrapper_with_mixin", + wraps=sql_tracking.patch_cursor_wrapper_with_mixin, ) - async def test_cursor_wrapper_async(self, mock_wrapper): + async def test_cursor_wrapper_async(self, mock_patch_cursor_wrapper): await sync_to_async(sql_call)() - self.assertEqual(mock_wrapper.call_count, 1) + self.assertIn( + mock_patch_cursor_wrapper.mock_calls, + [ + [call(CursorWrapper, sql_tracking.NormalCursorMixin)], + # CursorDebugWrapper is used if the test is called with `--debug-sql` + [call(CursorDebugWrapper, sql_tracking.NormalCursorMixin)], + ], + ) @patch( - "debug_toolbar.panels.sql.tracking.NormalCursorWrapper", - wraps=sql_tracking.NormalCursorWrapper, + "debug_toolbar.panels.sql.tracking.patch_cursor_wrapper_with_mixin", + wraps=sql_tracking.patch_cursor_wrapper_with_mixin, ) - async def test_cursor_wrapper_asyncio_ctx(self, mock_wrapper): + async def test_cursor_wrapper_asyncio_ctx(self, mock_patch_cursor_wrapper): self.assertTrue(sql_tracking.allow_sql.get()) await sync_to_async(sql_call)() @@ -116,7 +137,21 @@ async def task(): await asyncio.create_task(task()) # Because it was called in another context, it should not have affected ours self.assertTrue(sql_tracking.allow_sql.get()) - self.assertEqual(mock_wrapper.call_count, 1) + + self.assertIn( + mock_patch_cursor_wrapper.mock_calls, + [ + [ + call(CursorWrapper, sql_tracking.NormalCursorMixin), + call(CursorWrapper, sql_tracking.ExceptionCursorMixin), + ], + # CursorDebugWrapper is used if the test is called with `--debug-sql` + [ + call(CursorDebugWrapper, sql_tracking.NormalCursorMixin), + call(CursorDebugWrapper, sql_tracking.ExceptionCursorMixin), + ], + ], + ) def test_generate_server_timing(self): self.assertEqual(len(self.panel._queries), 0) From fefec8e9d0275e851d71157231f2cf6f7b965efe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 20:15:02 +0000 Subject: [PATCH 36/37] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-prettier: v3.0.0 → v3.0.1](https://github.com/pre-commit/mirrors-prettier/compare/v3.0.0...v3.0.1) - [github.com/astral-sh/ruff-pre-commit: v0.0.281 → v0.0.282](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.281...v0.0.282) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a81ecfbdc..001b07e34 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: - id: rst-backticks - id: rst-directive-colons - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0 + rev: v3.0.1 hooks: - id: prettier types_or: [javascript, css] @@ -39,7 +39,7 @@ repos: args: - --fix - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.281' + rev: 'v0.0.282' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 7ab6b0f4802f19a28e4141df0be70b4f0b28eae1 Mon Sep 17 00:00:00 2001 From: tschilling Date: Thu, 10 Aug 2023 20:25:08 -0500 Subject: [PATCH 37/37] Version 4.2.0 --- debug_toolbar/__init__.py | 2 +- docs/changes.rst | 3 +++ docs/conf.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/__init__.py b/debug_toolbar/__init__.py index 1a9cf7c93..dbe08451f 100644 --- a/debug_toolbar/__init__.py +++ b/debug_toolbar/__init__.py @@ -4,7 +4,7 @@ # Do not use pkg_resources to find the version but set it here directly! # see issue #1446 -VERSION = "4.1.0" +VERSION = "4.2.0" # Code that discovers files or modules in INSTALLED_APPS imports this module. urls = "debug_toolbar.urls", APP_NAME diff --git a/docs/changes.rst b/docs/changes.rst index 5b7aaf1c1..89f5bdc7e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,9 @@ Change log Pending ------- +4.2.0 (2023-08-10) +------------------ + * Adjusted app directories system check to allow for nested template loaders. * Switched from flake8, isort and pyupgrade to `ruff `__. diff --git a/docs/conf.py b/docs/conf.py index 219a2f440..7fa8e6fce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ copyright = copyright.format(datetime.date.today().year) # The full version, including alpha/beta/rc tags -release = "4.1.0" +release = "4.2.0" # -- General configuration ---------------------------------------------------