Consider query params when detecting duplicate queries (#995)

* (Factor out key function for duplicate queries)

* Consider the query params when detecting duplicate queries

* SQL query recording: Track the raw_params too

The duplicate query detection needs this, because `params` is not always
JSON-serialisable.

* (Rename variable to avoid shadowing)

* SQLPanel: Track duplicated queries both with and without params

* SQL panel template: Include counts of duplicates with params too

* Convert raw_params lists to hashable tuples

* Wording change: duplicate -> similar, for queries with differing params

* Wording: "duplicate with params" -> duplicate, for identical queries

* SQL panel: Add explanatory tooltip text to similar / duplicate query counts

Author: PiDelport

debug_toolbar/panels/sql/panel.py

@@ -141,7 +141,16 @@ def disable_instrumentation(self):
     def generate_stats(self, request, response):
         colors = contrasting_color_generator()
         trace_colors = defaultdict(lambda: next(colors))
+        query_similar = defaultdict(lambda: defaultdict(int))
         query_duplicates = defaultdict(lambda: defaultdict(int))
+
+        # The keys used to determine similar and duplicate queries.
+        def similar_key(query):
+            return query['raw_sql']
+
+        def duplicate_key(query):
+            return (query['raw_sql'], tuple(query['raw_params']))
+
         if self._queries:
             width_ratio_tally = 0
             factor = int(256.0 / (len(self._databases) * 2.5))
@@ -164,7 +173,8 @@ def generate_stats(self, request, response):
             trans_id = None
             i = 0
             for alias, query in self._queries:
-                query_duplicates[alias][query["raw_sql"]] += 1
+                query_similar[alias][similar_key(query)] += 1
+                query_duplicates[alias][duplicate_key(query)] += 1
 
                 trans_id = query.get('trans_id')
                 last_trans_id = trans_ids.get(alias)
@@ -209,10 +219,18 @@ def generate_stats(self, request, response):
             if trans_id:
                 self._queries[(i - 1)][1]['ends_trans'] = True
 
-        # Queries are duplicates only if there's as least 2 of them.
+        # Queries are similar / duplicates only if there's as least 2 of them.
         # Also, to hide queries, we need to give all the duplicate groups an id
         query_colors = contrasting_color_generator()
-        query_duplicates = {
+        query_similar_colors = {
+            alias: {
+                query: (similar_count, next(query_colors))
+                for query, similar_count in queries.items()
+                if similar_count >= 2
+            }
+            for alias, queries in query_similar.items()
+        }
+        query_duplicates_colors = {
             alias: {
                 query: (duplicate_count, next(query_colors))
                 for query, duplicate_count in queries.items()
@@ -223,15 +241,23 @@ def generate_stats(self, request, response):
 
         for alias, query in self._queries:
             try:
-                duplicates_count, color = query_duplicates[alias][query["raw_sql"]]
-                query["duplicate_count"] = duplicates_count
-                query["duplicate_color"] = color
+                (query["similar_count"], query["similar_color"]) = (
+                    query_similar_colors[alias][similar_key(query)]
+                )
+                (query["duplicate_count"], query["duplicate_color"]) = (
+                    query_duplicates_colors[alias][duplicate_key(query)]
+                )
             except KeyError:
                 pass
 
         for alias, alias_info in self._databases.items():
             try:
-                alias_info["duplicate_count"] = sum(e[0] for e in query_duplicates[alias].values())
+                alias_info["similar_count"] = sum(
+                    e[0] for e in query_similar_colors[alias].values()
+                )
+                alias_info["duplicate_count"] = sum(
+                    e[0] for e in query_duplicates_colors[alias].values()
+                )
             except KeyError:
                 pass
 

debug_toolbar/panels/sql/tracking.py

@@ -134,6 +134,7 @@ def _record(self, method, sql, params):
                 'duration': duration,
                 'raw_sql': sql,
                 'params': _params,
+                'raw_params': params,
                 'stacktrace': stacktrace,
                 'start_time': start_time,
                 'stop_time': stop_time,

debug_toolbar/templates/debug_toolbar/panels/sql.html

@@ -5,8 +5,15 @@
 			<li>
 				<strong class="djdt-label"><span data-background-color="rgb({{ info.rgb_color|join:", " }})" class="djdt-color">&#160;</span> {{ alias }}</strong>
 				<span class="djdt-info">{{ info.time_spent|floatformat:"2" }} ms ({% blocktrans count info.num_queries as num %}{{ num }} query{% plural %}{{ num }} queries{% endblocktrans %}
-				{% if info.duplicate_count %}
-					{% blocktrans with dupes=info.duplicate_count %}including {{ dupes }} duplicates{% endblocktrans %}
+				{% if info.similar_count %}
+					{% blocktrans with count=info.similar_count trimmed %}
+						including <abbr title="Similar queries are queries with the same SQL, but potentially different parameters.">{{ count }} similar</abbr>
+					{% endblocktrans %}
+					{% if info.duplicate_count %}
+						{% blocktrans with dupes=info.duplicate_count trimmed %}
+							and <abbr title="Duplicate queries are identical to each other: they execute exactly the same SQL and parameters.">{{ dupes }} duplicates</abbr>
+						{% endblocktrans %}
+					{% endif %}
 				{% endif %})</span>
 			</li>
 		{% endfor %}
@@ -35,6 +42,12 @@
 						<div class="djDebugSqlWrap">
 							<div class="djDebugSql">{{ query.sql|safe }}</div>
 						</div>
+						{% if query.similar_count %}
+							<strong>
+								<span data-background-color="{{ query.similar_color }}">&#160;</span>
+								{% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %}
+							</strong>
+						{% endif %}
 						{% if query.duplicate_count %}
 							<strong>
 								<span data-background-color="{{ query.duplicate_color }}">&#160;</span>