Added some debug functionality which makes it possible to see from where (python source file and line) an SQL query comes when looking at the sql query summary available at the bottom of pages in debug mode, on INTERNAL_IPS.

- Legacy-Id: 13188

ietf/static/ietf/css/ietf.css

@@ -824,3 +824,22 @@ blockquote {
     page-break-before: always;
   }
 }
+
+.modal-max {
+  width: 90%;
+}
+
+#debug-query-table .code {
+  background-color: #eee;
+  font-family: 'PT Mono', monospace;
+  font-size: 0.8em;
+  word-break: break-all;
+}
+
+#debug-query-table table {
+  max-width: 100%;
+}
+
+#debug-query-table .sql {
+  border-top: 1px solid #555;
+}

ietf/templates/debug.html

@@ -13,7 +13,7 @@
 	     onclick="$('#debug-query-table').toggleClass('hide');">Show</a>
 	{% endif %}
       </p>
-      <table class="table table-condensed table-striped tablesorter hide" id="debug-query-table">
+      <table class="table table-condensed tablesorter hide" id="debug-query-table">
 	<thead>
 	  <tr>
 	    <th data-header="sequence">#</th>
@@ -28,12 +28,52 @@
 	<tbody>
 	  {% with sql_queries|annotate_sql_queries as sql_query_info %}
 	      {% for query in sql_query_info %}
-		<tr>
+		<tr class="sql">
 		  <td>{{ forloop.counter }}</td>
 		  <td>{{ query.sql|expand_comma|escape }}</td>
 		  <td>{{ query.count }}</td>
 		  <td>{{ query.where }}</td>
-		  <td>{{ query.loc }}</td>
+		  <td>
+		     {{ query.loc }}
+		     {% if query.origin %}
+		     <button type="button" class="btn btn-primary btn-xs" data-toggle="modal" data-target="#modal-{{forloop.counter}}" >Origin</button>
+		     {% endif %}
+		     <div class="modal fade" id="modal-{{forloop.counter}}" tabindex="-1" role="dialog" aria-labelledby="modal-title-{{forloop.counter}}">
+		       <div class="modal-dialog modal-max" role="document">
+			 <div class="modal-content">
+			   <div class="modal-header">
+			     <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+			     <h4 class="modal-title" id="modal-title-{{forloop.counter}}">QuerySet Origin for Query #{{forloop.counter}}</h4>
+			   </div>
+			   <div class="modal-body">
+			     <table class="table table-condensed">
+				<thead>
+				<tr>
+				  <th>File (line)</th>
+				  <th>Method</th>
+				  <th>Code</th>
+				</tr>
+				</thead>
+				<tbody>
+				{% if query.origin %}
+				{% for origin in query.origin %}
+				<tr class="code">
+				   <td>{{ origin.1 }}({{ origin.2 }})</td>
+				   <td>{{ origin.6 }}()</td>
+				   <td>{{ origin.4.0 }}</td>
+				</tr>
+				{% endfor %}
+				{% endif %}
+				</tbody>
+			     </table>
+			   </div>
+			   <div class="modal-footer">
+			     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+			   </div>
+			 </div>
+		       </div>
+		     </div>
+		  </td>
 		  <td>{{ query.time }}</td>
 		  <td>{{ query.time_accum }}</td>
 		</tr>

patch/README

@@ -0,0 +1,10 @@
+
+This directory is for patches to the datatracker environment, not for patches
+to the datatracker itself (those are ephemeral, in the sense that they should
+not stay around as patches, they should be committed to the repository if they
+are of long-term value).
+
+As an example, trace-django-queryset-origin.patch is a patch to django 1.10
+which adds some meta-information to the query entries provided for debug
+purpose by django.template.context_processors.debug as sql_queries.
+

patch/trace-django-queryset-origin.patch

@@ -0,0 +1,196 @@
+--- ../x/env/lib/python2.7/site-packages/django/db/models/query.py	2017-04-07 15:10:29.426831000 -0700
++++ env/lib/python2.7/site-packages/django/db/models/query.py	2017-04-07 15:18:39.938521412 -0700
+@@ -5,6 +5,8 @@
+ import copy
+ import sys
+ import warnings
++import inspect
++import logging
+ from collections import OrderedDict, deque
+ 
+ from django.conf import settings
+@@ -34,6 +36,7 @@
+ # Pull into this namespace for backwards compatibility.
+ EmptyResultSet = sql.EmptyResultSet
+ 
++logger = logging.getLogger('django.db.backends')
+ 
+ class BaseIterable(object):
+     def __init__(self, queryset):
+@@ -51,6 +54,7 @@
+         compiler = queryset.query.get_compiler(using=db)
+         # Execute the query. This will also fill compiler.select, klass_info,
+         # and annotations.
++
+         results = compiler.execute_sql()
+         select, klass_info, annotation_col_map = (compiler.select, compiler.klass_info,
+                                                   compiler.annotation_col_map)
+@@ -174,6 +178,8 @@
+         self._known_related_objects = {}  # {rel_field, {pk: rel_obj}}
+         self._iterable_class = ModelIterable
+         self._fields = None
++        self._origin = []
++        self._djangodir = __file__[:(__file__.index('django')+len('django')+1)]
+ 
+     def as_manager(cls):
+         # Address the circular dependency between `Queryset` and `Manager`.
+@@ -316,6 +322,37 @@
+         combined.query.combine(other.query, sql.OR)
+         return combined
+ 
++    def _add_origin(self, depth=1):
++        if settings.DEBUG:
++            # get list of frame records.  Each is:
++            # [ frame, filename, lineno, function, code_context, index ]
++            stack = inspect.stack()
++            # caller stack record
++            method = stack[depth][3]
++            # look for the first stack entry which is not from django
++            i = 0
++            while i<len(stack) and stack[i][1].startswith(self._djangodir) and not stack[i][3] == 'render':
++                i += 1
++            if i<len(stack):
++                stack = stack[i:]
++                frame = stack[0][0]
++                function = stack[0][3]
++                if function == 'render' and 'context' in frame.f_locals:
++                    that = frame.f_locals['self']
++                    if hasattr(that, 'filename'):
++                        import debug
++                        debug.show('that.filename')
++                self._origin += [stack[0]+(method,), ]
++            else:
++                self._origin += [stack[2]+(method,), ]
++
++    def _log_origin(self):
++        if settings.DEBUG:
++            self._add_origin(depth=3)
++            for place in self._origin:
++                frame, filename, lineno, function, code_context, index, method = place[:7]
++                logger.debug("QuerySet: %s(%s) in %s: %s()\n %s", filename, lineno, function, method, code_context[index].rstrip())
++
+     ####################################
+     # METHODS THAT DO DATABASE QUERIES #
+     ####################################
+@@ -793,6 +830,7 @@
+         Returns a new QuerySet instance with the args ANDed to the existing
+         set.
+         """
++        self._add_origin()
+         return self._filter_or_exclude(False, *args, **kwargs)
+ 
+     def exclude(self, *args, **kwargs):
+@@ -800,6 +838,7 @@
+         Returns a new QuerySet instance with NOT (args) ANDed to the existing
+         set.
+         """
++        self._add_origin()
+         return self._filter_or_exclude(True, *args, **kwargs)
+ 
+     def _filter_or_exclude(self, negate, *args, **kwargs):
+@@ -824,6 +863,7 @@
+         This exists to support framework features such as 'limit_choices_to',
+         and usually it will be more natural to use other methods.
+         """
++        self._add_origin()
+         if isinstance(filter_obj, Q) or hasattr(filter_obj, 'add_to_query'):
+             clone = self._clone()
+             clone.query.add_q(filter_obj)
+@@ -836,6 +876,7 @@
+         Returns a new QuerySet instance that will select objects with a
+         FOR UPDATE lock.
+         """
++        self._add_origin()
+         obj = self._clone()
+         obj._for_write = True
+         obj.query.select_for_update = True
+@@ -855,6 +896,7 @@
+         if self._fields is not None:
+             raise TypeError("Cannot call select_related() after .values() or .values_list()")
+ 
++        self._add_origin()
+         obj = self._clone()
+         if fields == (None,):
+             obj.query.select_related = False
+@@ -874,6 +916,7 @@
+         prefetch is appended to. If prefetch_related(None) is called, the list
+         is cleared.
+         """
++        self._add_origin()
+         clone = self._clone()
+         if lookups == (None,):
+             clone._prefetch_related_lookups = []
+@@ -886,6 +929,7 @@
+         Return a query set in which the returned objects have been annotated
+         with extra data or aggregations.
+         """
++        self._add_origin()
+         annotations = OrderedDict()  # To preserve ordering of args
+         for arg in args:
+             # The default_alias property may raise a TypeError, so we use
+@@ -929,6 +973,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot reorder a query once a slice has been taken."
++        self._add_origin()
+         obj = self._clone()
+         obj.query.clear_ordering(force_empty=False)
+         obj.query.add_ordering(*field_names)
+@@ -940,6 +985,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot create distinct fields once a slice has been taken."
++        self._add_origin()
+         obj = self._clone()
+         obj.query.add_distinct_fields(*field_names)
+         return obj
+@@ -951,6 +997,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot change a query once a slice has been taken"
++        self._add_origin()
+         clone = self._clone()
+         clone.query.add_extra(select, select_params, where, params, tables, order_by)
+         return clone
+@@ -959,6 +1006,7 @@
+         """
+         Reverses the ordering of the QuerySet.
+         """
++        self._add_origin()
+         clone = self._clone()
+         clone.query.standard_ordering = not clone.query.standard_ordering
+         return clone
+@@ -973,6 +1021,7 @@
+         """
+         if self._fields is not None:
+             raise TypeError("Cannot call defer() after .values() or .values_list()")
++        self._add_origin()
+         clone = self._clone()
+         if fields == (None,):
+             clone.query.clear_deferred_loading()
+@@ -992,6 +1041,7 @@
+             # Can only pass None to defer(), not only(), as the rest option.
+             # That won't stop people trying to do this, so let's be explicit.
+             raise TypeError("Cannot pass None as an argument to only().")
++        self._add_origin()
+         clone = self._clone()
+         clone.query.add_immediate_loading(fields)
+         return clone
+@@ -1078,13 +1128,17 @@
+         clone._known_related_objects = self._known_related_objects
+         clone._iterable_class = self._iterable_class
+         clone._fields = self._fields
++        clone._origin = self._origin
+ 
+         clone.__dict__.update(kwargs)
+         return clone
+ 
+     def _fetch_all(self):
+         if self._result_cache is None:
++            self._log_origin()
+             self._result_cache = list(self.iterator())
++            if settings.DEBUG:
++                connections[self.db].queries_log[-1]['origin'] = self._origin
+         if self._prefetch_related_lookups and not self._prefetch_done:
+             self._prefetch_related_objects()
+