From acb8345a7796345cfb42f0ffc09e97278cf29b7f Mon Sep 17 00:00:00 2001
From: Henrik Levkowetz <henrik@levkowetz.com>
Date: Mon, 10 Apr 2017 17:27:11 +0000
Subject: [PATCH] Added some debug functionality which makes it possible to see
 from where (python source file and line) an SQL query comes when looking at
 the sql query summary available at the bottom of pages in debug mode, on
 INTERNAL_IPS.  - Legacy-Id: 13188

---
 ietf/static/ietf/css/ietf.css            |  19 +++
 ietf/templates/debug.html                |  46 +++++-
 patch/README                             |  10 ++
 patch/trace-django-queryset-origin.patch | 196 +++++++++++++++++++++++
 4 files changed, 268 insertions(+), 3 deletions(-)
 create mode 100644 patch/README
 create mode 100644 patch/trace-django-queryset-origin.patch

diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css
index 49b0f0f99..6a2911985 100644
--- a/ietf/static/ietf/css/ietf.css
+++ b/ietf/static/ietf/css/ietf.css
@@ -824,3 +824,22 @@ blockquote {
     page-break-before: always;
   }
 }
+
+.modal-max {
+  width: 90%;
+}
+
+#debug-query-table .code {
+  background-color: #eee;
+  font-family: 'PT Mono', monospace;
+  font-size: 0.8em;
+  word-break: break-all;
+}
+
+#debug-query-table table {
+  max-width: 100%;
+}
+
+#debug-query-table .sql {
+  border-top: 1px solid #555;
+}
diff --git a/ietf/templates/debug.html b/ietf/templates/debug.html
index 07e4dd7a5..2d8ab7e39 100644
--- a/ietf/templates/debug.html
+++ b/ietf/templates/debug.html
@@ -13,7 +13,7 @@
 	     onclick="$('#debug-query-table').toggleClass('hide');">Show</a>
 	{% endif %}
       </p>
-      <table class="table table-condensed table-striped tablesorter hide" id="debug-query-table">
+      <table class="table table-condensed tablesorter hide" id="debug-query-table">
 	<thead>
 	  <tr>
 	    <th data-header="sequence">#</th>
@@ -28,12 +28,52 @@
 	<tbody>
 	  {% with sql_queries|annotate_sql_queries as sql_query_info %}
 	      {% for query in sql_query_info %}
-		<tr>
+		<tr class="sql">
 		  <td>{{ forloop.counter }}</td>
 		  <td>{{ query.sql|expand_comma|escape }}</td>
 		  <td>{{ query.count }}</td>
 		  <td>{{ query.where }}</td>
-		  <td>{{ query.loc }}</td>
+		  <td>
+		     {{ query.loc }}
+		     {% if query.origin %}
+		     <button type="button" class="btn btn-primary btn-xs" data-toggle="modal" data-target="#modal-{{forloop.counter}}" >Origin</button>
+		     {% endif %}
+		     <div class="modal fade" id="modal-{{forloop.counter}}" tabindex="-1" role="dialog" aria-labelledby="modal-title-{{forloop.counter}}">
+		       <div class="modal-dialog modal-max" role="document">
+			 <div class="modal-content">
+			   <div class="modal-header">
+			     <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+			     <h4 class="modal-title" id="modal-title-{{forloop.counter}}">QuerySet Origin for Query #{{forloop.counter}}</h4>
+			   </div>
+			   <div class="modal-body">
+			     <table class="table table-condensed">
+				<thead>
+				<tr>
+				  <th>File (line)</th>
+				  <th>Method</th>
+				  <th>Code</th>
+				</tr>
+				</thead>
+				<tbody>
+				{% if query.origin %}
+				{% for origin in query.origin %}
+				<tr class="code">
+				   <td>{{ origin.1 }}({{ origin.2 }})</td>
+				   <td>{{ origin.6 }}()</td>
+				   <td>{{ origin.4.0 }}</td>
+				</tr>
+				{% endfor %}
+				{% endif %}
+				</tbody>
+			     </table>
+			   </div>
+			   <div class="modal-footer">
+			     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+			   </div>
+			 </div>
+		       </div>
+		     </div>
+		  </td>
 		  <td>{{ query.time }}</td>
 		  <td>{{ query.time_accum }}</td>
 		</tr>
diff --git a/patch/README b/patch/README
new file mode 100644
index 000000000..1b355b6db
--- /dev/null
+++ b/patch/README
@@ -0,0 +1,10 @@
+
+This directory is for patches to the datatracker environment, not for patches
+to the datatracker itself (those are ephemeral, in the sense that they should
+not stay around as patches, they should be committed to the repository if they
+are of long-term value).
+
+As an example, trace-django-queryset-origin.patch is a patch to django 1.10
+which adds some meta-information to the query entries provided for debug
+purpose by django.template.context_processors.debug as sql_queries.
+
diff --git a/patch/trace-django-queryset-origin.patch b/patch/trace-django-queryset-origin.patch
new file mode 100644
index 000000000..d23eb8357
--- /dev/null
+++ b/patch/trace-django-queryset-origin.patch
@@ -0,0 +1,196 @@
+--- ../x/env/lib/python2.7/site-packages/django/db/models/query.py	2017-04-07 15:10:29.426831000 -0700
++++ env/lib/python2.7/site-packages/django/db/models/query.py	2017-04-07 15:18:39.938521412 -0700
+@@ -5,6 +5,8 @@
+ import copy
+ import sys
+ import warnings
++import inspect
++import logging
+ from collections import OrderedDict, deque
+ 
+ from django.conf import settings
+@@ -34,6 +36,7 @@
+ # Pull into this namespace for backwards compatibility.
+ EmptyResultSet = sql.EmptyResultSet
+ 
++logger = logging.getLogger('django.db.backends')
+ 
+ class BaseIterable(object):
+     def __init__(self, queryset):
+@@ -51,6 +54,7 @@
+         compiler = queryset.query.get_compiler(using=db)
+         # Execute the query. This will also fill compiler.select, klass_info,
+         # and annotations.
++
+         results = compiler.execute_sql()
+         select, klass_info, annotation_col_map = (compiler.select, compiler.klass_info,
+                                                   compiler.annotation_col_map)
+@@ -174,6 +178,8 @@
+         self._known_related_objects = {}  # {rel_field, {pk: rel_obj}}
+         self._iterable_class = ModelIterable
+         self._fields = None
++        self._origin = []
++        self._djangodir = __file__[:(__file__.index('django')+len('django')+1)]
+ 
+     def as_manager(cls):
+         # Address the circular dependency between `Queryset` and `Manager`.
+@@ -316,6 +322,37 @@
+         combined.query.combine(other.query, sql.OR)
+         return combined
+ 
++    def _add_origin(self, depth=1):
++        if settings.DEBUG:
++            # get list of frame records.  Each is:
++            # [ frame, filename, lineno, function, code_context, index ]
++            stack = inspect.stack()
++            # caller stack record
++            method = stack[depth][3]
++            # look for the first stack entry which is not from django
++            i = 0
++            while i<len(stack) and stack[i][1].startswith(self._djangodir) and not stack[i][3] == 'render':
++                i += 1
++            if i<len(stack):
++                stack = stack[i:]
++                frame = stack[0][0]
++                function = stack[0][3]
++                if function == 'render' and 'context' in frame.f_locals:
++                    that = frame.f_locals['self']
++                    if hasattr(that, 'filename'):
++                        import debug
++                        debug.show('that.filename')
++                self._origin += [stack[0]+(method,), ]
++            else:
++                self._origin += [stack[2]+(method,), ]
++
++    def _log_origin(self):
++        if settings.DEBUG:
++            self._add_origin(depth=3)
++            for place in self._origin:
++                frame, filename, lineno, function, code_context, index, method = place[:7]
++                logger.debug("QuerySet: %s(%s) in %s: %s()\n %s", filename, lineno, function, method, code_context[index].rstrip())
++
+     ####################################
+     # METHODS THAT DO DATABASE QUERIES #
+     ####################################
+@@ -793,6 +830,7 @@
+         Returns a new QuerySet instance with the args ANDed to the existing
+         set.
+         """
++        self._add_origin()
+         return self._filter_or_exclude(False, *args, **kwargs)
+ 
+     def exclude(self, *args, **kwargs):
+@@ -800,6 +838,7 @@
+         Returns a new QuerySet instance with NOT (args) ANDed to the existing
+         set.
+         """
++        self._add_origin()
+         return self._filter_or_exclude(True, *args, **kwargs)
+ 
+     def _filter_or_exclude(self, negate, *args, **kwargs):
+@@ -824,6 +863,7 @@
+         This exists to support framework features such as 'limit_choices_to',
+         and usually it will be more natural to use other methods.
+         """
++        self._add_origin()
+         if isinstance(filter_obj, Q) or hasattr(filter_obj, 'add_to_query'):
+             clone = self._clone()
+             clone.query.add_q(filter_obj)
+@@ -836,6 +876,7 @@
+         Returns a new QuerySet instance that will select objects with a
+         FOR UPDATE lock.
+         """
++        self._add_origin()
+         obj = self._clone()
+         obj._for_write = True
+         obj.query.select_for_update = True
+@@ -855,6 +896,7 @@
+         if self._fields is not None:
+             raise TypeError("Cannot call select_related() after .values() or .values_list()")
+ 
++        self._add_origin()
+         obj = self._clone()
+         if fields == (None,):
+             obj.query.select_related = False
+@@ -874,6 +916,7 @@
+         prefetch is appended to. If prefetch_related(None) is called, the list
+         is cleared.
+         """
++        self._add_origin()
+         clone = self._clone()
+         if lookups == (None,):
+             clone._prefetch_related_lookups = []
+@@ -886,6 +929,7 @@
+         Return a query set in which the returned objects have been annotated
+         with extra data or aggregations.
+         """
++        self._add_origin()
+         annotations = OrderedDict()  # To preserve ordering of args
+         for arg in args:
+             # The default_alias property may raise a TypeError, so we use
+@@ -929,6 +973,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot reorder a query once a slice has been taken."
++        self._add_origin()
+         obj = self._clone()
+         obj.query.clear_ordering(force_empty=False)
+         obj.query.add_ordering(*field_names)
+@@ -940,6 +985,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot create distinct fields once a slice has been taken."
++        self._add_origin()
+         obj = self._clone()
+         obj.query.add_distinct_fields(*field_names)
+         return obj
+@@ -951,6 +997,7 @@
+         """
+         assert self.query.can_filter(), \
+             "Cannot change a query once a slice has been taken"
++        self._add_origin()
+         clone = self._clone()
+         clone.query.add_extra(select, select_params, where, params, tables, order_by)
+         return clone
+@@ -959,6 +1006,7 @@
+         """
+         Reverses the ordering of the QuerySet.
+         """
++        self._add_origin()
+         clone = self._clone()
+         clone.query.standard_ordering = not clone.query.standard_ordering
+         return clone
+@@ -973,6 +1021,7 @@
+         """
+         if self._fields is not None:
+             raise TypeError("Cannot call defer() after .values() or .values_list()")
++        self._add_origin()
+         clone = self._clone()
+         if fields == (None,):
+             clone.query.clear_deferred_loading()
+@@ -992,6 +1041,7 @@
+             # Can only pass None to defer(), not only(), as the rest option.
+             # That won't stop people trying to do this, so let's be explicit.
+             raise TypeError("Cannot pass None as an argument to only().")
++        self._add_origin()
+         clone = self._clone()
+         clone.query.add_immediate_loading(fields)
+         return clone
+@@ -1078,13 +1128,17 @@
+         clone._known_related_objects = self._known_related_objects
+         clone._iterable_class = self._iterable_class
+         clone._fields = self._fields
++        clone._origin = self._origin
+ 
+         clone.__dict__.update(kwargs)
+         return clone
+ 
+     def _fetch_all(self):
+         if self._result_cache is None:
++            self._log_origin()
+             self._result_cache = list(self.iterator())
++            if settings.DEBUG:
++                connections[self.db].queries_log[-1]['origin'] = self._origin
+         if self._prefetch_related_lookups and not self._prefetch_done:
+             self._prefetch_related_objects()
+