Added a distribution tab to visualize request time distributions over time

README.md

@@ -381,6 +381,14 @@ Silk currently generates two bits of code per request:
 
 Both are intended for use in replaying the request. The curl command can be used to replay via command-line and the python code can be used within a Django unit test or simply as a standalone script.
 
+## Distribution View
+
+By setting the configuration ``SILKY_DISTRIBUTION_TAB`` to ``True`` a distribution tab can be enabled with a visualisation for plotting the distribution of timings of requests. At present it is possible to group the requests by date or revision.
+
+<img src="https://raw.githubusercontent.com/jazzband/django-silk/master/screenshots/11.png" width="720px"/>
+
+You can click on the various groups to drill down to eventually see the actual requests that are running slowly. This visualisation can be useful for benchmarking an application over versions or time to see if performance improves or degrades.
+
 ## Configuration
 
 ### Authentication/Authorisation

project/example_app/views.py

@@ -9,7 +9,7 @@
 def index(request):
     @silk_profile()
     def do_something_long():
-        sleep(1.345)
+        sleep(0.345)
 
     with silk_profile(name='Why do this take so long?'):
         do_something_long()

project/project/settings.py

@@ -128,3 +128,5 @@
 # SILKY_AUTHENTICATION = True
 # SILKY_AUTHORISATION = True
 
+SILKY_DISTRIBUTION_TAB = True
+

silk/config.py

@@ -3,6 +3,7 @@
 from silk.singleton import Singleton
 
 
+
 def default_permissions(user):
     if user:
         return user.is_staff
@@ -29,6 +30,9 @@ class SilkyConfig(metaclass=Singleton):
         'SILKY_PYTHON_PROFILER_FUNC': None,
         'SILKY_STORAGE_CLASS': 'silk.storage.ProfilerResultStorage',
         'SILKY_MIDDLEWARE_CLASS': 'silk.middleware.SilkyMiddleware',
+        'SILKY_REVISION': '',
+        'SILKY_POST_PROCESS_REQUEST': lambda x: None,
+        'SILKY_DISTRIBUTION_TAB': False,
         'SILKY_JSON_ENSURE_ASCII': True,
         'SILKY_ANALYZE_QUERIES': False
     }

silk/migrations/0008_request_revision.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-12-08 12:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import silk.storage
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('silk', '0007_sqlquery_identifier'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='request',
+            name='revision',
+            field=models.TextField(blank=True, default=''),
+        ),
+    ]

silk/models.py

@@ -72,6 +72,7 @@ class Request(models.Model):
     meta_time_spent_queries = FloatField(null=True, blank=True)
     pyprofile = TextField(blank=True, default='')
     prof_file = FileField(max_length=300, blank=True, storage=silk_storage)
+    revision = TextField(blank=True, default='')
 
     # Useful method to create shortened copies of strings without losing start and end context
     # Used to ensure path and view_name don't exceed 190 characters
@@ -117,6 +118,30 @@ def time_spent_on_sql_queries(self):
         """
         return sum(x.time_taken for x in SQLQuery.objects.filter(request=self))
 
+    @classmethod
+    def get_date(cls, start_time):
+        return start_time.date()
+
+    @property
+    def date(self):
+        return self.get_date(self.start_time)
+
+    @classmethod
+    def get_hour(cls, start_time):
+        return start_time.strftime('%Y-%m-%d %H')
+
+    @property
+    def hour(self):
+        return self.get_hour(self.start_time)
+
+    @classmethod
+    def get_minute(cls, start_time):
+        return start_time.strftime('%Y-%m-%d %H:%M')
+
+    @property
+    def minute(self):
+        return self.get_minute(self.start_time)
+
     @property
     def headers(self):
         if self.encoded_headers:
@@ -182,6 +207,12 @@ def save(self, *args, **kwargs):
         if self.view_name and len(self.view_name) > 190:
             self.view_name = self._shorten(self.view_name)
 
+        config = SilkyConfig()
+
+        self.revision = config.SILKY_REVISION
+
+        config.SILKY_POST_PROCESS_REQUEST(self)
+
         super(Request, self).save(*args, **kwargs)
         Request.garbage_collect(force=False)
 

silk/request_filters.py

@@ -206,16 +206,21 @@ def __init__(self, value):
         super(MethodFilter, self).__init__(value, method=value)
 
 
-def filters_from_request(request):
+class RevisionFilter(BaseFilter):
+    def __init__(self, value):
+        super(RevisionFilter, self).__init__(value, revision=value)
+
+
+def filters_from_query_dict(query_dict):
     raw_filters = {}
-    for key in request.POST:
+    for key in query_dict:
         splt = key.split('-')
         if splt[0].startswith('filter'):
             ident = splt[1]
             typ = splt[2]
             if ident not in raw_filters:
                 raw_filters[ident] = {}
-            raw_filters[ident][typ] = request.POST[key]
+            raw_filters[ident][typ] = query_dict[key]
     filters = {}
     for ident, raw_filter in raw_filters.items():
         value = raw_filter.get('value', '')
@@ -228,4 +233,29 @@ def filters_from_request(request):
                 filters[ident] = f
             except FilterValidationError:
                 logger.warn('Validation error when processing filter %s(%s)' % (typ, value))
-    return filters
+
+
+def get_path_and_filters(request, session_key_request_filters):
+    path = request.GET.get('path', None)
+    raw_filters = request.session.get(session_key_request_filters, {})
+
+    filter_classes = set(x.__name__ for x in BaseFilter.__subclasses__())
+
+    def get_filter_class_parameter(filter_class):
+        without_filter = filter_class.replace('Filter', '')
+        result = without_filter[0].lower() + without_filter[1:]
+        return result
+
+    url_filters = {
+        filter_class: dict(typ=filter_class, value=url_filter)
+        for filter_class in filter_classes
+        for filter_class_parameter in [get_filter_class_parameter(filter_class)]
+        for url_filter in [request.GET.get(filter_class_parameter, None)]
+        if url_filter is not None
+    }
+
+    def make_filters(raw_filters):
+        return [BaseFilter.from_dict(x) for _, x in raw_filters.items()]
+
+    filters = make_filters(dict(raw_filters, **url_filters))
+    return path, raw_filters, filters

silk/static/silk/js/distribution.js

@@ -0,0 +1,177 @@
+/**
+ * Get the view name filter param dictionary for use in generating uri query parameters.
+ * @param viewName
+ */
+function viewNameFilter(viewName) {
+  return {
+    'viewName': viewName
+  }
+}
+
+/**
+ * Get the revision filter param dictionary for use in generating uri query parameters.
+ * @param revision The revision to filter by.
+ */
+function revisionFilter(revision) {
+  return {
+    'revision': revision
+  }
+}
+
+/**
+ * Get the date filter param dictionary for use in generating uri query parameters.
+ * @param date The date to filter by.
+ */
+function dateFilter(date) {
+  date = new Date(date);
+  return {
+    'afterDate': strftime('%Y/%m/%d 00:00', date),
+    'beforeDate': strftime('%Y/%m/%d 23:59', date)
+  }
+}
+
+/**
+ * Get the group-by parameter dictionary for use in generating uri query parameters.
+ * @param group The attribute which should be used for grouping distribution charts.
+ * @returns {{group-by: *}}
+ */
+function groupBy(group) {
+  return {'group-by': group}
+}
+
+// mapping of filter by parameters to function for generating filter query parameters
+var filterFunctions = {
+  'view_name': viewNameFilter,
+  'date': dateFilter,
+  'revision': revisionFilter
+};
+
+/**
+ * Make URI parameters from a dictionary.
+ * @param obj dictionary to convert to parameters.
+ * @returns {string}
+ */
+function makeURIParameters(obj) {
+  var str = [];
+  for (var p in obj) {
+    if (obj.hasOwnProperty(p)) {
+      str.push(encodeURIComponent(p) + "=" + encodeURIComponent(obj[p]));
+    }
+  }
+  return str.join("&");
+}
+
+/**
+ * Initialise the distribution chart.
+ * @param distributionUrl
+ */
+function initChart(distributionUrl) {
+
+  var inputURIParams = $.url('?');
+  var groupByParam = inputURIParams === undefined ? 'date' : inputURIParams.group_by || 'date';
+  var level = inputURIParams === undefined ? 0 : parseInt(inputURIParams.level) || 0;
+  var locationWithoutQuery = location.href.split("?")[0];
+  var filterParams = {};
+  var filterName = '';
+  var filterParamValue = '';
+
+  // parse the input uri parameters to get the filter to apply
+  for (var paramName in inputURIParams) {
+    if (paramName in filterFunctions) {
+      filterName = paramName;
+      var filter = filterFunctions[paramName];
+      filterParamValue = inputURIParams[paramName];
+      filterValue = filter(filterParamValue);
+      filterParams = Object.assign(filterParams, filterValue);
+    }
+  }
+
+  // generate the chart title
+  var viewTitleValue = 'all';
+  var groupTitleValue = 'all';
+  var groupTitle = groupByParam;
+  if (level === 1) {
+    groupTitle = filterName;
+    groupTitleValue = filterParamValue;
+  } else if (level === 2) {
+    viewTitleValue = filterParamValue;
+  }
+  var title = 'view: <i>' + viewTitleValue + '</i> | ' +
+    groupTitle + ': <i>' + groupTitleValue + '</i>'
+  ;
+  $('#title').html(title);
+
+  var groupParams = groupBy(groupByParam);
+  var outputURIParams = makeURIParameters(Object.assign(filterParams, groupParams));
+  var url = [distributionUrl, outputURIParams].join("?");
+
+  d3.csv(url, function (error, data) {
+
+    var maxValue = 0;
+
+    data.forEach(function (d) {
+      d.value = +d.value;
+      if (d.value > maxValue) {
+        maxValue = d.value;
+      }
+    });
+
+    var chart = makeDistroChart({
+      data: data,
+      xName: 'group',
+      yName: 'value',
+      axisLabels: {xAxis: null, yAxis: 'Time Taken (ms)'},
+      selector: "#chart-distro1",
+      chartSize: {height: window.innerHeight - 110, width: window.innerWidth},
+      constrainExtremes: false,
+      margin: {top: 20, right: 20, bottom: 80, left: 60}
+
+    });
+
+    for (const groupName in chart.groupObjs) {
+      const groupObj = chart.groupObjs[groupName];
+      groupObj.g
+        .on('mouseover.opacity', function () {
+          groupObj.g.transition().duration(300).attr('opacity', 0.5).style('cursor', 'pointer');
+        })
+        .on('mouseout.opacity', function () {
+          groupObj.g.transition().duration(300).attr('opacity', 1).style('cursor', 'default');
+          return false;
+        })
+        .on('click.opacity', function () {
+          var outputLinkParams = Object.assign({}, inputURIParams);
+          outputLinkParams[groupByParam] = groupName;
+          if (level === 0) {
+            outputLinkParams['group_by'] = 'view_name';
+            outputLinkParams['level'] = 1;
+            window.location = [locationWithoutQuery, makeURIParameters(outputLinkParams)].join('?');
+          }
+          else if (level === 1) {
+            outputLinkParams['group_by'] = filterName;
+            outputLinkParams['level'] = 2;
+            delete outputLinkParams[filterName];
+            window.location = [locationWithoutQuery, makeURIParameters(outputLinkParams)].join('?');
+          }
+          else {
+            // go to normal silk view
+            outputLinkParams = {
+                viewName: filterParamValue
+            };
+            if (groupParams['group-by'] === 'date') {
+              outputLinkParams['afterDate'] = strftime('%Y/%m/%d 00:00', new Date(groupName));
+              outputLinkParams['beforeDate'] = strftime('%Y/%m/%d 23:59', new Date(groupName));
+            }
+            else if (groupParams['group-by'] === 'revision') {
+              outputLinkParams['revision'] = groupName;
+            }
+            window.location = ['/silk/requests', makeURIParameters(outputLinkParams)].join('?');
+          }
+        })
+      ;
+    }
+
+    chart.renderBoxPlot();
+
+    // TODO: Maybe a linear regression to show a simple trend line?
+  });
+}