[CalendarServer-changes] [6133] CalendarServer/trunk/contrib/performance

[source_changes at macosforge.org]

Revision: 6133
          http://trac.macosforge.org/projects/calendarserver/changeset/6133
Author:   exarkun at twistedmatrix.com
Date:     2010-08-19 11:47:14 -0700 (Thu, 19 Aug 2010)
Log Message:
-----------
normalize statements before counting up times

Modified Paths:
--------------
    CalendarServer/trunk/contrib/performance/stats.py

Added Paths:
-----------
    CalendarServer/trunk/contrib/performance/test_stats.py

Modified: CalendarServer/trunk/contrib/performance/stats.py
===================================================================
--- CalendarServer/trunk/contrib/performance/stats.py	2010-08-19 18:08:19 UTC (rev 6132)
+++ CalendarServer/trunk/contrib/performance/stats.py	2010-08-19 18:47:14 UTC (rev 6133)
@@ -1,4 +1,6 @@
 
+import sqlparse
+
 NANO = 1000000000.0
 
 
@@ -45,20 +47,45 @@
 class SQLDuration(_Statistic):
     commands = ['summarize', 'statements']
 
+    def _substitute(self, expression, replacement):
+        try:
+            expression.tokens
+        except AttributeError:
+            return
+
+        for i, token in enumerate(expression.tokens):
+            if sqlparse.tokens.is_token_subtype(token.ttype, sqlparse.tokens.Literal):
+                expression.tokens[i] = replacement
+            elif token.is_whitespace():
+                expression.tokens[i] = sqlparse.sql.Token('Whitespace', ' ')
+            else:
+                self._substitute(token, replacement)
+
+
+    def normalize(self, sql):
+        (statement,) = sqlparse.parse(sql)
+        # Replace any literal values with placeholders
+        qmark = sqlparse.sql.Token('Operator', '?')
+        self._substitute(statement, qmark)
+        return sqlparse.format(statement.to_unicode().encode('ascii'))
+
+
     def summarize(self, data):
         statements = {}
         intervals = []
         for (sql, interval) in data:
+            sql = self.normalize(sql)
             intervals.append(interval)
             statements[sql] = statements.get(sql, 0) + 1
         for statement, count in statements.iteritems():
-            print count, ':', statement.replace('\n', ' ')
+            print count, ':', statement
         return _Statistic.summarize(self, intervals)
 
 
     def statements(self, data):
         statements = {}
         for (sql, interval) in data:
+            sql = self.normalize(sql)
             statements.setdefault(sql, []).append(interval)
         
         byTime = []
@@ -68,7 +95,7 @@
         byTime.reverse()
 
         for (time, statement) in byTime:
-            print time / NANO * 1000, 'ms:', statement.replace('\n', ' ')
+            print time / NANO * 1000, 'ms:', statement
 
 class Bytes(_Statistic):
     pass

Added: CalendarServer/trunk/contrib/performance/test_stats.py
===================================================================
--- CalendarServer/trunk/contrib/performance/test_stats.py	                        (rev 0)
+++ CalendarServer/trunk/contrib/performance/test_stats.py	2010-08-19 18:47:14 UTC (rev 6133)
@@ -0,0 +1,21 @@
+
+from unittest import TestCase
+
+from stats import SQLDuration
+
+class SQLDurationTests(TestCase):
+    def setUp(self):
+        self.stat = SQLDuration('foo')
+
+
+    def test_normalize_integer(self):
+        self.assertEquals(
+            self.stat.normalize('SELECT foo FROM bar WHERE 1'),
+            'SELECT foo FROM bar WHERE ?')
+        self.assertEquals(
+            self.stat.normalize('SELECT foo FROM bar WHERE x = 1'),
+            'SELECT foo FROM bar WHERE x = ?')
+        self.assertEquals(
+            self.stat.normalize('SELECT foo + 1 FROM bar'),
+            'SELECT foo + ? FROM bar')
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100819/6802f471/attachment-0001.html>