[CalendarServer-changes] [6568] CalendarServer/trunk/contrib/performance

[source_changes at macosforge.org]

Revision: 6568
          http://trac.macosforge.org/projects/calendarserver/changeset/6568
Author:   exarkun at twistedmatrix.com
Date:     2010-11-04 13:25:29 -0700 (Thu, 04 Nov 2010)
Log Message:
-----------
Extract SQL statement execution count from the already-collected data

Modified Paths:
--------------
    CalendarServer/trunk/contrib/performance/massupload.py
    CalendarServer/trunk/contrib/performance/report.py
    CalendarServer/trunk/contrib/performance/stats.py

Modified: CalendarServer/trunk/contrib/performance/massupload.py
===================================================================
--- CalendarServer/trunk/contrib/performance/massupload.py	2010-11-04 17:28:56 UTC (rev 6567)
+++ CalendarServer/trunk/contrib/performance/massupload.py	2010-11-04 20:25:29 UTC (rev 6568)
@@ -46,7 +46,22 @@
                         benchmark, param, statistic,
                         options['backend'], options['environment'],
                         samples)
+                    
+                    # This is somewhat hard-coded to the currently
+                    # collected stats.
+                    if statistic == 'SQL':
+                        stat, samples = select(
+                            raw, benchmark, param, 'execute')
+                        samples = stat.squash(samples, 'count')
+                        yield upload(
+                            reactor, 
+                            options['url'], options['project'],
+                            options['revision'], options['revision-date'],
+                            benchmark, param, statistic + 'count',
+                            options['backend'], options['environment'],
+                            samples)
 
+
     d = coiterate(go())
     d.addErrback(err, "Mass upload failed")
     reactor.callWhenRunning(d.addCallback, lambda ign: reactor.stop())

Modified: CalendarServer/trunk/contrib/performance/report.py
===================================================================
--- CalendarServer/trunk/contrib/performance/report.py	2010-11-04 17:28:56 UTC (rev 6567)
+++ CalendarServer/trunk/contrib/performance/report.py	2010-11-04 20:25:29 UTC (rev 6568)
@@ -9,9 +9,9 @@
         stat, samples = select(pickle.load(file(sys.argv[1])), *sys.argv[2:5])
         if len(sys.argv) == 5:
             print 'Samples'
-            print '\t' + '\n\t'.join(map(str, samples))
+            print '\t' + '\n\t'.join(map(str, stat.squash(samples)))
             print 'Commands'
             print '\t' + '\n\t'.join(stat.commands)
         else:
-            getattr(stat, sys.argv[5])(samples)
+            print getattr(stat, sys.argv[5])(samples, *sys.argv[6:])
 

Modified: CalendarServer/trunk/contrib/performance/stats.py
===================================================================
--- CalendarServer/trunk/contrib/performance/stats.py	2010-11-04 17:28:56 UTC (rev 6567)
+++ CalendarServer/trunk/contrib/performance/stats.py	2010-11-04 20:25:29 UTC (rev 6568)
@@ -52,7 +52,7 @@
         return '<Stat %r>' % (self.name,)
 
 
-    def squash(self, samples):
+    def squash(self, samples, mode=None):
         """
         Normalize the sample data into float values (one per sample)
         in seconds (I hope time is the only thing you measure).
@@ -61,11 +61,12 @@
 
 
     def summarize(self, data):
-        print self.name, 'mean', mean(data)
-        print self.name, 'median', median(data)
-        print self.name, 'stddev', stddev(data)
-        print self.name, 'median absolute deviation', mad(data)
-        print self.name, 'sum', sum(data)
+        return ''.join([
+                self.name, ' mean ', str(mean(data)), '\n',
+                self.name, ' median ', str(median(data)), '\n',
+                self.name, ' stddev ', str(stddev(data)), '\n',
+                self.name, ' median absolute deviation ', str(mad(data)), '\n',
+                self.name, ' sum ', str(sum(data)), '\n'])
 
 
     def write(self, basename, data):
@@ -113,12 +114,22 @@
         return sqlparse.format(statement.to_unicode().encode('ascii'))
 
 
-    def squash(self, samples):
-        times = []
+    def squash(self, samples, mode="duration"):
+        """
+        Summarize the execution of a number of SQL statements.
+
+        @param mode: C{"duration"} to squash the durations into the
+            result.  C{"count"} to squash the count of statements
+            executed into the result.
+        """
+        results = []
         for data in samples:
-            times.append(
-                sum([interval for (sql, interval) in data]) / NANO)
-        return times
+            if mode == "duration":
+                value = sum([interval for (sql, interval) in data]) / NANO
+            else:
+                value = len(data)
+            results.append(value)
+        return results
 
 
     def summarize(self, samples):
@@ -131,9 +142,10 @@
                 statements[sql] = statements.get(sql, 0) + 1
                 total += interval
             times.append(total / NANO * 1000)
-        for statement, count in statements.iteritems():
-            print count, ':', statement
-        return _Statistic.summarize(self, times)
+        return ''.join([
+                '%d: %s\n' % (count, statement)
+                for (statement, count) 
+                in statements.iteritems()]) + _Statistic.summarize(self, times)
 
 
     def statements(self, samples):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20101104/c0dabda2/attachment-0001.html>