Revision
6470
Author
exarkun@twistedmatrix.com
Date
2010-10-27 09:21:58 -0700 (Wed, 27 Oct 2010)

Log Message

Fixes and improvements to the benchmarking tools

  * benchmark.py: handle a particular kind of dtrace output misformatting and report it (so that maybe it can be fixed)
  * reupload.sh: script for pushing a pile of local pickled data into a codespeed instance
  * sample.sh: use the new massupload script to speed up the upload phase of benchmarking
  * massupload, massupload.py: an uploader that sends all stats from a particular run at once, saving the significant cost of pickle loading for each stat
  * upload.py: changes to facilitate massupload

Modified Paths

Added Paths

Diff

Modified: CalendarServer/trunk/contrib/performance/benchmark.py (6469 => 6470)


--- CalendarServer/trunk/contrib/performance/benchmark.py	2010-10-26 18:23:52 UTC (rev 6469)
+++ CalendarServer/trunk/contrib/performance/benchmark.py	2010-10-27 16:21:58 UTC (rev 6470)
@@ -113,7 +113,12 @@
 
 
     def _op_EXECUTE(self, cmd, rest):
-        which, when = rest.split(None, 1)
+        try:
+            which, when = rest.split(None, 1)
+        except ValueError:
+            print 'Bad EXECUTE line: %r' % (rest,)
+            return
+
         if which == 'SQL':
             self.sql = when
             return

Added: CalendarServer/trunk/contrib/performance/massupload (0 => 6470)


--- CalendarServer/trunk/contrib/performance/massupload	                        (rev 0)
+++ CalendarServer/trunk/contrib/performance/massupload	2010-10-27 16:21:58 UTC (rev 6470)
@@ -0,0 +1,3 @@
+#!/usr/bin/python
+from massupload import main
+main()
Property changes on: CalendarServer/trunk/contrib/performance/massupload
___________________________________________________________________

Added: svn:executable

Added: CalendarServer/trunk/contrib/performance/massupload.py (0 => 6470)


--- CalendarServer/trunk/contrib/performance/massupload.py	                        (rev 0)
+++ CalendarServer/trunk/contrib/performance/massupload.py	2010-10-27 16:21:58 UTC (rev 6470)
@@ -0,0 +1,53 @@
+import sys, pickle
+
+from twisted.internet import reactor
+from twisted.internet.task import coiterate
+from twisted.python.usage import UsageError
+from twisted.python.log import err
+
+from benchlib import select
+from upload import UploadOptions, upload
+
+class MassUploadOptions(UploadOptions):
+    optParameters = [
+        ("benchmarks", None, None, ""),
+        ("parameters", None, None, ""),
+        ("statistics", None, None, "")]
+
+    opt_statistic = None
+
+    def parseArgs(self, filename):
+        self['filename'] = filename
+        UploadOptions.parseArgs(self)
+
+
+def main():
+    options = MassUploadOptions()
+    try:
+        options.parseOptions(sys.argv[1:])
+    except UsageError, e:
+        print e
+        return 1
+
+    fname = options['filename']
+    raw = pickle.load(file(fname))
+
+    def go():
+        for benchmark in options['benchmarks'].split():
+            for param in options['parameters'].split():
+                for statistic in options['statistics'].split():
+                    stat, samples = select(
+                        raw, benchmark, param, statistic)
+                    samples = stat.squash(samples)
+                    yield upload(
+                        reactor, 
+                        options['url'], options['project'],
+                        options['revision'], options['revision-date'],
+                        benchmark, param, statistic,
+                        options['backend'], options['environment'],
+                        samples)
+
+    d = coiterate(go())
+    d.addErrback(err, "Mass upload failed")
+    reactor.callWhenRunning(d.addCallback, lambda ign: reactor.stop())
+    reactor.run()

Added: CalendarServer/trunk/contrib/performance/reupload.sh (0 => 6470)


--- CalendarServer/trunk/contrib/performance/reupload.sh	                        (rev 0)
+++ CalendarServer/trunk/contrib/performance/reupload.sh	2010-10-27 16:21:58 UTC (rev 6470)
@@ -0,0 +1,22 @@
+#!/bin/bash -x
+
+BENCHMARKS="event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event"
+
+for rev in 6446; do
+for f in eighth-try/r$rev-*; do
+    base=`basename $f`
+    revision=${base:1:4}
+    backend=${base:6:10}
+    date="`./svn-committime ~/Projects/CalendarServer/trunk $revision`"
+    for b in $BENCHMARKS; do
+	for p in 1 9 81; do
+	    for s in pagein pageout; do
+		./upload \
+		    --url http://localhost:8000/result/add/ \
+		    --revision $revision --revision-date "$date" \
+		    --environment nmosbuilder --backend $backend --statistic "$f,$b,$p,$s"
+	    done
+	done
+    done
+done
+done
Property changes on: CalendarServer/trunk/contrib/performance/reupload.sh
___________________________________________________________________

Added: svn:executable

Modified: CalendarServer/trunk/contrib/performance/sample.sh (6469 => 6470)


--- CalendarServer/trunk/contrib/performance/sample.sh	2010-10-26 18:23:52 UTC (rev 6469)
+++ CalendarServer/trunk/contrib/performance/sample.sh	2010-10-27 16:21:58 UTC (rev 6470)
@@ -10,7 +10,7 @@
 SOURCE=~/Projects/CalendarServer/trunk
 NUM_INSTANCES=2
 BENCHMARKS="event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event"
-STATISTICS=(HTTP SQL read write)
+STATISTICS=(HTTP SQL read write pagein pageout)
 ADDURL=http://localhost:8000/result/add/
 export PYTHONPATH=$PYTHONPATH:$SOURCE/../Twisted
 
@@ -52,17 +52,13 @@
   popd
   sudo PYTHONPATH=$PYTHONPATH ./benchmark --label r$REV-$backend --log-directory $LOGS $BENCHMARKS
   data=`echo -n r$REV-$backend*`
-  for p in 1 9 81; do
-    for b in $BENCHMARKS; do
-      for stat in "${STATISTICS[@]}"; do
-        sudo -v # Bump timestamp again
-        ./upload \
-            --url $ADDURL --revision $REV \
-            --revision-date "$DATE" --environment nmosbuilder \
-            --backend $backend --statistic "$data,$b,$p,$stat"
-      done
-    done
-  done
-
+  ./massupload \
+      --url $ADDURL --revision $REV \
+      --revision-date "$DATE" --environment nmosbuilder \
+      --backend $backend \
+      --benchmarks "$BENCHMARKS" \
+      --parameters "1 9 81" \
+      --statistics "${STATISTICS[*]}" \
+      $data
   mv $data $RESULTS
 done

Modified: CalendarServer/trunk/contrib/performance/upload.py (6469 => 6470)


--- CalendarServer/trunk/contrib/performance/upload.py	2010-10-26 18:23:52 UTC (rev 6469)
+++ CalendarServer/trunk/contrib/performance/upload.py	2010-10-27 16:21:58 UTC (rev 6470)
@@ -39,9 +39,9 @@
 
 
 
-def upload(reactor, url, project, revision, revision_date, benchmark,
-           executable, environment, result_value, result_date, std_dev,
-           max_value, min_value):
+def _upload(reactor, url, project, revision, revision_date, benchmark,
+            executable, environment, result_value, result_date, std_dev,
+            max_value, min_value):
     data = {
         'commitid': str(revision),
         'revision_date': revision_date,
@@ -70,6 +70,25 @@
     return d
 
 
+def upload(reactor, url, project, revision, revision_date, benchmark, param, statistic, backend, environment, samples):
+    d = _upload(
+        reactor,
+        url=url,
+        project=project,
+        revision=revision,
+        revision_date=revision_date,
+        benchmark='%s-%s-%s' % (benchmark, param, statistic),
+        executable='%s-backend' % (backend,),
+        environment=environment,
+        result_value=median(samples),
+        result_date=datetime.now(),
+        std_dev=mad(samples),  # Not really!
+        max_value=max(samples),
+        min_value=min(samples))
+    d.addErrback(err, "Upload failed")
+    return d
+
+
 def main():
     options = UploadOptions()
     try:
@@ -85,18 +104,10 @@
 
     d = upload(
         reactor,
-        url=options['url'],
-        project=options['project'],
-        revision=options['revision'],
-        revision_date=options['revision-date'],
-        benchmark='%s-%s-%s' % (benchmark, param, statistic),
-        executable='%s-backend' % (options['backend'],),
-        environment=options['environment'],
-        result_value=median(samples),
-        result_date=datetime.now(),
-        std_dev=mad(samples),  # Not really!
-        max_value=max(samples),
-        min_value=min(samples))
-    d.addErrback(err, "Upload failed")
+        options['url'], options['project'], options['revision'],
+        options['revision-date'],
+        benchmark, param, statistic,
+        options['backend'], options['environment'],
+        samples)
     reactor.callWhenRunning(d.addCallback, lambda ign: reactor.stop())
     reactor.run()