Revision: 6470 http://trac.macosforge.org/projects/calendarserver/changeset/6470 Author: exarkun@twistedmatrix.com Date: 2010-10-27 09:21:58 -0700 (Wed, 27 Oct 2010) Log Message: ----------- Fixes and improvements to the benchmarking tools * benchmark.py: handle a particular kind of dtrace output misformatting and report it (so that maybe it can be fixed) * reupload.sh: script for pushing a pile of local pickled data into a codespeed instance * sample.sh: use the new massupload script to speed up the upload phase of benchmarking * massupload, massupload.py: an uploader that sends all stats from a particular run at once, saving the significant cost of pickle loading for each stat * upload.py: changes to facilitate massupload Modified Paths: -------------- CalendarServer/trunk/contrib/performance/benchmark.py CalendarServer/trunk/contrib/performance/sample.sh CalendarServer/trunk/contrib/performance/upload.py Added Paths: ----------- CalendarServer/trunk/contrib/performance/massupload CalendarServer/trunk/contrib/performance/massupload.py CalendarServer/trunk/contrib/performance/reupload.sh Modified: CalendarServer/trunk/contrib/performance/benchmark.py =================================================================== --- CalendarServer/trunk/contrib/performance/benchmark.py 2010-10-26 18:23:52 UTC (rev 6469) +++ CalendarServer/trunk/contrib/performance/benchmark.py 2010-10-27 16:21:58 UTC (rev 6470) @@ -113,7 +113,12 @@ def _op_EXECUTE(self, cmd, rest): - which, when = rest.split(None, 1) + try: + which, when = rest.split(None, 1) + except ValueError: + print 'Bad EXECUTE line: %r' % (rest,) + return + if which == 'SQL': self.sql = when return Added: CalendarServer/trunk/contrib/performance/massupload =================================================================== --- CalendarServer/trunk/contrib/performance/massupload (rev 0) +++ CalendarServer/trunk/contrib/performance/massupload 2010-10-27 16:21:58 UTC (rev 6470) @@ -0,0 +1,3 @@ +#!/usr/bin/python +from massupload import main +main() Property changes on: CalendarServer/trunk/contrib/performance/massupload ___________________________________________________________________ Added: svn:executable + * Added: CalendarServer/trunk/contrib/performance/massupload.py =================================================================== --- CalendarServer/trunk/contrib/performance/massupload.py (rev 0) +++ CalendarServer/trunk/contrib/performance/massupload.py 2010-10-27 16:21:58 UTC (rev 6470) @@ -0,0 +1,53 @@ +import sys, pickle + +from twisted.internet import reactor +from twisted.internet.task import coiterate +from twisted.python.usage import UsageError +from twisted.python.log import err + +from benchlib import select +from upload import UploadOptions, upload + +class MassUploadOptions(UploadOptions): + optParameters = [ + ("benchmarks", None, None, ""), + ("parameters", None, None, ""), + ("statistics", None, None, "")] + + opt_statistic = None + + def parseArgs(self, filename): + self['filename'] = filename + UploadOptions.parseArgs(self) + + +def main(): + options = MassUploadOptions() + try: + options.parseOptions(sys.argv[1:]) + except UsageError, e: + print e + return 1 + + fname = options['filename'] + raw = pickle.load(file(fname)) + + def go(): + for benchmark in options['benchmarks'].split(): + for param in options['parameters'].split(): + for statistic in options['statistics'].split(): + stat, samples = select( + raw, benchmark, param, statistic) + samples = stat.squash(samples) + yield upload( + reactor, + options['url'], options['project'], + options['revision'], options['revision-date'], + benchmark, param, statistic, + options['backend'], options['environment'], + samples) + + d = coiterate(go()) + d.addErrback(err, "Mass upload failed") + reactor.callWhenRunning(d.addCallback, lambda ign: reactor.stop()) + reactor.run() Added: CalendarServer/trunk/contrib/performance/reupload.sh =================================================================== --- CalendarServer/trunk/contrib/performance/reupload.sh (rev 0) +++ CalendarServer/trunk/contrib/performance/reupload.sh 2010-10-27 16:21:58 UTC (rev 6470) @@ -0,0 +1,22 @@ +#!/bin/bash -x + +BENCHMARKS="event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event" + +for rev in 6446; do +for f in eighth-try/r$rev-*; do + base=`basename $f` + revision=${base:1:4} + backend=${base:6:10} + date="`./svn-committime ~/Projects/CalendarServer/trunk $revision`" + for b in $BENCHMARKS; do + for p in 1 9 81; do + for s in pagein pageout; do + ./upload \ + --url http://localhost:8000/result/add/ \ + --revision $revision --revision-date "$date" \ + --environment nmosbuilder --backend $backend --statistic "$f,$b,$p,$s" + done + done + done +done +done Property changes on: CalendarServer/trunk/contrib/performance/reupload.sh ___________________________________________________________________ Added: svn:executable + * Modified: CalendarServer/trunk/contrib/performance/sample.sh =================================================================== --- CalendarServer/trunk/contrib/performance/sample.sh 2010-10-26 18:23:52 UTC (rev 6469) +++ CalendarServer/trunk/contrib/performance/sample.sh 2010-10-27 16:21:58 UTC (rev 6470) @@ -10,7 +10,7 @@ SOURCE=~/Projects/CalendarServer/trunk NUM_INSTANCES=2 BENCHMARKS="event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event" -STATISTICS=(HTTP SQL read write) +STATISTICS=(HTTP SQL read write pagein pageout) ADDURL=http://localhost:8000/result/add/ export PYTHONPATH=$PYTHONPATH:$SOURCE/../Twisted @@ -52,17 +52,13 @@ popd sudo PYTHONPATH=$PYTHONPATH ./benchmark --label r$REV-$backend --log-directory $LOGS $BENCHMARKS data=`echo -n r$REV-$backend*` - for p in 1 9 81; do - for b in $BENCHMARKS; do - for stat in "${STATISTICS[@]}"; do - sudo -v # Bump timestamp again - ./upload \ - --url $ADDURL --revision $REV \ - --revision-date "$DATE" --environment nmosbuilder \ - --backend $backend --statistic "$data,$b,$p,$stat" - done - done - done - + ./massupload \ + --url $ADDURL --revision $REV \ + --revision-date "$DATE" --environment nmosbuilder \ + --backend $backend \ + --benchmarks "$BENCHMARKS" \ + --parameters "1 9 81" \ + --statistics "${STATISTICS[*]}" \ + $data mv $data $RESULTS done Modified: CalendarServer/trunk/contrib/performance/upload.py =================================================================== --- CalendarServer/trunk/contrib/performance/upload.py 2010-10-26 18:23:52 UTC (rev 6469) +++ CalendarServer/trunk/contrib/performance/upload.py 2010-10-27 16:21:58 UTC (rev 6470) @@ -39,9 +39,9 @@ -def upload(reactor, url, project, revision, revision_date, benchmark, - executable, environment, result_value, result_date, std_dev, - max_value, min_value): +def _upload(reactor, url, project, revision, revision_date, benchmark, + executable, environment, result_value, result_date, std_dev, + max_value, min_value): data = { 'commitid': str(revision), 'revision_date': revision_date, @@ -70,6 +70,25 @@ return d +def upload(reactor, url, project, revision, revision_date, benchmark, param, statistic, backend, environment, samples): + d = _upload( + reactor, + url=url, + project=project, + revision=revision, + revision_date=revision_date, + benchmark='%s-%s-%s' % (benchmark, param, statistic), + executable='%s-backend' % (backend,), + environment=environment, + result_value=median(samples), + result_date=datetime.now(), + std_dev=mad(samples), # Not really! + max_value=max(samples), + min_value=min(samples)) + d.addErrback(err, "Upload failed") + return d + + def main(): options = UploadOptions() try: @@ -85,18 +104,10 @@ d = upload( reactor, - url=options['url'], - project=options['project'], - revision=options['revision'], - revision_date=options['revision-date'], - benchmark='%s-%s-%s' % (benchmark, param, statistic), - executable='%s-backend' % (options['backend'],), - environment=options['environment'], - result_value=median(samples), - result_date=datetime.now(), - std_dev=mad(samples), # Not really! - max_value=max(samples), - min_value=min(samples)) - d.addErrback(err, "Upload failed") + options['url'], options['project'], options['revision'], + options['revision-date'], + benchmark, param, statistic, + options['backend'], options['environment'], + samples) reactor.callWhenRunning(d.addCallback, lambda ign: reactor.stop()) reactor.run()