[CalendarServer-changes] [1488] CalDAVTester/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Apr 13 14:03:31 PDT 2007
Revision: 1488
http://trac.macosforge.org/projects/calendarserver/changeset/1488
Author: cdaboo at apple.com
Date: 2007-04-13 14:03:30 -0700 (Fri, 13 Apr 2007)
Log Message:
-----------
Tools for performance load testing. The multiloader will ramp up a series of simulataneous client connections
via multiple threads/processes to load the server with requests. It measures the average server throughput and
the average response time.
Modified Paths:
--------------
CalDAVTester/trunk/src/perfinfo.py
Added Paths:
-----------
CalDAVTester/trunk/loader.py
CalDAVTester/trunk/multiloader.py
Added: CalDAVTester/trunk/loader.py
===================================================================
--- CalDAVTester/trunk/loader.py (rev 0)
+++ CalDAVTester/trunk/loader.py 2007-04-13 21:03:30 UTC (rev 1488)
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+#
+##
+# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+#
+# Runs a series of test suites inb parallel using a thread pool
+#
+
+import cPickle as pickle
+import getopt
+
+import sys
+
+from src.perfinfo import perfinfo
+
+def usage():
+ print """Usage: loader.py [options] <<script file>>
+Options:
+ -h Print this help and exit
+ -n Do not print out results
+ -o file Write raw results to file
+ -i num Numeric offset for test counter
+"""
+
+def runIt(script, silent=False):
+
+ pinfo = perfinfo.parseFile(script)
+
+ pinfo.doStart(silent)
+
+ allresults = pinfo.doLoadRamping()
+
+ pinfo.doEnd(silent)
+
+ return allresults
+
+if __name__ == "__main__":
+
+ output_file = None
+ no_results = False
+ offset = 0
+
+ options, args = getopt.getopt(sys.argv[1:], "hno:i:")
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ sys.exit(0)
+ elif option == "-n":
+ no_results = True
+ elif option == "-o":
+ output_file = value
+ elif option == "-i":
+ offset = int(value)
+ else:
+ print "Unrecognized option: %s" % (option,)
+ usage()
+ raise ValueError
+
+ perfinfoname = "scripts/performance/perfinfo.xml"
+ if len(args) > 0:
+ perfinfoname = args[0]
+
+ allresults = perfinfo.runIt("load ramping", perfinfoname, silent=True, offset=offset)
+
+ if output_file:
+ fd = open(output_file, "w")
+ fd.write(pickle.dumps(allresults))
+ fd.close()
+
+ if not no_results:
+ # Print out averaged results.
+ print "\n\nClients\tReqs/sec\tResponse (secs)"
+ print "====================================================================="
+ for raw, clients, reqs, resp in allresults:
+ for x in raw:
+ print x
+ print "%d\t%.1f\t%.3f" % (clients, reqs, resp,)
Property changes on: CalDAVTester/trunk/loader.py
___________________________________________________________________
Name: svn:executable
+ *
Added: CalDAVTester/trunk/multiloader.py
===================================================================
--- CalDAVTester/trunk/multiloader.py (rev 0)
+++ CalDAVTester/trunk/multiloader.py 2007-04-13 21:03:30 UTC (rev 1488)
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+#
+##
+# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+#
+# Runs a series of test suites inb parallel using a thread pool
+#
+
+import cPickle as pickle
+import getopt
+import time
+
+import os
+import sys
+
+def usage():
+ print """Usage: multiload.py [options] <<script file>>
+Options:
+ -h Print this help and exit
+ -p num Number of sub-process to create
+"""
+
+if __name__ == "__main__":
+
+ processes = 1
+
+ options, args = getopt.getopt(sys.argv[1:], "hp:")
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ sys.exit(0)
+ elif option == "-p":
+ processes = int(value)
+ else:
+ print "Unrecognized option: %s" % (option,)
+ usage()
+ raise ValueError
+
+ perfinfoname = "scripts/performance/perfinfo.xml"
+ if len(args) > 0:
+ perfinfoname = args[0]
+
+ pids = []
+ offset = 50
+ for i in range(processes):
+ output_file = "tmp.output.%s" % (i,)
+ if i == 0:
+ pids.append(os.spawnlp(os.P_NOWAIT, "python", "python", "loader.py", "-n", "-o", output_file, perfinfoname))
+ else:
+ pids.append(os.spawnlp(os.P_NOWAIT, "python", "python", "loader.py", "-n", "-o", output_file, "-i", "%d" % (i * offset), perfinfoname))
+ print "Created pid %s" % (pids[-1],)
+
+ while sum(pids) != 0:
+ try:
+ for i, pid in enumerate(pids):
+ wpid, sts = os.waitpid(pid, os.WNOHANG)
+ if wpid and os.WIFEXITED(sts):
+ pids[i] = 0
+ time.sleep(10)
+ except OSError:
+ break
+
+ print "All child process complete. Aggregating data now..."
+
+ raw = []
+ clients = []
+ for i in range(processes):
+ output_file = "tmp.output.%s" % (i,)
+ fd = open(output_file, "r")
+ s = fd.read()
+ fd.close()
+ result = pickle.loads(s)
+ if len(raw) == 0:
+ for j in range(len(result)):
+ raw.append([])
+ clients.append(result[j][1])
+ for j in range(len(result)):
+ raw[j].append(result[j][0])
+
+ allresults = []
+ for ctr, items in enumerate(raw):
+ aggregate = {}
+ for item in items:
+ for time, resp, num in item:
+ aggregate.setdefault(time, []).append((resp, num))
+
+ averaged = {}
+ for key, values in aggregate.iteritems():
+ average = 0.0
+ nums = 0
+ for resp, num in values:
+ average += resp * num
+ nums += num
+ average /= nums
+ averaged[key] = (average, nums,)
+
+ avkeys = averaged.keys()
+ avkeys.sort()
+ avkeys = avkeys[len(avkeys)/3:-len(avkeys)/3]
+ rawresults = []
+ average_time = 0.0
+ average_clients = 0.0
+ for i in avkeys:
+ rawresults.append((i, averaged[i][0], averaged[i][1]))
+ average_time += averaged[i][0]
+ average_clients += averaged[i][1]
+ average_time /= len(avkeys)
+ average_clients /= len(avkeys)
+
+ allresults.append((clients[ctr] * processes, average_clients, average_time,))
+
+ # Print out averaged results.
+ print "\n\nClients\tReqs/sec\tResponse (secs)"
+ print "====================================================================="
+ for clients, reqs, resp in allresults:
+ print "%d\t%.1f\t%.3f" % (clients, reqs, resp,)
Property changes on: CalDAVTester/trunk/multiloader.py
___________________________________________________________________
Name: svn:executable
+ *
Modified: CalDAVTester/trunk/src/perfinfo.py
===================================================================
--- CalDAVTester/trunk/src/perfinfo.py 2007-04-13 21:01:37 UTC (rev 1487)
+++ CalDAVTester/trunk/src/perfinfo.py 2007-04-13 21:03:30 UTC (rev 1488)
@@ -15,13 +15,20 @@
#
# DRI: Cyrus Daboo, cdaboo at apple.com
##
+from src.manager import manager
+from random import randrange
+from threading import Timer
+import time
"""
Class that encapsulates the server information for a CalDAV test run.
"""
import src.xmlDefs
+import xml.dom.minidom
+START_DELAY = 3.0
+
class perfinfo( object ):
"""
Maintains information about the performance test scenario.
@@ -39,6 +46,41 @@
self.endscript = ""
self.subsdict = {}
+ @classmethod
+ def runIt(cls, type, script, silent=False, offset=0):
+
+ if type not in ("load ramping",):
+ raise ValueError("Performance type '%s' not supported." % (type,))
+
+ pinfo = perfinfo.parseFile(script)
+
+ pinfo.doStart(silent)
+
+ if type == "load ramping":
+ allresults = pinfo.doLoadRamping(offset)
+
+ pinfo.doEnd(silent)
+
+ return allresults
+
+
+ @classmethod
+ def parseFile(cls, filename):
+ # Open and parse the server config file
+ fd = open(filename, "r")
+ doc = xml.dom.minidom.parse( fd )
+ fd.close()
+
+ # Verify that top-level element is correct
+ perfinfo_node = doc._get_documentElement()
+ if perfinfo_node._get_localName() != src.xmlDefs.ELEMENT_PERFINFO:
+ raise ValueError("Invalid configuration file: %s" % (filename,))
+ if not perfinfo_node.hasChildNodes():
+ raise ValueError("Invalid configuration file: %s" % (filename,))
+ pinfo = perfinfo()
+ pinfo.parseXML(perfinfo_node)
+ return pinfo
+
def parseXML( self, node ):
for child in node._get_childNodes():
if child._get_localName() == src.xmlDefs.ELEMENT_CLIENTS:
@@ -70,13 +112,21 @@
runs = None
for schild in child._get_childNodes():
if schild._get_localName() == src.xmlDefs.ELEMENT_CLIENTS:
- clients = int(schild.firstChild.data)
+ clist = schild.firstChild.data.split(",")
+ if len(clist) == 1:
+ clients = int(clist[0])
+ else:
+ clients = range(int(clist[0]), int(clist[1]) + 1, int(clist[2]))
elif schild._get_localName() == src.xmlDefs.ELEMENT_SPREAD:
spread = float(schild.firstChild.data)
elif schild._get_localName() == src.xmlDefs.ELEMENT_RUNS:
runs = int(schild.firstChild.data)
if spread and runs:
- self.tests.append((clients, spread, runs,))
+ if isinstance(clients, list):
+ for client in clients:
+ self.tests.append((client, spread, runs,))
+ else:
+ self.tests.append((clients, spread, runs,))
def parseSubstitutionsXML(self, node):
for child in node._get_childNodes():
@@ -90,3 +140,142 @@
value = schild.firstChild.data
if key and value:
self.subsdict[key] = value
+
+ @classmethod
+ def subs(cls, str, i):
+ if "%" in str:
+ return str % i
+ else:
+ return str
+
+ def doScript(self, script):
+ # Create argument list that varies for each threaded client. Basically use a separate
+ # server account for each client.
+ def runner(*args):
+ """
+ Test runner method.
+ @param *args:
+ """
+
+ if self.logging:
+ print "Start: %s" % (args[0]["moresubs"]["$userid1:"],)
+ try:
+ mgr = manager(level=manager.LOG_NONE)
+ result, timing = mgr.runWithOptions(*args[1:], **args[0])
+ if self.logging:
+ print "Done: %s" % (args[0]["moresubs"]["$userid1:"],)
+ except Exception, e:
+ print "Thread run exception: %s" % (str(e),)
+
+ args = []
+ for i in range(1, self.clients + 1):
+ moresubs = {}
+ for key, value in self.subsdict.iteritems():
+ moresubs[key] = self.subs(value, i)
+ args.append(({"moresubs": moresubs}, self.subs(self.serverinfo, i), "", [self.subs(script, i)]))
+ for arg in args:
+ runner(*arg)
+
+ def doStart(self, silent):
+ if self.startscript:
+ if not silent:
+ print "Runnning start script %s" % (self.startscript,)
+ self.doScript(self.startscript)
+
+ def doEnd(self, silent):
+ if self.endscript:
+ if not silent:
+ print "Runnning end script %s" % (self.endscript,)
+ self.doScript(self.endscript)
+
+ def doLoadRamping(self, offset = 0):
+ # Cummulative results
+ allresults = []
+
+ for test in self.tests:
+ failed = [False]
+ result = [0.0, 0.0, 0.0]
+ results = []
+
+ endtime = time.time() + START_DELAY + test[2]
+
+ def runner(*args):
+ """
+ Test runner method.
+ @param *args:
+ """
+
+ while(True):
+ if self.logging:
+ print "Start: %s" % (args[0]["moresubs"]["$userid1:"],)
+ try:
+ mgr = manager(level=manager.LOG_NONE)
+ result, timing = mgr.runWithOptions(*args[1:], **args[0])
+ if result > 0:
+ failed[0] = True
+ results.append((time.time(), timing))
+ if divmod(len(results), 10)[1] == 0:
+ print len(results)
+ if self.logging:
+ print "Done: %s %.3f" % (args[0]["moresubs"]["$userid1:"], timing,)
+ except Exception, e:
+ print "Thread run exception: %s" % (str(e),)
+ if time.time() > endtime:
+ break
+ #time.sleep(randrange(0, 100)/100.0 * test[1])
+
+ # Create argument list that varies for each threaded client. Basically use a separate
+ # server account for each client.
+ args = []
+ for i in range(1 + offset, test[0] + 1 + offset):
+ moresubs = {}
+ for key, value in self.subsdict.iteritems():
+ moresubs[key] = self.subs(value, i)
+ args.append(({"moresubs": moresubs}, self.subs(self.serverinfo, i), "", [self.subs(self.testinfo, i)]))
+
+ if self.threads:
+ # Run threads by queuing up a set of timers set to start 5 seconds + random time
+ # after thread is actually started. The random time is spread over the interval
+ # we are testing over. Wait for all threads to finish.
+ timers = []
+ for arg in args:
+ sleeper = START_DELAY + randrange(0, 100)/100.0 * test[1]
+ timers.append(Timer(sleeper, runner, arg))
+
+ for thread in timers:
+ thread.start( )
+
+ for thread in timers:
+ thread.join(None)
+ else:
+ # Just execute each client request one after the other.
+ for arg in args:
+ runner(*arg)
+
+ # Average over 1 sec intervals
+ bins = {}
+ for timestamp, timing in results:
+ bins.setdefault(int(timestamp), []).append(timing)
+ avbins = {}
+ for key, values in bins.iteritems():
+ average = 0.0
+ for i in values:
+ average += i
+ average /= len(values)
+ avbins[key] = (average, len(values),)
+
+ avkeys = avbins.keys()
+ avkeys.sort()
+ rawresults = []
+ average_time = 0.0
+ average_clients = 0.0
+ for i in avkeys:
+ rawresults.append((i, avbins[i][0], avbins[i][1]))
+ average_time += avbins[i][0]
+ average_clients += avbins[i][1]
+ average_time /= len(avkeys)
+ average_clients /= len(avkeys)
+
+ allresults.append((rawresults, test[0], average_clients, average_time,))
+
+ return allresults
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20070413/573df9c0/attachment.html
More information about the calendarserver-changes
mailing list