Revision: 1488 http://trac.macosforge.org/projects/calendarserver/changeset/1488 Author: cdaboo@apple.com Date: 2007-04-13 14:03:30 -0700 (Fri, 13 Apr 2007) Log Message: ----------- Tools for performance load testing. The multiloader will ramp up a series of simulataneous client connections via multiple threads/processes to load the server with requests. It measures the average server throughput and the average response time. Modified Paths: -------------- CalDAVTester/trunk/src/perfinfo.py Added Paths: ----------- CalDAVTester/trunk/loader.py CalDAVTester/trunk/multiloader.py Added: CalDAVTester/trunk/loader.py =================================================================== --- CalDAVTester/trunk/loader.py (rev 0) +++ CalDAVTester/trunk/loader.py 2007-04-13 21:03:30 UTC (rev 1488) @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +## +# Copyright (c) 2006-2007 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DRI: Cyrus Daboo, cdaboo@apple.com +## +# +# Runs a series of test suites inb parallel using a thread pool +# + +import cPickle as pickle +import getopt + +import sys + +from src.perfinfo import perfinfo + +def usage(): + print """Usage: loader.py [options] <<script file>> +Options: + -h Print this help and exit + -n Do not print out results + -o file Write raw results to file + -i num Numeric offset for test counter +""" + +def runIt(script, silent=False): + + pinfo = perfinfo.parseFile(script) + + pinfo.doStart(silent) + + allresults = pinfo.doLoadRamping() + + pinfo.doEnd(silent) + + return allresults + +if __name__ == "__main__": + + output_file = None + no_results = False + offset = 0 + + options, args = getopt.getopt(sys.argv[1:], "hno:i:") + + for option, value in options: + if option == "-h": + usage() + sys.exit(0) + elif option == "-n": + no_results = True + elif option == "-o": + output_file = value + elif option == "-i": + offset = int(value) + else: + print "Unrecognized option: %s" % (option,) + usage() + raise ValueError + + perfinfoname = "scripts/performance/perfinfo.xml" + if len(args) > 0: + perfinfoname = args[0] + + allresults = perfinfo.runIt("load ramping", perfinfoname, silent=True, offset=offset) + + if output_file: + fd = open(output_file, "w") + fd.write(pickle.dumps(allresults)) + fd.close() + + if not no_results: + # Print out averaged results. + print "\n\nClients\tReqs/sec\tResponse (secs)" + print "=====================================================================" + for raw, clients, reqs, resp in allresults: + for x in raw: + print x + print "%d\t%.1f\t%.3f" % (clients, reqs, resp,) Property changes on: CalDAVTester/trunk/loader.py ___________________________________________________________________ Name: svn:executable + * Added: CalDAVTester/trunk/multiloader.py =================================================================== --- CalDAVTester/trunk/multiloader.py (rev 0) +++ CalDAVTester/trunk/multiloader.py 2007-04-13 21:03:30 UTC (rev 1488) @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# +## +# Copyright (c) 2006-2007 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DRI: Cyrus Daboo, cdaboo@apple.com +## +# +# Runs a series of test suites inb parallel using a thread pool +# + +import cPickle as pickle +import getopt +import time + +import os +import sys + +def usage(): + print """Usage: multiload.py [options] <<script file>> +Options: + -h Print this help and exit + -p num Number of sub-process to create +""" + +if __name__ == "__main__": + + processes = 1 + + options, args = getopt.getopt(sys.argv[1:], "hp:") + + for option, value in options: + if option == "-h": + usage() + sys.exit(0) + elif option == "-p": + processes = int(value) + else: + print "Unrecognized option: %s" % (option,) + usage() + raise ValueError + + perfinfoname = "scripts/performance/perfinfo.xml" + if len(args) > 0: + perfinfoname = args[0] + + pids = [] + offset = 50 + for i in range(processes): + output_file = "tmp.output.%s" % (i,) + if i == 0: + pids.append(os.spawnlp(os.P_NOWAIT, "python", "python", "loader.py", "-n", "-o", output_file, perfinfoname)) + else: + pids.append(os.spawnlp(os.P_NOWAIT, "python", "python", "loader.py", "-n", "-o", output_file, "-i", "%d" % (i * offset), perfinfoname)) + print "Created pid %s" % (pids[-1],) + + while sum(pids) != 0: + try: + for i, pid in enumerate(pids): + wpid, sts = os.waitpid(pid, os.WNOHANG) + if wpid and os.WIFEXITED(sts): + pids[i] = 0 + time.sleep(10) + except OSError: + break + + print "All child process complete. Aggregating data now..." + + raw = [] + clients = [] + for i in range(processes): + output_file = "tmp.output.%s" % (i,) + fd = open(output_file, "r") + s = fd.read() + fd.close() + result = pickle.loads(s) + if len(raw) == 0: + for j in range(len(result)): + raw.append([]) + clients.append(result[j][1]) + for j in range(len(result)): + raw[j].append(result[j][0]) + + allresults = [] + for ctr, items in enumerate(raw): + aggregate = {} + for item in items: + for time, resp, num in item: + aggregate.setdefault(time, []).append((resp, num)) + + averaged = {} + for key, values in aggregate.iteritems(): + average = 0.0 + nums = 0 + for resp, num in values: + average += resp * num + nums += num + average /= nums + averaged[key] = (average, nums,) + + avkeys = averaged.keys() + avkeys.sort() + avkeys = avkeys[len(avkeys)/3:-len(avkeys)/3] + rawresults = [] + average_time = 0.0 + average_clients = 0.0 + for i in avkeys: + rawresults.append((i, averaged[i][0], averaged[i][1])) + average_time += averaged[i][0] + average_clients += averaged[i][1] + average_time /= len(avkeys) + average_clients /= len(avkeys) + + allresults.append((clients[ctr] * processes, average_clients, average_time,)) + + # Print out averaged results. + print "\n\nClients\tReqs/sec\tResponse (secs)" + print "=====================================================================" + for clients, reqs, resp in allresults: + print "%d\t%.1f\t%.3f" % (clients, reqs, resp,) Property changes on: CalDAVTester/trunk/multiloader.py ___________________________________________________________________ Name: svn:executable + * Modified: CalDAVTester/trunk/src/perfinfo.py =================================================================== --- CalDAVTester/trunk/src/perfinfo.py 2007-04-13 21:01:37 UTC (rev 1487) +++ CalDAVTester/trunk/src/perfinfo.py 2007-04-13 21:03:30 UTC (rev 1488) @@ -15,13 +15,20 @@ # # DRI: Cyrus Daboo, cdaboo@apple.com ## +from src.manager import manager +from random import randrange +from threading import Timer +import time """ Class that encapsulates the server information for a CalDAV test run. """ import src.xmlDefs +import xml.dom.minidom +START_DELAY = 3.0 + class perfinfo( object ): """ Maintains information about the performance test scenario. @@ -39,6 +46,41 @@ self.endscript = "" self.subsdict = {} + @classmethod + def runIt(cls, type, script, silent=False, offset=0): + + if type not in ("load ramping",): + raise ValueError("Performance type '%s' not supported." % (type,)) + + pinfo = perfinfo.parseFile(script) + + pinfo.doStart(silent) + + if type == "load ramping": + allresults = pinfo.doLoadRamping(offset) + + pinfo.doEnd(silent) + + return allresults + + + @classmethod + def parseFile(cls, filename): + # Open and parse the server config file + fd = open(filename, "r") + doc = xml.dom.minidom.parse( fd ) + fd.close() + + # Verify that top-level element is correct + perfinfo_node = doc._get_documentElement() + if perfinfo_node._get_localName() != src.xmlDefs.ELEMENT_PERFINFO: + raise ValueError("Invalid configuration file: %s" % (filename,)) + if not perfinfo_node.hasChildNodes(): + raise ValueError("Invalid configuration file: %s" % (filename,)) + pinfo = perfinfo() + pinfo.parseXML(perfinfo_node) + return pinfo + def parseXML( self, node ): for child in node._get_childNodes(): if child._get_localName() == src.xmlDefs.ELEMENT_CLIENTS: @@ -70,13 +112,21 @@ runs = None for schild in child._get_childNodes(): if schild._get_localName() == src.xmlDefs.ELEMENT_CLIENTS: - clients = int(schild.firstChild.data) + clist = schild.firstChild.data.split(",") + if len(clist) == 1: + clients = int(clist[0]) + else: + clients = range(int(clist[0]), int(clist[1]) + 1, int(clist[2])) elif schild._get_localName() == src.xmlDefs.ELEMENT_SPREAD: spread = float(schild.firstChild.data) elif schild._get_localName() == src.xmlDefs.ELEMENT_RUNS: runs = int(schild.firstChild.data) if spread and runs: - self.tests.append((clients, spread, runs,)) + if isinstance(clients, list): + for client in clients: + self.tests.append((client, spread, runs,)) + else: + self.tests.append((clients, spread, runs,)) def parseSubstitutionsXML(self, node): for child in node._get_childNodes(): @@ -90,3 +140,142 @@ value = schild.firstChild.data if key and value: self.subsdict[key] = value + + @classmethod + def subs(cls, str, i): + if "%" in str: + return str % i + else: + return str + + def doScript(self, script): + # Create argument list that varies for each threaded client. Basically use a separate + # server account for each client. + def runner(*args): + """ + Test runner method. + @param *args: + """ + + if self.logging: + print "Start: %s" % (args[0]["moresubs"]["$userid1:"],) + try: + mgr = manager(level=manager.LOG_NONE) + result, timing = mgr.runWithOptions(*args[1:], **args[0]) + if self.logging: + print "Done: %s" % (args[0]["moresubs"]["$userid1:"],) + except Exception, e: + print "Thread run exception: %s" % (str(e),) + + args = [] + for i in range(1, self.clients + 1): + moresubs = {} + for key, value in self.subsdict.iteritems(): + moresubs[key] = self.subs(value, i) + args.append(({"moresubs": moresubs}, self.subs(self.serverinfo, i), "", [self.subs(script, i)])) + for arg in args: + runner(*arg) + + def doStart(self, silent): + if self.startscript: + if not silent: + print "Runnning start script %s" % (self.startscript,) + self.doScript(self.startscript) + + def doEnd(self, silent): + if self.endscript: + if not silent: + print "Runnning end script %s" % (self.endscript,) + self.doScript(self.endscript) + + def doLoadRamping(self, offset = 0): + # Cummulative results + allresults = [] + + for test in self.tests: + failed = [False] + result = [0.0, 0.0, 0.0] + results = [] + + endtime = time.time() + START_DELAY + test[2] + + def runner(*args): + """ + Test runner method. + @param *args: + """ + + while(True): + if self.logging: + print "Start: %s" % (args[0]["moresubs"]["$userid1:"],) + try: + mgr = manager(level=manager.LOG_NONE) + result, timing = mgr.runWithOptions(*args[1:], **args[0]) + if result > 0: + failed[0] = True + results.append((time.time(), timing)) + if divmod(len(results), 10)[1] == 0: + print len(results) + if self.logging: + print "Done: %s %.3f" % (args[0]["moresubs"]["$userid1:"], timing,) + except Exception, e: + print "Thread run exception: %s" % (str(e),) + if time.time() > endtime: + break + #time.sleep(randrange(0, 100)/100.0 * test[1]) + + # Create argument list that varies for each threaded client. Basically use a separate + # server account for each client. + args = [] + for i in range(1 + offset, test[0] + 1 + offset): + moresubs = {} + for key, value in self.subsdict.iteritems(): + moresubs[key] = self.subs(value, i) + args.append(({"moresubs": moresubs}, self.subs(self.serverinfo, i), "", [self.subs(self.testinfo, i)])) + + if self.threads: + # Run threads by queuing up a set of timers set to start 5 seconds + random time + # after thread is actually started. The random time is spread over the interval + # we are testing over. Wait for all threads to finish. + timers = [] + for arg in args: + sleeper = START_DELAY + randrange(0, 100)/100.0 * test[1] + timers.append(Timer(sleeper, runner, arg)) + + for thread in timers: + thread.start( ) + + for thread in timers: + thread.join(None) + else: + # Just execute each client request one after the other. + for arg in args: + runner(*arg) + + # Average over 1 sec intervals + bins = {} + for timestamp, timing in results: + bins.setdefault(int(timestamp), []).append(timing) + avbins = {} + for key, values in bins.iteritems(): + average = 0.0 + for i in values: + average += i + average /= len(values) + avbins[key] = (average, len(values),) + + avkeys = avbins.keys() + avkeys.sort() + rawresults = [] + average_time = 0.0 + average_clients = 0.0 + for i in avkeys: + rawresults.append((i, avbins[i][0], avbins[i][1])) + average_time += avbins[i][0] + average_clients += avbins[i][1] + average_time /= len(avkeys) + average_clients /= len(avkeys) + + allresults.append((rawresults, test[0], average_clients, average_time,)) + + return allresults