[CalendarServer-changes] [9305] CalendarServer/trunk/contrib/performance/loadtest

Revision: 9305 http://trac.macosforge.org/projects/calendarserver/changeset/9305 Author: cdaboo@apple.com Date: 2012-05-29 08:23:08 -0700 (Tue, 29 May 2012) Log Message: ----------- More fine-grained request type logging including -small, -medium and -large tags for requests based on input size (number of attendees, resources etc). Added the ability to specify failure thresholds via the config plist and a new json data file. Modified Paths: -------------- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist CalendarServer/trunk/contrib/performance/loadtest/config.plist CalendarServer/trunk/contrib/performance/loadtest/ical.py CalendarServer/trunk/contrib/performance/loadtest/logger.py CalendarServer/trunk/contrib/performance/loadtest/population.py CalendarServer/trunk/contrib/performance/loadtest/profiles.py CalendarServer/trunk/contrib/performance/loadtest/sim.py CalendarServer/trunk/contrib/performance/loadtest/test_population.py CalendarServer/trunk/contrib/performance/loadtest/test_sim.py Added Paths: ----------- CalendarServer/trunk/contrib/performance/loadtest/thresholds.json Modified: CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-29 15:23:08 UTC (rev 9305) @@ -474,16 +474,52 @@ <array> <!-- ReportStatistics generates an end-of-run summary of the HTTP requests made, their timings, and their results. --> - <string>contrib.performance.loadtest.population.ReportStatistics</string> - + <dict> + <key>type</key> + <string>contrib.performance.loadtest.population.ReportStatistics</string> + <key>params</key> + <dict> + <!-- The thresholds for each request type --> + <key>thresholds</key> + <string>contrib/performance/loadtest/thresholds.json</string> + + <!-- The % of failures that constitute a failed test --> + <key>failCutoff</key> + <real>1.0</real> + </dict> + </dict> + <!-- RequestLogger generates a realtime log of all HTTP requests made during the load test. --> - <string>contrib.performance.loadtest.ical.RequestLogger</string> - + <dict> + <key>type</key> + <string>contrib.performance.loadtest.ical.RequestLogger</string> + <key>params</key> + <dict> + </dict> + </dict> + <!-- OperationLogger generates an end-of-run summary of the gross operations performed (logical operations which may span more than one HTTP request, such as inviting an attendee to an event). --> - <string>contrib.performance.loadtest.profiles.OperationLogger</string> + <dict> + <key>type</key> + <string>contrib.performance.loadtest.profiles.OperationLogger</string> + <key>params</key> + <dict> + <!-- The thresholds for each operation type --> + <key>thresholds</key> + <string>contrib/performance/loadtest/thresholds.json</string> + + <!-- The % of operations beyond the lag cut-off that constitute a failed test --> + <key>lagCutoff</key> + <real>1.0</real> + + <!-- The % of failures that constitute a failed test --> + <key>failCutoff</key> + <real>1.0</real> + </dict> + </dict> </array> </dict> </plist> Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-29 15:23:08 UTC (rev 9305) @@ -466,18 +466,54 @@ <!-- Define some log observers to report on the load test. --> <key>observers</key> <array> - <!-- ReportStatistics generates an end-of-run summary of the HTTP requests + <!-- ReportStatistics generates an end-of-run summary of the HTTP requests made, their timings, and their results. --> - <string>contrib.performance.loadtest.population.ReportStatistics</string> - - <!-- RequestLogger generates a realtime log of all HTTP requests made + <dict> + <key>type</key> + <string>contrib.performance.loadtest.population.ReportStatistics</string> + <key>params</key> + <dict> + <!-- The thresholds for each request type --> + <key>thresholdsPath</key> + <string>contrib/performance/loadtest/thresholds.json</string> + + <!-- The % of failures that constitute a failed test --> + <key>failCutoff</key> + <real>1.0</real> + </dict> + </dict> + + <!-- RequestLogger generates a realtime log of all HTTP requests made during the load test. --> - <string>contrib.performance.loadtest.ical.RequestLogger</string> - - <!-- OperationLogger generates an end-of-run summary of the gross operations - performed (logical operations which may span more than one HTTP request, + <dict> + <key>type</key> + <string>contrib.performance.loadtest.ical.RequestLogger</string> + <key>params</key> + <dict> + </dict> + </dict> + + <!-- OperationLogger generates an end-of-run summary of the gross operations + performed (logical operations which may span more than one HTTP request, such as inviting an attendee to an event). --> - <string>contrib.performance.loadtest.profiles.OperationLogger</string> + <dict> + <key>type</key> + <string>contrib.performance.loadtest.profiles.OperationLogger</string> + <key>params</key> + <dict> + <!-- The thresholds for each operation type --> + <key>thresholdsPath</key> + <string>contrib/performance/loadtest/thresholds.json</string> + + <!-- The % of operations beyond the lag cut-off that constitute a failed test --> + <key>lagCutoff</key> + <real>1.0</real> + + <!-- The % of failures that constitute a failed test --> + <key>failCutoff</key> + <real>1.0</real> + </dict> + </dict> </array> </dict> </plist> Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -421,7 +421,7 @@ @inlineCallbacks - def _proppatch(self, url, body): + def _proppatch(self, url, body, method_label=None): """ Issue a PROPPATCH on the chosen URL """ @@ -431,7 +431,8 @@ 'PROPPATCH', self.root + url[1:].encode('utf-8'), hdrs, - StringProducer(body) + StringProducer(body), + method_label=method_label, ) if response.code == MULTI_STATUS: body = yield readBody(response) @@ -473,6 +474,7 @@ location, self._STARTUP_WELL_KNOWN, allowedStatus=(MULTI_STATUS, MOVED_PERMANENTLY), + method_label="PROPFIND{well-known}", ) # Follow any redirect @@ -483,6 +485,7 @@ location, self._STARTUP_WELL_KNOWN, allowedStatus=(MULTI_STATUS), + method_label="PROPFIND{well-known}", ) returnValue(result[location]) @@ -498,6 +501,7 @@ _ignore_response, result = yield self._propfind( principalPath, self._STARTUP_PRINCIPAL_PROPFIND_INITIAL, + method_label="PROPFIND{find-principal}", ) returnValue(result[principalPath]) @@ -512,6 +516,7 @@ _ignore_response, result = yield self._propfind( self.principalURL, self._STARTUP_PRINCIPAL_PROPFIND, + method_label="PROPFIND{principal}", ) returnValue(result[self.principalURL]) @@ -647,7 +652,7 @@ self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': calendar.changeToken}, depth='1', otherTokens = True, - method_label="REPORT{sync}", + method_label="REPORT{sync}" if calendar.changeToken else "REPORT{sync-init}", ) changed = [] @@ -756,11 +761,18 @@ # Next do a REPORT on events that might have information # we don't know about. hrefs = "".join([self._POLL_CALENDAR_MULTIGET_REPORT_HREF % {'href': event} for event in events]) + + label_suffix = "small" + if len(hrefs) > 5: + label_suffix = "medium" + if len(hrefs) > 15: + label_suffix = "large" + return self._report( calendar, self._POLL_CALENDAR_MULTIGET_REPORT % {'hrefs': hrefs}, depth=None, - method_label="REPORT{multiget}", + method_label="REPORT{multiget-%s}" % (label_suffix,), ) @@ -834,9 +846,21 @@ # Patch calendar properties for cal in calendars: if cal.name != "inbox": - yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_COLOR) - yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_ORDER) - yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE) + yield self._proppatch( + cal.url, + self._STARTUP_PROPPATCH_CALENDAR_COLOR, + method_label="PROPPATCH{calendar}", + ) + yield self._proppatch( + cal.url, + self._STARTUP_PROPPATCH_CALENDAR_ORDER, + method_label="PROPPATCH{calendar}", + ) + yield self._proppatch( + cal.url, + self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE, + method_label="PROPPATCH{calendar}", + ) def _pollFirstTime2(self): @@ -848,6 +872,7 @@ _ignore_response, result = yield self._propfind( notificationURL, self._POLL_NOTIFICATION_PROPFIND, + method_label="PROPFIND{notification}", ) returnValue(result) @@ -858,6 +883,7 @@ notificationURL, self._POLL_NOTIFICATION_PROPFIND_D1, depth='1', + method_label="PROPFIND{notification-items}", ) returnValue(result) @@ -1055,6 +1081,12 @@ attendees.append(attendee) vevent.mainComponent().addProperty(attendee) + label_suffix = "small" + if len(attendees) > 5: + label_suffix = "medium" + if len(attendees) > 15: + label_suffix = "large" + # At last, upload the new event definition response = yield self._request( (NO_CONTENT, PRECONDITION_FAILED,), @@ -1064,7 +1096,7 @@ 'content-type': ['text/calendar'], 'if-match': [event.etag]}), StringProducer(vevent.getTextWithTimezones(includeTimezones=True)), - method_label="PUT{organizer}" + method_label="PUT{organizer-%s}" % (label_suffix,) ) # Finally, re-retrieve the event to update the etag @@ -1124,12 +1156,19 @@ headers.addRawHeader('if-schedule-tag-match', event.scheduleTag) okCodes = (NO_CONTENT, PRECONDITION_FAILED,) + attendees = list(vevent.mainComponent().properties('ATTENDEE')) + label_suffix = "small" + if len(attendees) > 5: + label_suffix = "medium" + if len(attendees) > 15: + label_suffix = "large" + response = yield self._request( okCodes, 'PUT', self.root + href[1:].encode('utf-8'), headers, StringProducer(vevent.getTextWithTimezones(includeTimezones=True)), - method_label="PUT{attendee}", + method_label="PUT{attendee-%s}" % (label_suffix,), ) self._updateEvent(response, href) @@ -1144,7 +1183,11 @@ self._removeEvent(href) response = yield self._request( - NO_CONTENT, 'DELETE', self.root + href[1:].encode('utf-8')) + NO_CONTENT, + 'DELETE', + self.root + href[1:].encode('utf-8'), + method_label="DELETE{event}", + ) returnValue(response) @@ -1153,13 +1196,21 @@ headers = Headers({ 'content-type': ['text/calendar'], }) + + attendees = list(vcalendar.mainComponent().properties('ATTENDEE')) + label_suffix = "small" + if len(attendees) > 5: + label_suffix = "medium" + if len(attendees) > 15: + label_suffix = "large" + response = yield self._request( CREATED, 'PUT', self.root + href[1:].encode('utf-8'), headers, StringProducer(vcalendar.getTextWithTimezones(includeTimezones=True)), - method_label="PUT{organizer}" if invite else None, + method_label="PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}", ) self._localUpdateEvent(response, href, vcalendar) @@ -1198,7 +1249,12 @@ @inlineCallbacks def _updateEvent(self, ignored, href): - response = yield self._request(OK, 'GET', self.root + href[1:].encode('utf-8')) + response = yield self._request( + OK, + 'GET', + self.root + href[1:].encode('utf-8'), + method_label="GET{event}", + ) headers = response.headers etag = headers.getRawHeaders('etag')[0] scheduleTag = headers.getRawHeaders('schedule-tag', [None])[0] @@ -1252,6 +1308,12 @@ end = end.getText() now = PyCalendarDateTime.getNowUTC().getText() + label_suffix = "small" + if len(users) > 5: + label_suffix = "medium" + if len(users) > 15: + label_suffix = "large" + response = yield self._request( OK, 'POST', outbox, Headers({ @@ -1267,7 +1329,7 @@ 'start': start, 'end': end, 'now': now}), - method_label="POST{fb}", + method_label="POST{fb}-%s" % (label_suffix,), ) body = yield readBody(response) returnValue(body) @@ -1545,6 +1607,7 @@ _ignore_response, result = yield self._propfind( '/principals/', self._STARTUP_PRINCIPAL_PROPFIND_INITIAL, + method_label="PROPFIND{find-principal}", ) returnValue(result['/principals/']) @@ -1554,8 +1617,16 @@ # Patch calendar properties for cal in calendars: if cal.name != "inbox": - yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_COLOR) - yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_ORDER) + yield self._proppatch( + cal.url, + self._STARTUP_PROPPATCH_CALENDAR_COLOR, + method_label="PROPPATCH{calendar}", + ) + yield self._proppatch( + cal.url, + self._STARTUP_PROPPATCH_CALENDAR_ORDER, + method_label="PROPPATCH{calendar}", + ) def _pollFirstTime2(self): Modified: CalendarServer/trunk/contrib/performance/loadtest/logger.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/logger.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/logger.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -60,6 +60,7 @@ for ctr, item in enumerate(self._thresholds): _ignore_threshold, fail_at = item + fail_at = fail_at.get(operation, fail_at["default"]) if thresholds[ctr] * 100.0 / count > fail_at: failure = True Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -26,6 +26,7 @@ from tempfile import mkdtemp from itertools import izip from datetime import datetime +import json import os from twisted.internet.defer import DeferredList @@ -334,39 +335,57 @@ """ # the response time thresholds to display together with failing % count threshold - _thresholds = ( - (0.5, 100.0), - ( 1, 100.0), - ( 3, 5.0), - ( 5, 1.0), - ( 10, 0.5), - ) + _thresholds_default = { + "requests":{ + "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0], + "thresholds":{ + "default":[ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + } + } + } _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure - _fields = [ - ('request', -20, '%-20s'), + _fields_init = [ + ('request', -25, '%-25s'), ('count', 8, '%8s'), ('failed', 8, '%8s'), ] - for threshold, _ignore_fail_at in _thresholds: - _fields.append(('>%g sec' % (threshold,), 10, '%10s')) - - _fields.extend([ + _fields_extend = [ ('mean', 8, '%8.4f'), ('median', 8, '%8.4f'), ('stddev', 8, '%8.4f'), ('STATUS', 8, '%8s'), - ]) + ] - def __init__(self): + def __init__(self, **params): self._perMethodTimes = {} self._users = set() self._clients = set() self._failed_clients = [] self._startTime = datetime.now() + # Load parameters from config + if "thresholdsPath" in params: + jsondata = json.load(open(params["thresholds"])) + if "thresholds" in params: + jsondata = params["thresholds"] + else: + jsondata = self._thresholds_default + self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]] + for ctr, item in enumerate(self._thresholds): + for k, v in jsondata["requests"]["thresholds"].items(): + item[1][k] = v[ctr] + + self._fields = self._fields_init[:] + for threshold, _ignore_fail_at in self._thresholds: + self._fields.append(('>%g sec' % (threshold,), 10, '%10s')) + self._fields.extend(self._fields_extend) + if "failCutoff" in params: + self._fail_cut_off = params["failCutoff"] + + def countUsers(self): return len(self._users) @@ -458,6 +477,7 @@ for ctr, item in enumerate(self._thresholds): threshold, fail_at = item + fail_at = fail_at.get(method, fail_at["default"]) checks.append( (overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,)) ) Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -21,6 +21,7 @@ from __future__ import division +import json import sys, random from uuid import uuid4 @@ -702,41 +703,61 @@ lagFormat = u'{lag %5.2f ms}' # the response time thresholds to display together with failing % count threshold - _thresholds = ( - (0.5, 100.0), - ( 1, 100.0), - ( 3, 100.0), - ( 5, 100.0), - ( 10, 100.0), - ) + _thresholds_default = { + "requests":{ + "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0], + "thresholds":{ + "default":[ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0], + } + } + } _lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure - _fields = [ - ('operation', -20, '%-20s'), + _fields_init = [ + ('operation', -25, '%-25s'), ('count', 8, '%8s'), ('failed', 8, '%8s'), ] - for threshold, _ignore_fail_at in _thresholds: - _fields.append(('>%g sec' % (threshold,), 10, '%10s')) - - _fields.extend([ + _fields_extend = [ ('mean', 8, '%8.4f'), ('median', 8, '%8.4f'), ('stddev', 8, '%8.4f'), ('avglag (ms)', 12, '%12.4f'), ('STATUS', 8, '%8s'), - ]) + ] - def __init__(self, outfile=None): + def __init__(self, outfile=None, **params): self._perOperationTimes = {} self._perOperationLags = {} if outfile is None: outfile = sys.stdout self._outfile = outfile + + # Load parameters from config + if "thresholdsPath" in params: + jsondata = json.load(open(params["thresholds"])) + if "thresholds" in params: + jsondata = params["thresholds"] + else: + jsondata = self._thresholds_default + self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]] + for ctr, item in enumerate(self._thresholds): + for k, v in jsondata["operations"]["thresholds"].items(): + item[1][k] = v[ctr] + + self._fields = self._fields_init[:] + for threshold, _ignore_fail_at in self._thresholds: + self._fields.append(('>%g sec' % (threshold,), 10, '%10s')) + self._fields.extend(self._fields_extend) + if "lagCutoff" in params: + self._lag_cut_off = params["lagCutoff"] + if "failCutoff" in params: + self._fail_cut_off = params["failCutoff"] + def observe(self, event): if event.get("type") == "operation": event = event.copy() Modified: CalendarServer/trunk/contrib/performance/loadtest/sim.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -292,8 +292,10 @@ observers = [] if 'observers' in config: - for observerName in config['observers']: - observers.append(namedAny(observerName)()) + for observer in config['observers']: + observerName = observer["type"] + observerParams = observer["params"] + observers.append(namedAny(observerName)(**observerParams)) records = [] if 'accounts' in config: Modified: CalendarServer/trunk/contrib/performance/loadtest/test_population.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -159,3 +159,134 @@ duration=2.5, user='user01', client_type="test", client_id="1234")) self.assertEqual([], logger.failures()) + + + def test_bucketRequest(self): + """ + PUT(xxx-large/medium/small} have different thresholds. Test that requests straddling + each of those are correctly determined to be failures or not. + """ + + _thresholds = { + "requests":{ + "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0], + "thresholds":{ + "default": [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PUT{organizer-small}": [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + "PUT{organizer-medium}":[ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5], + "PUT{organizer-large}": [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0], + } + } + } + + # -small below threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + self.assertEqual([], logger.failures()) + + # -small above 0.5 threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-small}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + self.assertEqual( + ["Greater than 50% PUT{organizer-small} exceeded 0.5 second response time"], + logger.failures() + ) + + # -medium below 0.5 threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=0.6, user='user01', client_type="test", client_id="1234")) + self.assertEqual( + [], + logger.failures() + ) + + # -medium above 1.0 threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-medium}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + self.assertEqual( + ["Greater than 50% PUT{organizer-medium} exceeded 1 second response time"], + logger.failures() + ) + + # -large below 1.0 threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=1.6, user='user01', client_type="test", client_id="1234")) + self.assertEqual( + [], + logger.failures() + ) + + # -large above 3.0 threshold + logger = ReportStatistics(thresholds=_thresholds) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=0.2, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=3.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=3.6, user='user01', client_type="test", client_id="1234")) + logger.observe(dict( + type='response', method='PUT{organizer-large}', success=True, + duration=3.6, user='user01', client_type="test", client_id="1234")) + self.assertEqual( + ["Greater than 50% PUT{organizer-large} exceeded 3 second response time"], + logger.failures() + ) + Modified: CalendarServer/trunk/contrib/performance/loadtest/test_sim.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-27 14:37:49 UTC (rev 9304) +++ CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-29 15:23:08 UTC (rev 9305) @@ -501,7 +501,8 @@ """ config = FilePath(self.mktemp()) config.setContent(writePlistToString({ - "observers": ["contrib.performance.loadtest.population.SimpleStatistics"]})) + "observers": {"type":"contrib.performance.loadtest.population.SimpleStatistics"} + })) sim = LoadSimulator.fromCommandLine(['--config', config.path]) self.assertEquals(len(sim.observers), 1) self.assertIsInstance(sim.observers[0], SimpleStatistics) Added: CalendarServer/trunk/contrib/performance/loadtest/thresholds.json =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/thresholds.json (rev 0) +++ CalendarServer/trunk/contrib/performance/loadtest/thresholds.json 2012-05-29 15:23:08 UTC (rev 9305) @@ -0,0 +1,53 @@ +{ + "requests": { + "limits" : [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0], + + "thresholds": { + "default" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + + "GET{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + + "PUT{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PUT{attendee-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + "PUT{attendee-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.0], + "PUT{attendee-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0], + "PUT{organizer-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + "PUT{organizer-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5], + "PUT{organizer-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0], + + "DELETE{event}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + + "POST{fb-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + "POST{fb-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5], + "POST{fb-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0], + + "PROPFIND{well-known}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{find-principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{home}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{calendar}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{notification}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "PROPFIND{notification-items}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + + "PROPPATCH{calendar}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + + "REPORT{pset}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{expand}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{psearch}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{sync-init}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{sync}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{vevent}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{vtodo}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{multiget-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0], + "REPORT{multiget-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5], + "REPORT{multiget-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0] + } + }, + "operations": { + "limits" : [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0], + + "thresholds": { + "default" : [ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0] + } + } +}