[CalendarServer-changes] [9305] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Tue May 29 08:23:08 PDT 2012
Revision: 9305
http://trac.macosforge.org/projects/calendarserver/changeset/9305
Author: cdaboo at apple.com
Date: 2012-05-29 08:23:08 -0700 (Tue, 29 May 2012)
Log Message:
-----------
More fine-grained request type logging including -small, -medium and -large tags for requests based on input size (number of
attendees, resources etc). Added the ability to specify failure thresholds via the config plist and a new json data file.
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
CalendarServer/trunk/contrib/performance/loadtest/config.plist
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/logger.py
CalendarServer/trunk/contrib/performance/loadtest/population.py
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
CalendarServer/trunk/contrib/performance/loadtest/sim.py
CalendarServer/trunk/contrib/performance/loadtest/test_population.py
CalendarServer/trunk/contrib/performance/loadtest/test_sim.py
Added Paths:
-----------
CalendarServer/trunk/contrib/performance/loadtest/thresholds.json
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-29 15:23:08 UTC (rev 9305)
@@ -474,16 +474,52 @@
<array>
<!-- ReportStatistics generates an end-of-run summary of the HTTP requests
made, their timings, and their results. -->
- <string>contrib.performance.loadtest.population.ReportStatistics</string>
-
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.population.ReportStatistics</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each request type -->
+ <key>thresholds</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+ </dict>
+ </dict>
+
<!-- RequestLogger generates a realtime log of all HTTP requests made
during the load test. -->
- <string>contrib.performance.loadtest.ical.RequestLogger</string>
-
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.ical.RequestLogger</string>
+ <key>params</key>
+ <dict>
+ </dict>
+ </dict>
+
<!-- OperationLogger generates an end-of-run summary of the gross operations
performed (logical operations which may span more than one HTTP request,
such as inviting an attendee to an event). -->
- <string>contrib.performance.loadtest.profiles.OperationLogger</string>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.profiles.OperationLogger</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each operation type -->
+ <key>thresholds</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The % of operations beyond the lag cut-off that constitute a failed test -->
+ <key>lagCutoff</key>
+ <real>1.0</real>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+ </dict>
+ </dict>
</array>
</dict>
</plist>
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-29 15:23:08 UTC (rev 9305)
@@ -466,18 +466,54 @@
<!-- Define some log observers to report on the load test. -->
<key>observers</key>
<array>
- <!-- ReportStatistics generates an end-of-run summary of the HTTP requests
+ <!-- ReportStatistics generates an end-of-run summary of the HTTP requests
made, their timings, and their results. -->
- <string>contrib.performance.loadtest.population.ReportStatistics</string>
-
- <!-- RequestLogger generates a realtime log of all HTTP requests made
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.population.ReportStatistics</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each request type -->
+ <key>thresholdsPath</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+ </dict>
+ </dict>
+
+ <!-- RequestLogger generates a realtime log of all HTTP requests made
during the load test. -->
- <string>contrib.performance.loadtest.ical.RequestLogger</string>
-
- <!-- OperationLogger generates an end-of-run summary of the gross operations
- performed (logical operations which may span more than one HTTP request,
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.ical.RequestLogger</string>
+ <key>params</key>
+ <dict>
+ </dict>
+ </dict>
+
+ <!-- OperationLogger generates an end-of-run summary of the gross operations
+ performed (logical operations which may span more than one HTTP request,
such as inviting an attendee to an event). -->
- <string>contrib.performance.loadtest.profiles.OperationLogger</string>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.profiles.OperationLogger</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each operation type -->
+ <key>thresholdsPath</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The % of operations beyond the lag cut-off that constitute a failed test -->
+ <key>lagCutoff</key>
+ <real>1.0</real>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+ </dict>
+ </dict>
</array>
</dict>
</plist>
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -421,7 +421,7 @@
@inlineCallbacks
- def _proppatch(self, url, body):
+ def _proppatch(self, url, body, method_label=None):
"""
Issue a PROPPATCH on the chosen URL
"""
@@ -431,7 +431,8 @@
'PROPPATCH',
self.root + url[1:].encode('utf-8'),
hdrs,
- StringProducer(body)
+ StringProducer(body),
+ method_label=method_label,
)
if response.code == MULTI_STATUS:
body = yield readBody(response)
@@ -473,6 +474,7 @@
location,
self._STARTUP_WELL_KNOWN,
allowedStatus=(MULTI_STATUS, MOVED_PERMANENTLY),
+ method_label="PROPFIND{well-known}",
)
# Follow any redirect
@@ -483,6 +485,7 @@
location,
self._STARTUP_WELL_KNOWN,
allowedStatus=(MULTI_STATUS),
+ method_label="PROPFIND{well-known}",
)
returnValue(result[location])
@@ -498,6 +501,7 @@
_ignore_response, result = yield self._propfind(
principalPath,
self._STARTUP_PRINCIPAL_PROPFIND_INITIAL,
+ method_label="PROPFIND{find-principal}",
)
returnValue(result[principalPath])
@@ -512,6 +516,7 @@
_ignore_response, result = yield self._propfind(
self.principalURL,
self._STARTUP_PRINCIPAL_PROPFIND,
+ method_label="PROPFIND{principal}",
)
returnValue(result[self.principalURL])
@@ -647,7 +652,7 @@
self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': calendar.changeToken},
depth='1',
otherTokens = True,
- method_label="REPORT{sync}",
+ method_label="REPORT{sync}" if calendar.changeToken else "REPORT{sync-init}",
)
changed = []
@@ -756,11 +761,18 @@
# Next do a REPORT on events that might have information
# we don't know about.
hrefs = "".join([self._POLL_CALENDAR_MULTIGET_REPORT_HREF % {'href': event} for event in events])
+
+ label_suffix = "small"
+ if len(hrefs) > 5:
+ label_suffix = "medium"
+ if len(hrefs) > 15:
+ label_suffix = "large"
+
return self._report(
calendar,
self._POLL_CALENDAR_MULTIGET_REPORT % {'hrefs': hrefs},
depth=None,
- method_label="REPORT{multiget}",
+ method_label="REPORT{multiget-%s}" % (label_suffix,),
)
@@ -834,9 +846,21 @@
# Patch calendar properties
for cal in calendars:
if cal.name != "inbox":
- yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_COLOR)
- yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_ORDER)
- yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE)
+ yield self._proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_COLOR,
+ method_label="PROPPATCH{calendar}",
+ )
+ yield self._proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_ORDER,
+ method_label="PROPPATCH{calendar}",
+ )
+ yield self._proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE,
+ method_label="PROPPATCH{calendar}",
+ )
def _pollFirstTime2(self):
@@ -848,6 +872,7 @@
_ignore_response, result = yield self._propfind(
notificationURL,
self._POLL_NOTIFICATION_PROPFIND,
+ method_label="PROPFIND{notification}",
)
returnValue(result)
@@ -858,6 +883,7 @@
notificationURL,
self._POLL_NOTIFICATION_PROPFIND_D1,
depth='1',
+ method_label="PROPFIND{notification-items}",
)
returnValue(result)
@@ -1055,6 +1081,12 @@
attendees.append(attendee)
vevent.mainComponent().addProperty(attendee)
+ label_suffix = "small"
+ if len(attendees) > 5:
+ label_suffix = "medium"
+ if len(attendees) > 15:
+ label_suffix = "large"
+
# At last, upload the new event definition
response = yield self._request(
(NO_CONTENT, PRECONDITION_FAILED,),
@@ -1064,7 +1096,7 @@
'content-type': ['text/calendar'],
'if-match': [event.etag]}),
StringProducer(vevent.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer}"
+ method_label="PUT{organizer-%s}" % (label_suffix,)
)
# Finally, re-retrieve the event to update the etag
@@ -1124,12 +1156,19 @@
headers.addRawHeader('if-schedule-tag-match', event.scheduleTag)
okCodes = (NO_CONTENT, PRECONDITION_FAILED,)
+ attendees = list(vevent.mainComponent().properties('ATTENDEE'))
+ label_suffix = "small"
+ if len(attendees) > 5:
+ label_suffix = "medium"
+ if len(attendees) > 15:
+ label_suffix = "large"
+
response = yield self._request(
okCodes,
'PUT',
self.root + href[1:].encode('utf-8'),
headers, StringProducer(vevent.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{attendee}",
+ method_label="PUT{attendee-%s}" % (label_suffix,),
)
self._updateEvent(response, href)
@@ -1144,7 +1183,11 @@
self._removeEvent(href)
response = yield self._request(
- NO_CONTENT, 'DELETE', self.root + href[1:].encode('utf-8'))
+ NO_CONTENT,
+ 'DELETE',
+ self.root + href[1:].encode('utf-8'),
+ method_label="DELETE{event}",
+ )
returnValue(response)
@@ -1153,13 +1196,21 @@
headers = Headers({
'content-type': ['text/calendar'],
})
+
+ attendees = list(vcalendar.mainComponent().properties('ATTENDEE'))
+ label_suffix = "small"
+ if len(attendees) > 5:
+ label_suffix = "medium"
+ if len(attendees) > 15:
+ label_suffix = "large"
+
response = yield self._request(
CREATED,
'PUT',
self.root + href[1:].encode('utf-8'),
headers,
StringProducer(vcalendar.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer}" if invite else None,
+ method_label="PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}",
)
self._localUpdateEvent(response, href, vcalendar)
@@ -1198,7 +1249,12 @@
@inlineCallbacks
def _updateEvent(self, ignored, href):
- response = yield self._request(OK, 'GET', self.root + href[1:].encode('utf-8'))
+ response = yield self._request(
+ OK,
+ 'GET',
+ self.root + href[1:].encode('utf-8'),
+ method_label="GET{event}",
+ )
headers = response.headers
etag = headers.getRawHeaders('etag')[0]
scheduleTag = headers.getRawHeaders('schedule-tag', [None])[0]
@@ -1252,6 +1308,12 @@
end = end.getText()
now = PyCalendarDateTime.getNowUTC().getText()
+ label_suffix = "small"
+ if len(users) > 5:
+ label_suffix = "medium"
+ if len(users) > 15:
+ label_suffix = "large"
+
response = yield self._request(
OK, 'POST', outbox,
Headers({
@@ -1267,7 +1329,7 @@
'start': start,
'end': end,
'now': now}),
- method_label="POST{fb}",
+ method_label="POST{fb}-%s" % (label_suffix,),
)
body = yield readBody(response)
returnValue(body)
@@ -1545,6 +1607,7 @@
_ignore_response, result = yield self._propfind(
'/principals/',
self._STARTUP_PRINCIPAL_PROPFIND_INITIAL,
+ method_label="PROPFIND{find-principal}",
)
returnValue(result['/principals/'])
@@ -1554,8 +1617,16 @@
# Patch calendar properties
for cal in calendars:
if cal.name != "inbox":
- yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_COLOR)
- yield self._proppatch(cal.url, self._STARTUP_PROPPATCH_CALENDAR_ORDER)
+ yield self._proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_COLOR,
+ method_label="PROPPATCH{calendar}",
+ )
+ yield self._proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_ORDER,
+ method_label="PROPPATCH{calendar}",
+ )
def _pollFirstTime2(self):
Modified: CalendarServer/trunk/contrib/performance/loadtest/logger.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/logger.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/logger.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -60,6 +60,7 @@
for ctr, item in enumerate(self._thresholds):
_ignore_threshold, fail_at = item
+ fail_at = fail_at.get(operation, fail_at["default"])
if thresholds[ctr] * 100.0 / count > fail_at:
failure = True
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -26,6 +26,7 @@
from tempfile import mkdtemp
from itertools import izip
from datetime import datetime
+import json
import os
from twisted.internet.defer import DeferredList
@@ -334,39 +335,57 @@
"""
# the response time thresholds to display together with failing % count threshold
- _thresholds = (
- (0.5, 100.0),
- ( 1, 100.0),
- ( 3, 5.0),
- ( 5, 1.0),
- ( 10, 0.5),
- )
+ _thresholds_default = {
+ "requests":{
+ "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds":{
+ "default":[ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ }
+ }
+ }
_fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
- _fields = [
- ('request', -20, '%-20s'),
+ _fields_init = [
+ ('request', -25, '%-25s'),
('count', 8, '%8s'),
('failed', 8, '%8s'),
]
- for threshold, _ignore_fail_at in _thresholds:
- _fields.append(('>%g sec' % (threshold,), 10, '%10s'))
-
- _fields.extend([
+ _fields_extend = [
('mean', 8, '%8.4f'),
('median', 8, '%8.4f'),
('stddev', 8, '%8.4f'),
('STATUS', 8, '%8s'),
- ])
+ ]
- def __init__(self):
+ def __init__(self, **params):
self._perMethodTimes = {}
self._users = set()
self._clients = set()
self._failed_clients = []
self._startTime = datetime.now()
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholds"]))
+ if "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["requests"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
+
def countUsers(self):
return len(self._users)
@@ -458,6 +477,7 @@
for ctr, item in enumerate(self._thresholds):
threshold, fail_at = item
+ fail_at = fail_at.get(method, fail_at["default"])
checks.append(
(overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,))
)
Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -21,6 +21,7 @@
from __future__ import division
+import json
import sys, random
from uuid import uuid4
@@ -702,41 +703,61 @@
lagFormat = u'{lag %5.2f ms}'
# the response time thresholds to display together with failing % count threshold
- _thresholds = (
- (0.5, 100.0),
- ( 1, 100.0),
- ( 3, 100.0),
- ( 5, 100.0),
- ( 10, 100.0),
- )
+ _thresholds_default = {
+ "requests":{
+ "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds":{
+ "default":[ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
+ }
+ }
+ }
_lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds
_fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
- _fields = [
- ('operation', -20, '%-20s'),
+ _fields_init = [
+ ('operation', -25, '%-25s'),
('count', 8, '%8s'),
('failed', 8, '%8s'),
]
- for threshold, _ignore_fail_at in _thresholds:
- _fields.append(('>%g sec' % (threshold,), 10, '%10s'))
-
- _fields.extend([
+ _fields_extend = [
('mean', 8, '%8.4f'),
('median', 8, '%8.4f'),
('stddev', 8, '%8.4f'),
('avglag (ms)', 12, '%12.4f'),
('STATUS', 8, '%8s'),
- ])
+ ]
- def __init__(self, outfile=None):
+ def __init__(self, outfile=None, **params):
self._perOperationTimes = {}
self._perOperationLags = {}
if outfile is None:
outfile = sys.stdout
self._outfile = outfile
+
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholds"]))
+ if "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["operations"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+ if "lagCutoff" in params:
+ self._lag_cut_off = params["lagCutoff"]
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
def observe(self, event):
if event.get("type") == "operation":
event = event.copy()
Modified: CalendarServer/trunk/contrib/performance/loadtest/sim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -292,8 +292,10 @@
observers = []
if 'observers' in config:
- for observerName in config['observers']:
- observers.append(namedAny(observerName)())
+ for observer in config['observers']:
+ observerName = observer["type"]
+ observerParams = observer["params"]
+ observers.append(namedAny(observerName)(**observerParams))
records = []
if 'accounts' in config:
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -159,3 +159,134 @@
duration=2.5, user='user01', client_type="test", client_id="1234"))
self.assertEqual([], logger.failures())
+
+
+ def test_bucketRequest(self):
+ """
+ PUT(xxx-large/medium/small} have different thresholds. Test that requests straddling
+ each of those are correctly determined to be failures or not.
+ """
+
+ _thresholds = {
+ "requests":{
+ "limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds":{
+ "default": [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PUT{organizer-small}": [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+ "PUT{organizer-medium}":[ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
+ "PUT{organizer-large}": [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+ }
+ }
+ }
+
+ # -small below threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual([], logger.failures())
+
+ # -small above 0.5 threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-small}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual(
+ ["Greater than 50% PUT{organizer-small} exceeded 0.5 second response time"],
+ logger.failures()
+ )
+
+ # -medium below 0.5 threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=0.6, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual(
+ [],
+ logger.failures()
+ )
+
+ # -medium above 1.0 threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-medium}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual(
+ ["Greater than 50% PUT{organizer-medium} exceeded 1 second response time"],
+ logger.failures()
+ )
+
+ # -large below 1.0 threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=1.6, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual(
+ [],
+ logger.failures()
+ )
+
+ # -large above 3.0 threshold
+ logger = ReportStatistics(thresholds=_thresholds)
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=0.2, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=3.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=3.6, user='user01', client_type="test", client_id="1234"))
+ logger.observe(dict(
+ type='response', method='PUT{organizer-large}', success=True,
+ duration=3.6, user='user01', client_type="test", client_id="1234"))
+ self.assertEqual(
+ ["Greater than 50% PUT{organizer-large} exceeded 3 second response time"],
+ logger.failures()
+ )
+
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_sim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-27 14:37:49 UTC (rev 9304)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-29 15:23:08 UTC (rev 9305)
@@ -501,7 +501,8 @@
"""
config = FilePath(self.mktemp())
config.setContent(writePlistToString({
- "observers": ["contrib.performance.loadtest.population.SimpleStatistics"]}))
+ "observers": {"type":"contrib.performance.loadtest.population.SimpleStatistics"}
+ }))
sim = LoadSimulator.fromCommandLine(['--config', config.path])
self.assertEquals(len(sim.observers), 1)
self.assertIsInstance(sim.observers[0], SimpleStatistics)
Added: CalendarServer/trunk/contrib/performance/loadtest/thresholds.json
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/thresholds.json (rev 0)
+++ CalendarServer/trunk/contrib/performance/loadtest/thresholds.json 2012-05-29 15:23:08 UTC (rev 9305)
@@ -0,0 +1,53 @@
+{
+ "requests": {
+ "limits" : [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+
+ "thresholds": {
+ "default" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+
+ "GET{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+
+ "PUT{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PUT{attendee-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+ "PUT{attendee-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.0],
+ "PUT{attendee-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+ "PUT{organizer-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+ "PUT{organizer-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
+ "PUT{organizer-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+
+ "DELETE{event}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+
+ "POST{fb-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+ "POST{fb-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
+ "POST{fb-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+
+ "PROPFIND{well-known}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{find-principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{home}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{calendar}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{notification}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{notification-items}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+
+ "PROPPATCH{calendar}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+
+ "REPORT{pset}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{expand}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{psearch}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{sync-init}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{sync}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{vevent}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{vtodo}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{multiget-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
+ "REPORT{multiget-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
+ "REPORT{multiget-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0]
+ }
+ },
+ "operations": {
+ "limits" : [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+
+ "thresholds": {
+ "default" : [ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]
+ }
+ }
+}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120529/838cae37/attachment-0001.html>
More information about the calendarserver-changes
mailing list