[CalendarServer-changes] [9712] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Wed Aug 15 18:37:13 PDT 2012
Revision: 9712
http://trac.macosforge.org/projects/calendarserver/changeset/9712
Author: cdaboo at apple.com
Date: 2012-08-15 18:37:11 -0700 (Wed, 15 Aug 2012)
Log Message:
-----------
Allow option to turn off invitee "clumping" in the RealisticInviter. Log simulation errors.
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
CalendarServer/trunk/contrib/performance/loadtest/config.plist
CalendarServer/trunk/contrib/performance/loadtest/population.py
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
CalendarServer/trunk/contrib/performance/loadtest/test_population.py
CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-08-16 01:37:11 UTC (rev 9712)
@@ -339,11 +339,18 @@
</dict>
</dict>
- <!-- Define the distribution of who will be invited to an event. Each
- set of credentials loaded by the load tester has an index; samples from this
- distribution will be added to that index to arrive at the index of some other
- credentials, which will be the target of the invitation. -->
- <key>inviteeDistanceDistribution</key>
+ <!-- Define the distribution of who will be invited to an event.
+
+ When inviteeClumping is turned on each invitee is based on a sample of
+ users "close to" the organizer based on account index. If the clumping
+ is too "tight" for the requested number of attendees, then invites for
+ those larger numbers will simply fail (the sim will report that situation).
+
+ When inviteeClumping is off invitees will be sampled across an entire
+ range of account indexes. In this case the distribution ought to be a
+ UniformIntegerDistribution with min=0 and max set to the number of accounts.
+ -->
+ <key>inviteeDistribution</key>
<dict>
<key>type</key>
<string>contrib.performance.stats.UniformIntegerDistribution</string>
@@ -358,6 +365,9 @@
</dict>
</dict>
+ <key>inviteeClumping</key>
+ <true/>
+
<!-- Define the distribution of how many attendees will be invited to an event.
LogNormal is the best fit to observed data.
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-08-16 01:37:11 UTC (rev 9712)
@@ -333,11 +333,18 @@
</dict>
</dict>
- <!-- Define the distribution of who will be invited to an event. Each
- set of credentials loaded by the load tester has an index; samples from this
- distribution will be added to that index to arrive at the index of some other
- credentials, which will be the target of the invitation. -->
- <key>inviteeDistanceDistribution</key>
+ <!-- Define the distribution of who will be invited to an event.
+
+ When inviteeClumping is turned on each invitee is based on a sample of
+ users "close to" the organizer based on account index. If the clumping
+ is too "tight" for the requested number of attendees, then invites for
+ those larger numbers will simply fail (the sim will report that situation).
+
+ When inviteeClumping is off invitees will be sampled across an entire
+ range of account indexes. In this case the distribution ought to be a
+ UniformIntegerDistribution with min=0 and max set to the number of accounts.
+ -->
+ <key>inviteeDistribution</key>
<dict>
<key>type</key>
<string>contrib.performance.stats.UniformIntegerDistribution</string>
@@ -352,6 +359,9 @@
</dict>
</dict>
+ <key>inviteeClumping</key>
+ <true/>
+
<!-- Define the distribution of how many attendees will be invited to an event.
LogNormal is the best fit to observed data.
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-08-16 01:37:11 UTC (rev 9712)
@@ -28,6 +28,7 @@
from datetime import datetime
from urllib2 import HTTPBasicAuthHandler
from urllib2 import HTTPDigestAuthHandler
+import collections
import json
import os
@@ -285,7 +286,12 @@
where.path,))
+ def _simFailure(self, reason, reactor):
+ if not self._stopped:
+ msg(type="sim-failure", reason=reason)
+
+
class SmoothRampUp(object):
def __init__(self, reactor, groups, groupSize, interval, clientsPerUser):
self.reactor = reactor
@@ -308,6 +314,8 @@
self.eventReceived(event)
elif event.get('type') == 'client-failure':
self.clientFailure(event)
+ elif event.get('type') == 'sim-failure':
+ self.simFailure(event)
def report(self, output):
@@ -322,8 +330,9 @@
class SimpleStatistics(StatisticsBase):
def __init__(self):
self._times = []
+ self._failures = collections.defaultdict(int)
+ self._simFailures = collections.defaultdict(int)
-
def eventReceived(self, event):
self._times.append(event['duration'])
if len(self._times) == 200:
@@ -335,9 +344,13 @@
def clientFailure(self, event):
- pass
+ self._failures[event] += 1
+ def simFailure(self, event):
+ self._simFailures[event] += 1
+
+
class ReportStatistics(StatisticsBase, SummarizingMixin):
"""
@@ -376,6 +389,7 @@
self._users = set()
self._clients = set()
self._failed_clients = []
+ self._failed_sim = collections.defaultdict(int)
self._startTime = datetime.now()
# Load parameters from config
@@ -411,6 +425,10 @@
return len(self._failed_clients)
+ def countSimFailures(self):
+ return len(self._failed_sim)
+
+
def eventReceived(self, event):
dataset = self._perMethodTimes.setdefault(event['method'], [])
dataset.append((event['success'], event['duration']))
@@ -422,6 +440,10 @@
self._failed_clients.append(event['reason'])
+ def simFailure(self, event):
+ self._failed_sim[event['reason']] += 1
+
+
def printMiscellaneous(self, output, items):
maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
fmt = "%"+maxColumnWidth+"s : %-s\n"
@@ -453,6 +475,9 @@
items['Failed clients'] = self.countClientFailures()
for ctr, reason in enumerate(self._failed_clients, 1):
items['Failure #%d' % (ctr,)] = reason
+ if self.countSimFailures() > 0:
+ for reason, count in self._failed_sim.items():
+ items['Failed operation'] = "%s : %d times" % (reason, count,)
self.printMiscellaneous(output, items)
output.write("\n")
self.printHeader(output, [
Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2012-08-16 01:37:11 UTC (rev 9712)
@@ -120,10 +120,25 @@
return passthrough
deferred.addBoth(finished)
return deferred
-
+ def _failedOperation(self, label, reason):
+ """
+ Helper to emit a log event when an operation fails.
+ """
+ msg(
+ type="operation",
+ phase="failed",
+ user=self._client.record.uid,
+ client_type=self._client.title,
+ client_id=self._client._client_id,
+ label=label,
+ reason=reason,
+ )
+ self._sim._simFailure("%s: %s" % (label, reason,), self._reactor)
+
+
class CannotAddAttendee(Exception):
"""
Indicates no new attendees can be invited to a particular event.
@@ -156,11 +171,11 @@
self,
enabled=True,
sendInvitationDistribution=NormalDistribution(600, 60),
- inviteeDistanceDistribution=UniformDiscreteDistribution(range(-10, 11))
+ inviteeDistribution=UniformDiscreteDistribution(range(-10, 11))
):
self.enabled = enabled
self._sendInvitationDistribution = sendInvitationDistribution
- self._inviteeDistanceDistribution = inviteeDistanceDistribution
+ self._inviteeDistribution = inviteeDistribution
def run(self):
@@ -180,7 +195,7 @@
for _ignore_i in range(10):
invitee = max(
- 0, self._number + self._inviteeDistanceDistribution.sample())
+ 0, self._number + self._inviteeDistribution.sample())
try:
record = self._sim.getUserRecord(invitee)
except IndexError:
@@ -291,7 +306,8 @@
self,
enabled=True,
sendInvitationDistribution=NormalDistribution(600, 60),
- inviteeDistanceDistribution=UniformDiscreteDistribution(range(-10, 11)),
+ inviteeDistribution=UniformDiscreteDistribution(range(-10, 11)),
+ inviteeClumping=True,
inviteeCountDistribution=LogNormalDistribution(1.2, 1.2),
eventStartDistribution=NearFutureDistribution(),
eventDurationDistribution=UniformDiscreteDistribution([
@@ -303,7 +319,8 @@
):
self.enabled = enabled
self._sendInvitationDistribution = sendInvitationDistribution
- self._inviteeDistanceDistribution = inviteeDistanceDistribution
+ self._inviteeDistribution = inviteeDistribution
+ self._inviteeClumping = inviteeClumping
self._inviteeCountDistribution = inviteeCountDistribution
self._eventStartDistribution = eventStartDistribution
self._eventDurationDistribution = eventDurationDistribution
@@ -326,8 +343,12 @@
invitees.add(att.value())
for _ignore_i in range(10):
- invitee = max(
- 0, self._number + self._inviteeDistanceDistribution.sample())
+
+ sample = self._inviteeDistribution.sample()
+ if self._inviteeClumping:
+ sample = self._number + sample
+ invitee = max(0, sample)
+
try:
record = self._sim.getUserRecord(invitee)
except IndexError:
@@ -401,6 +422,7 @@
try:
self._addAttendee(vevent, attendees)
except CannotAddAttendee:
+ self._failedOperation("invite", "Cannot add attendee")
return succeed(None)
href = '%s%s.ics' % (calendar.url, uid)
@@ -705,8 +727,9 @@
logger.
"""
formats = {
- u"start": u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
- u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
+ u"start" : u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
+ u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
+ u"failed": u"%(user)s x x x x x x x x x x x %(label)8s FAILED %(reason)s",
}
lagFormat = u'{lag %5.2f ms}'
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_population.py 2012-08-16 01:37:11 UTC (rev 9712)
@@ -57,8 +57,8 @@
def test_clientFailures(self):
"""
- L{ReportStatistics.countClients} returns the number of clients observed to
- have acted in the simulation.
+ L{ReportStatistics.countClientFailures} returns the number of clients observed to
+ have failed in the simulation.
"""
logger = ReportStatistics()
clients = ['c01', 'c02', 'c03']
@@ -68,6 +68,19 @@
self.assertEqual(len(clients), logger.countClientFailures())
+ def test_simFailures(self):
+ """
+ L{ReportStatistics.countSimFailures} returns the number of clients observed to
+ have caused an error in the simulation.
+ """
+ logger = ReportStatistics()
+ clients = ['c01', 'c02', 'c03']
+ for client in clients:
+ logger.observe(dict(
+ type='sim-failure', reason="testing %s" % (client,)))
+ self.assertEqual(len(clients), logger.countSimFailures())
+
+
def test_noFailures(self):
"""
If fewer than 1% of requests fail, fewer than 1% of requests take 5
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2012-08-15 21:42:51 UTC (rev 9711)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2012-08-16 01:37:11 UTC (rev 9712)
@@ -217,6 +217,7 @@
def __init__(self, number, serializePath):
self.serializePath = serializePath
os.mkdir(self.serializePath)
+ self.title = "StubClient"
self._events = {}
self._calendars = {}
self._pendingFailures = {}
@@ -421,7 +422,7 @@
_ignore_vevent, event, _ignore_calendar, client = self._simpleAccount(
userNumber, SIMPLE_EVENT)
inviter = Inviter(Clock(), self.sim, client, userNumber)
- inviter.setParameters(inviteeDistanceDistribution=Deterministic(1))
+ inviter.setParameters(inviteeDistribution=Deterministic(1))
inviter._invite()
attendees = tuple(event.component.mainComponent().properties('ATTENDEE'))
self.assertEquals(len(attendees), 1)
@@ -450,7 +451,7 @@
values = [selfNumber - selfNumber, otherNumber - selfNumber]
inviter = Inviter(Clock(), self.sim, client, selfNumber)
- inviter.setParameters(inviteeDistanceDistribution=SequentialDistribution(values))
+ inviter.setParameters(inviteeDistribution=SequentialDistribution(values))
inviter._invite()
attendees = tuple(event.component.mainComponent().properties('ATTENDEE'))
self.assertEquals(len(attendees), 1)
@@ -481,7 +482,7 @@
values = [inviteeNumber - selfNumber, anotherNumber - selfNumber]
inviter = Inviter(Clock(), self.sim, client, selfNumber)
- inviter.setParameters(inviteeDistanceDistribution=SequentialDistribution(values))
+ inviter.setParameters(inviteeDistribution=SequentialDistribution(values))
inviter._invite()
attendees = tuple(event.component.mainComponent().properties('ATTENDEE'))
self.assertEquals(len(attendees), 3)
@@ -507,7 +508,7 @@
selfNumber, INVITED_EVENT)
inviter = Inviter(Clock(), self.sim, client, selfNumber)
# Always return a user number which has already been invited.
- inviter.setParameters(inviteeDistanceDistribution=Deterministic(2 - selfNumber))
+ inviter.setParameters(inviteeDistribution=Deterministic(2 - selfNumber))
inviter._invite()
attendees = tuple(vevent.mainComponent().properties('ATTENDEE'))
self.assertEquals(len(attendees), 2)
@@ -609,7 +610,7 @@
client._calendars.update({calendar.url: calendar})
inviter = RealisticInviter(Clock(), self.sim, client, userNumber)
inviter.setParameters(
- inviteeDistanceDistribution=Deterministic(1),
+ inviteeDistribution=Deterministic(1),
inviteeCountDistribution=Deterministic(1)
)
inviter._invite()
@@ -638,7 +639,7 @@
inviter = RealisticInviter(Clock(), self.sim, client, selfNumber)
inviter.setParameters(
- inviteeDistanceDistribution=SequentialDistribution(values),
+ inviteeDistribution=SequentialDistribution(values),
inviteeCountDistribution=Deterministic(1)
)
inviter._invite()
@@ -668,7 +669,7 @@
inviter = RealisticInviter(Clock(), self.sim, client, selfNumber)
inviter.setParameters(
- inviteeDistanceDistribution=SequentialDistribution(values),
+ inviteeDistribution=SequentialDistribution(values),
inviteeCountDistribution=Deterministic(2)
)
inviter._invite()
@@ -697,7 +698,7 @@
client._calendars.update({calendar.url: calendar})
inviter = RealisticInviter(Clock(), self.sim, client, userNumber)
inviter.setParameters(
- inviteeDistanceDistribution=Deterministic(1),
+ inviteeDistribution=Deterministic(1),
inviteeCountDistribution=Deterministic(2)
)
inviter._invite()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120815/ee948205/attachment-0001.html>
More information about the calendarserver-changes
mailing list