[CalendarServer-changes] [15647] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jun 2 09:41:06 PDT 2016
Revision: 15647
http://trac.calendarserver.org//changeset/15647
Author: sagen at apple.com
Date: 2016-06-02 09:41:06 -0700 (Thu, 02 Jun 2016)
Log Message:
-----------
Sim fixes:
- subscribes to push notifications for shared-to-me calendars
- guards against re-entry due to multiple push notifications
- adds a random start time to the looping calls so they're not all in lock step
- adds client/instance information to user-agent
- always syncs the calendar home when a push arrives
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/clients.plist
CalendarServer/trunk/contrib/performance/loadtest/config.plist
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/population.py
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
Modified: CalendarServer/trunk/contrib/performance/loadtest/clients.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/clients.plist 2016-06-01 15:01:12 UTC (rev 15646)
+++ CalendarServer/trunk/contrib/performance/loadtest/clients.plist 2016-06-02 16:41:06 UTC (rev 15647)
@@ -38,7 +38,7 @@
<dict>
<!-- Name that appears in logs. -->
<key>title</key>
- <string>10.11a</string>
+ <string>main</string>
<!-- OS_X_10_11 can poll the calendar home at some interval. This is
in seconds. -->
@@ -76,7 +76,7 @@
<!-- Define the interval (in seconds) at which this profile will use
its client to create a new event. -->
<key>interval</key>
- <integer>120</integer>
+ <integer>180</integer>
<!-- Define how start times (DTSTART) for the randomly generated events
will be selected. This is an example of a "Distribution" parameter. The value
@@ -315,7 +315,7 @@
<!-- Define the interval (in seconds) at which this profile will use
its client to create a new event. -->
<key>interval</key>
- <integer>120</integer>
+ <integer>60</integer>
</dict>
</dict>
@@ -333,7 +333,7 @@
<!-- Define the interval (in seconds) at which this profile will use
its client to create a new event. -->
<key>interval</key>
- <integer>120</integer>
+ <integer>60</integer>
<!-- Define the description length distribution. -->
<key>descriptionLengthDistribution</key>
@@ -427,7 +427,7 @@
<!-- Define the interval (in seconds) at which this profile will share calendars. -->
<key>interval</key>
- <integer>300</integer>
+ <integer>1800</integer>
</dict>
</dict>
@@ -451,7 +451,7 @@
<dict>
<!-- mu gives the mean of the normal distribution (in seconds). -->
<key>mu</key>
- <integer>120</integer>
+ <integer>720</integer>
<!-- and sigma gives its standard deviation. -->
<key>sigma</key>
@@ -515,6 +515,10 @@
</dict>
</dict>
+ <!-- Percentage of time to do a lookup for invitee -->
+ <key>inviteeLookupPercentage</key>
+ <integer>50</integer>
+
<!-- Define how start times (DTSTART) for the randomly generated events
will be selected. This is an example of a "Distribution" parameter. The value
for most "Distribution" parameters are interchangeable and extensible. -->
@@ -765,7 +769,7 @@
<!-- Define the interval (in seconds) at which this profile will use
its client to refresh. -->
<key>interval</key>
- <integer>600</integer>
+ <integer>1800</integer>
</dict>
</dict>
@@ -807,7 +811,7 @@
<dict>
<!-- Name that appears in logs. -->
<key>title</key>
- <string>10.11b</string>
+ <string>idle</string>
<!-- OS_X_10_11 can poll the calendar home at some interval. This is
in seconds. -->
@@ -845,7 +849,7 @@
<!-- Determine the frequency at which this client configuration will
appear in the clients which are created by the load tester. -->
<key>weight</key>
- <integer>2</integer>
+ <integer>6</integer>
</dict>
</array>
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2016-06-01 15:01:12 UTC (rev 15646)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2016-06-02 16:41:06 UTC (rev 15647)
@@ -157,7 +157,7 @@
<!-- Number of clients each user is assigned to. -->
<!-- Set weight of clients to 1 if this is > 1. Number of clients must match this value if > 1. -->
<key>clientsPerUser</key>
- <integer>3</integer>
+ <integer>7</integer>
</dict>
</dict>
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2016-06-01 15:01:12 UTC (rev 15646)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2016-06-02 16:41:06 UTC (rev 15647)
@@ -509,6 +509,7 @@
serializePath,
record,
auth,
+ instanceNumber,
title=None,
calendarHomePollInterval=None,
supportPush=True,
@@ -516,6 +517,7 @@
):
self._client_id = str(uuid4())
+ self._instanceNumber = instanceNumber
self.reactor = reactor
@@ -580,7 +582,10 @@
self.xmpp = {}
self.ampPushKeys = {}
+ self.subscribedAmpPushKeys = set()
+ self._busyWithPush = False
+
# Keep track of push factories so we can unsubscribe at shutdown
self._pushFactories = []
@@ -599,7 +604,11 @@
Default is to add User-Agent, sub-classes should override to add other
client specific things, Accept etc.
"""
- headers.setRawHeaders('User-Agent', [self.USER_AGENT])
+ headers.setRawHeaders(
+ 'User-Agent', ["{} {} {}".format(
+ self.USER_AGENT, self.title, self._instanceNumber
+ )]
+ )
@inlineCallbacks
@@ -1017,6 +1026,17 @@
sharedByMe=isSharedByMe
))
+ # Also monitor shared-to-me calendars
+ if isShared and not isSharedByMe:
+ try:
+ pushkey = textProps[csxml.pushkey]
+ except KeyError:
+ pass
+ else:
+ if pushkey:
+ self.ampPushKeys[href] = pushkey
+
+
elif isNotifications:
textProps = results[href].getTextProperties()
notificationCollection = NotificationCollection(
@@ -1436,6 +1456,9 @@
if firstTime:
yield self._pollFirstTime2()
+ pushKeys = self.ampPushKeys.values()
+ self._monitorAmpPush(calendarHomeSet, pushKeys)
+
returnValue(True)
@@ -1580,23 +1603,39 @@
connect(GAIEndpoint(self.reactor, host, port), factory)
+
+ @inlineCallbacks
def _receivedPush(self, inboundID, dataChangedTimestamp, priority=5):
- for href, myId in self.ampPushKeys.iteritems():
- if inboundID == myId:
- self._checkCalendarsForEvents(href, push=True)
- break
- else:
- # somehow we are not subscribed to this id
- pass
+ if not self._busyWithPush:
+ self._busyWithPush = True
+ try:
+ for href, myId in self.ampPushKeys.iteritems():
+ if inboundID == myId:
+ yield self._checkCalendarsForEvents(self.calendarHomeHref, push=True)
+ break
+ else:
+ # somehow we are not subscribed to this id
+ pass
+ finally:
+ self._busyWithPush = False
def _monitorAmpPush(self, home, pushKeys):
"""
Start monitoring for AMP-based push notifications
"""
- AMPHub.subscribeToIDs(pushKeys, self._receivedPush)
+ # Only subscribe to keys we haven't previously subscribed to
+ subscribeTo = []
+ for pushKey in pushKeys:
+ if pushKey not in self.subscribedAmpPushKeys:
+ subscribeTo.append(pushKey)
+ self.subscribedAmpPushKeys.add(pushKey)
+ if subscribeTo:
+ AMPHub.subscribeToIDs(subscribeTo, self._receivedPush)
+
+
@inlineCallbacks
def _unsubscribePubSub(self):
for factory in self._pushFactories:
@@ -1950,7 +1989,7 @@
@inlineCallbacks
- def addInvite(self, href, component, attachmentSize=0):
+ def addInvite(self, href, component, attachmentSize=0, lookupPercentage=100):
"""
Add an event that is an invite - i.e., has attendees. We will do attendee lookups and freebusy
checks on each attendee to simulate what happens when an organizer creates a new invite.
@@ -1961,7 +2000,9 @@
for attendee in attendees:
if attendee.value() in (self.uuid, self.email):
continue
- yield self._attendeeAutoComplete(component, attendee)
+ choice = random.randint(1, 100)
+ if choice <= lookupPercentage:
+ yield self._attendeeAutoComplete(component, attendee)
# Now do a normal PUT
yield self.addEvent(href, component, invite=True, attachmentSize=attachmentSize)
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2016-06-01 15:01:12 UTC (rev 15646)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2016-06-02 16:41:06 UTC (rev 15647)
@@ -81,13 +81,13 @@
self.profileTypes = profileTypes
- def new(self, reactor, server, principalPathTemplate, serializationPath, userRecord, authInfo):
+ def new(self, reactor, server, principalPathTemplate, serializationPath, userRecord, authInfo, instanceNumber):
"""
Create a new instance of this client type.
"""
return self.clientType(
reactor, server, principalPathTemplate,
- serializationPath, userRecord, authInfo,
+ serializationPath, userRecord, authInfo, instanceNumber,
**self.clientParams
)
@@ -249,6 +249,7 @@
def add(self, numClients, clientsPerUser):
+ instanceNumber = 0
for _ignore_n in range(numClients):
number = self._nextUserNumber()
@@ -270,7 +271,9 @@
self.serializationPath,
record,
auth,
+ instanceNumber
)
+ instanceNumber += 1
self.clients.append(client)
d = client.run()
d.addErrback(self._clientFailure, reactor)
Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2016-06-01 15:01:12 UTC (rev 15646)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2016-06-02 16:41:06 UTC (rev 15647)
@@ -264,6 +264,7 @@
recurrenceDistribution=RecurrenceDistribution(False),
fileAttachPercentage=30,
fileSizeDistribution=NormalDistribution(1024, 1),
+ inviteeLookupPercentage=50,
):
self.enabled = enabled
self._sendInvitationDistribution = sendInvitationDistribution
@@ -275,6 +276,7 @@
self._recurrenceDistribution = recurrenceDistribution
self._fileAttachPercentage = fileAttachPercentage
self._fileSizeDistribution = fileSizeDistribution
+ self._inviteeLookupPercentage = inviteeLookupPercentage
def run(self):
@@ -381,7 +383,10 @@
attachmentSize = 0 # no attachment
href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addInvite(href, vcalendar, attachmentSize=attachmentSize)
+ d = self._client.addInvite(
+ href, vcalendar, attachmentSize=attachmentSize,
+ lookupPercentage=self._inviteeLookupPercentage
+ )
return self._newOperation("invite", d)
@@ -632,7 +637,12 @@
def run(self):
self._call = LoopingCall(self._addEvent)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _addEvent(self):
@@ -703,9 +713,15 @@
def run(self):
self._call = LoopingCall(self.action)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
+
def modifyEvent(self, href, vevent):
"""Overridden by subclasses"""
pass
@@ -820,7 +836,12 @@
def run(self):
self._call = LoopingCall(self.action)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
@inlineCallbacks
@@ -920,7 +941,12 @@
def run(self):
self._call = LoopingCall(self._updateEvent)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _initEvent(self):
@@ -1035,7 +1061,12 @@
def run(self):
self._call = LoopingCall(self._addTask)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _addTask(self):
@@ -1082,7 +1113,12 @@
def run(self):
self._call = LoopingCall(self._runQuery)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _runQuery(self):
@@ -1116,7 +1152,12 @@
def run(self):
self._call = LoopingCall(self._deepRefresh)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _deepRefresh(self):
@@ -1145,7 +1186,12 @@
def run(self):
self._call = LoopingCall(self._resetAccount)
self._call.clock = self._reactor
- return self._call.start(self._interval)
+ self._reactor.callLater(
+ self.random.randint(1, self._interval),
+ self._call.start,
+ self._interval
+ )
+ return Deferred()
def _resetAccount(self):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160602/4e008ff6/attachment-0001.html>
More information about the calendarserver-changes
mailing list