[CalendarServer-changes] [15648] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jun 2 20:52:46 PDT 2016
Revision: 15648
http://trac.calendarserver.org//changeset/15648
Author: sagen at apple.com
Date: 2016-06-02 20:52:46 -0700 (Thu, 02 Jun 2016)
Log Message:
-----------
The sim now uses sync report on homes instead of propfinds
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_11/poll_calendarhome_sync.request
CalendarServer/trunk/contrib/performance/loadtest/test_ical.py
CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
CalendarServer/trunk/contrib/performance/loadtest/test_sim.py
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2016-06-02 16:41:06 UTC (rev 15647)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2016-06-03 03:52:46 UTC (rev 15648)
@@ -493,6 +493,7 @@
_POLL_NOTIFICATION_PROPFIND_D1 = None
_NOTIFICATION_SYNC_REPORT = None
+ _CALENDARHOME_SYNC_REPORT = None
_USER_LIST_PRINCIPAL_PROPERTY_SEARCH = None
_POST_AVAILABILITY = None
@@ -540,6 +541,7 @@
self.calendarHomePollInterval = calendarHomePollInterval
self.calendarHomeHref = None
+ self.calendarHomeToken = ""
self.supportPush = supportPush
@@ -554,6 +556,7 @@
self.supportSync = self._SYNC_REPORT
self.supportNotificationSync = self._NOTIFICATION_SYNC_REPORT
+ self.supportCalendarHomeSync = self._CALENDARHOME_SYNC_REPORT
self.supportEnhancedAttendeeAutoComplete = self._CALENDARSERVER_PRINCIPAL_SEARCH_REPORT
@@ -626,7 +629,7 @@
method=method_label if method_label else method,
url=url,
user=self.record.uid,
- client_type=self.title,
+ client_type="({} {})".format(self.title, self._instanceNumber),
client_id=self._client_id,
)
@@ -648,7 +651,7 @@
responseBody=responseBody,
code=response.code,
user=self.record.uid,
- client_type=self.title,
+ client_type="({} {})".format(self.title, self._instanceNumber),
client_id=self._client_id,
duration=(after - before),
url=url,
@@ -852,13 +855,66 @@
depth='1',
method_label="PROPFIND{home}",
)
- calendars, notificationCollection = self._extractCalendars(
+ calendars, notificationCollection, calendarHomeToken = self._extractCalendars(
result, calendarHomeSet
)
- returnValue((calendars, notificationCollection, result,))
+ returnValue((calendars, notificationCollection, calendarHomeToken, result,))
+ @inlineCallbacks
+ def _calendarHomeSync(self, calendarHomeSet):
+ if not calendarHomeSet.endswith('/'):
+ calendarHomeSet = calendarHomeSet + '/'
+ result = yield self._report(
+ calendarHomeSet,
+ self._CALENDARHOME_SYNC_REPORT % {'sync-token': self.calendarHomeToken},
+ depth='1',
+ allowedStatus=(MULTI_STATUS, FORBIDDEN,),
+ otherTokens=True,
+ method_label="REPORT{sync-home}",
+ )
+ responseHrefs, others = result
+
+ calendars, notificationCollection, ignored = self._extractCalendars(
+ responseHrefs, calendarHomeSet
+ )
+
+ # if notificationCollection is None that means it hasn't changed, so
+ # we'll return the notificationCollection we already have
+ if not notificationCollection:
+ notificationCollection = self._notificationCollection
+
+ # by default we'll keep our old calendars...
+ newCalendars = {}
+ for href, oldCalendar in self._calendars.iteritems():
+ newCalendars[href] = oldCalendar
+
+ # ...but if a calendar shows up from _extractCalendars we know it's
+ # either new or modified, so replace our old copy with the new
+ for cal in calendars:
+ # print("SYNC DETECTED CHANGE IN", cal.url)
+ newCalendars[cal.url] = cal
+
+ # Now scan through the responses for 404s which let us know a collection
+ # has been removed
+ for responseHref in responseHrefs:
+ if responseHrefs[responseHref].getStatus() == 404:
+ if responseHref in newCalendars:
+ del newCalendars[responseHref]
+ # print("DELETED CALENDAR", responseHref)
+
+ # get the new token
+ newCalendarHomeToken = ""
+ for other in others:
+ if other.tag == davxml.sync_token:
+ newCalendarHomeToken = other.text
+ break
+
+ returnValue((newCalendars.values(), notificationCollection, newCalendarHomeToken))
+
+
+
def timeRangeQuery(self, url, start, end):
requestBody = """<?xml version="1.0" encoding="utf-8" ?>
@@ -899,7 +955,7 @@
@inlineCallbacks
def deepRefresh(self):
- calendars, notificationCollection, results = yield self._calendarHomePropfind(self.calendarHomeHref)
+ calendars, notificationCollection, calendarHomeToken, results = yield self._calendarHomePropfind(self.calendarHomeHref)
for calendar in calendars:
yield self._propfind(
calendar.url,
@@ -954,6 +1010,7 @@
"""
calendars = []
notificationCollection = None
+ calendarHomeToken = ""
changeTag = davxml.sync_token if self.supportSync else csxml.getctag
@@ -992,6 +1049,15 @@
if server and uri:
self.xmpp[href] = XMPPPush(server, uri, pushkey)
+ try:
+ calendarHomeToken = text[davxml.sync_token]
+ except KeyError:
+ pass
+
+
+ if results[href].getStatus() == 404:
+ continue
+
nodes = results[href].getNodeProperties()
isCalendar = False
isNotifications = False
@@ -1044,7 +1110,7 @@
textProps.get(changeTag, None)
)
- return calendars, notificationCollection
+ return calendars, notificationCollection, calendarHomeToken
def _updateCalendar(self, calendar, newToken, fetchEvents=True):
@@ -1102,7 +1168,7 @@
depth='1',
allowedStatus=(MULTI_STATUS, FORBIDDEN,),
otherTokens=True,
- method_label="REPORT{sync}" if calendar.changeToken else "REPORT{sync-init}",
+ method_label="REPORT{sync-coll}" if calendar.changeToken else "REPORT{sync-coll-init}",
)
if result is None:
if not fullSync:
@@ -1112,7 +1178,7 @@
self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': ''},
depth='1',
otherTokens=True,
- method_label="REPORT{sync}" if calendar.changeToken else "REPORT{sync-init}",
+ method_label="REPORT{sync-coll}" if calendar.changeToken else "REPORT{sync-coll-init}",
)
else:
raise IncorrectResponseCode((MULTI_STATUS,), None, None)
@@ -1288,7 +1354,7 @@
depth='1',
allowedStatus=(MULTI_STATUS, FORBIDDEN,),
otherTokens=True,
- method_label="REPORT{sync}" if oldToken else "REPORT{sync-init}",
+ method_label="REPORT{sync-notif}" if oldToken else "REPORT{sync-notif-init}",
)
if result is None:
if not fullSync:
@@ -1298,7 +1364,7 @@
self._NOTIFICATION_SYNC_REPORT % {'sync-token': ''},
depth='1',
otherTokens=True,
- method_label="REPORT{sync}" if oldToken else "REPORT{sync-init}",
+ method_label="REPORT{sync-notif}" if oldToken else "REPORT{sync-notif-init}",
)
else:
raise IncorrectResponseCode((MULTI_STATUS,), None, None)
@@ -1400,10 +1466,16 @@
returnValue(False)
self._checking.add(calendarHomeSet)
- calendars, notificationCollection, results = yield self._calendarHomePropfind(calendarHomeSet)
+ if not self.calendarHomeToken:
+ calendars, notificationCollection, newCalendarHomeToken, results = yield self._calendarHomePropfind(calendarHomeSet)
+ else:
+ calendars, notificationCollection, newCalendarHomeToken = yield self._calendarHomeSync(calendarHomeSet)
+ results = None
+ self.calendarHomeToken = newCalendarHomeToken
+
# First time operations
- if firstTime:
+ if firstTime and results:
yield self._pollFirstTime1(results[calendarHomeSet], calendars)
# Normal poll
@@ -1550,7 +1622,7 @@
type="operation",
phase="start",
user=self.record.uid,
- client_type=self.title,
+ client_type="({} {})".format(self.title, self._instanceNumber),
client_id=self._client_id,
label=label,
)
@@ -1573,7 +1645,7 @@
phase="end",
duration=after - before,
user=self.record.uid,
- client_type=self.title,
+ client_type="({} {})".format(self.title, self._instanceNumber),
client_id=self._client_id,
label=label,
success=success,
@@ -1717,6 +1789,8 @@
# Create dict for all the data we need to store
data = {
+ "homeToken": self.calendarHomeToken,
+ "attachmentsUrl": self._managed_attachments_server_url,
"principalURL": self.principalURL,
"calendars": [calendar.serialize() for calendar in sorted(self._calendars.values(), key=lambda x:x.name)],
"events": [event.serialize() for event in sorted(self._events.values(), key=lambda x:x.url)],
@@ -1749,6 +1823,8 @@
return
self.principalURL = data["principalURL"]
+ self.calendarHomeToken = data["homeToken"].encode("utf-8")
+ self._managed_attachments_server_url = data["attachmentsUrl"].encode("utf-8")
# Extract all the events first, then do the calendars (which reference the events)
for event in data["events"]:
@@ -2404,6 +2480,7 @@
_POLL_NOTIFICATION_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_notification_depth1_propfind')
_NOTIFICATION_SYNC_REPORT = loadRequestBody(_LOAD_PATH, 'notification_sync')
+ _CALENDARHOME_SYNC_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_sync')
_USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody('OS_X_10_7', 'user_list_principal_property_search')
_POST_AVAILABILITY = loadRequestBody('OS_X_10_7', 'post_availability')
Modified: CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_11/poll_calendarhome_sync.request
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_11/poll_calendarhome_sync.request 2016-06-02 16:41:06 UTC (rev 15647)
+++ CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_11/poll_calendarhome_sync.request 2016-06-03 03:52:46 UTC (rev 15648)
@@ -1,45 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<A:sync-collection xmlns:A="DAV:">
- <A:sync-token>data:,30_1122/8bbf7c540e5fca2cc3220f114a8164f7</A:sync-token>
+ <A:sync-token>%(sync-token)s</A:sync-token>
<A:sync-level>1</A:sync-level>
<A:prop>
- <C:publish-url xmlns:C="http://calendarserver.org/ns/"/>
- <B:supported-calendar-component-sets xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <B:schedule-calendar-transp xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <A:sync-token/>
+ <A:resourcetype/>
+ <A:displayname/>
+ <B:calendar-free-busy-set xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <B:default-alarm-vevent-date xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <C:allowed-sharing-modes xmlns:C="http://calendarserver.org/ns/"/>
+ <B:calendar-alarm xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <B:calendar-timezone xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <C:push-transports xmlns:C="http://calendarserver.org/ns/"/>
+ <B:supported-calendar-component-set xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <C:subscribed-strip-attachments xmlns:C="http://calendarserver.org/ns/"/>
+ <A:quota-available-bytes/>
+ <B:default-alarm-vevent-datetime xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <A:quota-used-bytes/>
+ <E:location-code xmlns:E="http://apple.com/ns/ical/"/>
+ <A:resource-id/>
+ <C:getctag xmlns:C="http://calendarserver.org/ns/"/>
+ <E:calendar-color xmlns:E="http://apple.com/ns/ical/"/>
<C:source xmlns:C="http://calendarserver.org/ns/"/>
- <B:calendar-description xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <D:location-code xmlns:D="http://apple.com/ns/ical/"/>
- <D:autoprovisioned xmlns:D="http://apple.com/ns/ical/"/>
- <A:quota-used-bytes/>
+ <E:autoprovisioned xmlns:E="http://apple.com/ns/ical/"/>
+ <A:current-user-privilege-set/>
<C:pre-publish-url xmlns:C="http://calendarserver.org/ns/"/>
- <D:calendar-order xmlns:D="http://apple.com/ns/ical/"/>
- <D:refreshrate xmlns:D="http://apple.com/ns/ical/"/>
- <C:subscribed-strip-attachments xmlns:C="http://calendarserver.org/ns/"/>
- <C:push-transports xmlns:C="http://calendarserver.org/ns/"/>
<B:schedule-default-calendar-URL xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <B:calendar-alarm xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <B:supported-calendar-component-set xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <B:schedule-calendar-transp xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <B:supported-calendar-component-sets xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <D:bulk-requests xmlns:D="http://me.com/_namespace/"/>
+ <E:calendar-order xmlns:E="http://apple.com/ns/ical/"/>
+ <E:refreshrate xmlns:E="http://apple.com/ns/ical/"/>
+ <C:subscribed-strip-todos xmlns:C="http://calendarserver.org/ns/"/>
+ <B:calendar-description xmlns:B="urn:ietf:params:xml:ns:caldav"/>
+ <E:language-code xmlns:E="http://apple.com/ns/ical/"/>
+ <C:publish-url xmlns:C="http://calendarserver.org/ns/"/>
+ <C:pushkey xmlns:C="http://calendarserver.org/ns/"/>
<A:owner/>
+ <A:supported-report-set/>
<A:add-member/>
+ <C:subscribed-strip-alarms xmlns:C="http://calendarserver.org/ns/"/>
<C:invite xmlns:C="http://calendarserver.org/ns/"/>
- <A:resource-id/>
- <E:bulk-requests xmlns:E="http://me.com/_namespace/"/>
- <B:calendar-timezone xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <A:supported-report-set/>
- <A:displayname/>
- <C:subscribed-strip-alarms xmlns:C="http://calendarserver.org/ns/"/>
- <D:language-code xmlns:D="http://apple.com/ns/ical/"/>
- <C:subscribed-strip-todos xmlns:C="http://calendarserver.org/ns/"/>
- <C:allowed-sharing-modes xmlns:C="http://calendarserver.org/ns/"/>
- <A:current-user-privilege-set/>
- <B:calendar-free-busy-set xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <A:sync-token/>
- <A:quota-available-bytes/>
- <C:pushkey xmlns:C="http://calendarserver.org/ns/"/>
- <B:default-alarm-vevent-date xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <A:resourcetype/>
- <B:default-alarm-vevent-datetime xmlns:B="urn:ietf:params:xml:ns:caldav"/>
- <D:calendar-color xmlns:D="http://apple.com/ns/ical/"/>
- <C:getctag xmlns:C="http://calendarserver.org/ns/"/>
</A:prop>
</A:sync-collection>
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_ical.py 2016-06-02 16:41:06 UTC (rev 15647)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_ical.py 2016-06-03 03:52:46 UTC (rev 15648)
@@ -1177,6 +1177,7 @@
serializePath,
self.record,
None,
+ 1
)
@@ -1241,7 +1242,7 @@
constructed from the data extracted from the response.
"""
home = "/calendars/__uids__/user01/"
- calendars, notificationCollection = self.client._extractCalendars(
+ calendars, notificationCollection, homeToken = self.client._extractCalendars(
self.client._parseMultiStatus(CALENDAR_HOME_PROPFIND_RESPONSE), home)
calendars.sort(key=lambda cal: cal.resourceType)
calendar, inbox = calendars
@@ -1558,6 +1559,9 @@
self.client._notificationCollection = NotificationCollection("/home/notification", "123")
+ self.client._managed_attachments_server_url = "attachmentsurl"
+ self.client.calendarHomeToken = "hometoken"
+
self.client.serialize()
self.assertTrue(os.path.exists(clientPath))
self.assertTrue(os.path.exists(indexPath))
@@ -1626,6 +1630,8 @@
}
],
"principalURL": null,
+ "homeToken": "hometoken",
+ "attachmentsUrl": "attachmentsurl",
"events": [
{
"url": "/home/calendar/1.ics",
@@ -1699,6 +1705,8 @@
"calendars": [
{
"changeToken": "321",
+ "homeToken": "321",
+ "attachmentsUrl": "https://example.com/attachments/",
"name": "calendar",
"shared": false,
"sharedByMe": false,
@@ -1740,6 +1748,8 @@
}
],
"principalURL": null,
+ "homeToken": "hometoken",
+ "attachmentsUrl": "attachmentsurl",
"events": [
{
"url": "/home/calendar/2.ics",
@@ -1787,6 +1797,8 @@
self.assertEqual(self.client._events["/home/inbox/i2.ics"].etag, "987.987")
self.assertEqual(self.client._events["/home/inbox/i2.ics"].getUID(), "00a79cad-857b-418e-a54a-340b5686d747")
self.assertEqual(str(self.client._events["/home/inbox/i2.ics"].component), cal2)
+ self.assertEqual(self.client.calendarHomeToken, "hometoken")
+ self.assertEqual(self.client._managed_attachments_server_url, "attachmentsurl")
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2016-06-02 16:41:06 UTC (rev 15647)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2016-06-03 03:52:46 UTC (rev 15648)
@@ -263,7 +263,7 @@
return succeed(None)
- def addInvite(self, href, vevent, attachmentSize=0):
+ def addInvite(self, href, vevent, attachmentSize=0, lookupPercentage=0):
return self.addEvent(href, vevent, attachmentSize=attachmentSize)
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_sim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2016-06-02 16:41:06 UTC (rev 15647)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2016-06-03 03:52:46 UTC (rev 15648)
@@ -175,7 +175,7 @@
profiles are not logged.
"""
class BrokenClient(object):
- def __init__(self, reactor, serverAddress, principalPathTemplate, serializationPath, userInfo, auth, runResult):
+ def __init__(self, reactor, serverAddress, principalPathTemplate, serializationPath, userInfo, auth, instanceNumber, runResult):
self._runResult = runResult
def run(self):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160602/1f451a02/attachment-0001.html>
More information about the calendarserver-changes
mailing list