[CalendarServer-changes] [6797] CalendarServer/trunk/contrib/performance
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jan 21 10:08:37 PST 2011
Revision: 6797
http://trac.macosforge.org/projects/calendarserver/changeset/6797
Author: exarkun at twistedmatrix.com
Date: 2011-01-21 10:08:36 -0800 (Fri, 21 Jan 2011)
Log Message:
-----------
Add a "user profile" which invites uses to events.
All clients started up by population.py are now given an "inviter" profile which (rapidly) adds invitees to a random event that it finds. Still to do is to add an "event creator" profile so that the inviter can find some events, as well as to allow tuning of the inviters parameters instead of hard-coding the loop interval and such.
The inviter also never un-invites anyone from an event, which would probably be a good thing to do.
Clients will also never accept or reject an invitation. A profile for that should be added as well.
The SnowLeopard iCal simulator also now has a method for adding an attendee to an event which implements the same behavior as the SnowLeopard iCal client.
The httpclient StringProducer is also changed to reject obviously bogus data (eg unicode).
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/httpclient.py
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/population.py
Added Paths:
-----------
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_post_availability.request
CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_user_list_principal_property_search.request
Modified: CalendarServer/trunk/contrib/performance/httpclient.py
===================================================================
--- CalendarServer/trunk/contrib/performance/httpclient.py 2011-01-20 21:29:03 UTC (rev 6796)
+++ CalendarServer/trunk/contrib/performance/httpclient.py 2011-01-21 18:08:36 UTC (rev 6797)
@@ -50,6 +50,9 @@
implements(IBodyProducer)
def __init__(self, body):
+ if not isinstance(body, str):
+ raise TypeError(
+ "StringProducer body must be str, not %r" % (type(body),))
self._body = body
self.length = len(self._body)
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-01-20 21:29:03 UTC (rev 6796)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-01-21 18:08:36 UTC (rev 6797)
@@ -15,18 +15,23 @@
#
##
+from uuid import uuid4
from operator import getitem
-from pprint import pprint
+from pprint import pformat
+from datetime import datetime
from xml.etree import ElementTree
ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
+from vobject import readComponents
+from vobject.icalendar import dateTimeToString
+
from twisted.python.log import addObserver, err, msg
from twisted.python.filepath import FilePath
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
from twisted.internet.task import LoopingCall
from twisted.web.http_headers import Headers
-from twisted.web.http import OK, MULTI_STATUS
+from twisted.web.http import OK, MULTI_STATUS, NO_CONTENT
from twisted.web.client import Agent
from protocol.webdav.propfindparser import PropFindParser
@@ -45,11 +50,13 @@
class Event(object):
- def __init__(self, url, etag):
+ def __init__(self, url, etag, vevent=None):
self.url = url
self.etag = etag
+ self.vevent = vevent
+
class Calendar(object):
def __init__(self, resourceType, name, url, ctag):
self.resourceType = resourceType
@@ -59,7 +66,13 @@
-class SnowLeopard(object):
+class BaseClient(object):
+ def addEventAttendee(self, href, attendee):
+ raise NotImplementedError("%r does not implement addEventAttendee" % (self.__class__,))
+
+
+
+class SnowLeopard(BaseClient):
"""
Implementation of the SnowLeopard iCal network behavior.
@@ -84,7 +97,11 @@
_CALENDAR_PROPFIND = loadRequestBody('sl_calendar_propfind')
_CALENDAR_REPORT = loadRequestBody('sl_calendar_report')
+ _USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody('sl_user_list_principal_property_search')
+ _POST_AVAILABILITY = loadRequestBody('sl_post_availability')
+ email = None
+
def __init__(self, reactor, host, port, user, auth):
self.reactor = reactor
self.agent = AuthHandlerAgent(Agent(self.reactor), auth)
@@ -102,9 +119,11 @@
self._events = {}
- def _request(self, expectedResponseCode, method, url, headers, body):
+ def _request(self, expectedResponseCode, method, url, headers=None, body=None):
+ if headers is None:
+ headers = Headers({})
headers.setRawHeaders('User-Agent', [self.USER_AGENT])
- msg(type="request", method=method, url=url)
+ msg(type="request", method=method, url=url, user=self.user)
d = self.agent.request(method, url, headers, body)
before = self.reactor.seconds()
def report(response):
@@ -251,14 +270,21 @@
if responseHref not in self._events:
self._events[responseHref] = Event(responseHref, None)
-
- if self._events[responseHref].etag != etag:
+
+ event = self._events[responseHref]
+ if event.etag != etag:
response = yield self._updateEvent(url, responseHref)
body = yield readBody(response)
- res = self._parseMultiStatus(body)[responseHref]
- etag = res.getTextProperties()[davxml.getetag]
- self._events[responseHref].etag = etag
+ self.eventChanged(responseHref, body)
+
+ def eventChanged(self, href, body):
+ event = self._events[href]
+ res = self._parseMultiStatus(body)[href]
+ text = res.getTextProperties()
+ event.etag = text[davxml.getetag]
+ event.vevent = list(readComponents(text[caldavxml.calendar_data]))[0]
+
def _updateEvent(self, calendar, event):
# Next do a REPORT on each event that might have information
@@ -318,6 +344,14 @@
principal = yield self._principalPropfind(self.user)
hrefs = principal.getHrefProperties()
+ # Remember our own email-like principal address
+ for principalURL in hrefs[caldavxml.calendar_user_address_set]:
+ if principalURL.toString().startswith("mailto:"):
+ self.email = principalURL.toString()
+ break
+ if self.email is None:
+ raise ValueError("Cannot operate without a mail-style principal URL")
+
# Do another kind of thing I guess
principalCollection = hrefs[davxml.principal_collection_set].toString()
(yield self._principalsReport(principalCollection))
@@ -363,11 +397,97 @@
yield Deferred()
+ def addEventAttendee(self, href, attendee):
+ name = attendee.params[u'CN'][0].encode('utf-8')
+ prefix = name[:4].lower()
+ email = attendee.params[u'EMAIL'][0].encode('utf-8')
+
+ event = self._events[href]
+ vevent = event.vevent
+
+ # First try to discover some names to supply to the
+ # auto-completion
+ d = self._request(
+ MULTI_STATUS, 'REPORT', self.root + 'principals/',
+ Headers({'content-type': ['text/xml']}),
+ StringProducer(self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
+ 'displayname': prefix,
+ 'email': prefix,
+ 'firstname': prefix,
+ 'lastname': prefix,
+ }))
+ d.addCallback(readBody)
+ def narrowed(ignored):
+ # Next just learn about the one name we selected.
+ d = self._request(
+ MULTI_STATUS, 'REPORT', self.root + 'principals/',
+ Headers({'content-type': ['text/xml']}),
+ StringProducer(self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
+ 'displayname': name,
+ 'email': name,
+ 'firstname': name,
+ 'lastname': name,
+ }))
+ d.addCallback(readBody)
+ return d
+ d.addCallback(narrowed)
+ def specific(ignored):
+ # Now learn about the attendee's availability
+ uid = vevent.contents[u'vevent'][0].contents[u'uid'][0].value
+ start = vevent.contents[u'vevent'][0].contents[u'dtstart'][0].value
+ end = vevent.contents[u'vevent'][0].contents[u'dtend'][0].value
+ now = datetime.now()
+ d = self._request(
+ OK, 'POST', self.root + 'calendars/__uids__/' + self.user + '/outbox/',
+ Headers({
+ 'content-type': ['text/calendar'],
+ 'originator': [self.email],
+ 'recipient': ['mailto:' + email]}),
+ StringProducer(self._POST_AVAILABILITY % {
+ 'attendee': 'mailto:' + email,
+ 'organizer': self.email,
+ 'vfreebusy-uid': str(uuid4()).upper(),
+ 'event-uid': uid.encode('utf-8'),
+ 'start': dateTimeToString(start, convertToUTC=True),
+ 'end': dateTimeToString(end, convertToUTC=True),
+ 'now': dateTimeToString(now, convertToUTC=True)}))
+ d.addCallback(readBody)
+ return d
+ d.addCallback(specific)
+ def availability(ignored):
+ # At last, upload the new event definition
+ vevent.contents['vevent'][0].contents.setdefault('attendee', []).append(attendee)
+ d = self._request(
+ NO_CONTENT, 'PUT', self.root + href[1:],
+ Headers({
+ 'content-type': ['text/calendar'],
+ 'if-match': [event.etag]}),
+ StringProducer(vevent.serialize()))
+ return d
+ d.addCallback(availability)
+ def check(ignored):
+ # Finally, re-retrieve the event
+ return self._request(OK, 'GET', self.root + href[1:])
+ d.addCallback(check)
+ def getETag(response):
+ etag = response.headers.getRawHeaders('etag')[0]
+ return readBody(response).addCallback(lambda body: (etag, body))
+ d.addCallback(getETag)
+ def record((etag, body)):
+ event = self._events[href]
+ event.etag = etag
+ event.vevent = list(readComponents(body))[0]
+ d.addCallback(record)
+ return d
+
+
+
class RequestLogger(object):
def observe(self, event):
if event.get("type") == "request":
- print event["method"], event["url"]
+ print event["user"], event["method"], event["url"]
+
def main():
from urllib2 import HTTPDigestAuthHandler
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-01-20 21:29:03 UTC (rev 6796)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-01-21 18:08:36 UTC (rev 6797)
@@ -24,6 +24,7 @@
from stats import mean, median, stddev, mad
from loadtest.ical import SnowLeopard, RequestLogger
+from loadtest.profiles import Inviter
class PopulationParameters(object):
@@ -83,10 +84,15 @@
self._user = 1
- def _nextUser(self):
+ def _nextUserNumber(self):
+ result = self._user
+ self._user += 1
+ return self._user
+
+
+ def _createUser(self, number):
from urllib2 import HTTPDigestAuthHandler
- user = "user%02d" % (self._user,)
- self._user += 1
+ user = "user%02d" % (number,)
auth = HTTPDigestAuthHandler()
auth.add_password(
realm="Test Realm",
@@ -98,9 +104,11 @@
def add(self, numClients):
for n in range(numClients):
- user, auth = self._nextUser()
+ number = self._nextUserNumber()
+ user, auth = self._createUser(number)
client = self._pop.next()(self.reactor, self.host, self.port, user, auth)
client.run()
+ Inviter(self.reactor, client, number).run()
print 'Now running', self._user, 'clients.'
@@ -189,6 +197,9 @@
from twisted.internet.task import LoopingCall
from twisted.python.log import addObserver
+ from twisted.python.failure import startDebugMode
+ startDebugMode()
+
report = ReportStatistics()
addObserver(SimpleStatistics().observe)
addObserver(report.observe)
Added: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py (rev 0)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2011-01-21 18:08:36 UTC (rev 6797)
@@ -0,0 +1,105 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+##
+
+"""
+Implementation of specific end-user behaviors.
+"""
+
+from random import choice, gauss
+
+from vobject.base import ContentLine
+from vobject.icalendar import VEvent
+
+from twisted.internet.defer import succeed
+from twisted.internet.task import LoopingCall
+
+
+class Inviter(object):
+ """
+ A Calendar user who invites and de-invites other users to events.
+ """
+ def __init__(self, reactor, client, userNumber):
+ self._reactor = reactor
+ self._client = client
+ self._number = userNumber
+
+
+ def run(self):
+ self._call = LoopingCall(self._invite)
+ self._call.clock = self._reactor
+ self._call.start(3)
+
+
+ def _addAttendee(self, event, attendees):
+ """
+ Create a new attendeed to add to the list of attendees for the
+ given event.
+ """
+ invitee = max(1, int(gauss(self._number, 30)))
+ if invitee == self._number:
+ # This will bias the distribution a little... it is the
+ # end of the world, probably.
+ invitee += 1
+ user = u'User %02d' % (invitee,)
+ email = u'user%02d at example.com' % (invitee,)
+ uuid = u'urn:uuid:user%02d' % (invitee,)
+
+ attendee = ContentLine(
+ name=u'ATTENDEE', params=[
+ [u'CN', user],
+ [u'CUTYPE', u'INDIVIDUAL'],
+ [u'EMAIL', email],
+ [u'PARTSTAT', u'NEEDS-ACTION'],
+ [u'ROLE', u'REQ-PARTICIPANT'],
+ [u'RSVP', u'TRUE'],
+ ],
+ value=uuid,
+ encoded=True)
+ attendee.parentBehavior = VEvent
+
+ return succeed(attendee)
+
+
+ def _invite(self):
+ """
+ Try to add a new attendee to an event, or perhaps remove an
+ existing attendee from an event.
+
+ @return: C{None} if there are no events to play with,
+ otherwise a L{Deferred} which fires when the attendee
+ change has been made.
+ """
+ # Pick an event at random
+ if self._client._events:
+ uuid = choice(self._client._events.keys())
+ event = self._client._events[uuid].vevent
+
+ # Find out who might attend
+ attendees = event.contents['vevent'][0].contents.get('attendee', [])
+
+ if len(attendees) < 2 or choice([False, True]):
+ d = self._addAttendee(event, attendees)
+ d.addCallback(
+ lambda attendee:
+ self._client.addEventAttendee(
+ uuid, attendee))
+ return d
+ else:
+ # XXX
+ # self._removeAttendee(attendees)
+ return
+
Added: CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_post_availability.request
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_post_availability.request (rev 0)
+++ CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_post_availability.request 2011-01-21 18:08:36 UTC (rev 6797)
@@ -0,0 +1,16 @@
+BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+VERSION:2.0
+METHOD:REQUEST
+PRODID:-//Apple Inc.//iCal 4.0.3//EN
+BEGIN:VFREEBUSY
+UID:%(vfreebusy-uid)s
+DTEND:%(end)s
+ATTENDEE:%(attendee)s
+DTSTART:%(start)s
+X-CALENDARSERVER-MASK-UID:%(event-uid)s
+DTSTAMP:%(now)s
+ORGANIZER:%(organizer)s
+SUMMARY:Availability for %(attendee)s
+END:VFREEBUSY
+END:VCALENDAR
Added: CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_user_list_principal_property_search.request
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_user_list_principal_property_search.request (rev 0)
+++ CalendarServer/trunk/contrib/performance/loadtest/request-data/sl_user_list_principal_property_search.request 2011-01-21 18:08:36 UTC (rev 6797)
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<x0:principal-property-search xmlns:x2="urn:ietf:params:xml:ns:caldav" xmlns:x0="DAV:" xmlns:x1="http://calendarserver.org/ns/" test="anyof"><x0:property-search><x0:prop><x0:displayname/></x0:prop><x0:match match-type="starts-with">%(displayname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:email-address-set/></x0:prop><x0:match match-type="starts-with">%(email)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:first-name/></x0:prop><x0:match match-type="starts-with">%(firstname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:last-name/></x0:prop><x0:match match-type="starts-with">%(lastname)s</x0:match></x0:property-search><x0:prop><x1:email-address-set/><x2:calendar-user-address-set/><x2:calendar-user-type/><x0:displayname/><x1:last-name/><x1:first-name/><x1:record-type/><x0:principal-URL/></x0:prop></x0:principal-property-search>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110121/de48a5cd/attachment-0001.html>
More information about the calendarserver-changes
mailing list