[CalendarServer-changes] [6868] CalendarServer/trunk/contrib/performance/loadtest

source_changes at macosforge.org source_changes at macosforge.org
Thu Feb 3 14:41:06 PST 2011


Revision: 6868
          http://trac.macosforge.org/projects/calendarserver/changeset/6868
Author:   exarkun at twistedmatrix.com
Date:     2011-02-03 14:41:05 -0800 (Thu, 03 Feb 2011)
Log Message:
-----------
Add a profile which creates new events; also fix a variety of bugs in the invite and accept profiles.

Modified Paths:
--------------
    CalendarServer/trunk/contrib/performance/loadtest/ical.py
    CalendarServer/trunk/contrib/performance/loadtest/population.py
    CalendarServer/trunk/contrib/performance/loadtest/profiles.py
    CalendarServer/trunk/contrib/performance/loadtest/test_ical.py
    CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py

Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py	2011-02-03 22:14:19 UTC (rev 6867)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py	2011-02-03 22:41:05 UTC (rev 6868)
@@ -24,14 +24,15 @@
 ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
 
 from vobject import readComponents
-from vobject.icalendar import dateTimeToString
+from vobject.base import ContentLine
+from vobject.icalendar import VEvent, dateTimeToString
 
 from twisted.python.log import addObserver, err, msg
 from twisted.python.filepath import FilePath
 from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
 from twisted.internet.task import LoopingCall
 from twisted.web.http_headers import Headers
-from twisted.web.http import OK, MULTI_STATUS, NO_CONTENT
+from twisted.web.http import OK, MULTI_STATUS, CREATED, NO_CONTENT
 from twisted.web.client import Agent
 
 from protocol.webdav.propfindparser import PropFindParser
@@ -75,6 +76,10 @@
     _events = None
     _calendars = None
 
+    def addEvent(self, href, vcalendar):
+        raise NotImplementedError("%r does not implement addEvent" % (self.__class__,))
+
+
     def addEventAttendee(self, href, attendee):
         raise NotImplementedError("%r does not implement addEventAttendee" % (self.__class__,))
 
@@ -98,7 +103,7 @@
 
     USER_AGENT = "DAVKit/4.0.3 (732); CalendarStore/4.0.3 (991); iCal/4.0.3 (1388); Mac OS X/10.6.4 (10F569)"
 
-    CALENDAR_HOME_POLL_INTERVAL = 15 # * 60
+    CALENDAR_HOME_POLL_INTERVAL = 60 * 3
 
     _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody('sl_startup_principal_propfind')
     _STARTUP_PRINCIPALS_REPORT = loadRequestBody('sl_startup_principals_report')
@@ -221,7 +226,9 @@
             StringProducer(self._STARTUP_PRINCIPAL_PROPFIND))
         d.addCallback(readBody)
         d.addCallback(self._parseMultiStatus)
-        d.addCallback(getitem, principalURL)
+        def get(result):
+            return result[principalURL]
+        d.addCallback(get)
         return d
 
 
@@ -374,14 +381,20 @@
     @inlineCallbacks
     def startup(self):
         # Orient ourselves, or something
-        principal = yield self._principalPropfind(self.user)
+        try:
+            principal = yield self._principalPropfind(self.user)
+        except:
+            err(None, "Startup principal PROPFIND failed")
+            return
+
         hrefs = principal.getHrefProperties()
 
         # Remember our own email-like principal address
         for principalURL in hrefs[caldavxml.calendar_user_address_set]:
-            if principalURL.toString().startswith("mailto:"):
+            if principalURL.toString().startswith(u"mailto:"):
                 self.email = principalURL.toString()
-                break
+            elif principalURL.toString().startswith(u"urn:"):
+                self.uuid = principalURL.toString()
         if self.email is None:
             raise ValueError("Cannot operate without a mail-style principal URL")
 
@@ -430,6 +443,30 @@
         yield Deferred()
 
 
+    def _makeSelfAttendee(self):
+        attendee = ContentLine(
+            name=u'ATTENDEE', params=[
+                [u'CN', self.user],
+                [u'CUTYPE', u'INDIVIDUAL'],
+                [u'PARTSTAT', u'ACCEPTED'],
+                ],
+            value=self.uuid,
+            encoded=True)
+        attendee.parentBehavior = VEvent
+        return attendee
+
+
+    def _makeSelfOrganizer(self):
+        organizer = ContentLine(
+            name=u'ORGANIZER', params=[
+                [u'CN', self.user],
+                ],
+            value=self.uuid,
+            encoded=True)
+        organizer.parentBehavior = VEvent
+        return organizer
+
+
     def addEventAttendee(self, href, attendee):
         name = attendee.params[u'CN'][0].encode('utf-8')
         prefix = name[:4].lower()
@@ -488,10 +525,19 @@
             return d
         d.addCallback(specific)
         def availability(ignored):
+            # If the event has no attendees, add ourselves as an attendee.
+            attendees = vevent.contents[u'vevent'][0].contents.setdefault(u'attendee', [])
+            if len(attendees) == 0:
+                # First add ourselves as a participant and as the
+                # organizer.  In the future for this event we should
+                # already have those roles.
+                attendees.append(self._makeSelfAttendee())
+                vevent.contents[u'vevent'][0].contents[u'organizer'] = [self._makeSelfOrganizer()]
+            attendees.append(attendee)
+
             # At last, upload the new event definition
-            vevent.contents['vevent'][0].contents.setdefault('attendee', []).append(attendee)
             d = self._request(
-                NO_CONTENT, 'PUT', self.root + href[1:],
+                NO_CONTENT, 'PUT', self.root + href[1:].encode('utf-8'),
                 Headers({
                         'content-type': ['text/calendar'],
                         'if-match': [event.etag]}),
@@ -518,18 +564,39 @@
             headers.addRawHeader('if-schedule-tag-match', event.scheduleTag)
 
         d = self._request(
-            NO_CONTENT, 'PUT', self.root + href[1:],
+            NO_CONTENT, 'PUT', self.root + href[1:].encode('utf-8'),
             headers, StringProducer(vevent.serialize()))
         d.addCallback(self._updateEvent, href)
         return d
 
 
+    def addEvent(self, href, vcalendar):
+        headers = Headers({
+                'content-type': ['text/calendar'],
+                })
+        d = self._request(
+            CREATED, 'PUT', self.root + href[1:].encode('utf-8'),
+            headers, StringProducer(vcalendar.serialize()))
+        d.addCallback(self._localUpdateEvent, href, vcalendar)
+        return d
+
+
+    def _localUpdateEvent(self, response, href, vcalendar):
+        headers = response.headers
+        etag = headers.getRawHeaders("etag", [None])[0]
+        scheduleTag = headers.getRawHeaders("schedule-tag", [None])[0]
+
+        event = Event(href, etag, vcalendar)
+        event.scheduleTag = scheduleTag
+        self._setEvent(href, event)
+
+
     def _updateEvent(self, ignored, href):
-        d = self._request(OK, 'GET', self.root + href[1:])
+        d = self._request(OK, 'GET', self.root + href[1:].encode('utf-8'))
         def getETag(response):
             headers = response.headers
             etag = headers.getRawHeaders('etag')[0]
-            scheduleTag = headers.getRawHeaders('schedule-tag')[0]
+            scheduleTag = headers.getRawHeaders('schedule-tag', [None])[0]
             return readBody(response).addCallback(
                 lambda body: (etag, scheduleTag, body))
         d.addCallback(getETag)

Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py	2011-02-03 22:14:19 UTC (rev 6867)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py	2011-02-03 22:41:05 UTC (rev 6868)
@@ -24,7 +24,7 @@
 
 from stats import mean, median, stddev, mad
 from loadtest.ical import SnowLeopard, RequestLogger
-from loadtest.profiles import Inviter, Accepter
+from loadtest.profiles import Eventer, Inviter, Accepter
 
 
 class PopulationParameters(object):
@@ -96,7 +96,7 @@
         auth = HTTPDigestAuthHandler()
         auth.add_password(
             realm="Test Realm",
-            uri="http://127.0.0.1:8008/",
+            uri="http://%s:%d/" % (self.host, self.port),
             user=user,
             passwd=user)
         return user, auth
@@ -108,6 +108,7 @@
             user, auth = self._createUser(number)
             client = self._pop.next()(self.reactor, self.host, self.port, user, auth)
             client.run()
+            Eventer(self.reactor, client, number).run()
             Inviter(self.reactor, client, number).run()
             Accepter(self.reactor, client, number).run()
         print 'Now running', self._user - 1, 'clients.'
@@ -214,8 +215,8 @@
         populator, parameters, reactor, '127.0.0.1', 8008)
 
     # Add some clients.
-    for i in range(50):
-        reactor.callLater(i, simulator.add, 1)
+    for i in range(10):
+        reactor.callLater(i * 30, simulator.add, 1)
     reactor.run()
     report.summarize()
 

Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py	2011-02-03 22:14:19 UTC (rev 6867)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py	2011-02-03 22:41:05 UTC (rev 6868)
@@ -20,8 +20,12 @@
 """
 
 import random
+from uuid import uuid4
 
-from vobject.base import ContentLine
+from datetime import datetime, timedelta
+
+from vobject import readComponents
+from vobject.base import Component, ContentLine
 from vobject.icalendar import VEvent
 
 from protocol.caldav.definitions import caldavxml
@@ -31,9 +35,10 @@
 from twisted.internet.task import LoopingCall
 
 
-class Inviter(object):
+class ProfileBase(object):
     """
-    A Calendar user who invites and de-invites other users to events.
+    Base class which provides some conveniences for profile
+    implementations.
     """
     random = random
 
@@ -43,11 +48,24 @@
         self._number = userNumber
 
 
+    def _calendarsOfType(self, calendarType):
+        return [
+            cal 
+            for cal 
+            in self._client._calendars.itervalues() 
+            if cal.resourceType == calendarType]
+
+
+
+class Inviter(ProfileBase):
+    """
+    A Calendar user who invites and de-invites other users to events.
+    """
     def run(self):
         self._call = LoopingCall(self._invite)
         self._call.clock = self._reactor
         # XXX Base this on something real
-        self._call.start(3)
+        self._call.start(20)
 
 
     def _addAttendee(self, event, attendees):
@@ -55,18 +73,18 @@
         Create a new attendee to add to the list of attendees for the
         given event.
         """
-        invitees = set([self._client.email[len('mailto:'):]])
+        invitees = set([u'urn:uuid:user%02d' % (self._number,)])
         for att in attendees:
-            invitees.add(att.params[u'EMAIL'][0])
+            invitees.add(att.value)
 
         while True:
-            invitee = max(1, int(self.random.gauss(self._number, 30)))
-            email = u'user%02d at example.com' % (invitee,)
-            if email not in invitees:
+            invitee = max(1, int(self.random.gauss(self._number, 3)))
+            uuid = u'urn:uuid:user%02d' % (invitee,)
+            if uuid not in invitees:
                 break
 
         user = u'User %02d' % (invitee,)
-        uuid = u'urn:uuid:user%02d' % (invitee,)
+        email = u'user%02d at example.com' % (invitee,)
 
         attendee = ContentLine(
             name=u'ATTENDEE', params=[
@@ -76,6 +94,7 @@
                 [u'PARTSTAT', u'NEEDS-ACTION'],
                 [u'ROLE', u'REQ-PARTICIPANT'],
                 [u'RSVP', u'TRUE'],
+                # [u'SCHEDULE-STATUS', u'1.2'],
                 ],
             value=uuid,
             encoded=True)
@@ -94,11 +113,7 @@
             change has been made.
         """
         # Find calendars which are eligible for invites
-        calendars = [
-            cal 
-            for cal 
-            in self._client._calendars.itervalues() 
-            if cal.resourceType == caldavxml.calendar]
+        calendars = self._calendarsOfType(caldavxml.calendar)
 
         while calendars:
             # Pick one at random from which to try to select an event
@@ -131,16 +146,13 @@
 
 
 
-class Accepter(object):
+class Accepter(ProfileBase):
     """
     A Calendar user who accepts invitations to events.
     """
-    random = random
 
     def __init__(self, reactor, client, userNumber):
-        self._reactor = reactor
-        self._client = client
-        self._number = userNumber
+        ProfileBase.__init__(self, reactor, client, userNumber)
         self._accepting = set()
 
 
@@ -189,3 +201,74 @@
         except KeyError:
             msg("Duplicated an attendee with no RSVP: %r" % (attendee,))
         return accepted
+
+
+
+class Eventer(ProfileBase):
+    """
+    A Calendar user who creates new events.
+    """
+    _eventTemplate = list(readComponents("""\
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.3//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:America/New_York
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0500
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:EDT
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0400
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:EST
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20101018T155431Z
+UID:C98AD237-55AD-4F7D-9009-0D355D835822
+DTEND;TZID=America/New_York:20101021T130000
+TRANSP:OPAQUE
+SUMMARY:Simple event
+DTSTART;TZID=America/New_York:20101021T120000
+DTSTAMP:20101018T155438Z
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+"""))[0]
+
+    def run(self):
+        self._call = LoopingCall(self._addEvent)
+        self._call.clock = self._reactor
+        # XXX Base this on something real
+        self._call.start(25)
+
+
+    def _addEvent(self):
+        calendars = self._calendarsOfType(caldavxml.calendar)
+
+        while calendars:
+            calendar = self.random.choice(calendars)
+            calendars.remove(calendar)
+
+            # Copy the template event and fill in some of its fields
+            # to make a new event to create on the calendar.
+            vcalendar = Component.duplicate(self._eventTemplate)
+            vevent = vcalendar.contents[u'vevent'][0]
+            tz = vevent.contents[u'created'][0].value.tzinfo
+            dtstamp = datetime.now(tz)
+            vevent.contents[u'created'][0].value = dtstamp
+            vevent.contents[u'dtstamp'][0].value = dtstamp
+            vevent.contents[u'dtstart'][0].value = dtstamp
+            vevent.contents[u'dtend'][0].value = dtstamp + timedelta(hours=1)
+            vevent.contents[u'uid'][0].value = unicode(uuid4())
+
+            href = '%s%s.ics' % (
+                calendar.url, vevent.contents[u'uid'][0].value)
+            return self._client.addEvent(href, vcalendar)

Modified: CalendarServer/trunk/contrib/performance/loadtest/test_ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_ical.py	2011-02-03 22:14:19 UTC (rev 6867)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_ical.py	2011-02-03 22:41:05 UTC (rev 6868)
@@ -16,7 +16,7 @@
 ##
 
 from vobject import readComponents
-from vobject.base import ContentLine
+from vobject.base import Component, ContentLine
 
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
@@ -863,7 +863,7 @@
         old = attendees[0]
         new = ContentLine.duplicate(old)
         new.params[u'CN'] = [u'Some Other Guy']
-        event = Event('/some/calendar/1234.ics', None, vevent)
+        event = Event(u'/some/calendar/1234.ics', None, vevent)
         self.client._events[event.url] = event
         self.client.changeEventAttendee(event.url, old, new)
 
@@ -873,6 +873,7 @@
         expectedResponseCode, method, url, headers, body = req
         self.assertEquals(method, 'PUT')
         self.assertEquals(url, 'http://127.0.0.1:80' + event.url)
+        self.assertIsInstance(url, str)
         self.assertEquals(headers.getRawHeaders('content-type'), ['text/calendar'])
 
         consumer = MemoryConsumer()
@@ -885,3 +886,69 @@
             self.assertEquals(attendees[1].params[u'CN'], [u'Some Other Guy'])
         finished.addCallback(cbFinished)
         return finished
+
+
+    def test_addEvent(self):
+        """
+        L{SnowLeopard.addEvent} PUTs the event passed to it to the
+        server and updates local state to reflect its existence.
+        """
+        requests = []
+        def request(*args):
+            result = Deferred()
+            requests.append((result, args))
+            return result
+        self.client._request = request
+
+        vcalendar = list(readComponents(EVENT))[0]
+        d = self.client.addEvent(u'/mumble/frotz.ics', vcalendar)
+
+        result, req = requests.pop(0)
+
+        # iCal PUTs the new VCALENDAR object.
+        expectedResponseCode, method, url, headers, body = req
+        self.assertEquals(method, 'PUT')
+        self.assertEquals(url, 'http://127.0.0.1:80/mumble/frotz.ics')
+        self.assertIsInstance(url, str)
+        self.assertEquals(headers.getRawHeaders('content-type'), ['text/calendar'])
+
+        consumer = MemoryConsumer()
+        finished = body.startProducing(consumer)
+        def cbFinished(ignored):
+            self.assertComponentsEqual(
+                list(readComponents(consumer.value()))[0],
+                vcalendar)
+        finished.addCallback(cbFinished)
+        return finished
+
+
+    def assertComponentsEqual(self, first, second):
+        self.assertEquals(first.name, second.name, "Component names not equal")
+        self.assertEquals(first.behavior, second.behavior, "Component behaviors not equal")
+
+        for k in first.contents:
+            if k not in second.contents:
+                self.fail("Content %r present in first but not second" % (k,))
+            self.assertEquals(
+                len(first.contents[k]), len(second.contents[k]), "Different length content %r" % (k,))
+            for (a, b) in zip(first.contents[k], second.contents[k]):
+                if isinstance(a, ContentLine):
+                    f = self.assertContentLinesEqual
+                elif isinstance(a, Component):
+                    f = self.assertComponentsEqual
+                else:
+                    f = self.assertEquals
+                f(a, b)
+        for k in second.contents:
+            if k not in first.contents:
+                self.fail("Content %r present in second but not first" % (k,))
+
+
+    def assertContentLinesEqual(self, first, second):
+        self.assertEquals(first.name, second.name, "ContentLine names not equal")
+        self.assertEquals(first.behavior, second.behavior, "ContentLine behaviors not equal")
+        self.assertEquals(first.value, second.value, "ContentLine values not equal")
+        self.assertEquals(first.params, second.params, "ContentLine params not equal")
+        self.assertEquals(
+            first.singletonparams, second.singletonparams,
+            "ContentLine singletonparams not equal")

Modified: CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py	2011-02-03 22:14:19 UTC (rev 6867)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py	2011-02-03 22:41:05 UTC (rev 6868)
@@ -26,7 +26,7 @@
 from twisted.trial.unittest import TestCase
 from twisted.internet.task import Clock
 
-from loadtest.profiles import Inviter, Accepter
+from loadtest.profiles import Eventer, Inviter, Accepter
 from loadtest.ical import Calendar, Event
 
 SIMPLE_EVENT = """\
@@ -171,6 +171,10 @@
         self.email = u"mailto:user%02d at example.com" % (number,)
 
 
+    def addEvent(self, href, vevent):
+        self._events[href] = Event(href, None, vevent)
+
+
     def addEventAttendee(self, href, attendee):
         vevent = self._events[href].vevent
         attendees = vevent.contents[u'vevent'][0].contents.setdefault(u'attendee', [])
@@ -476,3 +480,44 @@
         self.assertEquals(
             attendees[1].params[u'PARTSTAT'], [u'ACCEPTED'])
         self.assertNotIn(u'RSVP', attendees[1].params)
+
+
+
+class EventerTests(TestCase):
+    """
+    Tests for loadtest.profiles.Eventer, a profile which adds new
+    events on calendars.
+    """
+    def test_doNotAddEventOnInbox(self):
+        """
+        When the only calendar is a schedule inbox, no attempt is made
+        to add events on it.
+        """
+        calendar = Calendar(
+            caldavxml.schedule_inbox, u'inbox', u'/sched/inbox', None)
+        client = StubClient(21)
+        client._calendars.update({calendar.url: calendar})
+
+        eventer = Eventer(None, client, None)
+        eventer._addEvent()
+
+        self.assertEquals(client._events, {})
+
+
+    def test_addEvent(self):
+        """
+        When there is a normal calendar to add events to,
+        L{Eventer._addEvent} adds an event to it.
+        """
+        calendar = Calendar(
+            caldavxml.calendar, u'personal stuff', u'/cals/personal', None)
+        client = StubClient(31)
+        client._calendars.update({calendar.url: calendar})
+
+        eventer = Eventer(None, client, None)
+        eventer._addEvent()
+
+        self.assertEquals(len(client._events), 1)
+
+        # XXX Vary the event period/interval and the uid
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110203/ef84580b/attachment-0001.html>


More information about the calendarserver-changes mailing list