[CalendarServer-changes] [15046] CalendarServer/branches/users/sredmond
source_changes at macosforge.org
source_changes at macosforge.org
Mon Aug 17 13:29:08 PDT 2015
Revision: 15046
http://trac.calendarserver.org//changeset/15046
Author: sredmond at apple.com
Date: 2015-08-17 13:29:08 -0700 (Mon, 17 Aug 2015)
Log Message:
-----------
Creates private copy of (locally-modified) CalendarServer/trunk for client sim development
Modified Paths:
--------------
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py
CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py
Added Paths:
-----------
CalendarServer/branches/users/sredmond/clientsim/
Property changes on: CalendarServer/branches/users/sredmond/clientsim
___________________________________________________________________
Added: svn:ignore
+ .develop
.settings
.project
.pydevproject
*.egg-info
*.pyc
data
build
subprojects
_run
Added: svn:mergeinfo
+ /CalDAVTester/trunk:11193-11198
/CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/release/CalendarServer-5.1-dev:11846
/CalendarServer/branches/release/CalendarServer-5.2-dev:11972,12357-12358,12794,12814
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/cross-pod-sharing:12038-12191
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11607-11871
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/json:11622-11912
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/performance-tweaks:11824-11836
/CalendarServer/branches/users/cdaboo/pod2pod-migration:14338-14520
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/reverse-proxy-pods:11875-11900
/CalendarServer/branches/users/cdaboo/scheduling-queue-refresh:11783-12557
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/sharing-in-the-store:11935-12016
/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/gaya/cleanrevisions:12152-12334
/CalendarServer/branches/users/gaya/groupsharee2:13669-13773
/CalendarServer/branches/users/gaya/sharedgroupfixes:12120-12142
/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
/CalendarServer/branches/users/glyph/hang-fix:11465-11491
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
/CalendarServer/branches/users/glyph/whenNotProposed:11881-11897
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/move2who:12819-12860
/CalendarServer/branches/users/sagen/move2who-2:12861-12898
/CalendarServer/branches/users/sagen/move2who-3:12899-12913
/CalendarServer/branches/users/sagen/move2who-4:12914-13157
/CalendarServer/branches/users/sagen/move2who-5:13158-13163
/CalendarServer/branches/users/sagen/newcua:13309-13327
/CalendarServer/branches/users/sagen/newcua-1:13328-13330
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/recordtypes:13648-13656
/CalendarServer/branches/users/sagen/recordtypes-2:13657
/CalendarServer/branches/users/sagen/request-socket:14748-14767
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
/CalendarServer/branches/users/sagen/trashcan:14185-14269
/CalendarServer/branches/users/sagen/trashcan-2:14270-14324
/CalendarServer/branches/users/sagen/trashcan-3:14325-14450
/CalendarServer/branches/users/sagen/trashcan-4:14451-14471
/CalendarServer/branches/users/sagen/trashcan-5:14471-14555
/CalendarServer/branches/users/wsanchez/psycopg2cffi:14427-14439
/CalendarServer/branches/users/wsanchez/transations:5515-5593
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/accounts.csv 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv 2015-08-17 20:29:08 UTC (rev 15046)
@@ -96,4 +96,4 @@
user96,user96,User 96,user96 at example.com,10000000-0000-0000-0000-000000000096
user97,user97,User 97,user97 at example.com,10000000-0000-0000-0000-000000000097
user98,user98,User 98,user98 at example.com,10000000-0000-0000-0000-000000000098
-user99,user99,User 99,user99 at example.com,10000000-0000-0000-0000-000000000099
+user99,user99,User 99,user99 at example.com,10000000-0000-0000-0000-000000000099
\ No newline at end of file
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ampsim.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -56,7 +56,8 @@
from twext.enterprise.adbapi2 import Pickle
-from contrib.performance.loadtest.sim import _DirectoryRecord, LoadSimulator
+from contrib.performance.loadtest.sim import LoadSimulator
+from contrib.performance.loadtest.records import DirectoryRecord
class Configure(Command):
"""
@@ -79,7 +80,7 @@
class Account(Command):
"""
- This message represents a L{_DirectoryRecord} loaded by the manager process
+ This message represents a L{DirectoryRecord} loaded by the manager process
being relayed to a worker.
"""
arguments = [
@@ -106,7 +107,7 @@
@Account.responder
def account(self, **kw):
- self.records.append(_DirectoryRecord(**kw))
+ self.records.append(DirectoryRecord(**kw))
return {}
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -18,40 +18,28 @@
from caldavclientlibrary.protocol.caldav.definitions import caldavxml
from caldavclientlibrary.protocol.caldav.definitions import csxml
-from caldavclientlibrary.protocol.url import URL
from caldavclientlibrary.protocol.webdav.definitions import davxml
-from caldavclientlibrary.protocol.webdav.propfindparser import PropFindParser
+from caldavclientlibrary.protocol.url import URL
-from calendarserver.push.amppush import subscribeToIDs
-from calendarserver.tools.notifications import PubSubClientFactory
-
-from contrib.performance.httpauth import AuthHandlerAgent
-from contrib.performance.httpclient import StringProducer, readBody
+from contrib.performance.httpclient import readBody
from contrib.performance.loadtest.subscribe import Periodical
+from contrib.performance.loadtest.resources import Event, Calendar
+from contrib.performance.loadtest.requester import Requester, IncorrectResponseCode
+from contrib.performance.loadtest.push import PushMonitor
from pycalendar.datetime import DateTime
from pycalendar.duration import Duration
from pycalendar.timezone import Timezone
-from twext.internet.adaptendpoint import connect
-from twext.internet.gaiendpoint import GAIEndpoint
-from twisted.internet.ssl import ClientContextFactory
-
-from twisted.internet.defer import Deferred, inlineCallbacks, returnValue, \
- succeed
from twisted.internet.task import LoopingCall
-from twisted.python.filepath import FilePath
-from twisted.python.log import addObserver, err, msg
-from twisted.python.util import FancyEqMixin
-from twisted.web.client import Agent, ContentDecoderAgent, GzipDecoder, \
- _DeprecatedToCurrentPolicyForHTTPS
-from twisted.web.http import OK, MULTI_STATUS, CREATED, NO_CONTENT, PRECONDITION_FAILED, MOVED_PERMANENTLY, \
- FORBIDDEN, FOUND
+from twisted.internet.defer import succeed, Deferred, inlineCallbacks, returnValue
+from twisted.python.log import err, msg
+from twisted.web.http import OK, MULTI_STATUS, CREATED, NO_CONTENT, FORBIDDEN, PRECONDITION_FAILED, MOVED_PERMANENTLY, FOUND
from twisted.web.http_headers import Headers
from twistedcaldav.ical import Component, Property
-from urlparse import urlparse, urlunparse, urlsplit, urljoin
+from urlparse import urlparse, urlsplit
from uuid import uuid4
from xml.etree import ElementTree
@@ -59,193 +47,32 @@
import os
import random
-ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
+"""
+run
+ startup
+ deserialize
+ _startupPropfindWellKnown
+ _principalPropfindInitial
+ _extractPrincipalDetails
+ _checkCalendarsForEvents
+"""
-def loadRequestBody(clientType, label):
- return FilePath(__file__).sibling('request-data').child(clientType).child(label + '.request').getContent()
+ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
-SUPPORTED_REPORT_SET = '{DAV:}supported-report-set'
+SUPPORTED_REPORT_SET = davxml.supported_report_set.text
-class IncorrectResponseCode(Exception):
- """
- Raised when a response has a code other than the one expected.
+class Attendee(Property):
+ def __init__(self, ):
+ pass
- @ivar expected: The response codes which was expected.
- @type expected: C{tuple} of C{int}
- @ivar response: The response which was received
- @type response: L{twisted.web.client.Response}
- """
- def __init__(self, expected, response):
- self.expected = expected
- self.response = response
-
-
-
class MissingCalendarHome(Exception):
"""
Raised when the calendar home for a user is 404
"""
-
-class XMPPPush(object, FancyEqMixin):
- """
- This represents an XMPP PubSub location where push notifications for
- particular calendar home might be received.
- """
- compareAttributes = ('server', 'uri', 'pushkey')
-
- def __init__(self, server, uri, pushkey):
- self.server = server
- self.uri = uri
- self.pushkey = pushkey
-
-
-
-def u2str(data):
- return data.encode("utf-8") if type(data) is unicode else data
-
-
-
-class Event(object):
- def __init__(self, serializeBasePath, url, etag, component=None):
- self.serializeBasePath = serializeBasePath
- self.url = url
- self.etag = etag
- self.scheduleTag = None
- if component is not None:
- self.component = component
- self.uid = component.resourceUID() if component is not None else None
-
-
- def getUID(self):
- """
- Return the UID of the calendar resource.
- """
- return self.uid
-
-
- def serializePath(self):
- if self.serializeBasePath:
- calendar = os.path.join(self.serializeBasePath, self.url.split("/")[-2])
- if not os.path.exists(calendar):
- os.makedirs(calendar)
- return os.path.join(calendar, self.url.split("/")[-1])
- else:
- return None
-
-
- def serialize(self):
- """
- Create a dict of the data so we can serialize as JSON.
- """
-
- result = {}
- for attr in ("url", "etag", "scheduleTag", "uid",):
- result[attr] = getattr(self, attr)
- return result
-
-
- @staticmethod
- def deserialize(serializeLocation, data):
- """
- Convert dict (deserialized from JSON) into an L{Event}.
- """
-
- event = Event(serializeLocation, None, None)
- for attr in ("url", "etag", "scheduleTag", "uid",):
- setattr(event, attr, u2str(data[attr]))
- return event
-
-
- @property
- def component(self):
- """
- Data always read from disk - never cached in the object.
- """
- path = self.serializePath()
- if path and os.path.exists(path):
- f = open(path)
- comp = Component.fromString(f.read())
- f.close()
- return comp
- else:
- return None
-
-
- @component.setter
- def component(self, component):
- """
- Data always written to disk - never cached on the object.
- """
- path = self.serializePath()
- if path:
- if component is None:
- os.remove(path)
- else:
- f = open(path, "w")
- f.write(str(component))
- f.close()
- self.uid = component.resourceUID() if component is not None else None
-
-
- def removed(self):
- """
- Resource no longer exists on the server - remove associated data.
- """
- path = self.serializePath()
- if path and os.path.exists(path):
- os.remove(path)
-
-
-
-class Calendar(object):
- def __init__(self, resourceType, componentTypes, name, url, changeToken):
- self.resourceType = resourceType
- self.componentTypes = componentTypes
- self.name = name
- self.url = url
- self.changeToken = changeToken
- self.events = {}
-
-
- def serialize(self):
- """
- Create a dict of the data so we can serialize as JSON.
- """
-
- result = {}
- for attr in ("resourceType", "name", "url", "changeToken"):
- result[attr] = getattr(self, attr)
- result["componentTypes"] = list(sorted(self.componentTypes))
- result["events"] = sorted(self.events.keys())
- return result
-
-
- @staticmethod
- def deserialize(data, events):
- """
- Convert dict (deserialized from JSON) into an L{Calendar}.
- """
-
- calendar = Calendar(None, None, None, None, None)
- for attr in ("resourceType", "name", "url", "changeToken"):
- setattr(calendar, attr, u2str(data[attr]))
- calendar.componentTypes = set(map(u2str, data["componentTypes"]))
-
- for event in data["events"]:
- url = urljoin(calendar.url, event)
- if url in events:
- calendar.events[event] = events[url]
- else:
- # Ughh - an event is missing - force changeToken to empty to trigger full resync
- calendar.changeToken = ""
- return calendar
-
-
-
class BaseClient(object):
"""
Base interface for all simulated clients.
@@ -259,7 +86,7 @@
_client_id = None # Unique id for the client itself
- def _setEvent(self, href, event):
+ def _cacheEvent(self, href, event):
"""
Cache the provided event
"""
@@ -268,7 +95,7 @@
self._calendars[calendar + '/'].events[basePath] = event
- def _removeEvent(self, href):
+ def _invalidateEvent(self, href):
"""
Remove event from local cache.
"""
@@ -278,6 +105,21 @@
del self._calendars[calendar + '/'].events[basePath]
+ def _cacheCalendar(self, href, calendar):
+ """
+ Cache the provided L{Calendar}
+ """
+ self._calendars[href] = calendar
+
+
+ def _invalidateCalendar(self, href):
+ """
+ Remove calendar from the local cache
+ """
+ if href in self._calendars:
+ del self._calendars[href]
+
+
def addEvent(self, href, calendar):
"""
Called when a profile needs to add an event (no scheduling).
@@ -321,46 +163,29 @@
"""
raise NotImplementedError("%r does not implement changeEventAttendee" % (self.__class__,))
+ def addCalendar(self, href, calendar):
+ """
+ Called when a profile needs to add a new calendar.
+ """
+ raise NotImplementedError("%r does not implement addCalendar" % (self.__class__,))
+ def changeCalendar(self, href, calendar):
+ """
+ Called when a profile needs to change a calendar.
+ """
+ raise NotImplementedError("%r does not implement changeCalendar" % (self.__class__,))
-class _PubSubClientFactory(PubSubClientFactory):
- """
- Factory for XMPP pubsub functionality.
- """
- def __init__(self, client, *args, **kwargs):
- PubSubClientFactory.__init__(self, *args, **kwargs)
- self._client = client
+ def deleteCalendar(self, href):
+ """
+ Called when a profile needs to delete a calendar.
+ """
+ raise NotImplementedError("%r does not implement deleteCalendar" % (self.__class__,))
- def initFailed(self, reason):
- print('XMPP initialization failed', reason)
- def authFailed(self, reason):
- print('XMPP Authentication failed', reason)
- def handleMessageEventItems(self, iq):
- item = iq.firstChildElement().firstChildElement()
- if item:
- node = item.getAttribute("node")
- if node:
- url, _ignore_name, _ignore_kind = self.nodes.get(node, (None, None, None))
- if url is not None:
- self._client._checkCalendarsForEvents(url, push=True)
-
-
-
-class WebClientContextFactory(ClientContextFactory):
- """
- A web context factory which ignores the hostname and port and does no
- certificate verification.
- """
- def getContext(self, hostname, port):
- return ClientContextFactory.getContext(self)
-
-
-
class BaseAppleClient(BaseClient):
"""
Implementation of common OS X/iOS client behavior.
@@ -368,7 +193,8 @@
_client_type = "Generic"
- USER_AGENT = None # Override this for specific clients
+ # Override this for specific clients
+ USER_AGENT = None
# The default interval, used if none is specified in external
# configuration.
@@ -380,9 +206,6 @@
# Override and turn on if client supports Sync REPORT
_SYNC_REPORT = False
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
# Override and turn off if client does not support attendee lookups
_ATTENDEE_LOOKUPS = True
@@ -416,32 +239,23 @@
self,
reactor,
root,
- principalPathTemplate,
serializePath,
record,
auth,
title=None,
calendarHomePollInterval=None,
- supportPush=True,
supportAmpPush=True,
ampPushHost=None,
ampPushPort=62311,
):
-
self._client_id = str(uuid4())
self.reactor = reactor
- # The server might use gzip encoding
- agent = Agent(
- self.reactor,
- contextFactory=_DeprecatedToCurrentPolicyForHTTPS(WebClientContextFactory()),
+ self.requester = Requester(
+ root, self.getDefaultHeaders(), title,
+ record.uid, self._client_id, auth, self.reactor
)
- agent = ContentDecoderAgent(agent, [("gzip", GzipDecoder)])
- self.agent = AuthHandlerAgent(agent, auth)
-
- self.root = root
- self.principalPathTemplate = principalPathTemplate
self.record = record
self.title = title if title else self._client_type
@@ -450,208 +264,62 @@
calendarHomePollInterval = self.CALENDAR_HOME_POLL_INTERVAL
self.calendarHomePollInterval = calendarHomePollInterval
- self.supportPush = supportPush
+ if supportAmpPush:
+ if ampPushHost is None:
+ ampPushHost = urlparse(root)[1].split(":")[0]
+ self.monitor = PushMonitor(self.reactor, ampPushHost, ampPushPort, self.updateCalendarHomeFromPush)
+ else:
+ self.monitor = None
- self.supportAmpPush = supportAmpPush
- if ampPushHost is None:
- ampPushHost = urlparse(self.root)[1].split(":")[0]
- self.ampPushHost = ampPushHost
- self.ampPushPort = ampPushPort
-
self.serializePath = serializePath
self.supportSync = self._SYNC_REPORT
- # Keep track of the calendars on this account, keys are
- # Calendar URIs, values are Calendar instances.
- self._calendars = {}
-
# The principalURL found during discovery
self.principalURL = None
# The principal collection found during startup
self.principalCollection = None
+ # Keep track of the calendars on this account, keys are
+ # Calendar URIs, values are Calendar instances.
+ self._calendars = {}
+
# Keep track of the events on this account, keys are event
# URIs (which are unambiguous across different calendars
# because they start with the uri of the calendar they are
# part of), values are Event instances.
self._events = {}
- # Keep track of which calendar homes are being polled
- self._checking = set()
-
- # Keep track of XMPP parameters for calendar homes we encounter. This
- # dictionary has calendar home URLs as keys and XMPPPush instances as
- # values.
- self.xmpp = {}
-
- self.ampPushKeys = {}
-
- # Keep track of push factories so we can unsubscribe at shutdown
- self._pushFactories = []
-
# Allow events to go out into the world.
self.catalog = {
"eventChanged": Periodical(),
}
+ self._checking = set()
- def _addDefaultHeaders(self, headers):
- """
- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
- """
- headers.setRawHeaders('User-Agent', [self.USER_AGENT])
-
-
- @inlineCallbacks
- def _request(self, expectedResponseCodes, method, url, headers=None, body=None, method_label=None):
- """
- Execute a request and check against the expected response codes.
- """
- if type(expectedResponseCodes) is int:
- expectedResponseCodes = (expectedResponseCodes,)
- if headers is None:
- headers = Headers({})
- self._addDefaultHeaders(headers)
- msg(
- type="request",
- method=method_label if method_label else method,
- url=url,
- user=self.record.uid,
- client_type=self.title,
- client_id=self._client_id,
- )
-
- before = self.reactor.seconds()
- response = yield self.agent.request(method, url, headers, body)
-
- # XXX This is time to receive response headers, not time
- # to receive full response. Should measure the latter, if
- # not both.
- after = self.reactor.seconds()
-
- success = response.code in expectedResponseCodes
-
- msg(
- type="response",
- success=success,
- method=method_label if method_label else method,
- headers=headers,
- body=body,
- code=response.code,
- user=self.record.uid,
- client_type=self.title,
- client_id=self._client_id,
- duration=(after - before),
- url=url,
- )
-
- if success:
- returnValue(response)
-
- raise IncorrectResponseCode(expectedResponseCodes, response)
-
-
- def _parseMultiStatus(self, response, otherTokens=False):
- """
- Parse a <multistatus> - might need to return other top-level elements
- in the response - e.g. DAV:sync-token
- I{PROPFIND} request for the principal URL.
-
- @type response: C{str}
- @rtype: C{cls}
- """
- parser = PropFindParser()
- parser.parseData(response)
- if otherTokens:
- return (parser.getResults(), parser.getOthers(),)
- else:
- return parser.getResults()
-
_CALENDAR_TYPES = set([
caldavxml.calendar,
caldavxml.schedule_inbox,
])
- @inlineCallbacks
- def _propfind(self, url, body, depth='0', allowedStatus=(MULTI_STATUS,), method_label=None):
- """
- Issue a PROPFIND on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- if depth is not None:
- hdrs.addRawHeader('depth', depth)
- response = yield self._request(
- allowedStatus,
- 'PROPFIND',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
+ def getDefaultHeaders(self):
+ return {
+ 'User-Agent': [self.USER_AGENT],
+ 'Accept': ['*/*'],
+ 'Accept-Language': ['en-us'],
+ 'Accept-Encoding': ['gzip,deflate'],
+ 'Connection': ['keep-alive']
+ }
- body = yield readBody(response)
- result = self._parseMultiStatus(body) if response.code == MULTI_STATUS else None
-
- returnValue((response, result,))
-
-
@inlineCallbacks
- def _proppatch(self, url, body, method_label=None):
- """
- Issue a PROPPATCH on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- response = yield self._request(
- (OK, MULTI_STATUS,),
- 'PROPPATCH',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
- if response.code == MULTI_STATUS:
- body = yield readBody(response)
- result = self._parseMultiStatus(body)
- returnValue(result)
- else:
- returnValue(None)
-
-
- @inlineCallbacks
- def _report(self, url, body, depth='0', allowedStatus=(MULTI_STATUS,), otherTokens=False, method_label=None):
- """
- Issue a REPORT on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- if depth is not None:
- hdrs.addRawHeader('depth', depth)
- response = yield self._request(
- allowedStatus,
- 'REPORT',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
-
- body = yield readBody(response)
- result = self._parseMultiStatus(body, otherTokens) if response.code == MULTI_STATUS else None
-
- returnValue(result)
-
-
- @inlineCallbacks
def _startupPropfindWellKnown(self):
"""
Issue a PROPFIND on the /.well-known/caldav/ URL
"""
location = "/.well-known/caldav/"
- response, result = yield self._propfind(
+ response, result = yield self.requester.propfind(
location,
self._STARTUP_WELL_KNOWN,
allowedStatus=(MULTI_STATUS, MOVED_PERMANENTLY, FOUND,),
@@ -662,7 +330,7 @@
if response.code in (MOVED_PERMANENTLY, FOUND,):
location = response.headers.getRawHeaders("location")[0]
location = urlsplit(location)[2]
- response, result = yield self._propfind(
+ response, result = yield self.requester.propfind(
location,
self._STARTUP_WELL_KNOWN,
allowedStatus=(MULTI_STATUS),
@@ -678,8 +346,8 @@
Issue a PROPFIND on the /principals/users/<uid> URL to retrieve
the /principals/__uids__/<guid> principal URL
"""
- principalPath = self.principalPathTemplate % (user,)
- _ignore_response, result = yield self._propfind(
+ principalPath = '/principals/users/%s' % (user,)
+ _ignore_response, result = yield self.requester.propfind(
principalPath,
self._STARTUP_PRINCIPAL_PROPFIND_INITIAL,
method_label="PROPFIND{find-principal}",
@@ -694,7 +362,7 @@
user and return a L{Principal} instance constructed from the
response.
"""
- _ignore_response, result = yield self._propfind(
+ _ignore_response, result = yield self.requester.propfind(
self.principalURL,
self._STARTUP_PRINCIPAL_PROPFIND,
method_label="PROPFIND{principal}",
@@ -706,7 +374,7 @@
"""
Issue a principal-search-property-set REPORT against the chosen URL
"""
- return self._report(
+ return self.requester.report(
principalCollectionSet,
self._STARTUP_PRINCIPALS_REPORT,
allowedStatus=(OK,),
@@ -715,28 +383,14 @@
@inlineCallbacks
- def _calendarHomePropfind(self, calendarHomeSet):
- """
- Do the poll Depth:1 PROPFIND on the calendar home.
- """
- if not calendarHomeSet.endswith('/'):
- calendarHomeSet = calendarHomeSet + '/'
- _ignore_response, result = yield self._propfind(
- calendarHomeSet,
- self._POLL_CALENDARHOME_PROPFIND,
- depth='1',
- method_label="PROPFIND{home}",
- )
- calendars = self._extractCalendars(result, calendarHomeSet)
- returnValue((calendars, result,))
-
-
- @inlineCallbacks
def _extractPrincipalDetails(self):
# Using the actual principal URL, retrieve principal information
+ # XXX We could be recording more information here
principal = yield self._principalPropfind()
hrefs = principal.getHrefProperties()
+ # from pprint import pprint
+ # pprint(hrefs)
# Remember our outbox and ignore notifications
self.outbox = hrefs[caldavxml.schedule_outbox_URL].toString()
@@ -764,14 +418,114 @@
returnValue(principal)
+ def startup(self):
+ """
+ Overridden by subclasses of BaseAppleClient.
+ """
+ raise NotImplementedError
+ def calendarCheckLoop(self, calendarHome):
+ """
+ Periodically check the calendar home for changes to calendars.
+ """
+ pollCalendarHome = LoopingCall(
+ self.checkCalendarsForEvents, calendarHome)
+ return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
+
+ @inlineCallbacks
+ def updateCalendarHomeFromPush(self, calendarHomeSet):
+ """
+ Emulate the client behavior upon receiving a notification that the
+ given calendar home has changed.
+ """
+ # Todo - ensure that the self._checking set is properly cleared even if there is an error
+ self._checking.add(calendarHomeSet)
+ result = yield self._poll(calendarHomeSet, firstTime=False)
+
+ # Todo - should this be a returnValue?
+ yield self._newOperation("push", result)
+
+ @inlineCallbacks
+ def checkCalendarsForEvents(self, calendarHomeSet, firstTime=False):
+ """
+ The actions a client does when polling for changes, or in response to a
+ push notification of a change. There are some actions done on the first poll
+ we should emulate.
+ """
+
+ result = True
+ try:
+ result = yield self._newOperation("poll", self._poll(calendarHomeSet, firstTime))
+ finally:
+ if result:
+ try:
+ self._checking.remove(calendarHomeSet)
+ except KeyError:
+ pass
+ returnValue(result)
+
+ """
+ REFRESH UTILITIES
+ """
+
+ @inlineCallbacks
+ def _poll(self, calendarHomeSet, firstTime):
+ if calendarHomeSet in self._checking:
+ returnValue(False)
+ self._checking.add(calendarHomeSet)
+
+ calendars, results = yield self._calendarHomePropfind(calendarHomeSet)
+
+ # First time operations
+ if firstTime:
+ yield self._pollFirstTime1(results[calendarHomeSet], calendars)
+
+ # Normal poll
+ for cal in calendars:
+ newToken = cal.changeToken
+ if cal.url not in self._calendars:
+ # Calendar seen for the first time - reload it
+ self._calendars[cal.url] = cal
+ cal.changeToken = ""
+ yield self._updateCalendar(self._calendars[cal.url], newToken)
+ elif self._calendars[cal.url].changeToken != newToken:
+ # Calendar changed - reload it
+ yield self._updateCalendar(self._calendars[cal.url], newToken)
+
+ # When there is no sync REPORT, clients have to do a full PROPFIND
+ # on the notification collection because there is no ctag
+ if self.notificationURL is not None and not self.supportSync:
+ yield self._notificationPropfind(self.notificationURL)
+ yield self._notificationChangesPropfind(self.notificationURL)
+
+ # One time delegate expansion
+ if firstTime:
+ # yield self._pollFirstTime2()
+ pass
+
+ returnValue(True)
+
+ @inlineCallbacks
+ def _calendarHomePropfind(self, calendarHomeSet):
+ """
+ Do the poll Depth:1 PROPFIND on the calendar home.
+ """
+ if not calendarHomeSet.endswith('/'):
+ calendarHomeSet = calendarHomeSet + '/'
+ _ignore_response, result = yield self.requester.propfind(
+ calendarHomeSet,
+ self._POLL_CALENDARHOME_PROPFIND,
+ depth='1',
+ method_label="PROPFIND{home}",
+ )
+ calendars = self._extractCalendars(result, calendarHomeSet)
+ returnValue((calendars, result,))
+
+
def _extractCalendars(self, results, calendarHome=None):
"""
Parse a calendar home PROPFIND response and create local state
representing the calendars it contains.
-
- If XMPP push is enabled, also look for and record information about
- that from the response.
"""
calendars = []
for href in results:
@@ -785,18 +539,8 @@
pass
else:
if pushkey:
- self.ampPushKeys[href] = pushkey
+ self.monitor.addPushkey(href, pushkey)
- try:
- server = text[csxml.xmpp_server]
- uri = text[csxml.xmpp_uri]
- pushkey = text[csxml.pushkey]
- except KeyError:
- pass
- else:
- if server and uri:
- self.xmpp[href] = XMPPPush(server, uri, pushkey)
-
nodes = results[href].getNodeProperties()
for nodeType in nodes[davxml.resourcetype]:
if nodeType.tag in self._CALENDAR_TYPES:
@@ -819,6 +563,40 @@
return calendars
+
+
+ @inlineCallbacks
+ def _pollFirstTime1(self, homeNode, calendars):
+ # Detect sync report if needed
+ if self.supportSync:
+ nodes = homeNode.getNodeProperties()
+ syncnodes = nodes[davxml.supported_report_set].findall(
+ str(davxml.supported_report) + "/" +
+ str(davxml.report) + "/" +
+ str(davxml.sync_collection)
+ )
+ self.supportSync = len(syncnodes) != 0
+
+ # Patch calendar properties
+ for cal in calendars:
+ if cal.name != "inbox":
+ # yield self.requester.proppatch(
+ # cal.url,
+ # self._STARTUP_PROPPATCH_CALENDAR_COLOR,
+ # method_label="PROPPATCH{calendar}",
+ # )
+ yield self.requester.proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_ORDER,
+ method_label="PROPPATCH{calendar}",
+ )
+ yield self.requester.proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE,
+ method_label="PROPPATCH{calendar}",
+ )
+
+
def _updateCalendar(self, calendar, newToken):
"""
Update the local cached data for a calendar in an appropriate manner.
@@ -840,11 +618,11 @@
# the sim can fire a PUT between the PROPFIND and when process the removals.
old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
- _ignore_response, result = yield self._propfind(
+ _ignore_response, result = yield self.requester.propfind(
calendar.url,
self._POLL_CALENDAR_PROPFIND_D1,
+ method_label="PROPFIND{calendar}",
depth='1',
- method_label="PROPFIND{calendar}"
)
yield self._updateApplyChanges(calendar, result, old_hrefs)
@@ -868,7 +646,7 @@
# Get changes from sync REPORT (including the other nodes at the top-level
# which will have the new sync token.
fullSync = not calendar.changeToken
- result = yield self._report(
+ result = yield self.requester.report(
calendar.url,
self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': calendar.changeToken},
depth='1',
@@ -879,7 +657,7 @@
if result is None:
if not fullSync:
fullSync = True
- result = yield self._report(
+ result = yield self.requester.report(
calendar.url,
self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': ''},
depth='1',
@@ -905,13 +683,13 @@
# Differentiate a remove vs new/update result
if result[responseHref].getStatus() / 100 == 2:
if responseHref not in self._events:
- self._setEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
+ self._cacheEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
event = self._events[responseHref]
if event.etag != etag:
changed.append(responseHref)
elif result[responseHref].getStatus() == 404:
- self._removeEvent(responseHref)
+ self._invalidateEvent(responseHref)
yield self._updateChangedEvents(calendar, changed)
@@ -920,7 +698,7 @@
# Detect removed items and purge them
remove_hrefs = old_hrefs - set(changed)
for href in remove_hrefs:
- self._removeEvent(href)
+ self._invalidateEvent(href)
# Now update calendar to the new token taken from the report
for node in others:
@@ -951,7 +729,7 @@
continue
if responseHref not in self._events:
- self._setEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
+ self._cacheEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
event = self._events[responseHref]
if event.etag != etag:
@@ -963,7 +741,7 @@
# Detect removed items and purge them
remove_hrefs = old_hrefs - set(all_hrefs)
for href in remove_hrefs:
- self._removeEvent(href)
+ self._invalidateEvent(href)
@inlineCallbacks
@@ -979,9 +757,9 @@
changed = changed[self.MULTIGET_BATCH_SIZE:]
multistatus = yield self._eventReport(calendar.url, batchedHrefs)
- for responseHref in batchedHrefs:
+ for href in batchedHrefs:
try:
- res = multistatus[responseHref]
+ res = multistatus[href]
except KeyError:
# Resource might have been deleted
continue
@@ -993,18 +771,10 @@
except KeyError:
scheduleTag = None
body = text[caldavxml.calendar_data]
- self.eventChanged(responseHref, etag, scheduleTag, body)
+ component = Component.fromString(body)
+ self._updateEventCache(href, etag, scheduleTag, component)
- def eventChanged(self, href, etag, scheduleTag, body):
- event = self._events[href]
- event.etag = etag
- if scheduleTag is not None:
- event.scheduleTag = scheduleTag
- event.component = Component.fromString(body)
- self.catalog["eventChanged"].issue(href)
-
-
def _eventReport(self, calendar, events):
# Next do a REPORT on events that might have information
# we don't know about.
@@ -1018,7 +788,7 @@
if len(events) > 75:
label_suffix = "huge"
- return self._report(
+ return self.requester.report(
calendar,
self._POLL_CALENDAR_MULTIGET_REPORT % {'hrefs': hrefs},
depth=None,
@@ -1027,101 +797,8 @@
@inlineCallbacks
- def _checkCalendarsForEvents(self, calendarHomeSet, firstTime=False, push=False):
- """
- The actions a client does when polling for changes, or in response to a
- push notification of a change. There are some actions done on the first poll
- we should emulate.
- """
-
- result = True
- try:
- result = yield self._newOperation("push" if push else "poll", self._poll(calendarHomeSet, firstTime))
- finally:
- if result:
- try:
- self._checking.remove(calendarHomeSet)
- except KeyError:
- pass
- returnValue(result)
-
-
- @inlineCallbacks
- def _poll(self, calendarHomeSet, firstTime):
- if calendarHomeSet in self._checking:
- returnValue(False)
- self._checking.add(calendarHomeSet)
-
- calendars, results = yield self._calendarHomePropfind(calendarHomeSet)
-
- # First time operations
- if firstTime:
- yield self._pollFirstTime1(results[calendarHomeSet], calendars)
-
- # Normal poll
- for cal in calendars:
- newToken = cal.changeToken
- if cal.url not in self._calendars:
- # Calendar seen for the first time - reload it
- self._calendars[cal.url] = cal
- cal.changeToken = ""
- yield self._updateCalendar(self._calendars[cal.url], newToken)
- elif self._calendars[cal.url].changeToken != newToken:
- # Calendar changed - reload it
- yield self._updateCalendar(self._calendars[cal.url], newToken)
-
- # When there is no sync REPORT, clients have to do a full PROPFIND
- # on the notification collection because there is no ctag
- if self.notificationURL is not None and not self.supportSync:
- yield self._notificationPropfind(self.notificationURL)
- yield self._notificationChangesPropfind(self.notificationURL)
-
- # One time delegate expansion
- if firstTime:
- yield self._pollFirstTime2()
-
- returnValue(True)
-
-
- @inlineCallbacks
- def _pollFirstTime1(self, homeNode, calendars):
- # Detect sync report if needed
- if self.supportSync:
- nodes = homeNode.getNodeProperties()
- syncnodes = nodes[davxml.supported_report_set].findall(
- str(davxml.supported_report) + "/" +
- str(davxml.report) + "/" +
- str(davxml.sync_collection)
- )
- self.supportSync = len(syncnodes) != 0
-
- # Patch calendar properties
- for cal in calendars:
- if cal.name != "inbox":
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_COLOR,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_ORDER,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE,
- method_label="PROPPATCH{calendar}",
- )
-
-
- def _pollFirstTime2(self):
- return self._principalExpand(self.principalURL)
-
-
- @inlineCallbacks
def _notificationPropfind(self, notificationURL):
- _ignore_response, result = yield self._propfind(
+ _ignore_response, result = yield self.requester.propfind(
notificationURL,
self._POLL_NOTIFICATION_PROPFIND,
method_label="PROPFIND{notification}",
@@ -1131,7 +808,7 @@
@inlineCallbacks
def _notificationChangesPropfind(self, notificationURL):
- _ignore_response, result = yield self._propfind(
+ _ignore_response, result = yield self.requester.propfind(
notificationURL,
self._POLL_NOTIFICATION_PROPFIND_D1,
depth='1',
@@ -1139,10 +816,12 @@
)
returnValue(result)
+ def _pollFirstTime2(self):
+ return self._principalExpand(self.principalURL)
@inlineCallbacks
def _principalExpand(self, principalURL):
- result = yield self._report(
+ result = yield self.requester.report(
principalURL,
self._STARTUP_PRINCIPAL_EXPAND,
depth=None,
@@ -1151,19 +830,7 @@
returnValue(result)
- def startup(self):
- raise NotImplementedError
-
- def _calendarCheckLoop(self, calendarHome):
- """
- Periodically check the calendar home for changes to calendars.
- """
- pollCalendarHome = LoopingCall(
- self._checkCalendarsForEvents, calendarHome)
- return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
-
-
@inlineCallbacks
def _newOperation(self, label, deferred):
before = self.reactor.seconds()
@@ -1202,55 +869,7 @@
returnValue(result)
- def _monitorPubSub(self, home, params):
- """
- Start monitoring the
- """
- host, port = params.server.split(':')
- port = int(port)
-
- service, _ignore_stuff = params.uri.split('?')
- service = service.split(':', 1)[1]
-
- # XXX What is the domain of the 2nd argument supposed to be? The
- # hostname we use to connect, or the same as the email address in the
- # user record?
- factory = _PubSubClientFactory(
- self, "%s@%s" % (self.record.uid, host),
- self.record.password, service,
- {params.pushkey: (home, home, "Calendar home")}, False,
- sigint=False)
- self._pushFactories.append(factory)
- connect(GAIEndpoint(self.reactor, host, port), factory)
-
-
- def _receivedPush(self, inboundID, dataChangedTimestamp, priority=5):
- for href, id in self.ampPushKeys.iteritems():
- if inboundID == id:
- self._checkCalendarsForEvents(href, push=True)
- break
- else:
- # somehow we are not subscribed to this id
- pass
-
-
- def _monitorAmpPush(self, home, pushKeys):
- """
- Start monitoring for AMP-based push notifications
- """
- subscribeToIDs(
- self.ampPushHost, self.ampPushPort, pushKeys,
- self._receivedPush, self.reactor
- )
-
-
@inlineCallbacks
- def _unsubscribePubSub(self):
- for factory in self._pushFactories:
- yield factory.unsubscribeAll()
-
-
- @inlineCallbacks
def run(self):
"""
Emulate a CalDAV client.
@@ -1262,37 +881,28 @@
calendarHome = hrefs[caldavxml.calendar_home_set].toString()
if calendarHome is None:
raise MissingCalendarHome
- yield self._checkCalendarsForEvents(calendarHome, firstTime=True)
+ yield self.checkCalendarsForEvents(calendarHome, firstTime=True)
returnValue(calendarHome)
calendarHome = yield self._newOperation("startup: %s" % (self.title,), startup())
-
self.started = True
- # Start monitoring PubSub notifications, if possible.
- # _checkCalendarsForEvents populates self.xmpp if it finds
- # anything.
- if self.supportPush and calendarHome in self.xmpp:
- self._monitorPubSub(calendarHome, self.xmpp[calendarHome])
+ # Start monitoring AMP push notifications, if possible
+ if self.monitor and self.monitor.isSubscribedTo(calendarHome):
+ self.monitor.begin()
# Run indefinitely.
yield Deferred()
- elif self.supportAmpPush and calendarHome in self.ampPushKeys:
- pushKeys = self.ampPushKeys.values()
- self._monitorAmpPush(calendarHome, pushKeys)
- # Run indefinitely.
- yield Deferred()
else:
# This completes when the calendar home poll loop completes, which
# currently it never will except due to an unexpected error.
- yield self._calendarCheckLoop(calendarHome)
+ yield self.calendarCheckLoop(calendarHome)
def stop(self):
"""
Called before connections are closed, giving a chance to clean up
"""
-
self.serialize()
- return self._unsubscribePubSub()
+ return self.monitor.unsubscribeFromAll()
def serializeLocation(self):
@@ -1384,94 +994,67 @@
)
return organizer
+ def _getEventSizeDescription(self, numAttendees):
+ if numAttendees > 75:
+ return "huge"
+ if numAttendees > 20:
+ return "large"
+ if numAttendees > 5:
+ return "medium"
+ return "small"
+ """ literally wtf is this event stuff
+submitEvent(event: Event, )
+
+"""
@inlineCallbacks
def addEventAttendee(self, href, attendee):
+ individual = attendee.parameterValue('CUTYPE') == 'INDIVIDUAL'
event = self._events[href]
component = event.component
+ vevent = component.mainComponent()
+ query = attendee.parameterValue('CN')
+
+ from pprint import pprint
# Trigger auto-complete behavior
- yield self._attendeeAutoComplete(component, attendee)
+ matchingPrincipals = yield self._principalSearchReport(query, isAttendeeSearch=individual)
+ for k, v in matchingPrincipals.items():
+ pprint(k)
+ for prop, val in v.getNodeProperties().items():
+ print("%s %s" % (prop, val.__dict__))
+ for child in val._children:
+ print(child.text)
- # If the event has no attendees, add ourselves as an attendee.
- attendees = list(component.mainComponent().properties('ATTENDEE'))
- if len(attendees) == 0:
- # First add ourselves as a participant and as the
- # organizer. In the future for this event we should
- # already have those roles.
- component.mainComponent().addProperty(self._makeSelfOrganizer())
- component.mainComponent().addProperty(self._makeSelfAttendee())
- attendees.append(attendee)
- component.mainComponent().addProperty(attendee)
+ uuids = []
- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
+ for principal_url, propfindresult in matchingPrincipals.items():
+ props = propfindresult.getNodeProperties()
+ for cuaddr in props.get(caldavxml.calendar_user_address_set):
+ # print(cuaddr)
+ uuids.append(cuaddr.text)
+ break
- # At last, upload the new event definition
- response = yield self._request(
- (NO_CONTENT, PRECONDITION_FAILED,),
- 'PUT',
- self.root + href.encode('utf-8'),
- Headers({
- 'content-type': ['text/calendar'],
- 'if-match': [event.etag]}),
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer-%s}" % (label_suffix,)
- )
+ print(uuids)
- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
+ start = vevent.getStartDateUTC()
+ end = vevent.getEndDateUTC()
- @inlineCallbacks
- def _attendeeAutoComplete(self, component, attendee):
+ yield self.requestAvailability(start, end, uuids)
- if self._ATTENDEE_LOOKUPS:
- # Temporarily use some non-test names (some which will return
- # many results, and others which will return fewer) because the
- # test account names are all too similar
- # name = attendee.parameterValue('CN').encode("utf-8")
- # prefix = name[:4].lower()
- prefix = random.choice([
- "chris", "cyru", "dre", "eric", "morg",
- "well", "wilfr", "witz"
- ])
+ # # Do free-busy lookups
+ # if individual:
+ # # When adding individual attendees, we only look up the availability
+ # # of the specific attendee
+ # yield self.checkAvailability()
+ # else:
+ # # When adding a location, we look up the availability of each location
+ # # returned by the principal search.
+ # yield self.checkAvailability()
- email = attendee.value()
- if email.startswith("mailto:"):
- email = email[7:]
- elif attendee.hasParameter('EMAIL'):
- email = attendee.parameterValue('EMAIL').encode("utf-8")
- # First try to discover some names to supply to the
- # auto-completion
- yield self._report(
- self.principalCollection,
- self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
- 'displayname': prefix,
- 'email': prefix,
- 'firstname': prefix,
- 'lastname': prefix,
- },
- depth=None,
- method_label="REPORT{psearch}",
- )
-
- # Now learn about the attendee's availability
- yield self.requestAvailability(
- component.mainComponent().getStartDateUTC(),
- component.mainComponent().getEndDateUTC(),
- [self.email, u'mailto:' + email],
- [component.resourceUID()]
- )
-
-
@inlineCallbacks
def changeEventAttendee(self, href, oldAttendee, newAttendee):
event = self._events[href]
@@ -1480,149 +1063,197 @@
# Change the event to have the new attendee instead of the old attendee
component.mainComponent().removeProperty(oldAttendee)
component.mainComponent().addProperty(newAttendee)
- okCodes = NO_CONTENT
- headers = Headers({
- 'content-type': ['text/calendar'],
- })
+
+ headers = Headers()
if event.scheduleTag is not None:
headers.addRawHeader('if-schedule-tag-match', event.scheduleTag)
- okCodes = (NO_CONTENT, PRECONDITION_FAILED,)
-
+ event.component = component
attendees = list(component.mainComponent().properties('ATTENDEE'))
- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
+ label_suffix = self._getEventSizeDescription(len(attendees))
+ method_label = "PUT{attendee-%s}" % (label_suffix,)
- response = yield self._request(
- okCodes,
- 'PUT',
- self.root + href.encode('utf-8'),
- headers, StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{attendee-%s}" % (label_suffix,),
- )
+ yield self.putEvent(href, event, headers=headers, method_label=method_label, new=False)
- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
-
@inlineCallbacks
- def deleteEvent(self, href):
+ def addInvite(self, event):
"""
- Issue a DELETE for the given URL and remove local state
- associated with that event.
+ Add an event that is an invite - i.e., has attendees. Presumably the appropriate principal searches and
+ free-busy lookups have already been accounted for (in addEventAttendee)
"""
+ vevent = event.component.mainComponent()
+ # If the event has no attendees, add ourselves as an attendee.
+ attendees = list(vevent.properties('ATTENDEE'))
+ if len(attendees) == 0:
+ # First add ourselves as a participant and as the
+ # organizer. In the future for this event we should
+ # already have those roles.
+ vevent.addProperty(self._makeSelfOrganizer())
+ vevent.addProperty(self._makeSelfAttendee())
- self._removeEvent(href)
+ label_suffix = self._getEventSizeDescription(len(attendees))
+ method_label = "PUT{organizer-%s}" % (label_suffix,)
- response = yield self._request(
- NO_CONTENT,
- 'DELETE',
- self.root + href.encode('utf-8'),
- method_label="DELETE{event}",
- )
- returnValue(response)
+ yield self.updateEvent(event, method_label=method_label)
@inlineCallbacks
- def addEvent(self, href, component, invite=False):
+ def addEvent(self, href, event):
+ """
+ client.addEvent(
+ Event e
+ """
headers = Headers({
- 'content-type': ['text/calendar'],
+ 'if-none-match': ['*']
})
+ yield self.putEvent(
+ href,
+ event,
+ headers=headers,
+ method_label="PUT{event}"
+ )
- attendees = list(component.mainComponent().properties('ATTENDEE'))
- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
+ # attendees = list(component.mainComponent().properties('ATTENDEE'))
+ # label_suffix = self._getEventSizeDescription(len(attendees))
+ # method_label = "PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}"
- response = yield self._request(
- CREATED,
- 'PUT',
- self.root + href.encode('utf-8'),
- headers,
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}",
- )
- self._localUpdateEvent(response, href, component)
+ @inlineCallbacks
+ def updateEvent(self, event, method_label="PUT{event}"):
+ headers = Headers({
+ 'if-match': [event.etag]
+ })
+ yield self.putEvent(event.url, event, headers=headers, method_label=method_label)
@inlineCallbacks
- def addInvite(self, href, component):
+ def putEvent(self, href, event, headers=None, method_label=None):
"""
- Add an event that is an invite - i.e., has attendees. We will do attendee lookups and freebusy
- checks on each attendee to simulate what happens when an organizer creates a new invite.
+ PUT an event to the server
"""
+ if headers == None:
+ headers = Headers()
+ headers.addRawHeader('content-type', 'text/calendar')
- # Do lookup and free busy of each attendee (not self)
- attendees = list(component.mainComponent().properties('ATTENDEE'))
- for attendee in attendees:
- if attendee.value() in (self.uuid, self.email):
- continue
- yield self._attendeeAutoComplete(component, attendee)
+ okCodes = (CREATED, NO_CONTENT, PRECONDITION_FAILED)
- # Now do a normal PUT
- yield self.addEvent(href, component, invite=True)
+ # At last, upload the new event definition
+ response = yield self.requester.put(
+ okCodes,
+ href,
+ event.component,
+ headers=headers,
+ method_label=method_label
+ )
+ # If the server doesn't return an etag, it has changed the resource
+ # and we need to refetch it
+ if not response.headers.hasHeader('etag'):
+ yield self._refreshEvent(href)
+ else:
+ etag, scheduleTag = self.extractTags(response)
+ yield succeed(self._updateEventCache(href, etag=etag, scheduleTag=scheduleTag, component=event.component))
@inlineCallbacks
- def changeEvent(self, href):
+ def _refreshEvent(self, href):
+ """
+ Issues a GET to the specified href (representing an event that already exists on the server)
+ and uses the response to update local state associated with that event
+ """
+ response = yield self.requester.get(href, method_label="GET{event}")
+ etag, scheduleTag = self.extractTags(response)
+ body = yield readBody(response)
+ component = Component.fromString(body)
+ self._updateEventCache(href, etag=etag, scheduleTag=scheduleTag, component=component)
- event = self._events[href]
- component = event.component
+ def _updateEventCache(self, href, etag=None, scheduleTag=None, component=None):
+ """
+ Update local state associated with the event at href
+ """
- # At last, upload the new event definition
- response = yield self._request(
- (NO_CONTENT, PRECONDITION_FAILED,),
- 'PUT',
- self.root + href.encode('utf-8'),
- Headers({
- 'content-type': ['text/calendar'],
- 'if-match': [event.etag]
- }),
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{update}"
- )
+ if href in self._events:
+ event = self._events[href]
+ else: # This is a new resource
+ event = Event(self.serializeLocation(), href, None, None)
- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
+ if etag:
+ event.etag = etag
+ if scheduleTag:
+ event.scheduleTag = scheduleTag
+ if component:
+ event.component = component
+ if True: # XXX some other test
+ self.catalog["eventChanged"].issue(href)
+ self._cacheEvent(href, event)
- def _localUpdateEvent(self, response, href, component):
+ @inlineCallbacks
+ def deleteEvent(self, href):
+ """
+ Issue a DELETE for the given URL and remove local state
+ associated with that event.
+ """
+ self._invalidateEvent(href)
+ yield self.requester.delete(href, method_label="DELETE{event}")
+
+ def extractTags(self, response):
headers = response.headers
etag = headers.getRawHeaders("etag", [None])[0]
scheduleTag = headers.getRawHeaders("schedule-tag", [None])[0]
+ return etag, scheduleTag
- event = Event(self.serializeLocation(), href, etag, component)
- event.scheduleTag = scheduleTag
- self._setEvent(href, event)
+ # @inlineCallbacks
+ # def _attendeeAutoComplete(self, component, attendee):
+ # if self._ATTENDEE_LOOKUPS:
+ # # Temporarily use some non-test names (some which will return
+ # # many results, and others which will return fewer) because the
+ # # test account names are all too similar
+ # # name = attendee.parameterValue('CN').encode("utf-8")
+ # # prefix = name[:4].lower()
+ # prefix = random.choice([
+ # "chris", "cyru", "dre", "eric", "morg",
+ # "well", "wilfr", "witz"
+ # ])
- def updateEvent(self, href):
- return self._updateEvent(None, href)
+ # email = attendee.value()
+ # if email.startswith("mailto:"):
+ # email = email[7:]
+ # elif attendee.hasParameter('EMAIL'):
+ # email = attendee.parameterValue('EMAIL').encode("utf-8")
+ # # First try to discover some names to supply to the
+ # # auto-completion
+ # yield self.requester.report(
+ # self.principalCollection,
+ # self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
+ # 'displayname': prefix,
+ # 'email': prefix,
+ # 'firstname': prefix,
+ # 'lastname': prefix,
+ # },
+ # depth=None,
+ # method_label="REPORT{psearch}",
+ # )
+ # # Now learn about the attendee's availability
+ # yield self.requestAvailability(
+ # component.mainComponent().getStartDateUTC(),
+ # component.mainComponent().getEndDateUTC(),
+ # [self.email, u'mailto:' + email],
+ # [component.resourceUID()]
+ # )
+
@inlineCallbacks
- def _updateEvent(self, ignored, href):
- response = yield self._request(
- OK,
- 'GET',
- self.root + href.encode('utf-8'),
- method_label="GET{event}",
- )
- headers = response.headers
- etag = headers.getRawHeaders('etag')[0]
- scheduleTag = headers.getRawHeaders('schedule-tag', [None])[0]
- body = yield readBody(response)
- self.eventChanged(href, etag, scheduleTag, body)
+ def _principalSearchReport(self, query, isAttendeeSearch):
+ """ context = attendee if isAttendeeSearch else location """
+ context = "attendee" if isAttendeeSearch else "location"
+ tokens = query.split()
+ search = '\n'.join(["<C:search-token>%s</C:search-token>" % (token, ) for token in tokens])
+ body = self._CALENDARSERVER_PRINCIPAL_SEARCH_REPORT.format(context=context, searchTokens=search)
+ principals = yield self.requester.report('/principals/', body, depth=None)
+ print("Found some principals:")
+ returnValue(principals)
-
@inlineCallbacks
def requestAvailability(self, start, end, users, mask=set()):
"""
@@ -1644,8 +1275,6 @@
@return: A C{Deferred} which fires with a C{dict}. Keys in the dict
are user UUIDs (those requested) and values are something else.
"""
- outbox = self.root + self.outbox
-
if mask:
maskStr = u'\r\n'.join(['X-CALENDARSERVER-MASK-UID:' + uid
for uid in mask]) + u'\r\n'
@@ -1669,21 +1298,16 @@
end = end.getText()
now = DateTime.getNowUTC().getText()
- label_suffix = "small"
- if len(users) > 5:
- label_suffix = "medium"
- if len(users) > 20:
- label_suffix = "large"
- if len(users) > 75:
- label_suffix = "huge"
+ label_suffix = self._getEventSizeDescription(len(users))
- response = yield self._request(
- OK, 'POST', outbox,
- Headers({
- 'content-type': ['text/calendar'],
- 'originator': [self.email],
- 'recipient': [u', '.join(users).encode('utf-8')]}),
- StringProducer(self._POST_AVAILABILITY % {
+ headers = Headers({
+ 'content-type': ['text/calendar'],
+ 'originator': [self.email],
+ 'recipient': [u', '.join(users).encode('utf-8')]
+ })
+ response = yield self.requester.post(
+ self.outbox,
+ self._POST_AVAILABILITY % {
'attendees': attendeeStr,
'summary': (u'Availability for %s' % (', '.join(users),)).encode('utf-8'),
'organizer': self.email.encode('utf-8'),
@@ -1692,385 +1316,103 @@
'start': start,
'end': end,
'now': now,
- }),
+ },
+ headers=headers,
method_label="POST{fb-%s}" % (label_suffix,),
)
+
body = yield readBody(response)
returnValue(body)
-
-
-class OS_X_10_6(BaseAppleClient):
- """
- Implementation of the OS X 10.6 iCal network behavior.
-
- Anything OS X 10.6 iCal does on its own, or any particular
- network behaviors it takes in response to a user action, belong on
- this class.
-
- Usage-profile based behaviors ("the user modifies an event every
- 3.2 minutes") belong elsewhere.
- """
-
- _client_type = "OS X 10.6"
-
- USER_AGENT = "DAVKit/4.0.3 (732); CalendarStore/4.0.3 (991); iCal/4.0.3 (1388); Mac OS X/10.6.4 (10F569)"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 200
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = False
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = True
-
- # Request body data
- _LOAD_PATH = "OS_X_10_6"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PRINCIPAL_EXPAND = loadRequestBody(_LOAD_PATH, 'startup_principal_expand')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_TIMEZONE = loadRequestBody(_LOAD_PATH, 'startup_calendar_timezone_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
- _POLL_CALENDAR_SYNC_REPORT = None
- _POLL_NOTIFICATION_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_NOTIFICATION_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_notification_propfind_d1')
-
- _USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody(_LOAD_PATH, 'user_list_principal_property_search')
- _POST_AVAILABILITY = loadRequestBody(_LOAD_PATH, 'post_availability')
-
@inlineCallbacks
- def startup(self):
+ def postAttachment(self, href, content):
+ url = "{0}?{1}".format(href, "action=attachment-add")
+ filename = 'file-{}.txt'.format(len(content))
+ headers = Headers({
+ # 'Transfer-Encoding': ['Chunked'],
+ 'Content-Disposition': ['attachment; filename="{}"'.format(filename)]
+ })
+ l = len(content)
+ # lengthPrefix = hex(l)[2:].upper() # For some reason, this attachment is length-prefixed in hex
+ label_suffix = self._getEventSizeDescription(l / 1024)
+ # body = "{0}\n{1}\n0\n".format(lengthPrefix, content) # XXX There HAS to be a better way to do this
+ yield self.requester.post(
+ url,
+ content,
+ headers=headers,
+ method_label="POST{attach-%s}" % (label_suffix,)
+ )
- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = (yield self._extractPrincipalDetails())
- returnValue(principal)
-
-
-
-class OS_X_10_7(BaseAppleClient):
- """
- Implementation of the OS X 10.7 iCal network behavior.
- """
-
- _client_type = "OS X 10.7"
-
- USER_AGENT = "CalendarStore/5.0.2 (1166); iCal/5.0.2 (1571); Mac OS X/10.7.3 (11D50)"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 50
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = True
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = True
-
- # Request body data
- _LOAD_PATH = "OS_X_10_7"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PRINCIPAL_EXPAND = loadRequestBody(_LOAD_PATH, 'startup_principal_expand')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_TIMEZONE = loadRequestBody(_LOAD_PATH, 'startup_calendar_timezone_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
- _POLL_CALENDAR_SYNC_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_sync')
- _POLL_NOTIFICATION_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_NOTIFICATION_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_notification_propfind_d1')
-
- _USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody(_LOAD_PATH, 'user_list_principal_property_search')
- _POST_AVAILABILITY = loadRequestBody(_LOAD_PATH, 'post_availability')
-
-
- def _addDefaultHeaders(self, headers):
- """
- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
- """
-
- super(OS_X_10_7, self)._addDefaultHeaders(headers)
- headers.setRawHeaders('Accept', ['*/*'])
- headers.setRawHeaders('Accept-Language', ['en-us'])
- headers.setRawHeaders('Accept-Encoding', ['gzip,deflate'])
- headers.setRawHeaders('Connection', ['keep-alive'])
-
-
@inlineCallbacks
- def startup(self):
-
- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND well-known with redirect
- response = yield self._startupPropfindWellKnown()
- hrefs = response.getHrefProperties()
- if davxml.current_user_principal in hrefs:
- self.principalURL = hrefs[davxml.current_user_principal].toString()
- elif davxml.principal_URL in hrefs:
- self.principalURL = hrefs[davxml.principal_URL].toString()
- else:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = yield self._extractPrincipalDetails()
- returnValue(principal)
-
-
-
-class iOS_5(BaseAppleClient):
- """
- Implementation of the iOS 5 network behavior.
- """
-
- _client_type = "iOS 5"
-
- USER_AGENT = "iOS/5.1 (9B179) dataaccessd/1.0"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 50
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = False
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = True
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = False
-
- # Request body data
- _LOAD_PATH = "iOS_5"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_VEVENT_TR_QUERY = loadRequestBody(_LOAD_PATH, 'poll_calendar_vevent_tr_query')
- _POLL_CALENDAR_VTODO_QUERY = loadRequestBody(_LOAD_PATH, 'poll_calendar_vtodo_query')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
-
-
- def _addDefaultHeaders(self, headers):
+ def addCalendar(self, href, calendar_xml):
"""
- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
+ client.addCalendar(
+ '/calendars/__uids__/10000000-0000-0000-0000-000000000001/1C1A8475-2671-4B97-AC58-DD9777B5CD93/',
+ # <Component: 'BEGIN:VCALENDAR\r\n...END:VCALENDAR\r\n'>)
+ )
"""
+ response = yield self.requester.mkcalendar(
+ href,
+ calendar_xml,
+ method_label="MK{calendar}",
+ )
+ # self._cacheCalendar(href, calendar)
- super(iOS_5, self)._addDefaultHeaders(headers)
- headers.setRawHeaders('Accept', ['*/*'])
- headers.setRawHeaders('Accept-Language', ['en-us'])
- headers.setRawHeaders('Accept-Encoding', ['gzip,deflate'])
- headers.setRawHeaders('Connection', ['keep-alive'])
-
@inlineCallbacks
- def _pollFirstTime1(self, homeNode, calendars):
- # Patch calendar properties
- for cal in calendars:
- if cal.name != "inbox":
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_COLOR,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_ORDER,
- method_label="PROPPATCH{calendar}",
- )
+ def changeCalendar(self, href, body):
+ calendar = self._calendars[href]
+ headers = Headers({
+ 'content-type': ['text/xml']
+ })
- def _pollFirstTime2(self):
- # Nothing here
- return succeed(None)
+ # At last, upload the new event definition
+ response = yield self.requester.proppatch(
+ href,
+ body,
+ headers=headers,
+ method_label="PATCH{calendar}"
+ )
+ # Finally, re-retrieve the event to update the etag
+ # yield self._updateEvent(response, href)
- def _updateCalendar(self, calendar, newToken):
- """
- Update the local cached data for a calendar in an appropriate manner.
- """
- if calendar.name == "inbox":
- # Inbox is done as a PROPFIND Depth:1
- return self._updateCalendar_PROPFIND(calendar, newToken)
- elif "VEVENT" in calendar.componentTypes:
- # VEVENTs done as time-range VEVENT-only queries
- return self._updateCalendar_VEVENT(calendar, newToken)
- elif "VTODO" in calendar.componentTypes:
- # VTODOs done as VTODO-only queries
- return self._updateCalendar_VTODO(calendar, newToken)
-
-
@inlineCallbacks
- def _updateCalendar_VEVENT(self, calendar, newToken):
- """
- Sync all locally cached VEVENTs using a VEVENT-only time-range query.
- """
-
- # Grab old hrefs prior to the PROPFIND so we sync with the old state. We need this because
- # the sim can fire a PUT between the PROPFIND and when process the removals.
- old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
-
- now = DateTime.getNowUTC()
- now.setDateOnly(True)
- now.offsetMonth(-1) # 1 month back default
- result = yield self._report(
- calendar.url,
- self._POLL_CALENDAR_VEVENT_TR_QUERY % {"start-date": now.getText()},
- depth='1',
- method_label="REPORT{vevent}",
+ def postXML(self, href, xml):
+ headers = Headers({
+ 'content-type': ['text/xml']
+ })
+ response = yield self.requester.post(
+ href,
+ xml,
+ headers=headers,
+ method_label="SHARE{calendar}"
)
- yield self._updateApplyChanges(calendar, result, old_hrefs)
- # Now update calendar to the new token
- self._calendars[calendar.url].changeToken = newToken
-
-
@inlineCallbacks
- def _updateCalendar_VTODO(self, calendar, newToken):
+ def deleteCalendar(self, href):
"""
- Sync all locally cached VTODOs using a VTODO-only query.
+ Issue a DELETE for the given URL and remove local state
+ associated with that calendar.
+
+ Usage: client.deleteCalendar('/calendars/__uids__/<user-uid>/<calendar-uid>/')
"""
- # Grab old hrefs prior to the PROPFIND so we sync with the old state. We need this because
- # the sim can fire a PUT between the PROPFIND and when process the removals.
- old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
+ self._invalidateCalendar(href)
- result = yield self._report(
- calendar.url,
- self._POLL_CALENDAR_VTODO_QUERY,
- depth='1',
- method_label="REPORT{vtodo}",
- )
+ response = yield self.requester.delete(href, method_label="DELETE{calendar}")
+ returnValue(response)
- yield self._updateApplyChanges(calendar, result, old_hrefs)
- # Now update calendar to the new token
- self._calendars[calendar.url].changeToken = newToken
-
- @inlineCallbacks
- def startup(self):
-
- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND well-known with redirect
- response = yield self._startupPropfindWellKnown()
- hrefs = response.getHrefProperties()
- if davxml.current_user_principal in hrefs:
- self.principalURL = hrefs[davxml.current_user_principal].toString()
- elif davxml.principal_URL in hrefs:
- self.principalURL = hrefs[davxml.principal_URL].toString()
- else:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = yield self._extractPrincipalDetails()
- returnValue(principal)
-
-
-
-class RequestLogger(object):
- format = u"%(user)s request %(code)s%(success)s[%(duration)5.2f s] %(method)8s %(url)s"
- success = u"\N{CHECK MARK}"
- failure = u"\N{BALLOT X}"
-
- def observe(self, event):
- if event.get("type") == "response":
- formatArgs = dict(
- user=event['user'],
- method=event['method'],
- url=urlunparse(('', '') + urlparse(event['url'])[2:]),
- code=event['code'],
- duration=event['duration'],
- )
-
- if event['success']:
- formatArgs['success'] = self.success
- else:
- formatArgs['success'] = self.failure
- print((self.format % formatArgs).encode('utf-8'))
-
-
- def report(self, output):
- pass
-
-
- def failures(self):
- return []
-
-
-
def main():
from urllib2 import HTTPDigestAuthHandler
from twisted.internet import reactor
+ from twisted.python.log import addObserver
+ from contrib.performance.loadtest.logger import RequestLogger
+ from contrib.performance.loadtest.clients import OS_X_10_11
auth = HTTPDigestAuthHandler()
auth.add_password(
realm="Test Realm",
@@ -2080,14 +1422,17 @@
addObserver(RequestLogger().observe)
- from sim import _DirectoryRecord
- client = OS_X_10_6(
- reactor, 'http://127.0.0.1:8008/',
- _DirectoryRecord(
- u'user01', u'user01', u'User 01', u'user01 at example.org'),
- auth)
+ from contrib.performance.loadtest.records import DirectoryRecord
+ client = OS_X_10_11(
+ reactor,
+ 'http://127.0.0.1:8008/', # root
+ '/tmp/sim', # serializePath
+ DirectoryRecord(u'user01', u'user01', u'User 01', u'user01 at example.org', u'10000000-0000-0000-0000-000000000001'),
+ auth,
+ title='OS X 10.11 Client Simulator'
+ )
d = client.run()
- d.addErrback(err, "10.6 client run() problem")
+ d.addErrback(err, "10.11 client run() problem")
d.addCallback(lambda ignored: reactor.stop())
reactor.run()
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/logger.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -14,9 +14,27 @@
# limitations under the License.
#
##
+import json
+import collections
+import os
+import sys
+from datetime import datetime
-from contrib.performance.stats import mean, median, stddev
+from urlparse import urlparse, urlunparse
+from contrib.performance.stats import mean, median, stddev, mad
+
+class TerminalColors:
+ HEADER = '\033[95m'
+ OKBLUE = '\033[94m'
+ OKGREEN = '\033[92m'
+ WARNING = '\033[93m'
+ LIGHTBLUE = '\033[36m'
+ FAIL = '\033[91m'
+ ENDC = '\033[0m'
+ BOLD = '\033[1m'
+ UNDERLINE = '\033[4m'
+
class SummarizingMixin(object):
def printHeader(self, output, fields):
@@ -87,3 +105,467 @@
"""
for method, data in perOperationTimes:
self._printRow(output, formats, self._summarizeData(method, data))
+
+
+
+class RequestLogger(object):
+ format = u"%(user)s request %(code)s%(success)s[%(duration)5.2f s] %(method)8s %(url)s"
+ success = u"\N{CHECK MARK}"
+ failure = u"\N{BALLOT X}"
+
+ def observe(self, event):
+ if event.get("type") == "response":
+ formatArgs = dict(
+ user=event['user'],
+ method=event['method'],
+ url=urlunparse(('', '') + urlparse(event['url'])[2:]),
+ code=event['code'],
+ duration=event['duration'],
+ )
+
+ if event['success']:
+ formatArgs['success'] = self.success
+ start = TerminalColors.OKGREEN
+ else:
+ formatArgs['success'] = self.failure
+ start = TerminalColors.FAIL
+ print(start + (self.format % formatArgs).encode('utf-8') + TerminalColors.ENDC)
+
+
+ def report(self, output):
+ pass
+
+
+ def failures(self):
+ return []
+
+
+
+class OperationLogger(SummarizingMixin):
+ """
+ Profiles will initiate operations which may span multiple requests. Start
+ and stop log messages are emitted for these operations and logged by this
+ logger.
+ """
+ formats = {
+ u"start" : u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
+ u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
+ u"failed": u"%(user)s x x x x x x x x x x x %(label)8s FAILED %(reason)s",
+ }
+
+ lagFormat = u'{lag %5.2f ms}'
+
+ # the response time thresholds to display together with failing % count threshold
+ _thresholds_default = {
+ "operations": {
+ "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds": {
+ "default": [100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
+ }
+ }
+ }
+ _lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds
+ _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
+
+ _fields_init = [
+ ('operation', -25, '%-25s'),
+ ('count', 8, '%8s'),
+ ('failed', 8, '%8s'),
+ ]
+
+ _fields_extend = [
+ ('mean', 8, '%8.4f'),
+ ('median', 8, '%8.4f'),
+ ('stddev', 8, '%8.4f'),
+ ('avglag (ms)', 12, '%12.4f'),
+ ('STATUS', 8, '%8s'),
+ ]
+
+ def __init__(self, outfile=None, **params):
+ self._perOperationTimes = {}
+ self._perOperationLags = {}
+ if outfile is None:
+ outfile = sys.stdout
+ self._outfile = outfile
+
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholdsPath"]))
+ elif "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["operations"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+
+ if "lagCutoff" in params:
+ self._lag_cut_off = params["lagCutoff"]
+
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
+
+ def observe(self, event):
+ if event.get("type") == "operation":
+ event = event.copy()
+ lag = event.get('lag')
+ if lag is None:
+ event['lag'] = ''
+ else:
+ event['lag'] = self.lagFormat % (lag * 1000.0,)
+
+ self._outfile.write(
+ TerminalColors.LIGHTBLUE +
+ (self.formats[event[u'phase']] % event).encode('utf-8') + TerminalColors.ENDC + '\n')
+
+ if event[u'phase'] == u'end':
+ dataset = self._perOperationTimes.setdefault(event[u'label'], [])
+ dataset.append((event[u'success'], event[u'duration']))
+ elif lag is not None:
+ dataset = self._perOperationLags.setdefault(event[u'label'], [])
+ dataset.append(lag)
+
+
+ def _summarizeData(self, operation, data):
+ avglag = mean(self._perOperationLags.get(operation, [0.0])) * 1000.0
+ data = SummarizingMixin._summarizeData(self, operation, data)
+ return data[:-1] + (avglag,) + data[-1:]
+
+
+ def report(self, output):
+ output.write("\n")
+ self.printHeader(output, [
+ (label, width)
+ for (label, width, _ignore_fmt) in self._fields
+ ])
+ self.printData(
+ output,
+ [fmt for (label, width, fmt) in self._fields],
+ sorted(self._perOperationTimes.items())
+ )
+
+ _LATENCY_REASON = "Median %(operation)s scheduling lag greater than %(cutoff)sms"
+ _FAILED_REASON = "Greater than %(cutoff).0f%% %(operation)s failed"
+
+ def failures(self):
+ reasons = []
+
+ for operation, lags in self._perOperationLags.iteritems():
+ if median(lags) > self._lag_cut_off:
+ reasons.append(self._LATENCY_REASON % dict(
+ operation=operation.upper(), cutoff=self._lag_cut_off * 1000))
+
+ for operation, times in self._perOperationTimes.iteritems():
+ failures = len([success for (success, _ignore_duration) in times if not success])
+ if failures * 100.0 / len(times) > self._fail_cut_off:
+ reasons.append(self._FAILED_REASON % dict(
+ operation=operation.upper(), cutoff=self._fail_cut_off))
+
+ return reasons
+
+
+
+class StatisticsBase(object):
+ def observe(self, event):
+ if event.get('type') == 'response':
+ self.eventReceived(event)
+ elif event.get('type') == 'client-failure':
+ self.clientFailure(event)
+ elif event.get('type') == 'sim-failure':
+ self.simFailure(event)
+
+
+ def report(self, output):
+ pass
+
+
+ def failures(self):
+ return []
+
+
+
+class SimpleStatistics(StatisticsBase):
+ def __init__(self):
+ self._times = []
+ self._failures = collections.defaultdict(int)
+ self._simFailures = collections.defaultdict(int)
+
+
+ def eventReceived(self, event):
+ self._times.append(event['duration'])
+ if len(self._times) == 200:
+ print('mean:', mean(self._times))
+ print('median:', median(self._times))
+ print('stddev:', stddev(self._times))
+ print('mad:', mad(self._times))
+ del self._times[:100]
+
+
+ def clientFailure(self, event):
+ self._failures[event] += 1
+
+
+ def simFailure(self, event):
+ self._simFailures[event] += 1
+
+
+
+class ReportStatistics(StatisticsBase, SummarizingMixin):
+ """
+
+ @ivar _users: A C{set} containing all user UIDs which have been observed in
+ events. When generating the final report, the size of this set is
+ reported as the number of users in the simulation.
+
+ """
+
+ # the response time thresholds to display together with failing % count threshold
+ _thresholds_default = {
+ "requests": {
+ "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds": {
+ "default": [100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ }
+ }
+ }
+ _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
+
+ _fields_init = [
+ ('request', -25, '%-25s'),
+ ('count', 8, '%8s'),
+ ('failed', 8, '%8s'),
+ ]
+
+ _fields_extend = [
+ ('mean', 8, '%8.4f'),
+ ('median', 8, '%8.4f'),
+ ('stddev', 8, '%8.4f'),
+ ('QoS', 8, '%8.4f'),
+ ('STATUS', 8, '%8s'),
+ ]
+
+ def __init__(self, **params):
+ self._perMethodTimes = {}
+ self._users = set()
+ self._clients = set()
+ self._failed_clients = []
+ self._failed_sim = collections.defaultdict(int)
+ self._startTime = datetime.now()
+ self._expired_data = None
+
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholdsPath"]))
+ elif "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["requests"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+
+ if "benchmarksPath" in params:
+ self.benchmarks = json.load(open(params["benchmarksPath"]))
+ else:
+ self.benchmarks = {}
+
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
+
+ def observe(self, event):
+ if event.get('type') == 'sim-expired':
+ self.simExpired(event)
+ else:
+ super(ReportStatistics, self).observe(event)
+
+
+ def countUsers(self):
+ return len(self._users)
+
+
+ def countClients(self):
+ return len(self._clients)
+
+
+ def countClientFailures(self):
+ return len(self._failed_clients)
+
+
+ def countSimFailures(self):
+ return len(self._failed_sim)
+
+
+ def eventReceived(self, event):
+ dataset = self._perMethodTimes.setdefault(event['method'], [])
+ dataset.append((event['success'], event['duration']))
+ self._users.add(event['user'])
+ self._clients.add(event['client_id'])
+
+
+ def clientFailure(self, event):
+ self._failed_clients.append(event['reason'])
+
+
+ def simFailure(self, event):
+ self._failed_sim[event['reason']] += 1
+
+
+ def simExpired(self, event):
+ self._expired_data = event['reason']
+
+
+ def printMiscellaneous(self, output, items):
+ maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
+ fmt = "%" + maxColumnWidth + "s : %-s\n"
+ for k in sorted(items.iterkeys()):
+ output.write(fmt % (k.title(), items[k],))
+
+
+ def qos(self):
+ """
+ Determine a "quality of service" value that can be used for comparisons between runs. This value
+ is based on the percentage deviation of means of each request from a set of "benchmarks" for each
+ type of request.
+ """
+
+ # Get means for each type of method
+ means = {}
+ for method, results in self._perMethodTimes.items():
+ means[method] = mean([duration for success, duration in results if success])
+
+ # Determine percentage differences with weighting
+ differences = []
+ for method, value in means.items():
+ result = self.qos_value(method, value)
+ if result is not None:
+ differences.append(result)
+
+ return ("%-8.4f" % mean(differences)) if differences else "None"
+
+
+ def qos_value(self, method, value):
+ benchmark = self.benchmarks.get(method)
+ if benchmark is None:
+ return None
+ test_mean, weight = (benchmark["mean"], benchmark["weight"],)
+ return ((value / test_mean) - 1.0) * weight + 1.0
+
+
+ def _summarizeData(self, operation, data):
+ data = SummarizingMixin._summarizeData(self, operation, data)
+ value = self.qos_value(operation, data[-4])
+ if value is None:
+ value = 0.0
+ return data[:-1] + (value,) + data[-1:]
+
+
+ def report(self, output):
+ output.write("\n")
+ output.write("** REPORT **\n")
+ output.write("\n")
+ runtime = datetime.now() - self._startTime
+ cpu = os.times()
+ cpuUser = cpu[0] + cpu[2]
+ cpuSys = cpu[1] + cpu[3]
+ cpuTotal = cpuUser + cpuSys
+ runHours, remainder = divmod(runtime.seconds, 3600)
+ runMinutes, runSeconds = divmod(remainder, 60)
+ cpuHours, remainder = divmod(cpuTotal, 3600)
+ cpuMinutes, cpuSeconds = divmod(remainder, 60)
+ items = {
+ 'Users': self.countUsers(),
+ 'Clients': self.countClients(),
+ 'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
+ 'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
+ 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
+ 'QoS': self.qos(),
+ }
+ if self.countClientFailures() > 0:
+ items['Failed clients'] = self.countClientFailures()
+ for ctr, reason in enumerate(self._failed_clients, 1):
+ items['Failure #%d' % (ctr,)] = reason
+ if self.countSimFailures() > 0:
+ for reason, count in self._failed_sim.items():
+ items['Failed operation'] = "%s : %d times" % (reason, count,)
+ output.write("* Client\n")
+ self.printMiscellaneous(output, items)
+ output.write("\n")
+
+ if self._expired_data is not None:
+ items = {
+ "Req/sec" : "%.1f" % (self._expired_data[0],),
+ "Response": "%.1f (ms)" % (self._expired_data[1],),
+ "Slots": "%.2f" % (self._expired_data[2],),
+ "CPU": "%.1f%%" % (self._expired_data[3],),
+ }
+ output.write("* Server (Last 5 minutes)\n")
+ self.printMiscellaneous(output, items)
+ output.write("\n")
+ output.write("* Details\n")
+
+ self.printHeader(output, [
+ (label, width)
+ for (label, width, _ignore_fmt)
+ in self._fields
+ ])
+ self.printData(
+ output,
+ [fmt for (label, width, fmt) in self._fields],
+ sorted(self._perMethodTimes.items())
+ )
+
+ _FAILED_REASON = "Greater than %(cutoff)g%% %(method)s failed"
+
+ _REASON_1 = "Greater than %(cutoff)g%% %(method)s exceeded "
+ _REASON_2 = "%g second response time"
+
+ def failures(self):
+ # TODO
+ reasons = []
+
+ for (method, times) in self._perMethodTimes.iteritems():
+ failures = 0
+ overDurations = [0] * len(self._thresholds)
+
+ for success, duration in times:
+ if not success:
+ failures += 1
+ for ctr, item in enumerate(self._thresholds):
+ threshold, _ignore_fail_at = item
+ if duration > threshold:
+ overDurations[ctr] += 1
+
+ checks = [
+ (failures, self._fail_cut_off, self._FAILED_REASON),
+ ]
+
+ for ctr, item in enumerate(self._thresholds):
+ threshold, fail_at = item
+ fail_at = fail_at.get(method, fail_at["default"])
+ checks.append(
+ (overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,))
+ )
+
+ for count, cutoff, reason in checks:
+ if count * 100.0 / len(times) > cutoff:
+ reasons.append(reason % dict(method=method, cutoff=cutoff))
+
+ if self.countClientFailures() != 0:
+ reasons.append("Client failures: %d" % (self.countClientFailures(),))
+ if self.countSimFailures() != 0:
+ reasons.append("Overall failures: %d" % (self.countSimFailures(),))
+ return reasons
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -25,13 +25,10 @@
from tempfile import mkdtemp
from itertools import izip
-from datetime import datetime
+
from urllib2 import HTTPBasicAuthHandler
from urllib2 import HTTPDigestAuthHandler
from urllib2 import HTTPPasswordMgrWithDefaultRealm
-import collections
-import json
-import os
from twisted.internet.defer import DeferredList
from twisted.python.failure import Failure
@@ -41,10 +38,8 @@
from twistedcaldav.timezones import TimezoneCache
-from contrib.performance.stats import mean, median, stddev, mad
from contrib.performance.loadtest.trafficlogger import loggedReactor
-from contrib.performance.loadtest.logger import SummarizingMixin
-from contrib.performance.loadtest.ical import OS_X_10_6, RequestLogger
+
from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
@@ -64,10 +59,16 @@
def __call__(self, reactor, simulator, client, number):
- return self.profileType(reactor, simulator, client, number, **self.params)
+ base = self.profileType(**self.params)
+ base.setUp(reactor, simulator, client, number)
+ return base
+ def __repr__(self):
+ return "ProfileType(%s, params=%s)" % (self.profileType.__name__, self.params)
+
+
class ClientType(object, FancyEqMixin):
"""
@ivar clientType: An L{ICalendarClient} implementation
@@ -81,14 +82,15 @@
self.profileTypes = profileTypes
- def new(self, reactor, serverAddress, principalPathTemplate, serializationPath, userRecord, authInfo):
+ def new(self, reactor, serverAddress, serializationPath, userRecord, authInfo):
"""
Create a new instance of this client type.
"""
+ # print(self.clientType)
+ # print(self.clientParams)
return self.clientType(
- reactor, serverAddress, principalPathTemplate,
- serializationPath, userRecord, authInfo,
- **self.clientParams
+ reactor, serverAddress, serializationPath,
+ userRecord, authInfo, **self.clientParams
)
@@ -163,12 +165,13 @@
class CalendarClientSimulator(object):
def __init__(self, records, populator, parameters, reactor, server,
- principalPathTemplate, serializationPath, workerIndex=0, workerCount=1):
+ serializationPath, workerIndex=0, workerCount=1):
+ from pprint import pprint
+ pprint(records)
self._records = records
self.populator = populator
self.reactor = reactor
self.server = server
- self.principalPathTemplate = principalPathTemplate
self.serializationPath = serializationPath
self._pop = self.populator.populate(parameters)
self._user = 0
@@ -177,7 +180,7 @@
self.workerCount = workerCount
self.clients = []
- TimezoneCache.create()
+ # TimezoneCache.create()
def getUserRecord(self, index):
@@ -242,7 +245,6 @@
client = clientType.new(
reactor,
self.server,
- self.principalPathTemplate,
self.serializationPath,
self.getUserRecord(number),
auth,
@@ -252,6 +254,7 @@
d.addErrback(self._clientFailure, reactor)
for profileType in clientType.profileTypes:
+ print(profileType)
profile = profileType(reactor, self, client, number)
if profile.enabled:
d = profile.initialize()
@@ -265,7 +268,23 @@
# really used by much anyway.
msg(type="status", clientCount=self._user - 1)
+ # for i in range(numClients):
+ # for j in range(clientsPerUser):
+ # client = self._pop.next()
+ # # Reactor magic goes here
+ # client.setUp(
+ # self.reactor,
+ # self.server,
+ # self.serializationPath,
+ # self.getUserRecord(number),
+ # auth
+ # )
+ # for profile in client.profiles:
+ # profile.setUp(self.reactor, self, client, number)
+
+
+
def _dumpLogs(self, loggingReactor, reason):
path = FilePath(mkdtemp())
logstate = loggingReactor.getLogFiles()
@@ -278,6 +297,13 @@
return path
+ def _profileFailure(self, reason, profileType, reactor):
+ if not self._stopped:
+ where = self._dumpLogs(reactor, reason)
+ err(reason, "Profile stopped with error; recent traffic in %r" % (
+ where.path,))
+
+
def _clientFailure(self, reason, reactor):
if not self._stopped:
where = self._dumpLogs(reactor, reason)
@@ -288,13 +314,6 @@
msg(type="client-failure", reason="%s: %s" % (reason.type, reason.value,))
- def _profileFailure(self, reason, profileType, reactor):
- if not self._stopped:
- where = self._dumpLogs(reactor, reason)
- err(reason, "Profile stopped with error; recent traffic in %r" % (
- where.path,))
-
-
def _simFailure(self, reason, reactor):
if not self._stopped:
msg(type="sim-failure", reason=reason)
@@ -317,306 +336,6 @@
-class StatisticsBase(object):
- def observe(self, event):
- if event.get('type') == 'response':
- self.eventReceived(event)
- elif event.get('type') == 'client-failure':
- self.clientFailure(event)
- elif event.get('type') == 'sim-failure':
- self.simFailure(event)
-
-
- def report(self, output):
- pass
-
-
- def failures(self):
- return []
-
-
-
-class SimpleStatistics(StatisticsBase):
- def __init__(self):
- self._times = []
- self._failures = collections.defaultdict(int)
- self._simFailures = collections.defaultdict(int)
-
-
- def eventReceived(self, event):
- self._times.append(event['duration'])
- if len(self._times) == 200:
- print('mean:', mean(self._times))
- print('median:', median(self._times))
- print('stddev:', stddev(self._times))
- print('mad:', mad(self._times))
- del self._times[:100]
-
-
- def clientFailure(self, event):
- self._failures[event] += 1
-
-
- def simFailure(self, event):
- self._simFailures[event] += 1
-
-
-
-class ReportStatistics(StatisticsBase, SummarizingMixin):
- """
-
- @ivar _users: A C{set} containing all user UIDs which have been observed in
- events. When generating the final report, the size of this set is
- reported as the number of users in the simulation.
-
- """
-
- # the response time thresholds to display together with failing % count threshold
- _thresholds_default = {
- "requests": {
- "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
- "thresholds": {
- "default": [100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
- }
- }
- }
- _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
-
- _fields_init = [
- ('request', -25, '%-25s'),
- ('count', 8, '%8s'),
- ('failed', 8, '%8s'),
- ]
-
- _fields_extend = [
- ('mean', 8, '%8.4f'),
- ('median', 8, '%8.4f'),
- ('stddev', 8, '%8.4f'),
- ('QoS', 8, '%8.4f'),
- ('STATUS', 8, '%8s'),
- ]
-
- def __init__(self, **params):
- self._perMethodTimes = {}
- self._users = set()
- self._clients = set()
- self._failed_clients = []
- self._failed_sim = collections.defaultdict(int)
- self._startTime = datetime.now()
- self._expired_data = None
-
- # Load parameters from config
- if "thresholdsPath" in params:
- jsondata = json.load(open(params["thresholdsPath"]))
- elif "thresholds" in params:
- jsondata = params["thresholds"]
- else:
- jsondata = self._thresholds_default
- self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]]
- for ctr, item in enumerate(self._thresholds):
- for k, v in jsondata["requests"]["thresholds"].items():
- item[1][k] = v[ctr]
-
- self._fields = self._fields_init[:]
- for threshold, _ignore_fail_at in self._thresholds:
- self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
- self._fields.extend(self._fields_extend)
-
- if "benchmarksPath" in params:
- self.benchmarks = json.load(open(params["benchmarksPath"]))
- else:
- self.benchmarks = {}
-
- if "failCutoff" in params:
- self._fail_cut_off = params["failCutoff"]
-
-
- def observe(self, event):
- if event.get('type') == 'sim-expired':
- self.simExpired(event)
- else:
- super(ReportStatistics, self).observe(event)
-
-
- def countUsers(self):
- return len(self._users)
-
-
- def countClients(self):
- return len(self._clients)
-
-
- def countClientFailures(self):
- return len(self._failed_clients)
-
-
- def countSimFailures(self):
- return len(self._failed_sim)
-
-
- def eventReceived(self, event):
- dataset = self._perMethodTimes.setdefault(event['method'], [])
- dataset.append((event['success'], event['duration']))
- self._users.add(event['user'])
- self._clients.add(event['client_id'])
-
-
- def clientFailure(self, event):
- self._failed_clients.append(event['reason'])
-
-
- def simFailure(self, event):
- self._failed_sim[event['reason']] += 1
-
-
- def simExpired(self, event):
- self._expired_data = event['reason']
-
-
- def printMiscellaneous(self, output, items):
- maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
- fmt = "%" + maxColumnWidth + "s : %-s\n"
- for k in sorted(items.iterkeys()):
- output.write(fmt % (k.title(), items[k],))
-
-
- def qos(self):
- """
- Determine a "quality of service" value that can be used for comparisons between runs. This value
- is based on the percentage deviation of means of each request from a set of "benchmarks" for each
- type of request.
- """
-
- # Get means for each type of method
- means = {}
- for method, results in self._perMethodTimes.items():
- means[method] = mean([duration for success, duration in results if success])
-
- # Determine percentage differences with weighting
- differences = []
- for method, value in means.items():
- result = self.qos_value(method, value)
- if result is not None:
- differences.append(result)
-
- return ("%-8.4f" % mean(differences)) if differences else "None"
-
-
- def qos_value(self, method, value):
- benchmark = self.benchmarks.get(method)
- if benchmark is None:
- return None
- test_mean, weight = (benchmark["mean"], benchmark["weight"],)
- return ((value / test_mean) - 1.0) * weight + 1.0
-
-
- def _summarizeData(self, operation, data):
- data = SummarizingMixin._summarizeData(self, operation, data)
- value = self.qos_value(operation, data[-4])
- if value is None:
- value = 0.0
- return data[:-1] + (value,) + data[-1:]
-
-
- def report(self, output):
- output.write("\n")
- output.write("** REPORT **\n")
- output.write("\n")
- runtime = datetime.now() - self._startTime
- cpu = os.times()
- cpuUser = cpu[0] + cpu[2]
- cpuSys = cpu[1] + cpu[3]
- cpuTotal = cpuUser + cpuSys
- runHours, remainder = divmod(runtime.seconds, 3600)
- runMinutes, runSeconds = divmod(remainder, 60)
- cpuHours, remainder = divmod(cpuTotal, 3600)
- cpuMinutes, cpuSeconds = divmod(remainder, 60)
- items = {
- 'Users': self.countUsers(),
- 'Clients': self.countClients(),
- 'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
- 'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
- 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
- 'QoS': self.qos(),
- }
- if self.countClientFailures() > 0:
- items['Failed clients'] = self.countClientFailures()
- for ctr, reason in enumerate(self._failed_clients, 1):
- items['Failure #%d' % (ctr,)] = reason
- if self.countSimFailures() > 0:
- for reason, count in self._failed_sim.items():
- items['Failed operation'] = "%s : %d times" % (reason, count,)
- output.write("* Client\n")
- self.printMiscellaneous(output, items)
- output.write("\n")
-
- if self._expired_data is not None:
- items = {
- "Req/sec" : "%.1f" % (self._expired_data[0],),
- "Response": "%.1f (ms)" % (self._expired_data[1],),
- "Slots": "%.2f" % (self._expired_data[2],),
- "CPU": "%.1f%%" % (self._expired_data[3],),
- }
- output.write("* Server (Last 5 minutes)\n")
- self.printMiscellaneous(output, items)
- output.write("\n")
- output.write("* Details\n")
-
- self.printHeader(output, [
- (label, width)
- for (label, width, _ignore_fmt)
- in self._fields
- ])
- self.printData(
- output,
- [fmt for (label, width, fmt) in self._fields],
- sorted(self._perMethodTimes.items())
- )
-
- _FAILED_REASON = "Greater than %(cutoff)g%% %(method)s failed"
-
- _REASON_1 = "Greater than %(cutoff)g%% %(method)s exceeded "
- _REASON_2 = "%g second response time"
-
- def failures(self):
- # TODO
- reasons = []
-
- for (method, times) in self._perMethodTimes.iteritems():
- failures = 0
- overDurations = [0] * len(self._thresholds)
-
- for success, duration in times:
- if not success:
- failures += 1
- for ctr, item in enumerate(self._thresholds):
- threshold, _ignore_fail_at = item
- if duration > threshold:
- overDurations[ctr] += 1
-
- checks = [
- (failures, self._fail_cut_off, self._FAILED_REASON),
- ]
-
- for ctr, item in enumerate(self._thresholds):
- threshold, fail_at = item
- fail_at = fail_at.get(method, fail_at["default"])
- checks.append(
- (overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,))
- )
-
- for count, cutoff, reason in checks:
- if count * 100.0 / len(times) > cutoff:
- reasons.append(reason % dict(method=method, cutoff=cutoff))
-
- if self.countClientFailures() != 0:
- reasons.append("Client failures: %d" % (self.countClientFailures(),))
- if self.countSimFailures() != 0:
- reasons.append("Overall failures: %d" % (self.countSimFailures(),))
- return reasons
-
-
-
def main():
import random
@@ -626,6 +345,9 @@
from twisted.python.failure import startDebugMode
startDebugMode()
+ from contrib.performance.loadtest.clients import OS_X_10_6
+ from contrib.performance.loadtest.logger import ReportStatistics, SimpleStatistics, RequestLogger
+
report = ReportStatistics()
addObserver(SimpleStatistics().observe)
addObserver(report.observe)
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -21,9 +21,7 @@
from __future__ import division
-import json
import random
-import sys
from uuid import uuid4
from caldavclientlibrary.protocol.caldav.definitions import caldavxml
@@ -31,19 +29,21 @@
from twisted.python import context
from twisted.python.log import msg
from twisted.python.failure import Failure
-from twisted.internet.defer import Deferred, succeed, fail
+from twisted.internet.defer import Deferred, DeferredList, succeed, fail
from twisted.internet.task import LoopingCall
from twisted.web.http import PRECONDITION_FAILED
-from twistedcaldav.ical import Property, Component
+from twistedcaldav.ical import Property
-from contrib.performance.stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, mean, median
+from contrib.performance.stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, BernoulliDistribution
from contrib.performance.stats import LogNormalDistribution, RecurrenceDistribution
-from contrib.performance.loadtest.logger import SummarizingMixin
from contrib.performance.loadtest.ical import IncorrectResponseCode
+from contrib.performance.loadtest.resources import Calendar, Event
+from contrib.performance.loadtest.templates import eventTemplate, alarmTemplate, taskTemplate
from pycalendar.datetime import DateTime
from pycalendar.duration import Duration
+from pycalendar.value import Value
class ProfileBase(object):
"""
@@ -52,18 +52,53 @@
"""
random = random
- def __init__(self, reactor, simulator, client, userNumber, **params):
+ def __init__(self, enabled, interval, **params):
+ self.enabled = enabled
+ self._interval = interval
+ self.setParameters(**params)
+ self._initialized = False
+
+ def setUp(self, reactor, simulator, client, record):
self._reactor = reactor
self._sim = simulator
self._client = client
- self._number = userNumber
- self.setParameters(**params)
+ self._record = record
+ self._initialized = True
-
def setParameters(self):
pass
+ def run(self):
+ # def action(self):
+ # if self.enabled:
+ # return self.action()
+ print("Hello from run")
+ _call = LoopingCall(self.action)
+ _call.clock = self._reactor
+ return _call.start(self._interval)
+
+ # OR
+
+ # return self._loopWithDistribution(
+ # self._reactor, self._sendInvitationDistribution, self._invite)
+
+ # def _loopWithDistribution(reactor, distribution, function):
+ # result = Deferred()
+
+ # def repeat(ignored):
+ # reactor.callLater(distribution.sample(), iterate)
+
+ # def iterate():
+ # d = function()
+ # if d is not None:
+ # d.addCallbacks(repeat, result.errback)
+ # else:
+ # repeat(None)
+
+ # repeat(None)
+ # return result
+
def initialize(self):
"""
Called before the profile runs for real. Can be used to initialize client state.
@@ -90,6 +125,46 @@
return attendee.parameterValue('EMAIL') == self._client.email[len('mailto:'):]
+ def _getRandomCalendar(self):
+ """
+ Return a random L{Calendar} object from the current user
+ or C{None} if there are no calendars to work with
+ """
+ calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
+ if not calendars: # Oh no! There are no calendars to play with
+ return None
+ # Choose a random calendar
+ calendar = self.random.choice(calendars)
+ return calendar
+
+
+ def _getRandomEvent(self):
+ """
+ Return a random L{Event} object from the current user
+ or C{None} if there are no events to work with
+ """
+ calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+ if not calendar.events:
+ continue
+
+ events = calendar.events.keys()
+ while events:
+ href = self.random.choice(events)
+ events.remove(href)
+ event = calendar.events[href]
+ if not event.component:
+ continue
+ return event
+ return None
+
+
+ def _getRandomLocation(self):
+ pass
+
+
def _newOperation(self, label, deferred):
"""
Helper to emit a log event when a new operation is started and
@@ -155,47 +230,695 @@
"""
pass
+ """ Event-Interaction Profiles
+Event Creation - Eventer
+Event Changing - EventUpdaterBase
+ TitlerMixin
+ RelocaterMixin
+ ReschedulerMixin
+ RepeaterMixin
+ AlerterMixin
+ InviterMixin
+ NoterMixin
+ InviterMixin
+ LinkerMixin
+ AttacherMixin
+Event Deletion - EventerDeleter
+"""
+#####################
+# Eventer Behaviors #
+#####################
-def loopWithDistribution(reactor, distribution, function):
- result = Deferred()
+class EventerBase(ProfileBase):
+ """
+ Base profile for a calendar user who interacts with events
+ """
+ def setParameters(
+ self,
+ enabled=True,
+ interval=25,
+ **params
+ ):
+ self.enabled = enabled
+ self._interval = interval
+ self.setDistributions(**params)
- def repeat(ignored):
- reactor.callLater(distribution.sample(), iterate)
- def iterate():
- d = function()
- if d is not None:
- d.addCallbacks(repeat, result.errback)
+class Eventer(EventerBase):
+ """
+ A Calendar user who creates new events.
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self,
+ eventStartDistribution=NearFutureDistribution(),
+ eventDurationDistribution=UniformDiscreteDistribution([
+ 15 * 60, 30 * 60,
+ 45 * 60, 60 * 60,
+ 120 * 60
+ ])
+ ):
+ self._eventStartDistribution = eventStartDistribution
+ self._eventDurationDistribution = eventDurationDistribution
+
+ def _addEvent(self):
+ print "Hello a bit"
+
+ if not self._client.started:
+ return succeed(None)
+
+
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ print "Made it"
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStartDistribution.sample()
+ dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+# Could have better handling for not changing events once they're modified
+# esp re: repeating
+class EventUpdaterBase(EventerBase):
+ """Superclass of all event mixins.
+ Accepts two parameters
+ enabled: bool on or off
+ interval: distibution that generates integers representing delays
+ """
+ # COMPONENT_TYPE = None
+ def action(self):
+ event = self._getRandomEvent()
+ if not event:
+ return succeed(None)
+ component = event.component
+ vevent = component.mainComponent()
+
+ label = self.modifyEvent(event.url, vevent)
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ event.component = component
+ return self._client.updateEvent(event)
+ # d.addCallback(finish)
+
+ return self._newOperation(label, d)
+
+ def modifyEvent(self):
+ """Overriden by subclasses"""
+ pass
+
+class Titler(EventUpdaterBase):
+ def setDistributions(
+ self,
+ titleLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._titleLength = titleLengthDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._titleLength.sample()))
+ vevent.replaceProperty(Property("SUMMARY", "Event" + "." * (length - 5)))
+ return "update{title}"
+
+class Transparenter(EventUpdaterBase):
+ def setDistributions(
+ self,
+ transparentLikelihoodDistribution=BernoulliDistribution(0.95)
+ ):
+ self._transparentLikelihood = transparentLikelihoodDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ if self._transparentLikelihood.sample():
+ transparency = "TRANSPARENT"
else:
- repeat(None)
+ transparency = "OPAQUE"
+ vevent.replaceProperty(Property("TRANSP", transparency))
+ return "update{transp}"
- repeat(None)
- return result
+class Hider(EventUpdaterBase):
+ def setDistributions(
+ self,
+ publicLikelihoodDistribution=BernoulliDistribution(0.95)
+ ):
+ self._publicLikelihood = publicLikelihoodDistribution
+ def modifyEvent(self, _ignore_href, vevent):
+ if self._publicLikelihood.sample():
+ privacy = "PUBLIC"
+ else:
+ privacy = "CONFIDENTIAL"
+ vevent.replaceProperty(Property("X-CALENDARSERVER-ACCESS", privacy))
+ return "update{privacy}"
+class Noter(EventUpdaterBase):
+ def setDistributions(
+ self,
+ noteLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._noteLength = noteLengthDistribution
-class Inviter(ProfileBase):
+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._noteLength.sample()))
+ vevent.replaceProperty(Property("DESCRIPTION", "." * length))
+ return "update{notes}"
+
+class Linker(EventUpdaterBase):
+ def setDistributions(
+ self,
+ urlLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._urlLength = urlLengthDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._urlLength.sample()))
+ vevent.replaceProperty(Property("URL", 'https://bit.ly/' + '*' * length, valuetype=Value.VALUETYPE_URI))
+ return "update{url}"
+
+class Repeater(EventUpdaterBase):
+ def setDistributions(
+ self,
+ recurrenceDistribution=RecurrenceDistribution(False)
+ ):
+ self._recurrence = recurrenceDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ rrule = self._recurrence.sample()
+ if rrule is not None:
+ vevent.replaceProperty(Property(None, None, None, pycalendar=rrule))
+ return "update{rrule}"
+
+class Rescheduler(EventUpdaterBase):
+ def setDistributions(
+ self,
+ eventStartDistribution=NearFutureDistribution(),
+ eventDurationDistribution=UniformDiscreteDistribution([
+ 15 * 60, 30 * 60,
+ 45 * 60, 60 * 60,
+ 120 * 60
+ ])
+ ):
+ self._eventStart = eventStartDistribution
+ self._eventDuration = eventDurationDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ dtstart = self._eventStart.sample()
+ dtend = dtstart + Duration(seconds=self._eventDuration.sample())
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+ return "reschedule{event}"
+
+# class Alerter(EventUpdaterBase):
+# pass
+
+class Attacher(EventUpdaterBase):
+ def setDistributions(
+ self,
+ ):
+ # filesizeDistribution=NormalDistribution(24, 3),
+ # numAttachmentsDistribution=LogNormalDistribution(2, 1),
+ # attachLikelihoodDistribution=BernoulliDistribution(0.9)
+ # self._filesize = filesizeDistribution
+ # self._numAttachments = numAttachmentsDistribution
+ # self._attachLikelihood = attachLikelihoodDistribution
+ pass
+
+ def modifyEvent(self, href, vevent):
+ d = self._client.postAttachment(href, 'x' * 1024)
+ return "attach{files}"
+
+ def handleAttachments(self):
+ pass
+
+ # if True: # attachLikelihoodDistribution.sample():
+ # # size = max(0, int(self._filesize.sample()))
+ # numAttachments()
+ # self.attachFiles(event, filesizeDistribution.sample())
+ # else:
+ # pass
+
+ def attachFile(self, event):
+ # PUT new event information (nothing has actually changed)
+ # POST attachment (with Content-Disposition header, and response location)
+ # GET updated event
+ pass
+
+ def unattachFile(self):
+ pass
+
+class InviterBase(EventerBase):
"""
- A Calendar user who invites and de-invites other users to events.
+ Base profile for a calendar user that invites and deinvites other principals to events
"""
def setParameters(
self,
enabled=True,
sendInvitationDistribution=NormalDistribution(600, 60),
- inviteeDistribution=UniformDiscreteDistribution(range(-10, 11))
+ inviteeDistribution=UniformDiscreteDistribution(range(-10, 11)),
+ **params
):
self.enabled = enabled
self._sendInvitationDistribution = sendInvitationDistribution
self._inviteeDistribution = inviteeDistribution
+ if len(params) > 0:
+ pass
+ def getAttendees():
+ pass
- def run(self):
- return loopWithDistribution(
- self._reactor, self._sendInvitationDistribution, self._invite)
+ # def _invitePrincipal(self, ...):
+ # def _uninvitePrincipal(self, ...):
+
+
+
+ def _loopWithDistribution(reactor, distribution, function):
+ result = Deferred()
+
+ def repeat(ignored):
+ reactor.callLater(distribution.sample(), iterate)
+
+ def iterate():
+ d = function()
+ if d is not None:
+ d.addCallbacks(repeat, result.errback)
+ else:
+ repeat(None)
+
+ repeat(None)
+ return result
+
+ # def _didSelfOrganize(self, vevent):
+
+
+ # def _buildIndividualAttendee(self, commonName, record, ):
+
+ # # ATTENDEE;CN="Super User";CUTYPE=INDIVIDUAL;EMAIL="admin at example.com":mailto:admin at example.com
+ # # ATTENDEE;CN="User 04";CUTYPE=INDIVIDUAL;EMAIL="user04 at example.com":mailto:user04 at example.com
+
+ # role = ['REQ-PARTICIPANT', '']
+ # attendee = Property(
+ # name=u'ATTENDEE',
+ # value=cuaddr.encode("utf-8"),
+ # params={
+ # 'CN': commonName,
+ # 'CUTYPE': 'INDIVIDUAL',
+ # 'PARTSTAT': 'NEEDS-ACTION',
+ # 'ROLE': 'REQ-PARTICIPANT',
+ # 'RSVP': 'TRUE',
+ # },
+ # )
+ # return attendee
+
+ # def _buildLocationAttendee(self, cn, cuaddr):
+ # """
+ # Example Usage: profile._buildLocationAttendee("Location 01", "urn:uuid:...")
+ # """
+ # role = ['REQ-PARTICIPANT', '']
+ # attendee = Property(
+ # name=u'ATTENDEE',
+ # value=cuaddr.encode("utf-8"),
+ # params={
+ # 'CN': commonName,
+ # 'CUTYPE': 'ROOM',
+ # 'PARTSTAT': 'NEEDS-ACTION',
+ # 'ROLE': 'REQ-PARTICIPANT',
+ # 'RSVP': 'TRUE',
+ # },
+ # )
+ # return attendee
+
+ def _invite():
+ raise NotImplementedError
+
+ def _addAttendee():
+ raise NotImplementedError
+
+# TODO - invite groups
+# class Inviter(EventUpdaterBase): pass
+
+class Relocater(EventUpdaterBase):
+ def setParameters(
+ self,
+ ):
+ pass
+
+class AppleEventer(EventerBase):
+ """
+ Calendar user who makes events in the form of Apple logo
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self
+ ):
+ pass
+
+ def _addEvent(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStartDistribution.sample()
+ dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+class HappyEventer(EventerBase):
+ """
+ Calendar user who makes events in the form of Apple logo
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self
+ ):
+ self._eventStart = UniformDiscreteDistribution([
+ # August 3
+ DateTime(year=2015, month=8, day=3, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ # August 4
+ DateTime(year=2015, month=8, day=4, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ DateTime(year=2015, month=8, day=4, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+
+ # August 5
+ DateTime(year=2015, month=8, day=5, hours=13, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=13, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=17, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ # August 6
+ DateTime(year=2015, month=8, day=6, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ DateTime(year=2015, month=8, day=6, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+
+ # August 7
+ DateTime(year=2015, month=8, day=7, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ ])
+ self._eventDuration = UniformDiscreteDistribution([
+ 30 * 60,
+ 60 * 60
+ ])
+
+ def _addEvent(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStart.sample()
+ dtend = dtstart + Duration(seconds=self._eventDuration.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+
+
+# class EventDeleter(ProfileBase):
+
+
+
+""" TEST """
+# class Intern(object):
+# def __init__(self):
+# self.behaviors = [
+# Eventer(asdfjadsf),
+# Attacher(asjadsfjasdf),
+# Inviter(enabled=True, **params)
+# ]
+
+# def run(self):
+# deferreds = []
+# for behavior in self.behaviors:
+# deferreds.append(behavior.run())
+# return DeferredList(deferreds)
+
+
+
+####################
+# Class Hierarchy
+# ---------------
+# TaskBase
+# Tasker
+# TaskUpdaterBase
+# Titler
+# Alerter
+# Noter
+# Prioritizer
+# Completer
+# TaskDeleter
+####################
+class TaskBase(ProfileBase):
+ """
+ Base profile for a calendar user who interacts with tasks
+ """
+ def setParameters(self, enabled=True, interval=25, **params):
+ self.enabled = enabled
+ self._interval = interval
+
+class Tasker(TaskBase):
+ """
+ A Calendar user who creates new tasks.
+ """
+ def initialize(self):
+ self.action = self._addTask
+ return succeed(None)
+
+ def _addTask(self, title="Simple Task"):
+ if not self._client.started:
+ return succeed(None)
+
+ calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
+
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+
+ # Copy the template task and fill in some of its fields
+ # to make a new task to create on the calendar.
+ vcalendar = taskTemplate.duplicate()
+ vtodo = vcalendar.mainComponent()
+ uid = str(uuid4())
+ vtodo.replaceProperty(Property("UID", uid))
+ vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ d = self._client.addEvent(href, vcalendar)
+ return self._newOperation("create", d)
+
+
+class TaskUpdaterBase(TaskBase):
+ def action(self):
+ task = self._getRandomTask()
+ if not task:
+ return succeed(None)
+ component = task.component
+ vtodo = component.mainComponent()
+
+ label = self.modifyEvent(task.url, vtodo)
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ task.component = component
+ d = self._client.updateEvent(task)
+ return self._newOperation(label, d)
+
+ def modifyEvent(self):
+ """Overriden by subclasses"""
+ pass
+
+
+
+
+class TaskUpdater(TaskBase):
+ """
+ A Calendar user who creates and updates complex tasks with:
+ Mark as completed/not completed
+ Change name
+ Change priority
+ Change notes
+ Sets due dates
+ """
+
+ def setParameters(
+ self,
+ taskDueDistribution=NearFutureDistribution(),
+ ):
+ pass
+
+
+ def _addTask(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
+
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+
+ # Copy the template task and fill in some of its fields
+ # to make a new task to create on the calendar.
+ vcalendar = taskTemplate.duplicate()
+ vtodo = vcalendar.mainComponent()
+ uid = str(uuid4())
+ due = self._taskStartDistribution.sample()
+ vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("UID", uid))
+
+ # vtodo.replaceProperty(Property("SUMMARY", title))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ d = self._client.addEvent(href, vcalendar)
+ return self._newOperation("create", d)
+
+
+
+# class TaskTitlerMixin = TitlerMixin
+# class TaskAlerterMixin = AlerterMixin
+# self._taskStartDistribution = taskDueDistribution
+# vtodo.replaceProperty(Property("DUE", due))
+# class TaskNoterMixin = NoterMixin
+
+class Prioritizer(TaskBase):
+ PRIORITY_NONE = 0
+ PRIORITY_HIGH = 1
+ PRIORITY_MEDIUM = 5
+ PRIORITY_LOW = 9
+
+ def _setPriority(self, priority, vtodo):
+ if priority == self.PRIORITY_NONE:
+ vtodo.removeProperty("PRIORITY")
+ else:
+ vtodo.replaceProperty(Property("PRIORITY", priority))
+
+class Completer(TaskBase):
+ def _markTaskComplete(vtodo):
+ """ task is a Component representing a VTODO """
+ vtodo.replaceProperty(Property("COMPLETED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("PERCENT-COMPLETE", 100))
+ vtodo.replaceProperty(Property("STATUS", "COMPLETED"))
+
+
+ def _markTaskIncomplete(vtodo):
+ """ mark a VTODO as incomplete """
+ vtodo.removeProperty("COMPLETED")
+ vtodo.removeProperty("PERCENT-COMPLETE")
+ vtodo.replaceProperty(Property("STATUS", "NEEDS-ACTION"))
+
+# class TaskDeleter(ProfileBase): pass
+
+
+class Inviter(ProfileBase):
+
+ def initialize(self):
+ self.action = self.test
+ return succeed(None)
+
+ def test(self):
+ event = self._getRandomEvent()
+ if not event:
+ return succeed(None)
+ print("Found event: " + str(event))
+ href = event.url
+
+ attendee = Property(
+ name=u'ATTENDEE',
+ value='urn:uuid:30000000-0000-0000-0000-000000000002',
+ params={
+ 'CN': 'Location 02',
+ 'CUTYPE': 'ROOM',
+ 'PARTSTAT': 'NEEDS-ACTION',
+ 'ROLE': 'REQ-PARTICIPANT',
+ 'RSVP': 'TRUE',
+ },
+ )
+
+ d = self._client.addEventAttendee(href, attendee)
+
+ component = event.component
+ component.mainComponent().addProperty(attendee)
+ event.component = component
+
+ d2 = self._client.addInvite(event)
+ return self._newOperation("add attendee", DeferredList([d, d2]))
+
+
def _addAttendee(self, event, attendees):
"""
Create a new attendee to add to the list of attendees for the
@@ -280,7 +1003,8 @@
# Find out who might attend
attendees = tuple(component.properties('ATTENDEE'))
- d = self._addAttendee(event, attendees)
+ # d = self._addAttendee(event, attendees)
+ d = self._addLocation(event, "Location 05", "urn:uuid:30000000-0000-0000-0000-000000000005")
d.addCallbacks(
lambda attendee:
self._client.addEventAttendee(
@@ -297,25 +1021,6 @@
"""
A Calendar user who invites other users to new events.
"""
- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
-
def setParameters(
self,
enabled=True,
@@ -341,11 +1046,6 @@
self._recurrenceDistribution = recurrenceDistribution
- def run(self):
- return loopWithDistribution(
- self._reactor, self._sendInvitationDistribution, self._invite)
-
-
def _addAttendee(self, event, attendees):
"""
Create a new attendee to add to the list of attendees for the
@@ -413,7 +1113,7 @@
# Copy the template event and fill in some of its fields
# to make a new event to create on the calendar.
- vcalendar = self._eventTemplate.duplicate()
+ vcalendar = eventTemplate.duplicate()
vevent = vcalendar.mainComponent()
uid = str(uuid4())
dtstart = self._eventStartDistribution.sample()
@@ -444,7 +1144,9 @@
return self._newOperation("invite", d)
-
+##########################
+# Notification Behaviors #
+##########################
class Accepter(ProfileBase):
"""
A Calendar user who accepts invitations to events. As well as accepting requests, this
@@ -533,7 +1235,7 @@
# Download the event again and attempt to make the change
# to the attendee list again.
- d = self._client.updateEvent(href)
+ d = self._client._refreshEvent(href)
def cbUpdated(ignored):
d = change()
d.addErrback(scheduleError)
@@ -597,149 +1299,20 @@
-class Eventer(ProfileBase):
+class EventUpdater(EventerBase):
"""
- A Calendar user who creates new events.
- """
- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
- def setParameters(
- self,
- enabled=True,
- interval=25,
- eventStartDistribution=NearFutureDistribution(),
- eventDurationDistribution=UniformDiscreteDistribution([
- 15 * 60, 30 * 60,
- 45 * 60, 60 * 60,
- 120 * 60
- ]),
- recurrenceDistribution=RecurrenceDistribution(False),
- ):
- self.enabled = enabled
- self._interval = interval
- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
- self._recurrenceDistribution = recurrenceDistribution
-
-
- def run(self):
- self._call = LoopingCall(self._addEvent)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
-
-
- def _addEvent(self):
- if not self._client.started:
- return succeed(None)
-
- calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
-
- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = self._eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
- vevent.replaceProperty(Property("UID", uid))
-
- rrule = self._recurrenceDistribution.sample()
- if rrule is not None:
- vevent.addProperty(Property(None, None, None, pycalendar=rrule))
-
- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
-
-class EventUpdater(ProfileBase):
- """
A Calendar user who creates a new event, and then updates its alarm.
"""
- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-BEGIN:VALARM
-X-WR-ALARMUID:D9D1AC84-F629-4B9D-9B6B-4A6CA9A11FEF
-UID:D9D1AC84-F629-4B9D-9B6B-4A6CA9A11FEF
-DESCRIPTION:Event reminder
-TRIGGER:-PT8M
-ACTION:DISPLAY
-END:VALARM
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
- def setParameters(
- self,
- enabled=True,
- interval=25,
- eventStartDistribution=NearFutureDistribution(),
- eventDurationDistribution=UniformDiscreteDistribution([
- 15 * 60, 30 * 60,
- 45 * 60, 60 * 60,
- 120 * 60
- ]),
- recurrenceDistribution=RecurrenceDistribution(False),
- ):
- self.enabled = enabled
- self._interval = interval
- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
- self._recurrenceDistribution = recurrenceDistribution
-
-
def initialize(self):
"""
Called before the profile runs for real. Can be used to initialize client state.
@return: a L{Deferred} that fires when initialization is done
"""
+ self.action = self._updateEvent
return self._initEvent()
- def run(self):
- self._call = LoopingCall(self._updateEvent)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
-
-
def _initEvent(self):
if not self._client.started:
return succeed(None)
@@ -801,195 +1374,175 @@
d = self._client.changeEvent(event.url)
return self._newOperation("update", d)
+ # def _changeEventTitle(self, event, title):
+ # event.component = self._setEventTitle(event.component, title)
+ # event.
-class Tasker(ProfileBase):
+
+
+
+class EventDeleter(EventerBase):
"""
- A Calendar user who creates new tasks.
+ A calendar user who deletes events at random
"""
- _taskTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VTODO
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-SUMMARY:Simple task
-DUE;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-END:VTODO
-END:VCALENDAR
-""".replace("\n", "\r\n"))
+ def initialize(self):
+ self.action = self._deleteEvent
- def setParameters(
- self,
- enabled=True,
- interval=25,
- taskDueDistribution=NearFutureDistribution(),
- ):
+ def _deleteEvent(self):
+ event = self._getRandomEvent()
+ if event is None:
+ return succeed(None)
+ d = self._client.deleteEvent(event)
+ return self._newOperation("delete", d)
+
+
+
+
+
+######################
+# Calendar Behaviors #
+######################
+class CalendarBase(ProfileBase):
+ """
+ A calendar user who interacts with calendars
+ """
+ def initialize(self):
+ self.action = lambda: None
+ return succeed(None)
+
+
+ def setParameters(self, enabled=True, interval=25):
self.enabled = enabled
self._interval = interval
- self._taskStartDistribution = taskDueDistribution
- def run(self):
- self._call = LoopingCall(self._addTask)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
+class CalendarMaker(CalendarBase):
+ """ A Calendar user who adds new Calendars """
+ def initialize(self):
+ self.action = self._addCalendar
+ return succeed(None)
- def _addTask(self):
+
+ def _addCalendar(self):
if not self._client.started:
- return succeed(None)
+ return None
- calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
+ uid = str(uuid4())
- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
+ body = Calendar.buildCalendarXML(order=0, component_type="VEVENT", rgba_color='FB524FFF', name='Sample Calendar')
+ print("Making new calendar with uid: " + uid)
+ # XXX Just for testing! remove this soon
+ path = "/calendars/__uids__/" + self._client.record.guid + "/" + uid + "/"
+ d = self._client.addCalendar(path, body)
+ return self._newOperation("create", d)
- # Copy the template task and fill in some of its fields
- # to make a new task to create on the calendar.
- vcalendar = self._taskTemplate.duplicate()
- vtodo = vcalendar.mainComponent()
- uid = str(uuid4())
- due = self._taskStartDistribution.sample()
- vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DUE", due))
- vtodo.replaceProperty(Property("UID", uid))
- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
-class OperationLogger(SummarizingMixin):
+class CalendarUpdater(CalendarBase):
"""
- Profiles will initiate operations which may span multiple requests. Start
- and stop log messages are emitted for these operations and logged by this
- logger.
+ A calendar user who updates random calendars
"""
- formats = {
- u"start" : u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
- u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
- u"failed": u"%(user)s x x x x x x x x x x x %(label)8s FAILED %(reason)s",
- }
+ def initialize(self):
+ from collections import defaultdict
+ self.action = self._updateCalendar
+ self._calendarModCount = defaultdict(int) # Map from calendar href to count of modifications
+ return succeed(None)
- lagFormat = u'{lag %5.2f ms}'
+ def _updateCalendar(self):
+ if not self._client.started:
+ return None
- # the response time thresholds to display together with failing % count threshold
- _thresholds_default = {
- "operations": {
- "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
- "thresholds": {
- "default": [100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
- }
- }
- }
- _lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds
- _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
- _fields_init = [
- ('operation', -25, '%-25s'),
- ('count', 8, '%8s'),
- ('failed', 8, '%8s'),
- ]
+ self._calendarModCount[calendar.url] += 1
+ modcount = self._calendarModCount[calendar.url]
- _fields_extend = [
- ('mean', 8, '%8.4f'),
- ('median', 8, '%8.4f'),
- ('stddev', 8, '%8.4f'),
- ('avglag (ms)', 12, '%12.4f'),
- ('STATUS', 8, '%8s'),
- ]
+ colors = [
+ "#800000FF", # maroon
+ "#FF0000FF", # red
+ "#008000FF", # green
+ "#00FF00FF", # line
+ "#000080FF", # navy
+ "#0000FFFF", # blue
+ ]
+ color = colors[modcount % len(colors)]
+ self._client.setCalendarDisplayName(calendar, "Calendar ({mods})".format(mods=modcount))
+ self._client.setCalendarColor(calendar, color)
+ # choice = self.random.randint(0, 4)
+ # if choice == 0:
+ # self._client._
+ # return succeed(None)
- def __init__(self, outfile=None, **params):
- self._perOperationTimes = {}
- self._perOperationLags = {}
- if outfile is None:
- outfile = sys.stdout
- self._outfile = outfile
+ def randomUpdate(self):
+ pass
- # Load parameters from config
- if "thresholdsPath" in params:
- jsondata = json.load(open(params["thresholdsPath"]))
- elif "thresholds" in params:
- jsondata = params["thresholds"]
- else:
- jsondata = self._thresholds_default
- self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]]
- for ctr, item in enumerate(self._thresholds):
- for k, v in jsondata["operations"]["thresholds"].items():
- item[1][k] = v[ctr]
+class CalendarSharer(CalendarBase, InviterBase):
+ """
+ A calendar user who shares random calendars.
+ Even though the real client allows batch requests (e.g. 10 shares in one HTTP request),
+ we simplify life (TODO: keep it real) by having each HTTP request only add or remove one sharee.
+ """
- self._fields = self._fields_init[:]
- for threshold, _ignore_fail_at in self._thresholds:
- self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
- self._fields.extend(self._fields_extend)
+ def initialize(self):
+ self.action = self._shareCalendar
+ return succeed(None)
- if "lagCutoff" in params:
- self._lag_cut_off = params["lagCutoff"]
+ def _shareCalendar(self):
+ if not self._client.started:
+ return succeed(None)
- if "failCutoff" in params:
- self._fail_cut_off = params["failCutoff"]
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
+ # The decision of who to invite / uninvite should be made here
+ inv = random.randint(0, 1)
+ rem = random.randint(0, 1)
- def observe(self, event):
- if event.get("type") == "operation":
- event = event.copy()
- lag = event.get('lag')
- if lag is None:
- event['lag'] = ''
- else:
- event['lag'] = self.lagFormat % (lag * 1000.0,)
+ invRecord = self._sim.getUserRecord(inv)
+ remRecord = self._sim.getUserRecord(rem)
- self._outfile.write(
- (self.formats[event[u'phase']] % event).encode('utf-8') + '\n')
+ print("Sharing " + calendar.url)
+ self._inviteUser(calendar, invRecord)
+ # self._removeUser(calendar, remRecord)
- if event[u'phase'] == u'end':
- dataset = self._perOperationTimes.setdefault(event[u'label'], [])
- dataset.append((event[u'success'], event[u'duration']))
- elif lag is not None:
- dataset = self._perOperationLags.setdefault(event[u'label'], [])
- dataset.append(lag)
+ return succeed(None)
+ def _inviteUser(self, calendar, userRecord):
+ mailto = "mailto:{}".format(userRecord.email)
+ body = Calendar.addInviteeXML(mailto, calendar.name, readwrite=True)
+ d = self._client.postXML(calendar.url, body)
+ # print(body)
- def _summarizeData(self, operation, data):
- avglag = mean(self._perOperationLags.get(operation, [0.0])) * 1000.0
- data = SummarizingMixin._summarizeData(self, operation, data)
- return data[:-1] + (avglag,) + data[-1:]
+ def _removeUser(self, calendar, userRecord):
+ mailto = "mailto:{}".format(userRecord.email)
+ body = Calendar.removeInviteeXML(mailto)
- def report(self, output):
- output.write("\n")
- self.printHeader(output, [
- (label, width)
- for (label, width, _ignore_fmt) in self._fields
- ])
- self.printData(
- output,
- [fmt for (label, width, fmt) in self._fields],
- sorted(self._perOperationTimes.items())
- )
+ d = self._client.postXML(calendar.url, body)
+ # print(body)
- _LATENCY_REASON = "Median %(operation)s scheduling lag greater than %(cutoff)sms"
- _FAILED_REASON = "Greater than %(cutoff).0f%% %(operation)s failed"
- def failures(self):
- reasons = []
- for operation, lags in self._perOperationLags.iteritems():
- if median(lags) > self._lag_cut_off:
- reasons.append(self._LATENCY_REASON % dict(
- operation=operation.upper(), cutoff=self._lag_cut_off * 1000))
+class CalendarDeleter(CalendarBase):
+ """
+ A calendar user who deletes entire calendars
+ """
+ def initialize(self):
+ self.action = self._deleteCalendar
+ return succeed(None)
- for operation, times in self._perOperationTimes.iteritems():
- failures = len([success for (success, _ignore_duration) in times if not success])
- if failures * 100.0 / len(times) > self._fail_cut_off:
- reasons.append(self._FAILED_REASON % dict(
- operation=operation.upper(), cutoff=self._fail_cut_off))
+ def _deleteCalendar(self):
+ if not self._client.started:
+ return succeed(None)
- return reasons
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
+ print("Deleting " + calendar.url)
+ d = self._client.deleteCalendar(calendar.url)
+ return self._newOperation("delete", d)
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request 2015-08-17 20:29:08 UTC (rev 15046)
@@ -1,2 +1,2 @@
<?xml version="1.0" encoding="utf-8" ?>
-<x0:principal-property-search xmlns:x2="urn:ietf:params:xml:ns:caldav" xmlns:x0="DAV:" xmlns:x1="http://calendarserver.org/ns/" test="anyof"><x0:property-search><x0:prop><x0:displayname/></x0:prop><x0:match match-type="starts-with">%(displayname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:email-address-set/></x0:prop><x0:match match-type="starts-with">%(email)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:first-name/></x0:prop><x0:match match-type="starts-with">%(firstname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:last-name/></x0:prop><x0:match match-type="starts-with">%(lastname)s</x0:match></x0:property-search><x0:prop><x1:email-address-set/><x2:calendar-user-address-set/><x2:calendar-user-type/><x0:displayname/><x1:last-name/><x1:first-name/><x1:record-type/><x0:principal-URL/></x0:prop></x0:principal-property-search>
+<x0:principal-property-search xmlns:x2="urn:ietf:params:xml:ns:caldav" xmlns:x0="DAV:" xmlns:x1="http://calendarserver.org/ns/" test="anyof"><x0:property-search><x0:prop><x0:displayname/></x0:prop><x0:match match-type="starts-with">%(displayname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:email-address-set/></x0:prop><x0:match match-type="starts-with">%(email)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:first-name/></x0:prop><x0:match match-type="starts-with">%(firstname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:last-name/></x0:prop><x0:match match-type="starts-with">%(lastname)s</x0:match></x0:property-search><x0:prop><x1:email-address-set/><x2:calendar-user-address-set/><x2:calendar-user-type/><x0:displayname/><x1:last-name/><x1:first-name/><x1:record-type/><x0:principal-URL/></x0:prop></x0:principal-property-search>
\ No newline at end of file
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -45,58 +45,16 @@
from twisted.web.server import Site
-from contrib.performance.loadtest.ical import OS_X_10_6
+from contrib.performance.loadtest.clients import OS_X_10_6
from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
from contrib.performance.loadtest.population import (
Populator, ProfileType, ClientType, PopulationParameters, SmoothRampUp,
CalendarClientSimulator)
from contrib.performance.loadtest.webadmin import LoadSimAdminResource
-
-
-class _DirectoryRecord(object):
- def __init__(self, uid, password, commonName, email, guid):
- self.uid = uid
- self.password = password
- self.commonName = commonName
- self.email = email
- self.guid = guid
-
-
-
def safeDivision(value, total, factor=1):
return value * factor / total if total else 0
-
-
-def generateRecords(
- count, uidPattern="user%d", passwordPattern="user%d",
- namePattern="User %d", emailPattern="user%d at example.com",
- guidPattern="user%d"
-):
- for i in xrange(count):
- i += 1
- uid = uidPattern % (i,)
- password = passwordPattern % (i,)
- name = namePattern % (i,)
- email = emailPattern % (i,)
- guid = guidPattern % (i,)
- yield _DirectoryRecord(uid, password, name, email, guid)
-
-
-
-def recordsFromCSVFile(path):
- if path:
- pathObj = FilePath(path)
- else:
- pathObj = FilePath(__file__).sibling("accounts.csv")
- return [
- _DirectoryRecord(*line.decode('utf-8').split(u','))
- for line
- in pathObj.getContent().splitlines()]
-
-
-
class LagTrackingReactor(object):
"""
This reactor wraps another reactor and proxies all attribute
@@ -129,8 +87,9 @@
Command line configuration options for the load simulator.
"""
config = None
- _defaultConfig = FilePath(__file__).sibling("config.plist")
- _defaultClients = FilePath(__file__).sibling("clients.plist")
+ settings = FilePath(__file__).sibling("settings")
+ _defaultConfig = settings.child("config.plist")
+ _defaultClients = settings.child("clients.plist")
optParameters = [
("runtime", "t", None,
@@ -195,6 +154,8 @@
configFile.close()
try:
+ # from importlib import import_module
+ # client_config = import_module("contrib.performance.loadtest.settings.clients")
clientFile = self['clients'].open()
except IOError, e:
raise UsageError("--clients %s: %s" % (
@@ -202,13 +163,15 @@
try:
try:
client_config = readPlist(clientFile)
+ # self.config["clients"] = client_config.calendars_only# client_config["clients"]
self.config["clients"] = client_config["clients"]
if "arrivalInterval" in client_config:
self.config["arrival"]["params"]["interval"] = client_config["arrivalInterval"]
except ExpatError, e:
raise UsageError("--clients %s: %s" % (self['clients'].path, e))
finally:
- clientFile.close()
+ # clientFile.close()
+ pass
Arrival = namedtuple('Arrival', 'factory parameters')
@@ -224,17 +187,16 @@
@type arrival: L{Arrival}
@type parameters: L{PopulationParameters}
- @ivar records: A C{list} of L{_DirectoryRecord} instances giving
+ @ivar records: A C{list} of L{DirectoryRecord} instances giving
user information about the accounts on the server being put
under load.
"""
- def __init__(self, server, principalPathTemplate, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
+ def __init__(self, server, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
records=None, reactor=None, runtime=None, workers=None,
configTemplate=None, workerID=None, workerCount=1):
if reactor is None:
from twisted.internet import reactor
self.server = server
- self.principalPathTemplate = principalPathTemplate
self.webadminPort = webadminPort
self.serverStats = serverStats
self.serializationPath = serializationPath
@@ -270,6 +232,8 @@
"""
Create a L{LoadSimulator} from a parsed instance of a configuration
property list.
+
+ @type{config} L{Config} object
"""
workers = config.get("workers")
@@ -279,7 +243,6 @@
workerCount = config.get("workerCount", 1)
configTemplate = None
server = config.get('server', 'http://127.0.0.1:8008')
- principalPathTemplate = config.get('principalPathTemplate', '/principals/users/%s/')
serializationPath = None
if 'clientDataSerialization' in config:
@@ -306,6 +269,14 @@
parameters = PopulationParameters()
if 'clients' in config:
for clientConfig in config['clients']:
+ # parameters.addClient(
+ # clientConfig["weight"],
+ # ClientType(
+ # clientConfig["software"],
+ # clientConfig["params"],
+ # clientConfig["profiles"]
+ # )
+ # )
parameters.addClient(
clientConfig["weight"],
ClientType(
@@ -316,7 +287,8 @@
namedAny(profile["class"]),
cls._convertParams(profile["params"])
) for profile in clientConfig["profiles"]
- ]))
+ ])),
+
if not parameters.clients:
parameters.addClient(1,
ClientType(OS_X_10_6, {},
@@ -324,7 +296,6 @@
else:
# Manager / observer process.
server = ''
- principalPathTemplate = ''
serializationPath = None
arrival = None
parameters = None
@@ -359,7 +330,6 @@
return cls(
server,
- principalPathTemplate,
webadminPort,
serverStats,
serializationPath,
@@ -377,6 +347,104 @@
@classmethod
+ def fromConfigObject(cls, config, runtime=None, output=stdout):
+ workers = config['workers']
+ if workers is None:
+ # Client / place where the simulator actually runs configuration
+ workerID = config.get("workerID", 0)
+ workerCount = config.get("workerCount", 1)
+ configTemplate = None
+ server = config.get('server', 'http://127.0.0.1:8008')
+ serializationPath = None
+
+ serializationPath = config['serializationPath']
+
+ if 'arrival' in config:
+ arrival = Arrival(
+ namedAny(config['arrival']['factory']),
+ config['arrival']['params'])
+ else:
+ arrival = Arrival(
+ SmoothRampUp, dict(groups=10, groupSize=1, interval=3))
+
+ parameters = PopulationParameters()
+ if 'clients' in config:
+ for clientConfig in config['clients']:
+ parameters.addClient(
+ clientConfig["weight"],
+ ClientType(
+ clientConfig["software"],
+ clientConfig["params"],
+ clientConfig["profiles"]
+ )
+ )
+ # ClientType(
+ # namedAny(clientConfig["software"]),
+ # cls._convertParams(clientConfig["params"]),
+ # [
+ # ProfileType(
+ # namedAny(profile["class"]),
+ # cls._convertParams(profile["params"])
+ # ) for profile in clientConfig["profiles"]
+ # ]))
+ if not parameters.clients:
+ parameters.addClient(1,
+ ClientType(OS_X_10_6, {},
+ [Eventer, Inviter, Accepter]))
+ else:
+ # Manager / observer process.
+ server = ''
+ serializationPath = None
+ arrival = None
+ parameters = None
+ workerID = 0
+ configTemplate = config
+ workerCount = 1
+
+ # webadminPort =
+ webadminPort = None
+ if 'webadmin' in config:
+ if config['webadmin']['enabled']:
+ webadminPort = config['webadmin']['HTTPPort']
+
+ serverStats = None
+ if 'serverStats' in config:
+ if config['serverStats']['enabled']:
+ serverStats = config['serverStats']
+ serverStats['server'] = config['server'] if 'server' in config else ''
+
+ observers = []
+ if 'observers' in config:
+ for observer in config['observers']:
+ observerName = observer["type"]
+ observerParams = observer["params"]
+ observers.append(namedAny(observerName)(**observerParams))
+
+ records = []
+ if 'accounts' in config:
+ loader = config['accounts']['loader']
+ params = config['accounts']['params']
+ records.extend(namedAny(loader)(**params))
+ output.write("Loaded {0} accounts.\n".format(len(records)))
+
+ return cls(
+ server,
+ webadminPort,
+ serverStats,
+ serializationPath,
+ arrival,
+ parameters,
+ observers=observers,
+ records=records,
+ runtime=runtime,
+ reactor=reactor,
+ workers=workers,
+ configTemplate=configTemplate,
+ workerID=workerID,
+ workerCount=workerCount,
+ )
+
+ @classmethod
def _convertParams(cls, params):
"""
Find parameter values which should be more structured than plistlib is
@@ -415,7 +483,6 @@
self.parameters,
self.reactor,
self.server,
- self.principalPathTemplate,
self.serializationPath,
self.workerID,
self.workerCount,
@@ -423,6 +490,7 @@
def createArrivalPolicy(self):
+ # print(self.arrival.parameters)
return self.arrival.factory(self.reactor, **self.arrival.parameters)
@@ -431,16 +499,14 @@
Return a list of L{SimService} subclasses for C{attachServices} to
instantiate and attach to the reactor.
"""
- if self.workers is not None:
- return [
- ObserverService,
- WorkerSpawnerService,
- ReporterService,
- ]
+ if self.workers:
+ PrimaryService = WorkerSpawnerService
+ else:
+ PrimaryService = SimulatorService
return [
ObserverService,
- SimulatorService,
ReporterService,
+ PrimaryService
]
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/standard-configs/events-only.plist 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist 2015-08-17 20:29:08 UTC (rev 15046)
@@ -29,23 +29,23 @@
<dict>
- <!-- Here is a Lion iCal simulator. -->
+ <!-- Here is an El Captian iCal simulator. -->
<key>software</key>
- <string>contrib.performance.loadtest.ical.OS_X_10_7</string>
+ <string>contrib.performance.loadtest.ical.OS_X_10_11</string>
<!-- Arguments to use to initialize the client instance. -->
<key>params</key>
<dict>
<!-- Name that appears in logs. -->
<key>title</key>
- <string>10.7</string>
+ <string>10.11</string>
<!-- Client can poll the calendar home at some interval. This is
in seconds. -->
<key>calendarHomePollInterval</key>
- <integer>300000</integer>
+ <integer>30</integer>
- <!-- If the server advertises xmpp push, OS X 10.6 can wait for notifications
+ <!-- If the server advertises xmpp push, OS X 10.11 can wait for notifications
about calendar home changes instead of polling for them periodically. If
this option is true, then look for the server advertisement for xmpp push
and use it if possible. Still fall back to polling if there is no xmpp push
@@ -268,269 +268,6 @@
</dict>
</dict>
- <!-- This profile invites some number of new attendees to new events. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.RealisticInviter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define the frequency at which new invitations will be sent out. -->
- <key>sendInvitationDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mu gives the mean of the normal distribution (in seconds). -->
- <key>mu</key>
- <integer>10</integer>
-
- <!-- and sigma gives its standard deviation. -->
- <key>sigma</key>
- <integer>5</integer>
- </dict>
- </dict>
-
- <!-- Define the distribution of who will be invited to an event.
-
- When inviteeClumping is turned on each invitee is based on a sample of
- users "close to" the organizer based on account index. If the clumping
- is too "tight" for the requested number of attendees, then invites for
- those larger numbers will simply fail (the sim will report that situation).
-
- When inviteeClumping is off invitees will be sampled across an entire
- range of account indexes. In this case the distribution ought to be a
- UniformIntegerDistribution with min=0 and max set to the number of accounts.
- -->
- <key>inviteeDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.UniformIntegerDistribution</string>
- <key>params</key>
- <dict>
- <!-- The minimum value (inclusive) of the uniform distribution. -->
- <key>min</key>
- <integer>0</integer>
- <!-- The maximum value (exclusive) of the uniform distribution. -->
- <key>max</key>
- <integer>99</integer>
- </dict>
- </dict>
-
- <key>inviteeClumping</key>
- <true/>
-
- <!-- Define the distribution of how many attendees will be invited to an event.
-
- LogNormal is the best fit to observed data.
-
-
- For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
- mode should typically be 1, and mean whatever matches the user behavior.
- Our typical mean is 6.
- -->
- <key>inviteeCountDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>1</integer>
- <!-- mean - average-->
- <key>median</key>
- <integer>6</integer>
- <!-- maximum -->
- <key>maximum</key>
- <real>100</real>
- </dict>
- </dict>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile accepts invitations to events, handles cancels, and
- handles replies received. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Accepter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define how long to wait after seeing a new invitation before
- accepting it.
-
- For LogNormal "mode" is the peak, "median" is the 50% cummulative value
- (i.e., half of the user have accepted by that time).
- -->
- <key>acceptDelayDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>300</integer>
- <!-- median - 50% done-->
- <key>median</key>
- <integer>1800</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- A task-creating profile, which will periodically create
- new tasks at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Tasker</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new task. -->
- <key>interval</key>
- <integer>300</integer>
-
- <!-- Define how due times (DUE) for the randomly generated tasks
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>taskDueDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
- </dict>
- </dict>
-
</array>
<!-- Determine the frequency at which this client configuration will
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/webadmin.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -39,6 +39,7 @@
HEAD = """\
<html>
<head>
+<meta http-equiv="refresh" content="1" >
<style type="text/css">
body {color:#000000;}
h1 h2 h3 {color:#333333;}
@@ -80,9 +81,11 @@
def __init__(self, loadsim):
self.loadsim = loadsim
self.token = str(uuid.uuid4())
+ self.count = 0
def render_GET(self, request):
+ self.count += 1
return self._renderReport()
@@ -109,5 +112,5 @@
html = self.HEAD + self.BODY_RESULTS_STOPPED
return html % (None, report.getvalue(), ms)
else:
- html = self.HEAD + self.BODY_RESULTS
+ html = self.HEAD + "<h3>Request #" + str(self.count) + "</h3>" + self.BODY_RESULTS
return html % (self.token, report.getvalue(), ms)
Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py
===================================================================
--- CalendarServer/trunk/contrib/performance/stats.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py 2015-08-17 20:29:08 UTC (rev 15046)
@@ -14,26 +14,30 @@
# limitations under the License.
##
+"""
+Implementation of a statistics library for Calendar performance analysis.
+Exports:
+
+_Statistic object
+ Duration
+ SQLDuration
+ Bytes
+"""
+
from __future__ import print_function
-from math import log, sqrt
-from time import mktime
-import random
import sqlparse
-from pycalendar.datetime import DateTime
-from pycalendar.duration import Duration as PyDuration
-from pycalendar.icalendar.property import Property
-from pycalendar.timezone import Timezone
-
-from zope.interface import Interface, implements
-from twisted.python.util import FancyEqMixin
-
-
NANO = 1000000000.0
def mean(samples):
+ """
+ Computes the mean value of a list of numbers
+ @type samples list of numerics
+
+ Raises ZeroDivisionError if `samples` is empty
+ """
return sum(samples) / len(samples)
@@ -48,13 +52,6 @@
-def stddev(samples):
- m = mean(samples)
- variance = sum([datum ** 2 for datum in residuals(samples, m)]) / len(samples)
- return variance ** 0.5
-
-
-
def mad(samples):
"""
Return the median absolute deviation of the given data set.
@@ -65,6 +62,13 @@
+def stddev(samples):
+ m = mean(samples)
+ variance = sum([datum ** 2 for datum in residuals(samples, m)]) / len(samples)
+ return variance ** 0.5
+
+
+
class _Statistic(object):
commands = ['summarize']
@@ -221,272 +225,3 @@
def summarize(self, samples):
return _Statistic.summarize(self, self.squash(samples))
-
-
-
-def quantize(data):
- """
- Given some continuous data, quantize it into appropriately sized
- discrete buckets (eg, as would be suitable for constructing a
- histogram of the values).
- """
- # buckets = {}
- return []
-
-
-
-class IPopulation(Interface):
- def sample(): #@NoSelf
- pass
-
-
-
-class UniformDiscreteDistribution(object, FancyEqMixin):
- """
-
- """
- implements(IPopulation)
-
- compareAttributes = ['_values']
-
- def __init__(self, values, randomize=True):
- self._values = values
- self._randomize = randomize
- self._refill()
-
-
- def _refill(self):
- self._remaining = self._values[:]
- if self._randomize:
- random.shuffle(self._remaining)
-
-
- def sample(self):
- if not self._remaining:
- self._refill()
- return self._remaining.pop()
-
-
-
-class LogNormalDistribution(object, FancyEqMixin):
- """
- """
- implements(IPopulation)
-
- compareAttributes = ['_mu', '_sigma', '_maximum']
-
- def __init__(self, mu=None, sigma=None, mean=None, mode=None, median=None, maximum=None):
-
- if mu is not None and sigma is not None:
- scale = 1.0
- elif not (mu is None and sigma is None):
- raise ValueError("mu and sigma must both be defined or both not defined")
- elif mode is None:
- raise ValueError("When mu and sigma are not defined, mode must be defined")
- elif median is not None:
- scale = mode
- median /= mode
- mode = 1.0
- mu = log(median)
- sigma = sqrt(log(median) - log(mode))
- elif mean is not None:
- scale = mode
- mean /= mode
- mode = 1.0
- mu = log(mean) + log(mode) / 2.0
- sigma = sqrt(log(mean) - log(mode) / 2.0)
- else:
- raise ValueError("When using mode one of median or mean must be defined")
-
- self._mu = mu
- self._sigma = sigma
- self._scale = scale
- self._maximum = maximum
-
-
- def sample(self):
- result = self._scale * random.lognormvariate(self._mu, self._sigma)
- if self._maximum is not None and result > self._maximum:
- for _ignore in range(10):
- result = self._scale * random.lognormvariate(self._mu, self._sigma)
- if result <= self._maximum:
- break
- else:
- raise ValueError("Unable to generate LogNormalDistribution sample within required range")
- return result
-
-
-
-class FixedDistribution(object, FancyEqMixin):
- """
- """
- implements(IPopulation)
-
- compareAttributes = ['_value']
-
- def __init__(self, value):
- self._value = value
-
-
- def sample(self):
- return self._value
-
-
-
-class NearFutureDistribution(object, FancyEqMixin):
- compareAttributes = ['_offset']
-
- def __init__(self):
- self._offset = LogNormalDistribution(7, 0.8)
-
-
- def sample(self):
- now = DateTime.getNowUTC()
- now.offsetSeconds(int(self._offset.sample()))
- return now
-
-
-
-class NormalDistribution(object, FancyEqMixin):
- compareAttributes = ['_mu', '_sigma']
-
- def __init__(self, mu, sigma):
- self._mu = mu
- self._sigma = sigma
-
-
- def sample(self):
- # Only return positive values or zero
- v = random.normalvariate(self._mu, self._sigma)
- while v < 0:
- v = random.normalvariate(self._mu, self._sigma)
- return v
-
-
-
-class UniformIntegerDistribution(object, FancyEqMixin):
- compareAttributes = ['_min', '_max']
-
- def __init__(self, min, max):
- self._min = min
- self._max = max
-
-
- def sample(self):
- return int(random.uniform(self._min, self._max))
-
-
-NUM_WEEKDAYS = 7
-
-class WorkDistribution(object, FancyEqMixin):
- compareAttributes = ["_daysOfWeek", "_beginHour", "_endHour"]
-
- _weekdayNames = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"]
-
- def __init__(self, daysOfWeek=["mon", "tue", "wed", "thu", "fri"], beginHour=8, endHour=17, tzname="UTC"):
- self._daysOfWeek = [self._weekdayNames.index(day) for day in daysOfWeek]
- self._beginHour = beginHour
- self._endHour = endHour
- self._tzname = tzname
- self._helperDistribution = NormalDistribution(
- # Mean 6 workdays in the future
- 60 * 60 * 8 * 6,
- # Standard deviation of 4 workdays
- 60 * 60 * 8 * 4)
- self.now = DateTime.getNow
-
-
- def astimestamp(self, dt):
- return mktime(dt.timetuple())
-
-
- def _findWorkAfter(self, when):
- """
- Return a two-tuple of the start and end of work hours following
- C{when}. If C{when} falls within work hours, then the start time will
- be equal to when.
- """
- # Find a workday that follows the timestamp
- weekday = when.getDayOfWeek()
- for i in range(NUM_WEEKDAYS):
- day = when + PyDuration(days=i)
- if (weekday + i) % NUM_WEEKDAYS in self._daysOfWeek:
- # Joy, a day on which work might occur. Find the first hour on
- # this day when work may start.
- day.setHHMMSS(self._beginHour, 0, 0)
- begin = day
- end = begin.duplicate()
- end.setHHMMSS(self._endHour, 0, 0)
- if end > when:
- return begin, end
-
-
- def sample(self):
- offset = PyDuration(seconds=int(self._helperDistribution.sample()))
- beginning = self.now(Timezone(tzid=self._tzname))
- while offset:
- start, end = self._findWorkAfter(beginning)
- if end - start > offset:
- result = start + offset
- result.setMinutes(result.getMinutes() // 15 * 15)
- result.setSeconds(0)
- return result
- offset.setDuration(offset.getTotalSeconds() - (end - start).getTotalSeconds())
- beginning = end
-
-
-
-class RecurrenceDistribution(object, FancyEqMixin):
- compareAttributes = ["_allowRecurrence", "_weights"]
-
- _model_rrules = {
- "none": None,
- "daily": "RRULE:FREQ=DAILY",
- "weekly": "RRULE:FREQ=WEEKLY",
- "monthly": "RRULE:FREQ=MONTHLY",
- "yearly": "RRULE:FREQ=YEARLY",
- "dailylimit": "RRULE:FREQ=DAILY;COUNT=14",
- "weeklylimit": "RRULE:FREQ=WEEKLY;COUNT=4",
- "workdays": "RRULE:FREQ=DAILY;BYDAY=MO,TU,WE,TH,FR"
- }
-
- def __init__(self, allowRecurrence, weights={}):
- self._allowRecurrence = allowRecurrence
- self._rrules = []
- if self._allowRecurrence:
- for rrule, count in sorted(weights.items(), key=lambda x: x[0]):
- for _ignore in range(count):
- self._rrules.append(self._model_rrules[rrule])
- self._helperDistribution = UniformIntegerDistribution(0, len(self._rrules) - 1)
-
-
- def sample(self):
-
- if self._allowRecurrence:
- index = self._helperDistribution.sample()
- rrule = self._rrules[index]
- if rrule:
- prop = Property.parseText(rrule)
- return prop
-
- return None
-
-if __name__ == '__main__':
-
- from collections import defaultdict
- mu = 1.5
- sigma = 1.22
- distribution = LogNormalDistribution(mu, sigma, 100)
- result = defaultdict(int)
- for i in range(100000):
- s = int(distribution.sample())
- if s > 300:
- continue
- result[s] += 1
-
- total = 0
- for k, v in sorted(result.items(), key=lambda x: x[0]):
- print("%d\t%.5f" % (k, float(v) / result[1]))
- total += k * v
-
- print("Average: %.2f" % (float(total) / sum(result.values()),))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150817/65885432/attachment-0001.html>
More information about the calendarserver-changes
mailing list