Revision: 9300 http://trac.macosforge.org/projects/calendarserver/changeset/9300 Author: cdaboo@apple.com Date: 2012-05-25 06:22:21 -0700 (Fri, 25 May 2012) Log Message: ----------- Sim now follows redirect on .well-known and allows for a principal path to be specified to enable it to work with other servers. PYTHONPATH now set in sim script. Modified Paths: -------------- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist CalendarServer/trunk/contrib/performance/loadtest/config.plist CalendarServer/trunk/contrib/performance/loadtest/ical.py CalendarServer/trunk/contrib/performance/loadtest/population.py CalendarServer/trunk/contrib/performance/loadtest/sim.py CalendarServer/trunk/contrib/performance/loadtest/test_ical.py CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py CalendarServer/trunk/contrib/performance/loadtest/test_sim.py CalendarServer/trunk/sim Modified: CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2012-05-25 13:22:21 UTC (rev 9300) @@ -36,6 +36,10 @@ <key>server</key> <string>https://127.0.0.1:8443/</string> + <!-- The template URI for doing initial principal lookup on. --> + <key>principalPathTemplate</key> + <string>/principals/users/%s/</string> + <!-- Configure Admin Web UI. --> <key>webadmin</key> <dict> Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2012-05-25 13:22:21 UTC (rev 9300) @@ -23,6 +23,10 @@ <key>server</key> <string>https://127.0.0.1:8443/</string> + <!-- The template URI for doing initial principal lookup on. --> + <key>principalPathTemplate</key> + <string>/principals/users/%s/</string> + <!-- Configure Admin Web UI. --> <key>webadmin</key> <dict> Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -17,7 +17,7 @@ import random from uuid import uuid4 -from urlparse import urlparse, urlunparse +from urlparse import urlparse, urlunparse, urlsplit from xml.etree import ElementTree @@ -268,7 +268,7 @@ email = None - def __init__(self, reactor, root, record, auth, calendarHomePollInterval=None, supportPush=True, + def __init__(self, reactor, root, principalPathTemplate, record, auth, calendarHomePollInterval=None, supportPush=True, supportAmpPush=True, ampPushHost="localhost", ampPushPort=62311): self._client_id = str(uuid4()) @@ -276,6 +276,7 @@ self.reactor = reactor self.agent = AuthHandlerAgent(Agent(self.reactor), auth) self.root = root + self.principalPathTemplate = principalPathTemplate self.record = record if calendarHomePollInterval is None: @@ -414,11 +415,13 @@ StringProducer(body), method_label=method_label, ) + body = yield readBody(response) result = self._parseMultiStatus(body) - returnValue(result) + returnValue((response, result,)) + @inlineCallbacks def _proppatch(self, url, body): """ @@ -461,39 +464,44 @@ returnValue(result) + @inlineCallbacks def _startupPropfindWellKnown(self): """ Issue a PROPFIND on the /.well-known/caldav/ URL """ - return self._propfind( - '/.well-known/caldav/', + + location = "/.well-known/caldav/" + response, result = yield self._propfind( + location, self._STARTUP_WELL_KNOWN, allowedStatus=(MULTI_STATUS, MOVED_PERMANENTLY), ) + + # Follow any redirect + if response.code == MOVED_PERMANENTLY: + location = response.headers.getRawHeaders("location")[0] + location = urlsplit(location)[2] + response, result = yield self._propfind( + location, + self._STARTUP_WELL_KNOWN, + allowedStatus=(MULTI_STATUS), + ) + + returnValue(result[location]) + - - def _startupPropfindRoot(self): - """ - Issue a PROPFIND on the / URL - """ - return self._propfind( - '/', - self._STARTUP_WELL_KNOWN, - ) - - @inlineCallbacks def _principalPropfindInitial(self, user): """ Issue a PROPFIND on the /principals/users/<uid> URL to retrieve the /principals/__uids__/<guid> principal URL """ - principalURL = '/principals/users/' + user + '/' - result = yield self._propfind( - '/principals/users/' + user + '/', + principalPath = self.principalPathTemplate % (user,) + _ignore_response, result = yield self._propfind( + principalPath, self._STARTUP_PRINCIPAL_PROPFIND_INITIAL, ) - returnValue(result[principalURL]) + returnValue(result[principalPath]) @inlineCallbacks @@ -503,7 +511,7 @@ user and return a L{Principal} instance constructed from the response. """ - result = yield self._propfind( + _ignore_response, result = yield self._propfind( self.principalURL, self._STARTUP_PRINCIPAL_PROPFIND, ) @@ -529,7 +537,7 @@ """ if not calendarHomeSet.endswith('/'): calendarHomeSet = calendarHomeSet + '/' - result = yield self._propfind( + _ignore_response, result = yield self._propfind( calendarHomeSet, self._POLL_CALENDARHOME_PROPFIND, depth='1', @@ -613,7 +621,7 @@ # the sim can fire a PUT between the PROPFIND and when process the removals. old_hrefs = set([calendar.url + child for child in calendar.events.keys()]) - result = yield self._propfind( + _ignore_response, result = yield self._propfind( calendar.url, self._POLL_CALENDAR_PROPFIND_D1, depth='1', @@ -839,7 +847,7 @@ @inlineCallbacks def _notificationPropfind(self, notificationURL): - result = yield self._propfind( + _ignore_response, result = yield self._propfind( notificationURL, self._POLL_NOTIFICATION_PROPFIND, ) @@ -848,7 +856,7 @@ @inlineCallbacks def _notificationChangesPropfind(self, notificationURL): - result = yield self._propfind( + _ignore_response, result = yield self._propfind( notificationURL, self._POLL_NOTIFICATION_PROPFIND_D1, depth='1', @@ -1328,7 +1336,7 @@ @inlineCallbacks def startup(self): - # PROPFIND /principals/users/<uid> to retrieve /principals/__uids__/<guid> + # PROPFIND principal path to retrieve actual principal-URL response = yield self._principalPropfindInitial(self.record.uid) hrefs = response.getHrefProperties() self.principalURL = hrefs[davxml.principal_URL].toString() @@ -1429,16 +1437,18 @@ @inlineCallbacks def startup(self): - # PROPFIND well-known - ignore - yield self._startupPropfindWellKnown() - - # PROPFIND / - ignore - yield self._startupPropfindRoot() - - # PROPFIND /principals/users/<uid> to retrieve /principals/__uids__/<guid> - response = yield self._principalPropfindInitial(self.record.uid) + # PROPFIND well-known with redirect + response = yield self._startupPropfindWellKnown() hrefs = response.getHrefProperties() - self.principalURL = hrefs[davxml.principal_URL].toString() + if davxml.current_user_principal in hrefs: + self.principalURL = hrefs[davxml.current_user_principal].toString() + elif davxml.principal_URL in hrefs: + self.principalURL = hrefs[davxml.principal_URL].toString() + else: + # PROPFIND principal path to retrieve actual principal-URL + response = yield self._principalPropfindInitial(self.record.uid) + hrefs = response.getHrefProperties() + self.principalURL = hrefs[davxml.principal_URL].toString() # Using the actual principal URL, retrieve principal information principal = yield self._principalPropfind() @@ -1534,7 +1544,7 @@ Issue a PROPFIND on the /principals/ URL to retrieve the /principals/__uids__/<guid> principal URL """ - result = yield self._propfind( + _ignore_response, result = yield self._propfind( '/principals/', self._STARTUP_PRINCIPAL_PROPFIND_INITIAL, ) @@ -1622,16 +1632,18 @@ @inlineCallbacks def startup(self): - # PROPFIND well-known - ignore - yield self._startupPropfindWellKnown() - - # PROPFIND / - ignore - yield self._startupPropfindRoot() - - # PROPFIND /principals/ to retrieve /principals/__uids__/<guid> - response = yield self._principalPropfindInitial() + # PROPFIND well-known with redirect + response = yield self._startupPropfindWellKnown() hrefs = response.getHrefProperties() - self.principalURL = hrefs[davxml.current_user_principal].toString() + if davxml.current_user_principal in hrefs: + self.principalURL = hrefs[davxml.current_user_principal].toString() + elif davxml.principal_URL in hrefs: + self.principalURL = hrefs[davxml.principal_URL].toString() + else: + # PROPFIND principal path to retrieve actual principal-URL + response = yield self._principalPropfindInitial(self.record.uid) + hrefs = response.getHrefProperties() + self.principalURL = hrefs[davxml.principal_URL].toString() # Using the actual principal URL, retrieve principal information principal = yield self._principalPropfind() Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -76,12 +76,12 @@ self.profileTypes = profileTypes - def new(self, reactor, serverAddress, userRecord, authInfo): + def new(self, reactor, serverAddress, principalPathTemplate, userRecord, authInfo): """ Create a new instance of this client type. """ return self.clientType( - reactor, serverAddress, userRecord, authInfo, **self.clientParams) + reactor, serverAddress, principalPathTemplate, userRecord, authInfo, **self.clientParams) @@ -155,11 +155,12 @@ class CalendarClientSimulator(object): def __init__(self, records, populator, parameters, reactor, server, - workerIndex=0, workerCount=1): + principalPathTemplate, workerIndex=0, workerCount=1): self._records = records self.populator = populator self.reactor = reactor self.server = server + self.principalPathTemplate = principalPathTemplate self._pop = self.populator.populate(parameters) self._user = 0 self._stopped = False @@ -224,7 +225,8 @@ reactor = loggedReactor(self.reactor) client = clientType.new( - reactor, self.server, self.getUserRecord(number), auth) + reactor, self.server, self.principalPathTemplate, self.getUserRecord(number), auth + ) self.clients.append(client) d = client.run() d.addErrback(self._clientFailure, reactor) Modified: CalendarServer/trunk/contrib/performance/loadtest/sim.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/sim.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -193,12 +193,13 @@ user information about the accounts on the server being put under load. """ - def __init__(self, server, webadminPort, arrival, parameters, observers=None, + def __init__(self, server, principalPathTemplate, webadminPort, arrival, parameters, observers=None, records=None, reactor=None, runtime=None, workers=None, configTemplate=None, workerID=None, workerCount=1): if reactor is None: from twisted.internet import reactor self.server = server + self.principalPathTemplate = principalPathTemplate self.webadminPort = webadminPort self.arrival = arrival self.parameters = parameters @@ -241,11 +242,15 @@ workerCount = config.get("workerCount", 1) configTemplate = None server = 'http://127.0.0.1:8008/' + principalPathTemplate = "/principals/users/%s/" webadminPort = None if 'server' in config: server = config['server'] + if 'principalPathTemplate' in config: + principalPathTemplate = config['principalPathTemplate'] + if 'webadmin' in config: if config['webadmin']['enabled']: webadminPort = config['webadmin']['HTTPPort'] @@ -277,6 +282,7 @@ else: # Manager / observer process. server = '' + principalPathTemplate = '' webadminPort = None arrival = None parameters = None @@ -296,7 +302,7 @@ records.extend(namedAny(loader)(**params)) output.write("Loaded {0} accounts.\n".format(len(records))) - return cls(server, webadminPort, arrival, parameters, observers=observers, + return cls(server, principalPathTemplate, webadminPort, arrival, parameters, observers=observers, records=records, runtime=runtime, reactor=reactor, workers=workers, configTemplate=configTemplate, workerID=workerID, workerCount=workerCount) @@ -337,7 +343,7 @@ populator = Populator(Random()) return CalendarClientSimulator( self.records, populator, self.parameters, self.reactor, self.server, - self.workerID, self.workerCount + self.principalPathTemplate, self.workerID, self.workerCount ) Modified: CalendarServer/trunk/contrib/performance/loadtest/test_ical.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/test_ical.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/test_ical.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -1154,7 +1154,9 @@ TimezoneCache.create() self.record = _DirectoryRecord( u"user91", u"user91", u"User 91", u"user91@example.org") - self.client = OS_X_10_6(None, "http://127.0.0.1/", self.record, None) + self.client = OS_X_10_6( + None, "http://127.0.0.1/", "/principals/users/%s/", self.record, None + ) def interceptRequests(self): Modified: CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -316,7 +316,7 @@ """ def setUp(self): self.sim = CalendarClientSimulator( - AnyUser(), Populator(None), None, None, None) + AnyUser(), Populator(None), None, None, None, None) def _simpleAccount(self, userNumber, eventText): @@ -516,7 +516,7 @@ """ def setUp(self): self.sim = CalendarClientSimulator( - AnyUser(), Populator(None), None, None, None) + AnyUser(), Populator(None), None, None, None, None) def _simpleAccount(self, userNumber, eventText): @@ -685,7 +685,7 @@ """ def setUp(self): self.sim = CalendarClientSimulator( - AnyUser(), Populator(None), None, None, None) + AnyUser(), Populator(None), None, None, None, None) def test_enabled(self): @@ -949,7 +949,7 @@ """ def setUp(self): self.sim = CalendarClientSimulator( - AnyUser(), Populator(None), None, None, None) + AnyUser(), Populator(None), None, None, None, None) def test_enabled(self): Modified: CalendarServer/trunk/contrib/performance/loadtest/test_sim.py =================================================================== --- CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/contrib/performance/loadtest/test_sim.py 2012-05-25 13:22:21 UTC (rev 9300) @@ -117,7 +117,7 @@ """ calsim = CalendarClientSimulator( [self._user('alice'), self._user('bob'), self._user('carol')], - Populator(None), None, None, 'http://example.org:1234/') + Populator(None), None, None, 'http://example.org:1234/', None) users = sorted([ calsim._createUser(0)[0], calsim._createUser(1)[0], @@ -133,7 +133,7 @@ """ calsim = CalendarClientSimulator( [self._user('alice')], - Populator(None), None, None, 'http://example.org:1234/') + Populator(None), None, None, 'http://example.org:1234/', None) user, auth = calsim._createUser(0) self.assertEqual( auth.passwd.find_user_password('Test Realm', 'http://example.org:1234/')[1], @@ -146,7 +146,7 @@ profiles are not logged. """ class BrokenClient(object): - def __init__(self, reactor, serverAddress, userInfo, auth, runResult): + def __init__(self, reactor, serverAddress, principalPathTemplate, userInfo, auth, runResult): self._runResult = runResult def run(self): @@ -171,7 +171,7 @@ BrokenClient, {'runResult': clientRunResult}, [ProfileType(BrokenProfile, {'runResult': profileRunResult})])) sim = CalendarClientSimulator( - [self._user('alice')], Populator(None), params, None, 'http://example.com:1234/') + [self._user('alice')], Populator(None), params, None, 'http://example.com:1234/', None) sim.add(1, 1) sim.stop() clientRunResult.errback(RuntimeError("Some fictional client problem")) @@ -248,7 +248,7 @@ exc = self.assertRaises( SystemExit, StubSimulator.main, ['--config', config.path]) self.assertEquals( - exc.args, (StubSimulator(None, None, None, None).run(),)) + exc.args, (StubSimulator(None, None, None, None, None).run(),)) def test_createSimulator(self): @@ -259,7 +259,7 @@ """ server = 'http://127.0.0.7:1243/' reactor = object() - sim = LoadSimulator(server, None, None, None, reactor=reactor) + sim = LoadSimulator(server, None, None, None, None, reactor=reactor) calsim = sim.createSimulator() self.assertIsInstance(calsim, CalendarClientSimulator) self.assertIsInstance(calsim.reactor, LagTrackingReactor) @@ -439,7 +439,7 @@ reactor = object() sim = LoadSimulator( - None, None, Arrival(FakeArrival, {'x': 3, 'y': 2}), None, reactor=reactor) + None, None, None, Arrival(FakeArrival, {'x': 3, 'y': 2}), None, reactor=reactor) arrival = sim.createArrivalPolicy() self.assertIsInstance(arrival, FakeArrival) self.assertIdentical(arrival.reactor, sim.reactor) @@ -516,6 +516,7 @@ observers = [Observer()] sim = LoadSimulator( "http://example.com:123/", + "/principals/users/%s/", None, Arrival(lambda reactor: NullArrival(), {}), None, observers, reactor=Reactor()) Modified: CalendarServer/trunk/sim =================================================================== --- CalendarServer/trunk/sim 2012-05-24 22:18:24 UTC (rev 9299) +++ CalendarServer/trunk/sim 2012-05-25 13:22:21 UTC (rev 9300) @@ -23,4 +23,6 @@ source "${wd}/support/shell.sh"; +export PYTHONPATH="$("${wd}/run" -p)"; + exec "${wd}/contrib/performance/sim" "$@";