[CalendarServer-changes] [7498] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Thu May 19 10:07:07 PDT 2011
Revision: 7498
http://trac.macosforge.org/projects/calendarserver/changeset/7498
Author: exarkun at twistedmatrix.com
Date: 2011-05-19 10:07:06 -0700 (Thu, 19 May 2011)
Log Message:
-----------
Give each client/profile group a logged reactor and dump the traffic logs if the client or any of the profiles has a problem
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/population.py
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
CalendarServer/trunk/contrib/performance/loadtest/test_trafficlogger.py
CalendarServer/trunk/contrib/performance/loadtest/trafficlogger.py
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-05-19 13:37:05 UTC (rev 7497)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-05-19 17:07:06 UTC (rev 7498)
@@ -466,7 +466,7 @@
"""
pollCalendarHome = LoopingCall(
self._checkCalendarsForEvents, calendarHome)
- pollCalendarHome.start(self.calendarHomePollInterval, now=False)
+ return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
def _newOperation(self, label, deferred):
before = self.reactor.seconds()
@@ -492,11 +492,12 @@
hrefs = principal.getHrefProperties()
calendarHome = hrefs[caldavxml.calendar_home_set].toString()
yield self._checkCalendarsForEvents(calendarHome)
- self._calendarCheckLoop(calendarHome)
- yield self._newOperation("startup", startup())
+ returnValue(calendarHome)
+ calendarHome = yield self._newOperation("startup", startup())
- # XXX Oops, should probably stop sometime.
- yield Deferred()
+ # This completes when the calendar home poll loop completes, which
+ # currently it never will except due to an unexpected error.
+ yield self._calendarCheckLoop(calendarHome)
def _makeSelfAttendee(self):
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-05-19 13:37:05 UTC (rev 7497)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-05-19 17:07:06 UTC (rev 7498)
@@ -20,12 +20,15 @@
certain usage parameters.
"""
+from tempfile import mkdtemp
from itertools import izip
+from twisted.python.filepath import FilePath
from twisted.python.util import FancyEqMixin
from twisted.python.log import msg, err
from stats import mean, median, stddev, mad
+from loadtest.trafficlogger import loggedReactor
from loadtest.logger import SummarizingMixin
from loadtest.ical import SnowLeopard, RequestLogger
from loadtest.profiles import Eventer, Inviter, Accepter
@@ -147,25 +150,41 @@
user, auth = self._createUser(number)
clientType = self._pop.next()
+ reactor = loggedReactor(self.reactor)
client = clientType.clientType(
- self.reactor, self.host, self.port, user, auth)
+ reactor, self.host, self.port, user, auth)
d = client.run()
- d.addCallbacks(self._clientSuccess, self._clientFailure)
+ d.addErrback(self._clientFailure, reactor)
for profileType in clientType.profileTypes:
- profileType(self.reactor, client, number).run()
+ d = profileType(reactor, client, number).run()
+ d.addErrback(self._profileFailure, profileType, reactor)
msg(type="status", clientCount=self._user - 1)
- def _clientSuccess(self, result):
- pass
+ def _dumpLogs(self, loggingReactor):
+ path = FilePath(mkdtemp())
+ logstate = loggingReactor.getLogFiles()
+ for i, log in enumerate(logstate.finished):
+ path.child('%03.log' % (i,)).setContent(log.getvalue())
+ for i, log in enumerate(logstate.active, i):
+ path.child('%03.log' % (i,)).setContent(log.getvalue())
+ return path
- def _clientFailure(self, reason):
- err(reason, "Client stopped with error")
+ def _clientFailure(self, reason, reactor):
+ where = self._dumpLogs(reactor)
+ err(reason, "Client stopped with error; recent traffic in %r" % (
+ where.path,))
+ def _profileFailure(self, reason, profileType, reactor):
+ where = self._dumpLogs(reactor)
+ err(reason, "Profile stopped with error; recent traffic in %r" % (
+ where.path,))
+
+
class SmoothRampUp(object):
def __init__(self, reactor, groups, groupSize, interval):
self.reactor = reactor
Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2011-05-19 13:37:05 UTC (rev 7497)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2011-05-19 17:07:06 UTC (rev 7498)
@@ -33,7 +33,7 @@
from twisted.python import context
from twisted.python.log import msg
from twisted.python.failure import Failure
-from twisted.internet.defer import succeed, fail
+from twisted.internet.defer import Deferred, succeed, fail
from twisted.internet.task import LoopingCall
from twisted.web.http import PRECONDITION_FAILED
@@ -113,7 +113,7 @@
self._call = LoopingCall(self._invite)
self._call.clock = self._reactor
# XXX Base this on something real
- self._call.start(20)
+ return self._call.start(20)
def _addAttendee(self, event, attendees):
@@ -215,6 +215,8 @@
def run(self):
self._subscription = self._client.catalog["eventChanged"].subscribe(self.eventChanged)
+ # TODO: Propagate errors from eventChanged and _acceptInvitation to this Deferred
+ return Deferred()
def eventChanged(self, href):
@@ -335,7 +337,7 @@
self._call = LoopingCall(self._addEvent)
self._call.clock = self._reactor
# XXX Base this on something real
- self._call.start(25)
+ return self._call.start(25)
def _addEvent(self):
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_trafficlogger.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_trafficlogger.py 2011-05-19 13:37:05 UTC (rev 7497)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_trafficlogger.py 2011-05-19 17:07:06 UTC (rev 7498)
@@ -113,6 +113,36 @@
self.assertEqual(proto.data, "foo")
+ def test_getLogFiles(self):
+ """
+ The reactor returned by L{loggedReactor} has a C{getLogFiles} method
+ which returns a L{logstate} instance containing the active and
+ completed log files tracked by the logging wrapper.
+ """
+ wrapped = ClientFactory()
+ wrapped.protocol = Discard
+ reactor = MemoryReactor()
+ logged = loggedReactor(reactor)
+ logged.connectTCP('127.0.0.1', 1234, wrapped)
+ factory = reactor.tcpClients[0][2]
+
+ finished = factory.buildProtocol(None)
+ finished.makeConnection(StringTransport())
+ finished.dataReceived('finished')
+ finished.connectionLost(None)
+
+ active = factory.buildProtocol(None)
+ active.makeConnection(StringTransport())
+ active.dataReceived('active')
+
+ logs = logged.getLogFiles()
+ self.assertEqual(1, len(logs.finished))
+ self.assertIn('finished', logs.finished[0].getvalue())
+ self.assertEqual(1, len(logs.active))
+ self.assertIn('active', logs.active[0].getvalue())
+
+
+
class TrafficLoggingFactoryTests(TestCase):
"""
Tests for L{_TrafficLoggingFactory}.
@@ -121,6 +151,7 @@
self.wrapped = ClientFactory()
self.wrapped.protocol = Discard
self.factory = _TrafficLoggingFactory(self.wrapped)
+
def test_receivedBytesLogged(self):
"""
Modified: CalendarServer/trunk/contrib/performance/loadtest/trafficlogger.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/trafficlogger.py 2011-05-19 13:37:05 UTC (rev 7497)
+++ CalendarServer/trunk/contrib/performance/loadtest/trafficlogger.py 2011-05-19 17:07:06 UTC (rev 7498)
@@ -22,7 +22,9 @@
__all__ = ['loggedReactor']
+from weakref import ref
from StringIO import StringIO
+from collections import namedtuple
from zope.interface import providedBy
@@ -30,6 +32,10 @@
from twisted.internet.interfaces import IReactorCore, IReactorTime, IReactorTCP
from twisted.protocols.policies import WrappingFactory, TrafficLoggingProtocol
+
+logstate = namedtuple('logstate', 'active finished')
+
+
def loggedReactor(reactor):
"""
Construct and return a wrapper around the given C{reactor} which provides
@@ -52,9 +58,30 @@
A mixin for a reactor wrapper which defines C{connectTCP} so as to cause
traffic to be logged.
"""
+ _factories = None
+
+ @property
+ def factories(self):
+ if self._factories is None:
+ self._factories = []
+ return self._factories
+
+
+ def getLogFiles(self):
+ active = []
+ finished = []
+ for factoryref in self.factories:
+ factory = factoryref()
+ active.extend(factory.logs)
+ finished.extend(factory.finishedLogs)
+ return logstate(active, finished)
+
+
def connectTCP(self, host, port, factory, *args, **kwargs):
+ wrapper = _TrafficLoggingFactory(factory)
+ self.factories.append(ref(wrapper, self.factories.remove))
return self._reactor.connectTCP(
- host, port, _TrafficLoggingFactory(factory), *args, **kwargs)
+ host, port, wrapper, *args, **kwargs)
class _TrafficLoggingFactory(WrappingFactory):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110519/8ef6cda3/attachment-0001.html>
More information about the calendarserver-changes
mailing list