[CalendarServer-changes] [7749] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jul 8 13:20:18 PDT 2011
Revision: 7749
http://trac.macosforge.org/projects/calendarserver/changeset/7749
Author: exarkun at twistedmatrix.com
Date: 2011-07-08 13:20:17 -0700 (Fri, 08 Jul 2011)
Log Message:
-----------
Check latency and success rate on operations, fail the test run if thresholds are exceeded.
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/ical.py
CalendarServer/trunk/contrib/performance/loadtest/population.py
CalendarServer/trunk/contrib/performance/loadtest/profiles.py
CalendarServer/trunk/contrib/performance/loadtest/sim.py
CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
Modified: CalendarServer/trunk/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-07-08 19:14:57 UTC (rev 7748)
+++ CalendarServer/trunk/contrib/performance/loadtest/ical.py 2011-07-08 20:20:17 UTC (rev 7749)
@@ -862,6 +862,10 @@
pass
+ def failures(self):
+ return []
+
+
def main():
from urllib2 import HTTPDigestAuthHandler
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-07-08 19:14:57 UTC (rev 7748)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2011-07-08 20:20:17 UTC (rev 7749)
@@ -245,7 +245,11 @@
pass
+ def failures(self):
+ return []
+
+
class SimpleStatistics(StatisticsBase):
def __init__(self):
self._times = []
Modified: CalendarServer/trunk/contrib/performance/loadtest/profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2011-07-08 19:14:57 UTC (rev 7748)
+++ CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2011-07-08 20:20:17 UTC (rev 7749)
@@ -19,7 +19,9 @@
Implementation of specific end-user behaviors.
"""
-import random
+from __future__ import division
+
+import sys, random
from uuid import uuid4
from datetime import datetime, timedelta
@@ -37,7 +39,7 @@
from twisted.internet.task import LoopingCall
from twisted.web.http import PRECONDITION_FAILED
-from stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, mean
+from stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, mean, median
from loadtest.logger import SummarizingMixin
from loadtest.ical import IncorrectResponseCode
@@ -429,9 +431,12 @@
('avglag (ms)', 8, '%8.4f'),
]
- def __init__(self):
+ def __init__(self, outfile=None):
self._perOperationTimes = {}
self._perOperationLags = {}
+ if outfile is None:
+ outfile = sys.stdout
+ self._outfile = outfile
def observe(self, event):
@@ -441,7 +446,10 @@
event['lag'] = ''
else:
event['lag'] = self.lagFormat % (lag * 1000.0,)
- print (self.formats[event[u'phase']] % event).encode('utf-8')
+
+ self._outfile.write(
+ (self.formats[event[u'phase']] % event).encode('utf-8') + '\n')
+
if event[u'phase'] == u'end':
dataset = self._perOperationTimes.setdefault(event[u'label'], [])
dataset.append((event[u'success'], event[u'duration']))
@@ -464,6 +472,27 @@
[fmt for (label, width, fmt) in self._fields],
sorted(self._perOperationTimes.items()))
- # TODO
- # Check for >1 sec mean/median/something scheduling latency
- # >1% operation failure rate
+ _LATENCY_REASON = "Median %(operation)s scheduling lag greater than %(cutoff)sms"
+ _FAILED_REASON = "Greater than %(cutoff).0f%% %(operation)s failed"
+
+ def failures(self):
+ reasons = []
+
+ # Maximum allowed median scheduling latency, seconds
+ lagCutoff = 1.0
+
+ # Maximum allowed ratio of failed operations
+ failCutoff = 0.01
+
+ for operation, lags in self._perOperationLags.iteritems():
+ if median(lags) > lagCutoff:
+ reasons.append(self._LATENCY_REASON % dict(
+ operation=operation.upper(), cutoff=lagCutoff * 1000))
+
+ for operation, times in self._perOperationTimes.iteritems():
+ failures = len([success for (success, duration) in times if not success])
+ if failures / len(times) > failCutoff:
+ reasons.append(self._FAILED_REASON % dict(
+ operation=operation.upper(), cutoff=failCutoff * 100))
+
+ return reasons
Modified: CalendarServer/trunk/contrib/performance/loadtest/sim.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2011-07-08 19:14:57 UTC (rev 7748)
+++ CalendarServer/trunk/contrib/performance/loadtest/sim.py 2011-07-08 20:20:17 UTC (rev 7749)
@@ -273,7 +273,14 @@
if self.runtime is not None:
self.reactor.callLater(self.runtime, self.reactor.stop)
self.reactor.run()
+ failures = []
for obs in self.observers:
obs.report()
+ failures.extend(obs.failures())
+ if failures:
+ print 'FAIL'
+ print '\n'.join(failures)
+ else:
+ print 'PASS'
main = LoadSimulator.main
Modified: CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2011-07-08 19:14:57 UTC (rev 7748)
+++ CalendarServer/trunk/contrib/performance/loadtest/test_profiles.py 2011-07-08 20:20:17 UTC (rev 7749)
@@ -19,6 +19,8 @@
Tests for loadtest.profiles.
"""
+from StringIO import StringIO
+
from vobject import readComponents
from caldavclientlibrary.protocol.caldav.definitions import caldavxml
@@ -29,7 +31,7 @@
from twisted.web.http import NO_CONTENT, PRECONDITION_FAILED
from twisted.web.client import Response
-from loadtest.profiles import Eventer, Inviter, Accepter
+from loadtest.profiles import Eventer, Inviter, Accepter, OperationLogger
from loadtest.population import Populator, CalendarClientSimulator
from loadtest.ical import IncorrectResponseCode, Calendar, Event, BaseClient
from loadtest.sim import _DirectoryRecord
@@ -677,3 +679,58 @@
self.assertEquals(len(client._events), 1)
# XXX Vary the event period/interval and the uid
+
+
+
+class OperationLoggerTests(TestCase):
+ """
+ Tests for L{OperationLogger}.
+ """
+ def test_noFailures(self):
+ """
+ If the median lag is below 1 second and the failure rate is below 1%,
+ L{OperationLogger.failures} returns an empty list.
+ """
+ logger = OperationLogger(outfile=StringIO())
+ logger.observe(dict(
+ type='operation', phase='start', user='user01',
+ label='testing', lag=0.5))
+ logger.observe(dict(
+ type='operation', phase='end', user='user01',
+ duration=0.35, label='testing', success=True))
+ self.assertEqual([], logger.failures())
+
+
+ def test_lagLimitExceeded(self):
+ """
+ If the median scheduling lag for any operation in the simulation
+ exceeds 1 second, L{OperationLogger.failures} returns a list containing
+ a string describing that issue.
+ """
+ logger = OperationLogger(outfile=StringIO())
+ for lag in [100.0, 1100.0, 1200.0]:
+ logger.observe(dict(
+ type='operation', phase='start', user='user01',
+ label='testing', lag=lag))
+ self.assertEqual(
+ ["Median TESTING scheduling lag greater than 1000.0ms"],
+ logger.failures())
+
+
+ def test_failureLimitExceeded(self):
+ """
+ If the failure rate for any operation exceeds 1%,
+ L{OperationLogger.failures} returns a list containing a string
+ describing that issue.
+ """
+ logger = OperationLogger(outfile=StringIO())
+ for i in range(98):
+ logger.observe(dict(
+ type='operation', phase='end', user='user01',
+ duration=0.25, label='testing', success=True))
+ logger.observe(dict(
+ type='operation', phase='end', user='user01',
+ duration=0.25, label='testing', success=False))
+ self.assertEqual(
+ ["Greater than 1% TESTING failed"],
+ logger.failures())
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110708/0b0c67f7/attachment-0001.html>
More information about the calendarserver-changes
mailing list