<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[15081] CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/15081">15081</a></dd>
<dt>Author</dt> <dd>sredmond@apple.com</dd>
<dt>Date</dt> <dd>2015-08-31 13:37:45 -0700 (Mon, 31 Aug 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Adds Python configuration and a lot more</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestampsimpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestconfigpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/config.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestdistributionspy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/distributions.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtesticalpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestloggerpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpopulationpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestprofilespy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpushpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/push.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrecordspy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/records.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrequesterpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/requester.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestresourcespy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/resources.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsaltsettingsplistclientsplist">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/alt-settings/plist/clients.plist</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsclientspy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/clients.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsconfigpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/config.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsimpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_distributionspy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_distributions.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_pushpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_push.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_resourcespy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_resources.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestampsimpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -46,7 +46,6 @@
</span><span class="cx"> exit(0)
</span><span class="cx"> runmain()
</span><span class="cx">
</span><del>-
</del><span class="cx"> from copy import deepcopy
</span><span class="cx">
</span><span class="cx"> from plistlib import writePlistToString, readPlistFromString
</span><span class="lines">@@ -58,14 +57,16 @@
</span><span class="cx">
</span><span class="cx"> from contrib.performance.loadtest.sim import LoadSimulator
</span><span class="cx"> from contrib.performance.loadtest.records import DirectoryRecord
</span><ins>+from contrib.performance.loadtest.config import Config
</ins><span class="cx">
</span><span class="cx"> class Configure(Command):
</span><span class="cx"> """
</span><span class="cx"> Configure this worker process with the text of an XML property list.
</span><span class="cx"> """
</span><del>- arguments = [("plist", String())]
</del><ins>+ arguments = [("cfg", Pickle())]
</ins><span class="cx"> # Pass OSError exceptions through, presenting the exception message to the user.
</span><del>- errors = {OSError: 'OSError'}
</del><ins>+ # errors = {OSError: 'OSError'}
+ errors = {Exception: 'Exception'}
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -112,11 +113,16 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @Configure.responder
</span><del>- def config(self, plist):
</del><ins>+ def config(self, cfg):
</ins><span class="cx"> from sys import stderr
</span><del>- cfg = readPlistFromString(plist)
</del><ins>+ # cfg = readPlistFromString(plist)
+ config = Config.deserializeFromWorker(cfg)
+ with open('logs.txt', 'a') as f:
+ f.write('here')
+ f.write(str(config.__dict__))
+
</ins><span class="cx"> addObserver(self.emit)
</span><del>- sim = LoadSimulator.fromConfig(cfg)
</del><ins>+ sim = LoadSimulator.fromConfigObject(config)
</ins><span class="cx"> sim.records = self.records
</span><span class="cx"> sim.attachServices(stderr)
</span><span class="cx"> return {}
</span><span class="lines">@@ -161,26 +167,29 @@
</span><span class="cx"> email=record.email,
</span><span class="cx"> guid=record.guid)
</span><span class="cx">
</span><del>- workerConfig = deepcopy(self.loadsim.configTemplate)
- # The list of workers is for the manager only; the workers themselves
- # know they're workers because they _don't_ receive this list.
- del workerConfig["workers"]
- # The manager loads the accounts via the configured loader, then sends
- # them out to the workers (right above), which look at the state at an
- # instance level and therefore don't need a globally-named directory
- # record loader.
- del workerConfig["accounts"]
</del><ins>+ workerConfig = self.loadsim.configTemplate(self.whichWorker, self.numWorkers)
+ dupe = deepcopy(workerConfig)
+ del dupe['records']
+ print dupe
+ # # The list of workers is for the manager only; the workers themselves
+ # # know they're workers because they _don't_ receive this list.
+ # del workerConfig["workers"]
+ # # The manager loads the accounts via the configured loader, then sends
+ # # them out to the workers (right above), which look at the state at an
+ # # instance level and therefore don't need a globally-named directory
+ # # record loader.
+ # del workerConfig["accounts"]
</ins><span class="cx">
</span><del>- workerConfig["workerID"] = self.whichWorker
- workerConfig["workerCount"] = self.numWorkers
- workerConfig["observers"] = []
- workerConfig.pop("accounts", None)
</del><ins>+ # workerConfig["workerID"] = self.whichWorker
+ # workerConfig["workerCount"] = self.numWorkers
+ # workerConfig["observers"] = []
+ # workerConfig.pop("accounts", None)
</ins><span class="cx">
</span><del>- plist = writePlistToString(workerConfig)
</del><ins>+ # plist = writePlistToString(workerConfig)
</ins><span class="cx"> self.output.write("Initiating worker configuration\n")
</span><span class="cx"> def completed(x):
</span><span class="cx"> self.output.write("Worker configuration complete.\n")
</span><del>- self.callRemote(Configure, plist=plist).addCallback(completed)
</del><ins>+ self.callRemote(Configure, cfg=workerConfig).addCallback(completed)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @LogMessage.responder
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/config.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/config.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/config.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,69 +1,127 @@
</span><span class="cx"> from importlib import import_module
</span><span class="cx">
</span><ins>+from twisted.python.log import msg
</ins><span class="cx"> from contrib.performance.loadtest.logger import ReportStatistics, RequestLogger, OperationLogger
</span><del>-from contrib.performance.loadtest.sim import recordsFromCSVFile
</del><ins>+from contrib.performance.loadtest.records import recordsFromCSVFile
+from contrib.performance.loadtest.population import ClientFactory, PopulationParameters
</ins><span class="cx">
</span><del>-DEFAULTS = {
- server = "https://127.0.0.1:8443"
</del><ins>+class Config(object):
+ def __init__(self):
+ pass
</ins><span class="cx">
</span><del>- accounts = recordsFromCSVFile("contrib/performance/loadtest/accounts.csv")
</del><ins>+ def populateFrom(self, serverConfig, clientConfig, usePlist=False):
+ # If there is a list of workers, then this process is *not* a worker
+ isManager = serverConfig.get('workers') is not None
</ins><span class="cx">
</span><del>- _requestLogger = RequestLogger()
- _operationLogger = OperationLogger(
- thresholdsPath="contrib/performance/loadtest/thresholds.json",
- lagCutoff=1.0,
- failCutoff=1.0
- )
- _statisticsReporter = ReportStatistics(
- thresholdsPath="contrib/performance/loadtest/thresholds.json",
- benchmarksPath="contrib/performance/loadtest/benchmarks.json",
- failCutoff=1.0
- )
</del><ins>+ if usePlist:
+ # If the supplied files are plists, we need to convert the named objects into real Python objects.
+ # The ensuing hacky code is why I recommend we remove support for plist-based configuration
+ workers = config['workers']
+ if not isManager:
+ # Client / place where the simulator actually runs configuration
+ workerID = config.get("workerID", 0)
+ workerCount = config.get("workerCount", 1)
+ configTemplate = None
+ server = config.get('server', 'http://127.0.0.1:8008')
+ serializationPath = None
</ins><span class="cx">
</span><del>- arrival = SmoothRampUp(
- groups=2,
- groupSize=1,
- interval=3,
- clientsPerUser=1
- )
-}
</del><ins>+ serializationPath = config['serializationPath']
</ins><span class="cx">
</span><del>-class Config(object):
</del><ins>+ if 'arrival' in config:
+ arrival = Arrival(
+ namedAny(config['arrival']['factory']),
+ config['arrival']['params'])
+ else:
+ arrival = Arrival(
+ SmoothRampUp, dict(groups=10, groupSize=1, interval=3))
</ins><span class="cx">
</span><ins>+ parameters = PopulationParameters()
+ if 'clients' in config:
+ for clientConfig in config['clients']:
+ parameters.addClient(
+ clientConfig["weight"],
+ ClientType(
+ clientConfig["software"],
+ clientConfig["params"],
+ clientConfig["profiles"]
+ )
+ )
+ # ClientType(
+ # namedAny(clientConfig["software"]),
+ # cls._convertParams(clientConfig["params"]),
+ # [
+ # ProfileType(
+ # namedAny(profile["class"]),
+ # cls._convertParams(profile["params"])
+ # ) for profile in clientConfig["profiles"]
+ # ]))
+ if not parameters.clients:
+ parameters.addClient(1,
+ ClientType(OS_X_10_6, {},
+ [Eventer, Inviter, Accepter]))
+ else:
+ # Manager / observer process.
+ server = ''
+ serializationPath = None
+ arrival = None
+ parameters = None
+ workerID = 0
+ configTemplate = config
+ workerCount = 1
</ins><span class="cx">
</span><del>- def __init__(self, serverConfigFile, clientConfigFile):
- # These are modules
- serverConfigModule = import_module(serverConfigFile)
- clientConfigModule = import_module(clientConfigFile)
</del><ins>+ # webadminPort =
+ webadminPort = None
+ if 'webadmin' in config:
+ if config['webadmin']['enabled']:
+ webadminPort = config['webadmin']['HTTPPort']
</ins><span class="cx">
</span><del>- self.clients = clientConfigModule.clientConfiguration
- self.workers = workers
- self.configTemplate = configTemplate
- self.workerID = workerID
- self.workerCount = workerCount
</del><ins>+ serverStats = None
+ if 'serverStats' in config:
+ if config['serverStats']['enabled']:
+ serverStats = config['serverStats']
+ serverStats['server'] = config['server'] if 'server' in config else ''
</ins><span class="cx">
</span><del>- self.server = serverConfig.get('server')
- self.webadminPort = serverConfig.get('webadminPort')
- self.serverStats = serverConfig.get('serverStatsPort')
- self.serializationPath = serverConfig.get('serializationPath')
- self.arrival = serverConfig.get('arrival')
- self.observers = serverConfig.get('observers')
- self.records = serverConfig.get('records')
- self.workers = serverConfig.get('workers')
</del><ins>+ observers = []
+ if 'observers' in config:
+ for observer in config['observers']:
+ observerName = observer["type"]
+ observerParams = observer["params"]
+ observers.append(namedAny(observerName)(**observerParams))
</ins><span class="cx">
</span><del>- self.buildParameters()
</del><ins>+ records = []
+ if 'accounts' in config:
+ loader = config['accounts']['loader']
+ params = config['accounts']['params']
+ records.extend(namedAny(loader)(**params))
+ output.write("Loaded {0} accounts.\n".format(len(records)))
</ins><span class="cx">
</span><del>- def buildParameters(self):
- self.parameters = PopulationParameters()
- for client in self.clients:
- self.parameters.addClient(
</del><ins>+ else:
+ # Python configuration - super easy! Look! It's great!
+ self.webadminPort = serverConfig.get('webadminPort')
+ self.serverStats = serverConfig.get('serverStatsPort')
+ self.observers = serverConfig.get('observers') # Workers shouldn't need this
+ self.workers = serverConfig.get('workers')
+
+ self.server = serverConfig.get('server')
+ self.serializationPath = serverConfig.get('serializationPath')
+ self.arrival = serverConfig.get('arrival')
+ self.records = serverConfig.get('records')
+ self.workerID = serverConfig.get('workerID', 0)
+ self.workerCount = serverConfig.get('workerCount', 1)
+ self.parameters = self.buildParameters(clientConfig)
+
+ def buildParameters(self, clients):
+ parameters = PopulationParameters()
+ for client in clients:
+ parameters.addClient(
</ins><span class="cx"> client["weight"],
</span><del>- ClientType(
</del><ins>+ ClientFactory(
</ins><span class="cx"> client["software"],
</span><span class="cx"> client["params"],
</span><span class="cx"> client["profiles"]
</span><span class="cx"> )
</span><span class="cx"> )
</span><ins>+ return parameters
</ins><span class="cx">
</span><span class="cx"> def buildSerializationPath(self):
</span><span class="cx"> if self.serializationPath:
</span><span class="lines">@@ -75,7 +133,54 @@
</span><span class="cx"> print("Please consult the clientDataSerialization stanza of contrib/performance/loadtest/config.plist")
</span><span class="cx"> raise
</span><span class="cx">
</span><del>- def get(self, attr):
- if hasattr(self, attr):
- return getattr(self, attr)
- return DEFAULTS.get(attr, None)
</del><ins>+ def serializeForWorker(self, workerID, workerCount):
+ if not self.workers: # If we are workers, don't try to be a manager
+ return {}
+ # print "Trying to serialize for worker #" + str(workerID)
+ # print "My info, btw is " + str(self.__dict__)
+ info = {
+ 'webadminPort': '',
+ 'serverStats': '',
+ 'workers': [],
+ 'observers': [],
+ 'workerID': workerID,
+ 'workerCount': workerCount,
+ # Workers need some information to work correctly
+ 'server': self.server,
+ 'serializationPath': self.serializationPath,
+ 'arrival': self.arrival,
+ 'records': self.records,
+ 'parameters': self.parameters
+ }
+ return info
+
+ @classmethod
+ def deserializeFromWorker(cls, info):
+ base = cls()
+ base.__dict__.update(info)
+ return base
+
+ # Goodness, how awkward is this code? If we dropped support for plists, we could do away with it
+ @classmethod
+ def _convertParams(cls, params):
+ """
+ Find parameter values which should be more structured than plistlib is
+ capable of constructing and replace them with the more structured form.
+
+ Specifically, find keys that end with C{"Distribution"} and convert
+ them into some kind of distribution object using the associated
+ dictionary of keyword arguments.
+ """
+ for k, v in params.iteritems():
+ if k.endswith('Distribution'): # Goodness how fragile
+ params[k] = cls._convertDistribution(v)
+ return params
+
+
+ @classmethod
+ def _convertDistribution(cls, value):
+ """
+ Construct and return a new distribution object using the type and
+ params specified by C{value}.
+ """
+ return namedAny(value['type'])(**value['params'])
</ins><span class="cx">\ No newline at end of file
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestdistributionspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/distributions.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/distributions.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/distributions.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -26,14 +26,36 @@
</span><span class="cx"> Sampling from this distribution must *not* change the underlying behavior of a distribution
</span><span class="cx">
</span><span class="cx"> Distributions (all of which implement IDistribution):
</span><del>- UniformDiscreteDistribution
- LogNormalDistribution
- FixedDistribution
- NearFutureDistribution
- NormalDistribution
- UniformIntegerDistribution
- WorkDistribution
- RecurrenceDistribution
</del><ins>+ # Discrete Distributions / Finite Support
+ Bernoulli
+ Binomial
+ Rademacher
+ Fixed
+ UniformDiscrete
+ UniformInteger
+
+ # Discrete Distributions / Infinite Support (> 0)
+ Poisson
+ Geometric
+
+ LogNormal
+
+ Normal
+ UniformReal
+ Triangular
+ Beta
+ ChiSquared
+ Exponential
+ Gamma
+
+ # CalendarServer Specific
+ NearFuture
+ Work
+ Recurrence
+
+# TODO
+Implement simple ones through large ones
+Squeeze / pinch
</ins><span class="cx"> """
</span><span class="cx"> from math import log, sqrt
</span><span class="cx"> from time import mktime
</span><span class="lines">@@ -54,9 +76,7 @@
</span><span class="cx"> def sample(): #@NoSelf
</span><span class="cx"> pass
</span><span class="cx">
</span><del>-# class Bounded
</del><span class="cx">
</span><del>-
</del><span class="cx"> class UniformDiscreteDistribution(object, FancyEqMixin):
</span><span class="cx"> """
</span><span class="cx">
</span><span class="lines">@@ -65,22 +85,11 @@
</span><span class="cx">
</span><span class="cx"> compareAttributes = ['_values']
</span><span class="cx">
</span><del>- def __init__(self, values, randomize=True):
</del><ins>+ def __init__(self, values):
</ins><span class="cx"> self._values = values
</span><del>- self._randomize = randomize
- self._refill()
</del><span class="cx">
</span><del>-
- def _refill(self):
- self._remaining = self._values[:]
- if self._randomize:
- random.shuffle(self._remaining)
-
-
</del><span class="cx"> def sample(self):
</span><del>- if not self._remaining:
- self._refill()
- return self._remaining.pop()
</del><ins>+ return random.choice(self._values)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -218,7 +227,20 @@
</span><span class="cx"> return 1 if random.random() <= self._p else 0
</span><span class="cx">
</span><span class="cx">
</span><ins>+class RademacherDistribution(object, FancyEqMixin):
+ """
+ Takes value 1 with probability 1/2 and value -1 with probability 1/2
+ """
+ def __init__(self):
+ """
+ """
+ self._d = BernoulliDistribution(proportion=0.5)
</ins><span class="cx">
</span><ins>+ def sample(self):
+ return [-1, 1][self._d.sample()]
+
+
+
</ins><span class="cx"> class BinomialDistribution(object, FancyEqMixin):
</span><span class="cx"> compareAttributes = ["_successProbability", "_numTrials"]
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtesticalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -249,7 +249,6 @@
</span><span class="cx"> ampPushPort=62311,
</span><span class="cx"> ):
</span><span class="cx"> self._client_id = str(uuid4())
</span><del>-
</del><span class="cx"> self.reactor = reactor
</span><span class="cx">
</span><span class="cx"> self.requester = Requester(
</span><span class="lines">@@ -432,6 +431,7 @@
</span><span class="cx"> self.checkCalendarsForEvents, calendarHome)
</span><span class="cx"> return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
</span><span class="cx">
</span><ins>+ ### TODO this doesn't seem to always work
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def updateCalendarHomeFromPush(self, calendarHomeSet):
</span><span class="cx"> """
</span><span class="lines">@@ -538,8 +538,8 @@
</span><span class="cx"> except KeyError:
</span><span class="cx"> pass
</span><span class="cx"> else:
</span><del>- if pushkey:
- self.monitor.addPushkey(href, pushkey)
</del><ins>+ if pushkey and self.monitor:
+ self.monitor.addPushkey(pushkey, href)
</ins><span class="cx">
</span><span class="cx"> nodes = results[href].getNodeProperties()
</span><span class="cx"> for nodeType in nodes[davxml.resourcetype]:
</span><span class="lines">@@ -888,7 +888,7 @@
</span><span class="cx">
</span><span class="cx"> # Start monitoring AMP push notifications, if possible
</span><span class="cx"> if self.monitor and self.monitor.isSubscribedTo(calendarHome):
</span><del>- self.monitor.begin()
</del><ins>+ yield self.monitor.begin()
</ins><span class="cx"> # Run indefinitely.
</span><span class="cx"> yield Deferred()
</span><span class="cx"> else:
</span><span class="lines">@@ -902,6 +902,8 @@
</span><span class="cx"> Called before connections are closed, giving a chance to clean up
</span><span class="cx"> """
</span><span class="cx"> self.serialize()
</span><ins>+ if not self.monitor:
+ return succeed(None)
</ins><span class="cx"> return self.monitor.end()
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestloggerpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -107,7 +107,23 @@
</span><span class="cx"> self._printRow(output, formats, self._summarizeData(method, data))
</span><span class="cx">
</span><span class="cx">
</span><ins>+class MessageLogger(object):
+ def observe(self, event):
+ if event.get("type") == "log":
+ import random
+ identifier = random.random()
+ print(TerminalColors.WARNING + str(identifier) + '/' + event.get('val') + ':' + event.get('text') + TerminalColors.ENDC)
</ins><span class="cx">
</span><ins>+
+class EverythingLogger(object):
+ def observe(self, event):
+ # if event.get("type") == "response":
+ # from pprint import pprint
+ # pprint(event)
+ pass
+
+
+
</ins><span class="cx"> class RequestLogger(object):
</span><span class="cx"> format = u"%(user)s request %(code)s%(success)s[%(duration)5.2f s] %(method)8s %(url)s"
</span><span class="cx"> success = u"\N{CHECK MARK}"
</span><span class="lines">@@ -129,7 +145,7 @@
</span><span class="cx"> else:
</span><span class="cx"> formatArgs['success'] = self.failure
</span><span class="cx"> start = TerminalColors.FAIL
</span><del>- print(start + (self.format % formatArgs).encode('utf-8') + TerminalColors.ENDC)
</del><ins>+ print(start + (self.format % formatArgs).encode('utf-8') + "from Logger w/ id: " + str(id(self)) + TerminalColors.ENDC)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def report(self, output):
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpopulationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -43,51 +43,51 @@
</span><span class="cx"> from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
</span><span class="cx">
</span><span class="cx">
</span><del>-class ProfileType(object, FancyEqMixin):
- """
- @ivar profileType: A L{ProfileBase} subclass, or an L{ICalendarUserProfile}
- implementation.
</del><ins>+# class ProfileType(object, FancyEqMixin):
+# """
+# @ivar profileType: A L{ProfileBase} subclass
+# @type profileType: C{type}
</ins><span class="cx">
</span><del>- @ivar params: A C{dict} which will be passed to C{profileType} as keyword
- arguments to create a new profile instance.
- """
- compareAttributes = ("profileType", "params")
</del><ins>+# @ivar params: A C{dict} which will be passed to C{profileType} as keyword
+# arguments to create a new profile instance.
+# """
+# compareAttributes = ("profileType", "params")
</ins><span class="cx">
</span><del>- def __init__(self, profileType, params):
- self.profileType = profileType
- self.params = params
</del><ins>+# def __init__(self, profileType, params):
+# self.profileType = profileType
+# self.params = params
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __call__(self, reactor, simulator, client, number):
- base = self.profileType(**self.params)
- base.setUp(reactor, simulator, client, number)
- return base
</del><ins>+# def __call__(self, reactor, simulator, client, number):
+# base = self.profileType(**self.params)
+# base.setUp(reactor, simulator, client, number)
+# return base
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __repr__(self):
- return "ProfileType(%s, params=%s)" % (self.profileType.__name__, self.params)
</del><ins>+# def __repr__(self):
+# return "ProfileType(%s, params=%s)" % (self.profileType.__name__, self.params)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-class ClientType(object, FancyEqMixin):
</del><ins>+class ClientFactory(object, FancyEqMixin):
</ins><span class="cx"> """
</span><del>- @ivar clientType: An L{ICalendarClient} implementation
</del><ins>+ @ivar clientType: An L{BaseAppleClient} subclass
+ @ivar params: A C{dict} which will be passed to C{clientType} as keyword
+ arguements to create a new client
</ins><span class="cx"> @ivar profileTypes: A list of L{ProfileType} instances
</span><span class="cx"> """
</span><span class="cx"> compareAttributes = ("clientType", "profileTypes")
</span><span class="cx">
</span><del>- def __init__(self, clientType, clientParams, profileTypes):
</del><ins>+ def __init__(self, clientType, clientParams, profiles):
</ins><span class="cx"> self.clientType = clientType
</span><span class="cx"> self.clientParams = clientParams
</span><del>- self.profileTypes = profileTypes
</del><ins>+ self.profiles = profiles
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def new(self, reactor, serverAddress, serializationPath, userRecord, authInfo):
</span><span class="cx"> """
</span><span class="cx"> Create a new instance of this client type.
</span><span class="cx"> """
</span><del>- # print(self.clientType)
- # print(self.clientParams)
</del><span class="cx"> return self.clientType(
</span><span class="cx"> reactor, serverAddress, serializationPath,
</span><span class="cx"> userRecord, authInfo, **self.clientParams
</span><span class="lines">@@ -120,60 +120,67 @@
</span><span class="cx"> self.clients.append((weight, clientType))
</span><span class="cx">
</span><span class="cx">
</span><del>- def clientTypes(self):
- """
- Return a list of two-tuples giving the weights and types of
- clients in the population.
- """
- return self.clients
</del><ins>+ def clientGenerator(self):
+ while True:
+ for (weight, clientFactory) in self.clients:
+ for _ignore_i in xrange(weight):
+ yield clientFactory
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ # def clientTypes(self):
+ # """
+ # Return a list of two-tuples giving the weights and types of
+ # clients in the population.
+ # """
+ # return self.clients
</ins><span class="cx">
</span><del>-class Populator(object):
- """
- @ivar userPattern: A C{str} giving a formatting pattern to use to
- construct usernames. The string will be interpolated with a
- single integer, the incrementing counter of how many users
- have thus far been "used".
</del><span class="cx">
</span><del>- @ivar passwordPattern: Similar to C{userPattern}, but for
- passwords.
- """
- def __init__(self, random):
- self._random = random
</del><span class="cx">
</span><span class="cx">
</span><del>- def _cycle(self, elements):
- while True:
- for (weight, value) in elements:
- for _ignore_i in range(weight):
- yield value
</del><span class="cx">
</span><ins>+# class Populator(object):
+# """
+# """
+# def __init__(self):
+# self._random = random
</ins><span class="cx">
</span><del>- def populate(self, parameters):
- """
- Generate individuals such as might be randomly selected from a
- population with the given parameters.
</del><span class="cx">
</span><del>- @type parameters: L{PopulationParameters}
- @rtype: generator of L{ClientType} instances
- """
- for (clientType,) in izip(self._cycle(parameters.clientTypes())):
- yield clientType
</del><ins>+# def _cycle(self, elements):
+# while True:
+# for (weight, value) in elements:
+# for _ignore_i in range(weight):
+# yield value
</ins><span class="cx">
</span><span class="cx">
</span><ins>+# def populate(self, parameters):
+# """
+# Generate individuals such as might be randomly selected from a
+# population with the given parameters.
</ins><span class="cx">
</span><ins>+# @type parameters: L{PopulationParameters}
+# @rtype: generator of L{ClientType} instances
+# """
+# for (clientType,) in izip(self._cycle(parameters.clientTypes())):
+# yield clientType
+
+
+
</ins><span class="cx"> class CalendarClientSimulator(object):
</span><del>- def __init__(self, records, populator, parameters, reactor, server,
</del><ins>+ def __init__(self, records, parameters, reactor, server,
</ins><span class="cx"> serializationPath, workerIndex=0, workerCount=1):
</span><ins>+ import random
+ # i = random.randint(0, 1000)
+ # with open('log%d.txt'.format(i), 'a') as f:
+ # f.write('wtf')
+ val = random.random()
+ msg(type="log", text="In create client sim", val=str(val))
</ins><span class="cx"> from pprint import pprint
</span><span class="cx"> pprint(records)
</span><span class="cx"> self._records = records
</span><del>- self.populator = populator
</del><span class="cx"> self.reactor = reactor
</span><span class="cx"> self.server = server
</span><span class="cx"> self.serializationPath = serializationPath
</span><del>- self._pop = self.populator.populate(parameters)
</del><ins>+ self._populator = parameters.clientGenerator()
</ins><span class="cx"> self._user = 0
</span><span class="cx"> self._stopped = False
</span><span class="cx"> self.workerIndex = workerIndex
</span><span class="lines">@@ -228,63 +235,74 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def add(self, numClients, clientsPerUser):
</span><del>- for _ignore_n in range(numClients):
</del><ins>+ # for _ignore_n in range(numClients):
+ # number = self._nextUserNumber()
+
+ # for _ignore_peruser in range(clientsPerUser):
+ # clientType = self._populator.next()
+ # if (number % self.workerCount) != self.workerIndex:
+ # # If we're in a distributed work scenario and we are worker N,
+ # # we have to skip all but every Nth request (since every node
+ # # runs the same arrival policy).
+ # continue
+
+ # _ignore_user, auth = self._createUser(number)
+
+ # reactor = loggedReactor(self.reactor)
+ # client = clientType.new(
+ # reactor,
+ # self.server,
+ # self.serializationPath,
+ # self.getUserRecord(number),
+ # auth,
+ # )
+ # self.clients.append(client)
+ # d = client.run()
+ # d.addErrback(self._clientFailure, reactor)
+
+ # for profileType in clientType.profileTypes:
+ # print(profileType)
+ # profile = profileType(reactor, self, client, number)
+ # if profile.enabled:
+ # d = profile.initialize()
+ # def _run(result):
+ # d2 = profile.run()
+ # d2.addErrback(self._profileFailure, profileType, reactor)
+ # return d2
+ # d.addCallback(_run)
+
+ for i in range(numClients):
</ins><span class="cx"> number = self._nextUserNumber()
</span><del>-
- for _ignore_peruser in range(clientsPerUser):
- clientType = self._pop.next()
</del><ins>+ # What user are we representing?
+ for j in range(clientsPerUser):
</ins><span class="cx"> if (number % self.workerCount) != self.workerIndex:
</span><span class="cx"> # If we're in a distributed work scenario and we are worker N,
</span><span class="cx"> # we have to skip all but every Nth request (since every node
</span><span class="cx"> # runs the same arrival policy).
</span><span class="cx"> continue
</span><ins>+ clientFactory = self._populator.next()
</ins><span class="cx">
</span><span class="cx"> _ignore_user, auth = self._createUser(number)
</span><ins>+ reactor = loggedReactor(self.reactor)
</ins><span class="cx">
</span><del>- reactor = loggedReactor(self.reactor)
- client = clientType.new(
- reactor,
</del><ins>+ client = clientFactory.new(
+ self.reactor,
</ins><span class="cx"> self.server,
</span><span class="cx"> self.serializationPath,
</span><span class="cx"> self.getUserRecord(number),
</span><del>- auth,
</del><ins>+ auth
</ins><span class="cx"> )
</span><ins>+ import random
</ins><span class="cx"> self.clients.append(client)
</span><del>- d = client.run()
- d.addErrback(self._clientFailure, reactor)
</del><ins>+ client.run().addErrback(self._clientFailure, reactor)
+ for profileTemplate in clientFactory.profiles:
+ print("Templated interval is:" + str(profileTemplate._interval))
+ profile = profileTemplate.duplicate()
+ print("I see the interval as:" + str(profile._interval))
+ profile.setUp(self.reactor, self, client, number)
+ profile.run().addErrback(self._profileFailure, reactor)
</ins><span class="cx">
</span><del>- for profileType in clientType.profileTypes:
- print(profileType)
- profile = profileType(reactor, self, client, number)
- if profile.enabled:
- d = profile.initialize()
- def _run(result):
- d2 = profile.run()
- d2.addErrback(self._profileFailure, profileType, reactor)
- return d2
- d.addCallback(_run)
</del><span class="cx">
</span><del>- # XXX this status message is prone to be slightly inaccurate, but isn't
- # really used by much anyway.
- msg(type="status", clientCount=self._user - 1)
-
- # for i in range(numClients):
- # for j in range(clientsPerUser):
- # client = self._pop.next()
- # # Reactor magic goes here
- # client.setUp(
- # self.reactor,
- # self.server,
- # self.serializationPath,
- # self.getUserRecord(number),
- # auth
- # )
- # for profile in client.profiles:
- # profile.setUp(self.reactor, self, client, number)
-
-
-
-
</del><span class="cx"> def _dumpLogs(self, loggingReactor, reason):
</span><span class="cx"> path = FilePath(mkdtemp())
</span><span class="cx"> logstate = loggingReactor.getLogFiles()
</span><span class="lines">@@ -297,7 +315,7 @@
</span><span class="cx"> return path
</span><span class="cx">
</span><span class="cx">
</span><del>- def _profileFailure(self, reason, profileType, reactor):
</del><ins>+ def _profileFailure(self, reason, reactor):
</ins><span class="cx"> if not self._stopped:
</span><span class="cx"> where = self._dumpLogs(reactor, reason)
</span><span class="cx"> err(reason, "Profile stopped with error; recent traffic in %r" % (
</span><span class="lines">@@ -321,17 +339,16 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> class SmoothRampUp(object):
</span><del>- def __init__(self, reactor, groups, groupSize, interval, clientsPerUser):
- self.reactor = reactor
</del><ins>+ def __init__(self, groups, groupSize, interval, clientsPerUser):
</ins><span class="cx"> self.groups = groups
</span><span class="cx"> self.groupSize = groupSize
</span><span class="cx"> self.interval = interval
</span><span class="cx"> self.clientsPerUser = clientsPerUser
</span><span class="cx">
</span><span class="cx">
</span><del>- def run(self, simulator):
</del><ins>+ def run(self, reactor, simulator):
</ins><span class="cx"> for i in range(self.groups):
</span><del>- self.reactor.callLater(
</del><ins>+ reactor.callLater(
</ins><span class="cx"> self.interval * i, simulator.add, self.groupSize, self.clientsPerUser)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestprofilespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -23,6 +23,7 @@
</span><span class="cx">
</span><span class="cx"> import random
</span><span class="cx"> from uuid import uuid4
</span><ins>+from numbers import Number
</ins><span class="cx">
</span><span class="cx"> from caldavclientlibrary.protocol.caldav.definitions import caldavxml
</span><span class="cx">
</span><span class="lines">@@ -37,7 +38,7 @@
</span><span class="cx">
</span><span class="cx"> from contrib.performance.loadtest.distributions import (
</span><span class="cx"> NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, BernoulliDistribution,
</span><del>- LogNormalDistribution, RecurrenceDistribution
</del><ins>+ LogNormalDistribution, RecurrenceDistribution, FixedDistribution
</ins><span class="cx"> )
</span><span class="cx"> from contrib.performance.loadtest.ical import IncorrectResponseCode
</span><span class="cx"> from contrib.performance.loadtest.resources import Calendar, Event
</span><span class="lines">@@ -47,6 +48,23 @@
</span><span class="cx"> from pycalendar.duration import Duration
</span><span class="cx"> from pycalendar.value import Value
</span><span class="cx">
</span><ins>+def _loopWithDistribution(reactor, distribution, function):
+ result = Deferred()
+ print(distribution)
+
+ def repeat(ignored):
+ reactor.callLater(distribution.sample(), iterate)
+
+ def iterate():
+ d = function()
+ if d is not None:
+ d.addCallbacks(repeat, result.errback)
+ else:
+ repeat(None)
+
+ repeat(None)
+ return result
+
</ins><span class="cx"> class ProfileBase(object):
</span><span class="cx"> """
</span><span class="cx"> Base class which provides some conveniences for profile
</span><span class="lines">@@ -55,11 +73,19 @@
</span><span class="cx"> random = random
</span><span class="cx">
</span><span class="cx"> def __init__(self, enabled, interval, **params):
</span><ins>+ print("Creating new profile: %s" % (self.__class__.__name__,))
</ins><span class="cx"> self.enabled = enabled
</span><ins>+ if isinstance(interval, Number):
+ interval = FixedDistribution(interval)
</ins><span class="cx"> self._interval = interval
</span><del>- self.setParameters(**params)
</del><ins>+ print "**" + str(self._interval)
+ self._params = params
+ self.setDistributions(**params)
</ins><span class="cx"> self._initialized = False
</span><span class="cx">
</span><ins>+ def duplicate(self):
+ return type(self)(enabled=self.enabled, interval=self._interval, **self._params)
+
</ins><span class="cx"> def setUp(self, reactor, simulator, client, record):
</span><span class="cx"> self._reactor = reactor
</span><span class="cx"> self._sim = simulator
</span><span class="lines">@@ -67,49 +93,19 @@
</span><span class="cx"> self._record = record
</span><span class="cx"> self._initialized = True
</span><span class="cx">
</span><del>- def setParameters(self):
</del><ins>+ def setDistributions(self):
</ins><span class="cx"> pass
</span><span class="cx">
</span><ins>+ def _wrapper(self):
+ if not self.enabled:
+ return succeed(None)
+ if not self._client.started:
+ return succeed(None)
+ return self.action()
+
</ins><span class="cx"> def run(self):
</span><del>- # def action(self):
- # if self.enabled:
- # return self.action()
</del><ins>+ return _loopWithDistribution(self._reactor, self._interval, self._wrapper)
</ins><span class="cx">
</span><del>- print("Hello from run")
- _call = LoopingCall(self.action)
- _call.clock = self._reactor
- return _call.start(self._interval)
-
- # OR
-
- # return self._loopWithDistribution(
- # self._reactor, self._sendInvitationDistribution, self._invite)
-
- # def _loopWithDistribution(reactor, distribution, function):
- # result = Deferred()
-
- # def repeat(ignored):
- # reactor.callLater(distribution.sample(), iterate)
-
- # def iterate():
- # d = function()
- # if d is not None:
- # d.addCallbacks(repeat, result.errback)
- # else:
- # repeat(None)
-
- # repeat(None)
- # return result
-
- def initialize(self):
- """
- Called before the profile runs for real. Can be used to initialize client state.
-
- @return: a L{Deferred} that fires when initialization is done
- """
- return succeed(None)
-
-
</del><span class="cx"> def _calendarsOfType(self, calendarType, componentType):
</span><span class="cx"> return [
</span><span class="cx"> cal
</span><span class="lines">@@ -127,12 +123,12 @@
</span><span class="cx"> return attendee.parameterValue('EMAIL') == self._client.email[len('mailto:'):]
</span><span class="cx">
</span><span class="cx">
</span><del>- def _getRandomCalendar(self):
</del><ins>+ def _getRandomCalendarOfType(self, component_type):
</ins><span class="cx"> """
</span><span class="cx"> Return a random L{Calendar} object from the current user
</span><span class="cx"> or C{None} if there are no calendars to work with
</span><span class="cx"> """
</span><del>- calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
</del><ins>+ calendars = self._calendarsOfType(caldavxml.calendar, component_type)
</ins><span class="cx"> if not calendars: # Oh no! There are no calendars to play with
</span><span class="cx"> return None
</span><span class="cx"> # Choose a random calendar
</span><span class="lines">@@ -140,12 +136,12 @@
</span><span class="cx"> return calendar
</span><span class="cx">
</span><span class="cx">
</span><del>- def _getRandomEvent(self):
</del><ins>+ def _getRandomEventOfType(self, component_type):
</ins><span class="cx"> """
</span><span class="cx"> Return a random L{Event} object from the current user
</span><span class="cx"> or C{None} if there are no events to work with
</span><span class="cx"> """
</span><del>- calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
</del><ins>+ calendars = self._calendarsOfType(caldavxml.calendar, component_type)
</ins><span class="cx"> while calendars:
</span><span class="cx"> calendar = self.random.choice(calendars)
</span><span class="cx"> calendars.remove(calendar)
</span><span class="lines">@@ -232,50 +228,42 @@
</span><span class="cx"> """
</span><span class="cx"> pass
</span><span class="cx">
</span><del>- """ Event-Interaction Profiles
-Event Creation - Eventer
-Event Changing - EventUpdaterBase
- TitlerMixin
- RelocaterMixin
- ReschedulerMixin
- RepeaterMixin
- AlerterMixin
- InviterMixin
- NoterMixin
- InviterMixin
- LinkerMixin
- AttacherMixin
-Event Deletion - EventerDeleter
-"""
</del><span class="cx">
</span><span class="cx"> #####################
</span><del>-# Eventer Behaviors #
</del><ins>+# Eventer Hierarchy #
+# ----------------- #
+# EventBase #
+# Eventer #
+# EventUpdaterBase#
+# Titler #
+# Noter #
+# Linker #
+# Relocater #
+# Repeater #
+# Rescheduler #
+# ~Alerter~ #
+# Attacher #
+# InviterBase #
+# Inviter #
+# Relocater #
+# EventDeleter #
</ins><span class="cx"> #####################
</span><span class="cx">
</span><del>-class EventerBase(ProfileBase):
</del><ins>+class EventBase(ProfileBase):
</ins><span class="cx"> """
</span><span class="cx"> Base profile for a calendar user who interacts with events
</span><span class="cx"> """
</span><del>- def setParameters(
- self,
- enabled=True,
- interval=25,
- **params
- ):
- self.enabled = enabled
- self._interval = interval
- self.setDistributions(**params)
</del><ins>+ def _getRandomCalendar(self):
+ return self._getRandomCalendarOfType('VEVENT')
</ins><span class="cx">
</span><ins>+ def _getRandomEvent(self):
+ return self._getRandomEventOfType('VEVENT')
</ins><span class="cx">
</span><span class="cx">
</span><del>-class Eventer(EventerBase):
</del><ins>+class Eventer(EventBase):
</ins><span class="cx"> """
</span><span class="cx"> A Calendar user who creates new events.
</span><span class="cx"> """
</span><del>- def initialize(self):
- self.action = self._addEvent
- return succeed(None)
-
</del><span class="cx"> def setDistributions(
</span><span class="cx"> self,
</span><span class="cx"> eventStartDistribution=NearFutureDistribution(),
</span><span class="lines">@@ -285,30 +273,25 @@
</span><span class="cx"> 120 * 60
</span><span class="cx"> ])
</span><span class="cx"> ):
</span><del>- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
</del><ins>+ """
+ @param eventStartDistribution: Generates datetimes at which an event starts
+ @param eventDurationDistribution: Generates length of event (in seconds)
+ """
+ self._eventStart = eventStartDistribution
+ self._eventDuration = eventDurationDistribution
</ins><span class="cx">
</span><span class="cx"> def _addEvent(self):
</span><del>- print "Hello a bit"
-
- if not self._client.started:
- return succeed(None)
-
-
-
</del><ins>+ # Choose a random calendar on which to add an event
</ins><span class="cx"> calendar = self._getRandomCalendar()
</span><span class="cx"> if not calendar:
</span><span class="cx"> return succeed(None)
</span><span class="cx">
</span><del>- print "Made it"
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
</del><ins>+ # Form a new event by modifying fields of the template event
</ins><span class="cx"> vcalendar = eventTemplate.duplicate()
</span><span class="cx"> vevent = vcalendar.mainComponent()
</span><span class="cx"> uid = str(uuid4())
</span><del>- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
</del><ins>+ dtstart = self._eventStart.sample()
+ dtend = dtstart + Duration(seconds=self._eventDuration.sample())
</ins><span class="cx">
</span><span class="cx"> vevent.replaceProperty(Property("UID", uid))
</span><span class="cx"> vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
</span><span class="lines">@@ -319,17 +302,18 @@
</span><span class="cx"> href = '%s%s.ics' % (calendar.url, uid)
</span><span class="cx"> event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
</span><span class="cx"> d = self._client.addEvent(href, event)
</span><del>- return self._newOperation("create", d)
</del><ins>+ return self._newOperation("create{event}", d)
</ins><span class="cx">
</span><ins>+ action = _addEvent
+
</ins><span class="cx"> # Could have better handling for not changing events once they're modified
</span><span class="cx"> # esp re: repeating
</span><del>-class EventUpdaterBase(EventerBase):
</del><ins>+class EventUpdaterBase(EventBase):
</ins><span class="cx"> """Superclass of all event mixins.
</span><span class="cx"> Accepts two parameters
</span><span class="cx"> enabled: bool on or off
</span><span class="cx"> interval: distibution that generates integers representing delays
</span><span class="cx"> """
</span><del>- # COMPONENT_TYPE = None
</del><span class="cx"> def action(self):
</span><span class="cx"> event = self._getRandomEvent()
</span><span class="cx"> if not event:
</span><span class="lines">@@ -346,7 +330,7 @@
</span><span class="cx">
</span><span class="cx"> return self._newOperation(label, d)
</span><span class="cx">
</span><del>- def modifyEvent(self):
</del><ins>+ def modifyEvent(self, href, vevent):
</ins><span class="cx"> """Overriden by subclasses"""
</span><span class="cx"> pass
</span><span class="cx">
</span><span class="lines">@@ -450,19 +434,21 @@
</span><span class="cx"> return "reschedule{event}"
</span><span class="cx">
</span><span class="cx"> # class Alerter(EventUpdaterBase):
</span><ins>+# component.replaceProperty(Property("ACKNOWLEDGED", DateTime.getNowUTC()))
</ins><span class="cx"> # pass
</span><span class="cx">
</span><span class="cx"> class Attacher(EventUpdaterBase):
</span><span class="cx"> def setDistributions(
</span><span class="cx"> self,
</span><ins>+ filesizeDistribution=NormalDistribution(24, 3),
+ numAttachmentsDistribution=LogNormalDistribution(2, 1),
+ attachLikelihoodDistribution=BernoulliDistribution(0.9),
+
</ins><span class="cx"> ):
</span><del>- # filesizeDistribution=NormalDistribution(24, 3),
- # numAttachmentsDistribution=LogNormalDistribution(2, 1),
- # attachLikelihoodDistribution=BernoulliDistribution(0.9)
- # self._filesize = filesizeDistribution
</del><ins>+ self._filesize = filesizeDistribution
</ins><span class="cx"> # self._numAttachments = numAttachmentsDistribution
</span><span class="cx"> # self._attachLikelihood = attachLikelihoodDistribution
</span><del>- pass
</del><ins>+ # pass
</ins><span class="cx">
</span><span class="cx"> def modifyEvent(self, href, vevent):
</span><span class="cx"> d = self._client.postAttachment(href, 'x' * 1024)
</span><span class="lines">@@ -487,405 +473,52 @@
</span><span class="cx"> def unattachFile(self):
</span><span class="cx"> pass
</span><span class="cx">
</span><del>-class InviterBase(EventerBase):
- """
- Base profile for a calendar user that invites and deinvites other principals to events
- """
- def setParameters(
</del><ins>+class InviterBase(EventUpdaterBase):
+ def setDistributions(
</ins><span class="cx"> self,
</span><del>- enabled=True,
</del><ins>+ numInvitees=NormalDistribution(7, 2),
</ins><span class="cx"> sendInvitationDistribution=NormalDistribution(600, 60),
</span><del>- inviteeDistribution=UniformDiscreteDistribution(range(-10, 11)),
- **params
</del><ins>+ numInviteesDistribution=UniformDiscreteDistribution(range(-10, 11))
</ins><span class="cx"> ):
</span><span class="cx"> self.enabled = enabled
</span><span class="cx"> self._sendInvitationDistribution = sendInvitationDistribution
</span><del>- self._inviteeDistribution = inviteeDistribution
- if len(params) > 0:
- pass
</del><ins>+ self._numInvitees = inviteeDistribution
</ins><span class="cx">
</span><del>- def getAttendees():
</del><ins>+ def _findUninvitedRecord(self, vevent):
</ins><span class="cx"> pass
</span><span class="cx">
</span><del>- # def _invitePrincipal(self, ...):
</del><ins>+ def _addAttendee(self, some_id):
+ attendeeProp = self._buildAttendee()
+ self._client.attendeeAutocomplete
</ins><span class="cx">
</span><del>- # def _uninvitePrincipal(self, ...):
-
-
-
-
- def _loopWithDistribution(reactor, distribution, function):
- result = Deferred()
-
- def repeat(ignored):
- reactor.callLater(distribution.sample(), iterate)
-
- def iterate():
- d = function()
- if d is not None:
- d.addCallbacks(repeat, result.errback)
- else:
- repeat(None)
-
- repeat(None)
- return result
-
</del><span class="cx"> # def _didSelfOrganize(self, vevent):
</span><span class="cx">
</span><ins>+ # TODO handle alternate roles
+ def _buildAttendee(self, commonname, cuaddr, isIndividual=True, isRequired=False):
+ return Property(
+ name=u'ATTENDEE',
+ value=cuaddr.encode("utf-8"),
+ params={
+ 'CN': commonname,
+ 'CUTYPE': 'INDIVIDUAL' if isIndividual else 'ROOM',
+ 'PARTSTAT': 'NEEDS-ACTION',
+ 'ROLE': 'REQ-PARTICIPANT',
+ 'RSVP': 'TRUE',
+ },
+ )
</ins><span class="cx">
</span><del>- # def _buildIndividualAttendee(self, commonName, record, ):
</del><ins>+ def _getAttendees(self, vevent):
+ return vevent.properties('ATTENDEE')
</ins><span class="cx">
</span><del>- # # ATTENDEE;CN="Super User";CUTYPE=INDIVIDUAL;EMAIL="admin@example.com":mailto:admin@example.com
- # # ATTENDEE;CN="User 04";CUTYPE=INDIVIDUAL;EMAIL="user04@example.com":mailto:user04@example.com
-
- # role = ['REQ-PARTICIPANT', '']
- # attendee = Property(
- # name=u'ATTENDEE',
- # value=cuaddr.encode("utf-8"),
- # params={
- # 'CN': commonName,
- # 'CUTYPE': 'INDIVIDUAL',
- # 'PARTSTAT': 'NEEDS-ACTION',
- # 'ROLE': 'REQ-PARTICIPANT',
- # 'RSVP': 'TRUE',
- # },
- # )
- # return attendee
-
- # def _buildLocationAttendee(self, cn, cuaddr):
- # """
- # Example Usage: profile._buildLocationAttendee("Location 01", "urn:uuid:...")
- # """
- # role = ['REQ-PARTICIPANT', '']
- # attendee = Property(
- # name=u'ATTENDEE',
- # value=cuaddr.encode("utf-8"),
- # params={
- # 'CN': commonName,
- # 'CUTYPE': 'ROOM',
- # 'PARTSTAT': 'NEEDS-ACTION',
- # 'ROLE': 'REQ-PARTICIPANT',
- # 'RSVP': 'TRUE',
- # },
- # )
- # return attendee
-
</del><span class="cx"> def _invite():
</span><span class="cx"> raise NotImplementedError
</span><span class="cx">
</span><span class="cx"> def _addAttendee():
</span><span class="cx"> raise NotImplementedError
</span><span class="cx">
</span><del>-# TODO - invite groups
-# class Inviter(EventUpdaterBase): pass
</del><ins>+class Inviter(InviterBase):
</ins><span class="cx">
</span><del>-class Relocater(EventUpdaterBase):
- def setParameters(
- self,
- ):
- pass
-
-class AppleEventer(EventerBase):
- """
- Calendar user who makes events in the form of Apple logo
- """
</del><span class="cx"> def initialize(self):
</span><del>- self.action = self._addEvent
- return succeed(None)
-
- def setDistributions(
- self
- ):
- pass
-
- def _addEvent(self):
- if not self._client.started:
- return succeed(None)
-
- calendar = self._getRandomCalendar()
- if not calendar:
- return succeed(None)
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
-
- vevent.replaceProperty(Property("UID", uid))
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
-
- href = '%s%s.ics' % (calendar.url, uid)
- event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
- d = self._client.addEvent(href, event)
- return self._newOperation("create", d)
-
-class HappyEventer(EventerBase):
- """
- Calendar user who makes events in the form of Apple logo
- """
- def initialize(self):
- self.action = self._addEvent
- return succeed(None)
-
- def setDistributions(
- self
- ):
- self._eventStart = UniformDiscreteDistribution([
- # August 3
- DateTime(year=2015, month=8, day=3, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=3, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=3, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=3, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
-
- # August 4
- DateTime(year=2015, month=8, day=4, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=4, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=4, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=4, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
-
- DateTime(year=2015, month=8, day=4, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=4, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=4, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
-
- # August 5
- DateTime(year=2015, month=8, day=5, hours=13, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=5, hours=13, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=5, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=5, hours=17, minutes=30, seconds=0, tzid=None, utcoffset=None),
-
- # August 6
- DateTime(year=2015, month=8, day=6, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=6, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=6, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=6, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
-
- DateTime(year=2015, month=8, day=6, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=6, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=6, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
-
- # August 7
- DateTime(year=2015, month=8, day=7, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=7, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=7, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
- DateTime(year=2015, month=8, day=7, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
- ])
- self._eventDuration = UniformDiscreteDistribution([
- 30 * 60,
- 60 * 60
- ])
-
- def _addEvent(self):
- if not self._client.started:
- return succeed(None)
-
- calendar = self._getRandomCalendar()
- if not calendar:
- return succeed(None)
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStart.sample()
- dtend = dtstart + Duration(seconds=self._eventDuration.sample())
-
- vevent.replaceProperty(Property("UID", uid))
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
-
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
-
- href = '%s%s.ics' % (calendar.url, uid)
- event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
- d = self._client.addEvent(href, event)
- return self._newOperation("create", d)
-
-
-
-# class EventDeleter(ProfileBase):
-
-
-
-""" TEST """
-# class Intern(object):
-# def __init__(self):
-# self.behaviors = [
-# Eventer(asdfjadsf),
-# Attacher(asjadsfjasdf),
-# Inviter(enabled=True, **params)
-# ]
-
-# def run(self):
-# deferreds = []
-# for behavior in self.behaviors:
-# deferreds.append(behavior.run())
-# return DeferredList(deferreds)
-
-
-
-####################
-# Class Hierarchy
-# ---------------
-# TaskBase
-# Tasker
-# TaskUpdaterBase
-# Titler
-# Alerter
-# Noter
-# Prioritizer
-# Completer
-# TaskDeleter
-####################
-class TaskBase(ProfileBase):
- """
- Base profile for a calendar user who interacts with tasks
- """
- def setParameters(self, enabled=True, interval=25, **params):
- self.enabled = enabled
- self._interval = interval
-
-class Tasker(TaskBase):
- """
- A Calendar user who creates new tasks.
- """
- def initialize(self):
- self.action = self._addTask
- return succeed(None)
-
- def _addTask(self, title="Simple Task"):
- if not self._client.started:
- return succeed(None)
-
- calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
-
- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
-
- # Copy the template task and fill in some of its fields
- # to make a new task to create on the calendar.
- vcalendar = taskTemplate.duplicate()
- vtodo = vcalendar.mainComponent()
- uid = str(uuid4())
- vtodo.replaceProperty(Property("UID", uid))
- vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
-
- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
-class TaskUpdaterBase(TaskBase):
- def action(self):
- task = self._getRandomTask()
- if not task:
- return succeed(None)
- component = task.component
- vtodo = component.mainComponent()
-
- label = self.modifyEvent(task.url, vtodo)
- vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
-
- task.component = component
- d = self._client.updateEvent(task)
- return self._newOperation(label, d)
-
- def modifyEvent(self):
- """Overriden by subclasses"""
- pass
-
-
-
-
-class TaskUpdater(TaskBase):
- """
- A Calendar user who creates and updates complex tasks with:
- Mark as completed/not completed
- Change name
- Change priority
- Change notes
- Sets due dates
- """
-
- def setParameters(
- self,
- taskDueDistribution=NearFutureDistribution(),
- ):
- pass
-
-
- def _addTask(self):
- if not self._client.started:
- return succeed(None)
-
- calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
-
- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
-
- # Copy the template task and fill in some of its fields
- # to make a new task to create on the calendar.
- vcalendar = taskTemplate.duplicate()
- vtodo = vcalendar.mainComponent()
- uid = str(uuid4())
- due = self._taskStartDistribution.sample()
- vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("UID", uid))
-
- # vtodo.replaceProperty(Property("SUMMARY", title))
-
- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
-
-# class TaskTitlerMixin = TitlerMixin
-# class TaskAlerterMixin = AlerterMixin
-# self._taskStartDistribution = taskDueDistribution
-# vtodo.replaceProperty(Property("DUE", due))
-# class TaskNoterMixin = NoterMixin
-
-class Prioritizer(TaskBase):
- PRIORITY_NONE = 0
- PRIORITY_HIGH = 1
- PRIORITY_MEDIUM = 5
- PRIORITY_LOW = 9
-
- def _setPriority(self, priority, vtodo):
- vtodo.replaceProperty(Property("PRIORITY", priority))
-
-class Completer(TaskBase):
- def _markTaskComplete(vtodo):
- """ task is a Component representing a VTODO """
- vtodo.replaceProperty(Property("COMPLETED", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("PERCENT-COMPLETE", 100))
- vtodo.replaceProperty(Property("STATUS", "COMPLETED"))
-
-
- def _markTaskIncomplete(vtodo):
- """ mark a VTODO as incomplete """
- vtodo.removeProperty("COMPLETED")
- vtodo.removeProperty("PERCENT-COMPLETE")
- vtodo.replaceProperty(Property("STATUS", "NEEDS-ACTION"))
-
-# class TaskDeleter(ProfileBase): pass
-
-
-class Inviter(ProfileBase):
-
- def initialize(self):
</del><span class="cx"> self.action = self.test
</span><span class="cx"> return succeed(None)
</span><span class="cx">
</span><span class="lines">@@ -1014,135 +647,198 @@
</span><span class="cx"> # Oops, either no events or no calendars to play with.
</span><span class="cx"> return succeed(None)
</span><span class="cx">
</span><ins>+class Relocater(InviterBase):
+ def setDistributions(
+ self,
+ ):
+ pass
</ins><span class="cx">
</span><ins>+class EventDeleter(EventBase):
+ """
+ A calendar user who deletes events at random
+ """
+ def _deleteEvent(self):
+ event = self._getRandomEvent()
+ if event is None:
+ return succeed(None)
+ d = self._client.deleteEvent(event.url)
+ return self._newOperation("delete{event}", d)
</ins><span class="cx">
</span><del>-class RealisticInviter(ProfileBase):
</del><ins>+ action = _deleteEvent
+
+
+
+
+""" TEST """
+# class Intern(object):
+# def __init__(self):
+# self.behaviors = [
+# Eventer(asdfjadsf),
+# Attacher(asjadsfjasdf),
+# Inviter(enabled=True, **params)
+# ]
+
+# def run(self):
+# deferreds = []
+# for behavior in self.behaviors:
+# deferreds.append(behavior.run())
+# return DeferredList(deferreds)
+
+
+
+####################
+# Tasker Hierarchy #
+# ---------------- #
+# TaskBase #
+# Tasker #
+# TaskDeleter #
+# TaskUpdaterBase#
+# Titler #
+# Noter #
+# Prioritizer #
+# Completer #
+# Alerter #
+####################
+
+
+class TaskBase(ProfileBase):
</ins><span class="cx"> """
</span><del>- A Calendar user who invites other users to new events.
</del><ins>+ Base profile for a calendar user who interacts with tasks
</ins><span class="cx"> """
</span><del>- def setParameters(
- self,
- enabled=True,
- sendInvitationDistribution=NormalDistribution(600, 60),
- inviteeDistribution=UniformDiscreteDistribution(range(-10, 11)),
- inviteeClumping=True,
- inviteeCountDistribution=LogNormalDistribution(1.2, 1.2),
- eventStartDistribution=NearFutureDistribution(),
- eventDurationDistribution=UniformDiscreteDistribution([
- 15 * 60, 30 * 60,
- 45 * 60, 60 * 60,
- 120 * 60
- ]),
- recurrenceDistribution=RecurrenceDistribution(False),
- ):
- self.enabled = enabled
- self._sendInvitationDistribution = sendInvitationDistribution
- self._inviteeDistribution = inviteeDistribution
- self._inviteeClumping = inviteeClumping
- self._inviteeCountDistribution = inviteeCountDistribution
- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
- self._recurrenceDistribution = recurrenceDistribution
</del><ins>+ def _getRandomCalendar(self):
+ return self._getRandomCalendarOfType('VTODO')
</ins><span class="cx">
</span><ins>+ def _getRandomEvent(self):
+ return self._getRandomEventOfType('VTODO')
</ins><span class="cx">
</span><del>- def _addAttendee(self, event, attendees):
- """
- Create a new attendee to add to the list of attendees for the
- given event.
- """
- selfRecord = self._sim.getUserRecord(self._number)
- invitees = set([u'mailto:%s' % (selfRecord.email,)])
- for att in attendees:
- invitees.add(att.value())
</del><span class="cx">
</span><del>- for _ignore_i in range(10):
</del><ins>+class Tasker(TaskBase):
+ """
+ A Calendar user who creates new tasks.
+ """
+ def _addTask(self):
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
</ins><span class="cx">
</span><del>- sample = self._inviteeDistribution.sample()
- if self._inviteeClumping:
- sample = self._number + sample
- invitee = max(0, sample)
</del><ins>+ # Form a new event by modifying fields of the template event
+ vcalendar = taskTemplate.duplicate()
+ vtodo = vcalendar.mainComponent()
+ uid = str(uuid4())
</ins><span class="cx">
</span><del>- try:
- record = self._sim.getUserRecord(invitee)
- except IndexError:
- continue
- cuaddr = u'mailto:%s' % (record.email,)
- if cuaddr not in invitees:
- break
- else:
- raise CannotAddAttendee("Can't find uninvited user to invite.")
</del><ins>+ vtodo.replaceProperty(Property("UID", uid))
+ vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
</ins><span class="cx">
</span><del>- attendee = Property(
- name=u'ATTENDEE',
- value=cuaddr.encode("utf-8"),
- params={
- 'CN': record.commonName,
- 'CUTYPE': 'INDIVIDUAL',
- 'PARTSTAT': 'NEEDS-ACTION',
- 'ROLE': 'REQ-PARTICIPANT',
- 'RSVP': 'TRUE',
- },
- )
</del><ins>+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create{task}", d)
</ins><span class="cx">
</span><del>- event.addProperty(attendee)
- attendees.append(attendee)
</del><ins>+ action = _addTask
</ins><span class="cx">
</span><ins>+class TaskDeleter(TaskBase):
+ def _deleteTask(self):
+ event = self._getRandomEvent()
+ if event is None:
+ return succeed(None)
</ins><span class="cx">
</span><del>- def _invite(self):
- """
- Try to add a new event, or perhaps remove an
- existing attendee from an event.
</del><ins>+ d = self._client.deleteEvent(event.url)
+ return self._newOperation("delete{task}", d)
</ins><span class="cx">
</span><del>- @return: C{None} if there are no events to play with,
- otherwise a L{Deferred} which fires when the attendee
- change has been made.
- """
</del><ins>+ action = _deleteTask
</ins><span class="cx">
</span><del>- if not self._client.started:
</del><ins>+
+class TaskUpdaterBase(TaskBase):
+ def action(self):
+ task = self._getRandomEvent()
+ if not task:
</ins><span class="cx"> return succeed(None)
</span><ins>+ component = task.component
+ vtodo = component.mainComponent()
</ins><span class="cx">
</span><del>- # Find calendars which are eligible for invites
- calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
</del><ins>+ label = self.modifyEvent(task.url, vtodo)
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
</ins><span class="cx">
</span><del>- while calendars:
- # Pick one at random from which to try to create an event
- # to modify.
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
</del><ins>+ task.component = component
+ d = self._client.updateEvent(task, method_label="update{task}")
+ return self._newOperation(label, d)
</ins><span class="cx">
</span><del>- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
- vevent.replaceProperty(Property("UID", uid))
</del><ins>+ def modifyEvent(self, href, vtodo):
+ """Overriden by subclasses"""
+ pass
</ins><span class="cx">
</span><del>- rrule = self._recurrenceDistribution.sample()
- if rrule is not None:
- vevent.addProperty(Property(None, None, None, pycalendar=rrule))
</del><span class="cx">
</span><del>- vevent.addProperty(self._client._makeSelfOrganizer())
- vevent.addProperty(self._client._makeSelfAttendee())
</del><ins>+class TaskTitler(TaskUpdaterBase, Titler):
+ """
+ Changes the SUMMARY of a random VTODO
+ """
+ def modifyEvent(self, _ignore_href, vtodo):
+ vtodo.replaceProperty(Property("SUMMARY", "." * 5))
+ return "update{title}"
</ins><span class="cx">
</span><del>- attendees = list(vevent.properties('ATTENDEE'))
- for _ignore in range(int(self._inviteeCountDistribution.sample())):
- try:
- self._addAttendee(vevent, attendees)
- except CannotAddAttendee:
- self._failedOperation("invite", "Cannot add attendee")
- return succeed(None)
</del><ins>+class TaskNoter(TaskUpdaterBase, Noter):
+ """
+ Changes the NOTES of a random VTODO
+ """
+ def modifyEvent(self, _ignore_href, vtodo):
+ vtodo.replaceProperty(Property("DESCRIPTION", "." * 5))
+ return "update{notes}"
</ins><span class="cx">
</span><del>- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addInvite(href, vcalendar)
- return self._newOperation("invite", d)
</del><span class="cx">
</span><ins>+# class TaskAlerterMixin = AlerterMixin (alarm AND due)
+# self._taskStartDistribution = taskDueDistribution
+# vtodo.replaceProperty(Property("DUE", due))
</ins><span class="cx">
</span><ins>+
+class Prioritizer(TaskUpdaterBase):
+ PRIORITY_NONE = 0
+ PRIORITY_HIGH = 1
+ PRIORITY_MEDIUM = 5
+ PRIORITY_LOW = 9
+
+ def setDistributions(
+ self,
+ priorityDistribution=UniformDiscreteDistribution([
+ PRIORITY_NONE, PRIORITY_LOW, PRIORITY_MEDIUM, PRIORITY_HIGH
+ ])
+ ):
+ self._priority = priorityDistribution
+
+ def modifyEvent(self, _ignore_href, vtodo):
+ self._setPriority(vtodo, self._priority.sample())
+
+
+ def _setPriority(self, vtodo, priority):
+ """ Set the PRIORITY of a VTODO """
+ vtodo.replaceProperty(Property("PRIORITY", priority))
+
+class Completer(TaskUpdaterBase):
+ def setDistributions(
+ self,
+ completeLikelihood=BernoulliDistribution(0.9)
+ ):
+ self._complete = completeLikelihood
+
+ def modifyEvent(self, _ignore_href, vtodo):
+ if self._complete.sample():
+ self._markTaskComplete(vtodo)
+ else:
+ self._markTaskIncomplete(vtodo)
+
+ def _markTaskComplete(self, vtodo):
+ """ Mark a VTODO as complete """
+ vtodo.replaceProperty(Property("COMPLETED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("PERCENT-COMPLETE", 100))
+ vtodo.replaceProperty(Property("STATUS", "COMPLETED"))
+
+ def _markTaskIncomplete(self, vtodo):
+ """ Mark a VTODO as incomplete """
+ vtodo.removeProperty("COMPLETED")
+ vtodo.removeProperty("PERCENT-COMPLETE")
+ vtodo.replaceProperty(Property("STATUS", "NEEDS-ACTION"))
+
+
</ins><span class="cx"> ##########################
</span><span class="cx"> # Notification Behaviors #
</span><span class="cx"> ##########################
</span><span class="lines">@@ -1151,7 +847,7 @@
</span><span class="cx"> A Calendar user who accepts invitations to events. As well as accepting requests, this
</span><span class="cx"> will also remove cancels and replies.
</span><span class="cx"> """
</span><del>- def setParameters(
</del><ins>+ def setDistributions(
</ins><span class="cx"> self,
</span><span class="cx"> enabled=True,
</span><span class="cx"> acceptDelayDistribution=NormalDistribution(1200, 60)
</span><span class="lines">@@ -1298,107 +994,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-class EventUpdater(EventerBase):
- """
- A Calendar user who creates a new event, and then updates its alarm.
- """
- def initialize(self):
- """
- Called before the profile runs for real. Can be used to initialize client state.
-
- @return: a L{Deferred} that fires when initialization is done
- """
- self.action = self._updateEvent
- return self._initEvent()
-
-
- def _initEvent(self):
- if not self._client.started:
- return succeed(None)
-
- # If it already exists, don't re-create
- calendar = self._calendarsOfType(caldavxml.calendar, "VEVENT")[0]
- if calendar.events:
- events = [event for event in calendar.events.values() if event.url.endswith("event_to_update.ics")]
- if events:
- return succeed(None)
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = self._eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
- vevent.replaceProperty(Property("UID", uid))
-
- rrule = self._recurrenceDistribution.sample()
- if rrule is not None:
- vevent.addProperty(Property(None, None, None, pycalendar=rrule))
-
- href = '%s%s' % (calendar.url, "event_to_update.ics")
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
- def _updateEvent(self):
- """
- Try to add a new attendee to an event, or perhaps remove an
- existing attendee from an event.
-
- @return: C{None} if there are no events to play with,
- otherwise a L{Deferred} which fires when the attendee
- change has been made.
- """
-
- if not self._client.started:
- return succeed(None)
-
- # If it does not exist, try to create it
- calendar = self._calendarsOfType(caldavxml.calendar, "VEVENT")[0]
- if not calendar.events:
- return self._initEvent()
- events = [event for event in calendar.events.values() if event.url.endswith("event_to_update.ics")]
- if not events:
- return self._initEvent()
- event = events[0]
-
- # Add/update the ACKNOWLEDGED property
- component = event.component.mainComponent()
- component.replaceProperty(Property("ACKNOWLEDGED", DateTime.getNowUTC()))
- d = self._client.changeEvent(event.url)
- return self._newOperation("update", d)
-
- # def _changeEventTitle(self, event, title):
- # event.component = self._setEventTitle(event.component, title)
- # event.
-
-
-
-
-
-class EventDeleter(EventerBase):
- """
- A calendar user who deletes events at random
- """
- def initialize(self):
- self.action = self._deleteEvent
-
- def _deleteEvent(self):
- event = self._getRandomEvent()
- if event is None:
- return succeed(None)
- d = self._client.deleteEvent(event)
- return self._newOperation("delete", d)
-
-
-
-
-
</del><span class="cx"> ######################
</span><span class="cx"> # Calendar Behaviors #
</span><span class="cx"> ######################
</span><span class="lines">@@ -1411,7 +1006,7 @@
</span><span class="cx"> return succeed(None)
</span><span class="cx">
</span><span class="cx">
</span><del>- def setParameters(self, enabled=True, interval=25):
</del><ins>+ def setDistributions(self, enabled=True, interval=25):
</ins><span class="cx"> self.enabled = enabled
</span><span class="cx"> self._interval = interval
</span><span class="cx">
</span><span class="lines">@@ -1419,26 +1014,26 @@
</span><span class="cx">
</span><span class="cx"> class CalendarMaker(CalendarBase):
</span><span class="cx"> """ A Calendar user who adds new Calendars """
</span><del>- def initialize(self):
- self.action = self._addCalendar
- return succeed(None)
</del><span class="cx">
</span><del>-
</del><span class="cx"> def _addCalendar(self):
</span><del>- if not self._client.started:
- return None
</del><ins>+ print "Adding a calendar"
+ # if not self._client.started:
+ # return None
</ins><span class="cx">
</span><del>- uid = str(uuid4())
</del><ins>+ # uid = str(uuid4())
</ins><span class="cx">
</span><del>- body = Calendar.buildCalendarXML(order=0, component_type="VEVENT", rgba_color='FB524FFF', name='Sample Calendar')
- print("Making new calendar with uid: " + uid)
- # XXX Just for testing! remove this soon
- path = "/calendars/__uids__/" + self._client.record.guid + "/" + uid + "/"
- d = self._client.addCalendar(path, body)
</del><ins>+ # body = Calendar.buildCalendarXML(order=0, component_type="VEVENT", rgba_color='FB524FFF', name='Sample Calendar')
+ # print("Making new calendar with uid: " + uid)
+ # # XXX Just for testing! remove this soon
+ # path = "/calendars/__uids__/" + self._client.record.guid + "/" + uid + "/"
+ # d = self._client.addCalendar(path, body)
+ d = succeed('calendar created')
</ins><span class="cx"> return self._newOperation("create", d)
</span><span class="cx">
</span><ins>+ action = _addCalendar
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class CalendarUpdater(CalendarBase):
</span><span class="cx"> """
</span><span class="cx"> A calendar user who updates random calendars
</span><span class="lines">@@ -1545,3 +1140,16 @@
</span><span class="cx"> print("Deleting " + calendar.url)
</span><span class="cx"> d = self._client.deleteCalendar(calendar.url)
</span><span class="cx"> return self._newOperation("delete", d)
</span><ins>+
+if __name__ == '__main__':
+ class TestProfile(ProfileBase):
+ def sayHello(self):
+ print("Hello!")
+ action = sayHello
+
+ from twisted.internet import reactor
+
+ profile = TestProfile(enabled=True, interval=1)
+ profile.setUp(reactor, None, None, None)
+ profile.run()
+ reactor.run()
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpushpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/push.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/push.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/push.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,6 +1,10 @@
</span><del>-from calendarserver.push.amppush import subscribeToIDs
-from twisted.internet.defer import succeed
</del><ins>+import uuid
</ins><span class="cx">
</span><ins>+from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.internet.defer import inlineCallbacks
+
+from calendarserver.push.amppush import SubscribeToID, UnsubscribeFromID, AMPPushClientFactory
+
</ins><span class="cx"> class PushMonitor(object):
</span><span class="cx"> """
</span><span class="cx"> Watchguard that monitors push notifications (AMP Push)
</span><span class="lines">@@ -21,71 +25,107 @@
</span><span class="cx"> @param ampPushPort: AMP port to connect to (e.g. 62311)
</span><span class="cx"> @type ampPushPort: integer
</span><span class="cx"> @param callback: a one-argument function that is fired
</span><del>- with a calendar hrefupon receipt of a push notification
</del><ins>+ with a calendar href upon receipt of a push notification
</ins><span class="cx"> for that resource
</span><span class="cx"> @type callback: one-argument callable
</span><span class="cx"> """
</span><ins>+
+ if reactor is None:
+ from twisted.internet import reactor
+
</ins><span class="cx"> self._reactor = reactor
</span><span class="cx"> self._ampPushHost = ampPushHost
</span><span class="cx"> self._ampPushPort = ampPushPort
</span><span class="cx">
</span><span class="cx"> # Keep track of AMP parameters for calendar homes we encounter. This
</span><del>- # dictionary has calendar home URLs as keys and pushkeys as
- # values.
</del><ins>+ # dictionary has pushkeys as keys and calendar home URLs as values.
</ins><span class="cx"> self._ampPushkeys = {}
</span><span class="cx">
</span><ins>+ self._callback = callback
+
+ self._token = str(uuid.uuid4()) # Unique token for this monitor
+ self._endpoint = TCP4ClientEndpoint(self._reactor, self._ampPushHost, self._ampPushPort)
+ self._factory = AMPPushClientFactory(self._receivedAMPPush)
+ self._connected = False
+
+ @inlineCallbacks
</ins><span class="cx"> def begin(self):
</span><span class="cx"> """
</span><span class="cx"> Start monitoring for AMP-based push notifications
</span><span class="cx"> """
</span><del>- self._subscribeToIDs(self.ampPushkeys)
</del><ins>+ self._protocol = yield self._endpoint.connect(self._factory)
+ self._connected = True
+ pushkeys = self._ampPushkeys.keys()
+ yield self._subscribeToPushkeys(pushkeys)
</ins><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def end(self):
</span><span class="cx"> """
</span><del>- Finish monitoring push notifications. Any other cleanup should be done here
</del><ins>+ Finish monitoring push notifications.
</ins><span class="cx"> """
</span><del>- self._unsubscribeFromAll()
</del><ins>+ pushkeys = self._ampPushkeys.keys()
+ self._ampPushkeys = {}
+ yield self._unsubscribeFromPushkeys(pushkeys)
</ins><span class="cx">
</span><del>- def _subscribeToIDs(self, ids):
</del><ins>+ # Close the connection between client and server
+ yield self._protocol.transport.loseConnection()
+ self._connected = False
</ins><span class="cx">
</span><del>- subscribeToIDs(
- self._ampPushHost,
- self._ampPushPort,
- ids,
- self._receivedAmpPush,
- self._reactor
- )
</del><span class="cx">
</span><del>- def _receivedAMPPush(self, inboundID, dataChangedTimestamp, priority=5):
- print("-" * 64)
- print("{} received a PUSH with ID={}, timestamp={}, priority={}".format(self.record.commonName, inboundID, dataChangedTimestamp, priority))
- print("By the way, my AMP keys are {}".format(self._ampPushkeys))
- print("-" * 64)
</del><ins>+ def addPushkey(self, pushkey, href):
+ """
+ Register a pushkey associated with a specific calendar href.
</ins><span class="cx">
</span><del>- for href, calendar_id in self.ampPushkeys.iteritems():
- if inboundID == calendar_id:
- self.callback(href)
- break
- else:
- # Somehow we are not subscribed to this inboundID
- print("*" * 16 + "Oh no - we're not subscribed to " + str(inboundID) + " but we received a notification anyway!")
- pass
</del><ins>+ @param pushkey: AMP pushkey returned by the server, used to listen to notifications
+ @type pushkey: C{str}
+ @param href: href of calendar home set. When the server triggers a push for the
+ associated pushkey, the callback will be fired with this href
+ @type href: C{str}
</ins><span class="cx">
</span><del>- def _unsubscribeFromAll(self):
- # For now, the server doesn't support unsubscribing from pushkeys, so we simply
- # "forget" about our registered pushkeys
- self._ampPushkeys = {}
</del><ins>+ Example Usage:
+ monitor.addPushkey('/CalDAV/localhost/<uid>', '/calendars/__uids__/<uid>')
+ """
+ self._ampPushkeys[pushkey] = href
+ if self._connected:
+ return self._subscribeToPushkey(pushkey)
</ins><span class="cx">
</span><ins>+ def removePushkey(self, pushkey):
+ """
+ Unregister the calendar home associated with the specified pushkey
+ """
+ if pushkey in self._ampPushkeys:
+ del self._ampPushkeys[pushkey]
+ if self._connected:
+ return self._unsubscribeFromPushkey(pushkey)
</ins><span class="cx">
</span><del>- def addPushkey(self, href, pushkey):
- self._ampPushkeys[href] = pushkey
- self.subscribeToIDs()
</del><ins>+ def isSubscribedTo(self, href):
+ """
+ Returns true if and only if the given calendar href is actively being monitored
+ """
+ return href in self._ampPushkeys.itervalues()
</ins><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _subscribeToPushkeys(self, pushkeys):
+ for pushkey in pushkeys:
+ yield self._subscribeToPushkey(pushkey)
</ins><span class="cx">
</span><del>- def removePushkey(self, pushkey):
- # if self.ampPushkeys.has_value(pushkey):
- # del self.ampPushKeys
- pass
</del><ins>+ @inlineCallbacks
+ def _unsubscribeFromPushkeys(self, pushkeys):
+ for pushkey in pushkeys:
+ yield self._unsubscribeFromPushkey(pushkey)
</ins><span class="cx">
</span><del>- def isSubscribedTo(self, href):
- return href in self.ampPushkeys
</del><ins>+ def _subscribeToPushkey(self, pushkey):
+ return self._protocol.callRemote(SubscribeToID, token=self._token, id=pushkey)
+
+ def _unsubscribeFromPushkey(self, pushkey):
+ return self._protocol.callRemote(UnsubscribeFromID, id=pushkey)
+
+
+ def _receivedAMPPush(self, inboundID, dataChangedTimestamp, priority=5):
+ if inboundID in self._ampPushkeys:
+ # Only react if we're tracking this pushkey
+ href = self._ampPushkeys[inboundID]
+ self._callback(href)
+ else:
+ # Somehow we are not subscribed to this pushkey
+ pass
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrecordspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/records.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/records.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/records.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -11,6 +11,19 @@
</span><span class="cx"> def __repr__(self):
</span><span class="cx"> return "Record(%s:%s %s %s %s)" % (self.uid, self.password, self.commonName, self.email, self.guid)
</span><span class="cx">
</span><ins>+# def generateRecords(
+# count, uidPattern="user%d", passwordPattern="user%d",
+# namePattern="User %d", emailPattern="user%d@example.com",
+# guidPattern="user%d"
+# ):
+# for i in xrange(count):
+# i += 1
+# uid = uidPattern % (i,)
+# password = passwordPattern % (i,)
+# name = namePattern % (i,)
+# email = emailPattern % (i,)
+# guid = guidPattern % (i,)
+# yield DirectoryRecord(uid, password, name, email, guid)
</ins><span class="cx">
</span><span class="cx"> def recordsFromCSVFile(path):
</span><span class="cx"> if path:
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrequesterpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/requester.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/requester.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/requester.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -24,7 +24,7 @@
</span><span class="cx"> class Requester(object):
</span><span class="cx"> """
</span><span class="cx"> Utility to create requests on behalf of a client. Public methods are:
</span><del>- method url body headers status method_label
</del><ins>+ method url body headers status method_label
</ins><span class="cx"> ------------------------------------------------------------------------
</span><span class="cx"> GET req ---
</span><span class="cx"> POST req req
</span><span class="lines">@@ -100,6 +100,7 @@
</span><span class="cx"> client_id=self._client_id,
</span><span class="cx"> )
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> before = self._reactor.seconds()
</span><span class="cx"> response = yield self._agent.request(method, url, headers, StringProducer(body) if body else None)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestresourcespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/resources.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/resources.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/resources.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -114,13 +114,6 @@
</span><span class="cx"> self.url = url
</span><span class="cx"> self.changeToken = changeToken
</span><span class="cx"> self.events = {}
</span><del>- # print("----\nNew Calendar")
- # print("Resource Type: ", self.resourceType)
- # print("Component Types: ", self.componentTypes)
- # print("Name: ", self.name)
- # print("URL: ", self.url)
- # print("Change Token: ", self.changeToken)
- # print("Events: ", self.events)
</del><span class="cx">
</span><span class="cx">
</span><span class="cx"> def serialize(self):
</span><span class="lines">@@ -156,20 +149,7 @@
</span><span class="cx"> calendar.changeToken = ""
</span><span class="cx"> return calendar
</span><span class="cx">
</span><del>- @staticmethod
- def buildCalendarXML(order=0, component_type='VEVENT', rgba_color='FB524FFF', name='Simple Calendar'):
- # TODO add timezone information
</del><span class="cx">
</span><del>- # MakeCalendar(None, '/', name, )
-
- # body = _STARTUP_CREATE_CALENDAR.format(
- # order=order,
- # component_type=component_type,
- # color=rgba_color,
- # name=name)
- # return body
- return ""
-
</del><span class="cx"> @staticmethod
</span><span class="cx"> def addInviteeXML(uid, summary, readwrite=True):
</span><span class="cx"> return AddInvitees(None, '/', [uid], readwrite, summary=summary).request_data.text
</span><span class="lines">@@ -231,11 +211,3 @@
</span><span class="cx"> el = ElementTree.Element(qn)
</span><span class="cx"> el.text = order
</span><span class="cx"> return Calendar._buildPropPatchXML(el)
</span><del>-
- # @inlineCallbacks
- # def setCalendarProperty(self, calendar):
-
- # def do_stuff(...):
- # body =
- # yield self.requester.proppatch(href, body, method_label="PROPPATCH{calendar}")
- #
</del><span class="cx">\ No newline at end of file
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsaltsettingsplistclientsplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/alt-settings/plist/clients.plist (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/alt-settings/plist/clients.plist 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/alt-settings/plist/clients.plist 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -38,7 +38,7 @@
</span><span class="cx"> <dict>
</span><span class="cx"> <!-- Name that appears in logs. -->
</span><span class="cx"> <key>title</key>
</span><del>- <string>10.11 Intern</string>
</del><ins>+ <string>10.11</string>
</ins><span class="cx">
</span><span class="cx"> <!-- OS_X_10_7 can poll the calendar home at some interval. This is
</span><span class="cx"> in seconds. -->
</span><span class="lines">@@ -64,7 +64,7 @@
</span><span class="cx"> <array>
</span><span class="cx"> <dict>
</span><span class="cx"> <key>class</key>
</span><del>- <string>contrib.performance.loadtest.profiles.Inviter</string>
</del><ins>+ <string>contrib.performance.loadtest.profiles.Eventer</string>
</ins><span class="cx">
</span><span class="cx"> <key>params</key>
</span><span class="cx"> <dict>
</span><span class="lines">@@ -74,7 +74,7 @@
</span><span class="cx"> <!-- Define the interval (in seconds) at which this profile will use
</span><span class="cx"> its client to create a new event. -->
</span><span class="cx"> <key>interval</key>
</span><del>- <integer>1</integer>
</del><ins>+ <integer>2</integer>
</ins><span class="cx"> </dict>
</span><span class="cx"> </dict>
</span><span class="cx"> </array>
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsclientspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/clients.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/clients.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/clients.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,35 +1,26 @@
</span><span class="cx"> from contrib.performance.loadtest.clients import iOS_5, OS_X_10_6, OS_X_10_7, OS_X_10_11
</span><del>-from contrib.performance.loadtest.profiles import CalendarMaker, CalendarUpdater, CalendarSharer, CalendarDeleter
-from contrib.performance.loadtest.population import ProfileType
</del><ins>+from contrib.performance.loadtest.profiles import (
+ Eventer, EventDeleter,
+ Titler,
+ Tasker, TaskDeleter,
+ TaskTitler, TaskNoter, Completer, Prioritizer,
</ins><span class="cx">
</span><del>-from preset_distributions import STANDARD_WORK_DISTRIBUTION, LOW_RECURRENCE_DISTRIBUTION, MEDIUM_RECURRENCE_DISTRIBUTION
</del><ins>+ CalendarMaker, CalendarUpdater, CalendarSharer, CalendarDeleter
+)
+from contrib.performance.loadtest.distributions import FixedDistribution, BernoulliDistribution
</ins><span class="cx">
</span><del>-# We have to roll our own deep copy method because you can't deep copy Twisted's reactor
-class ClientFactory(object):
</del><span class="cx">
</span><del>- def __init__(self, client, weight):
- pass
</del><ins>+from preset_distributions import STANDARD_WORK_DISTRIBUTION, LOW_RECURRENCE_DISTRIBUTION, MEDIUM_RECURRENCE_DISTRIBUTION
</ins><span class="cx">
</span><del>- @staticmethod
- def _duplicateClient(client):
- return type(client)(
- # some params
- )
-
- def new(reactor, ):
- pass
-
-class ProfileFactory(object):
- def __init__(self, profile):
- pass
-
- @staticmethod
- def _duplicateProfile(profile):
- return type(profile)()
-
-calendars_only = [
</del><ins>+config = [
</ins><span class="cx"> {
</span><span class="cx"> "software": OS_X_10_11,
</span><ins>+ # title="10.11",
+ # calendarHomePollInterval=5,
+ # supportAmpPush=True,
+ # ampPushHost="localhost",
+ # ampPushPort62311
+ # )
</ins><span class="cx"> "params": {
</span><span class="cx"> "title": "10.11",
</span><span class="cx"> "calendarHomePollInterval": 5,
</span><span class="lines">@@ -38,33 +29,41 @@
</span><span class="cx"> "ampPushPort": 62311
</span><span class="cx"> },
</span><span class="cx"> "profiles": [
</span><del>- ProfileType(CalendarMaker, dict(enabled=True, interval=15)),
</del><ins>+ # Eventer(enabled=True, interval=0.01, eventStartDistribution=STANDARD_WORK_DISTRIBUTION),
+ # Titler(enabled=True, interval=1, titleLengthDistribution=FixedDistribution(10)),
</ins><span class="cx">
</span><del>- # CalendarMaker(enabled=True, interval=15),
</del><ins>+ # Tasker(enabled=False, interval=1),
+ # Completer(enabled=True, interval=0.5, completeLikelihood=BernoulliDistribution(0.5)),
+ # Prioritizer(enabled=True, interval=0.1),
+ TaskTitler(enabled=True, interval=1),
+ TaskNoter(enabled=True, interval=1),
+ # TaskDeleter(enabled=True, interval=1),
+
+ # CalendarMaker(enabled=True, interval=1),
</ins><span class="cx"> # CalendarUpdater(enabled=True, interval=5),
</span><span class="cx"> # CalendarSharer(enabled=True, interval=30),
</span><span class="cx"> # CalendarDeleter(false=True, interval=30)
</span><span class="cx"> ],
</span><del>- "weight": 1
</del><ins>+ "weight": 3
</ins><span class="cx"> }
</span><span class="cx"> ]
</span><span class="cx">
</span><del>-# TBD what about multiple weights?
-calendars_only_ideal = [
- OS_X_10_11(
- title="10.11",
- calendarHomePollInterval=5,
- supportAmpPush=True,
- ampPushHost="localhost",
- ampPushPort=62311,
- profiles=[
- CalendarMaker(enabled=True, interval=15),
- # CalendarUpdater(enabled=True, interval=5),
- # CalendarSharer(enabled=False, interval=30),
- # CalendarDeleter(enabled=False, interval=30)
- ]
- )
-]
</del><ins>+# # TBD what about multiple weights?
+# calendars_only_ideal = [
+# OS_X_10_11(
+# title="10.11",
+# calendarHomePollInterval=5,
+# supportAmpPush=True,
+# ampPushHost="localhost",
+# ampPushPort=62311,
+# profiles=[
+# CalendarMaker(enabled=True, interval=15),
+# # CalendarUpdater(enabled=True, interval=5),
+# # CalendarSharer(enabled=False, interval=30),
+# # CalendarDeleter(enabled=False, interval=30)
+# ]
+# )
+# ]
</ins><span class="cx">
</span><span class="cx"> # event_updates_only = [
</span><span class="cx"> # {
</span><span class="lines">@@ -111,7 +110,3 @@
</span><span class="cx"> # "weight": 1
</span><span class="cx"> # }
</span><span class="cx"> # ]
</span><del>-
-
-# clientConfiguration = calendars_only
-# __all__ = [clientConfiguration]
</del></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsettingsconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/config.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/config.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/settings/config.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,22 +1,13 @@
</span><del>-# Generally, the defaults are good enough for us.
</del><ins>+from contrib.performance.loadtest.settings.defaults import arrival, requestLogger, operationLogger, statisticsReporter, accounts
</ins><span class="cx">
</span><del>-config = dict(
- server=server,
- webadminPort=8080,
- serverStatsPort=8100,
- serializationPath='/tmp/sim',
- arrival=arrival,
- observers=[_requestLogger, _operationLogger, _statisticsReporter],
- records=accounts
-)
</del><ins>+from contrib.performance.loadtest.logger import EverythingLogger, MessageLogger
</ins><span class="cx">
</span><del>-config_dist = dict(
- server=server,
- webadminPort=8080,
- serverStatsPort=8100,
- serializationPath='/tmp/sim',
- arrival=arrival,
- observers=[_requestLogger, _operationLogger, _statisticsReporter],
- records=accounts,
- workers=["./bin/python contrib/performance/loadtest/ampsim.py"] * 6,
-)
</del><span class="cx">\ No newline at end of file
</span><ins>+config = {
+ "server": 'https://127.0.0.1:8443',
+ "webadminPort": 8080,
+ "serverStatsPort": {'server': 'localhost', 'Port': 8100},
+ "serializationPath": '/tmp/sim',
+ "arrival": arrival,
+ "observers": [requestLogger, operationLogger, statisticsReporter, EverythingLogger(), MessageLogger()],
+ "records": accounts
+}
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsimpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -48,8 +48,9 @@
</span><span class="cx"> from contrib.performance.loadtest.clients import OS_X_10_6
</span><span class="cx"> from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
</span><span class="cx"> from contrib.performance.loadtest.population import (
</span><del>- Populator, ProfileType, ClientType, PopulationParameters, SmoothRampUp,
</del><ins>+ ClientFactory, PopulationParameters, SmoothRampUp,
</ins><span class="cx"> CalendarClientSimulator)
</span><ins>+from contrib.performance.loadtest.config import Config
</ins><span class="cx"> from contrib.performance.loadtest.webadmin import LoadSimAdminResource
</span><span class="cx">
</span><span class="cx"> def safeDivision(value, total, factor=1):
</span><span class="lines">@@ -88,19 +89,30 @@
</span><span class="cx"> """
</span><span class="cx"> config = None
</span><span class="cx"> settings = FilePath(__file__).sibling("settings")
</span><del>- _defaultConfig = settings.child("config.plist")
- _defaultClients = settings.child("clients.plist")
</del><ins>+ # plists = settings.child('alt-settings').child('plist')
+ # _defaultConfig = plists.child("config.plist")
+ # _defaultClients = plists.child("clients.plist")
+ # _defaultConfig = settings.child('settings.config')
+ # _defaultClients = settings.child('settings.clients')
+ _defaultConfig = 'contrib.performance.loadtest.settings.config'
+ _defaultClients = 'contrib.performance.loadtest.settings.clients'
</ins><span class="cx">
</span><ins>+ optFlags = [
+ ("debug", "d", "Enable Deferred and Failure debugging"),
+ ("debug-deferred", None, "Enable only Deferred debugging"),
+ ("debug-failure", None, "Enable only Failure debugging"),
+ ("use-plist", None, "Interpret configuration files as XML property lists (default false)")
+ ]
+
</ins><span class="cx"> optParameters = [
</span><span class="cx"> ("runtime", "t", None,
</span><span class="cx"> "Specify the limit (seconds) on the time to run the simulation.",
</span><span class="cx"> int),
</span><span class="cx"> ("config", None, _defaultConfig,
</span><del>- "Configuration plist file name from which to read simulation parameters.",
- FilePath),
</del><ins>+ "Configuration plist file name from which to read simulation parameters."),
</ins><span class="cx"> ("clients", None, _defaultClients,
</span><del>- "Configuration plist file name from which to read client parameters.",
- FilePath),
</del><ins>+ "Configuration plist file name from which to read client parameters."),
+ ("logfile", "l", '-', FilePath)
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -140,40 +152,48 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def postOptions(self):
</span><del>- try:
- configFile = self['config'].open()
- except IOError, e:
- raise UsageError("--config %s: %s" % (
- self['config'].path, e.strerror))
- try:
</del><ins>+ """
+ Convert the given configuration files to dictionaries, respectively in
+ self.config and self.clients
+ """
+ configPath = self["config"]
+ clientsPath = self["clients"]
+ if self['use-plist']:
</ins><span class="cx"> try:
</span><del>- self.config = readPlist(configFile)
</del><ins>+ with open(configPath) as configFile: # Could raise an IOError
+ self.config = readPlist(configFile) # Could raise an ExpatError
+ except IOError, e:
+ raise UsageError("--config %s: %s" % (configPath.path, e.strerror))
</ins><span class="cx"> except ExpatError, e:
</span><del>- raise UsageError("--config %s: %s" % (self['config'].path, e))
- finally:
- configFile.close()
</del><ins>+ raise UsageError("--config %s: %s" % (configPath.path, e))
</ins><span class="cx">
</span><del>- try:
- # from importlib import import_module
- # client_config = import_module("contrib.performance.loadtest.settings.clients")
- clientFile = self['clients'].open()
- except IOError, e:
- raise UsageError("--clients %s: %s" % (
- self['clients'].path, e.strerror))
- try:
</del><span class="cx"> try:
</span><del>- client_config = readPlist(clientFile)
- # self.config["clients"] = client_config.calendars_only# client_config["clients"]
- self.config["clients"] = client_config["clients"]
- if "arrivalInterval" in client_config:
- self.config["arrival"]["params"]["interval"] = client_config["arrivalInterval"]
</del><ins>+ with open(clientsPath) as clientFile: # Could raise an IOError
+ self.clients = readPlist(clientFile) # Could raise an ExpatError
+ except IOError, e:
+ raise UsageError("--clients %s: %s" % (clientsPath.path, e.strerror))
</ins><span class="cx"> except ExpatError, e:
</span><del>- raise UsageError("--clients %s: %s" % (self['clients'].path, e))
- finally:
- # clientFile.close()
- pass
</del><ins>+ raise UsageError("--clients %s: %s" % (clientsPath.path, e))
</ins><span class="cx">
</span><ins>+ self.config["clients"] = self.clients["clients"]
</ins><span class="cx">
</span><ins>+ else:
+ from importlib import import_module
+ try:
+ config = import_module(configPath) # Could raise ImportError
+ self.config = config.config # Could raise attribute error
+ except (ImportError, AttributeError), e:
+ raise UsageError("--config %s: %s" % (configPath, e))
+ try:
+ clients = import_module(clientsPath) # Could raise ImportError
+ self.clients = clients.config # Could raise attribute error
+ except (ImportError, AttributeError), e:
+ raise UsageError("--clients %s: %s" % (clientsPath, e))
+
+ self.configObj = Config()
+ self.configObj.populateFrom(self.config, self.clients)
+
+
</ins><span class="cx"> Arrival = namedtuple('Arrival', 'factory parameters')
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -194,6 +214,8 @@
</span><span class="cx"> def __init__(self, server, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
</span><span class="cx"> records=None, reactor=None, runtime=None, workers=None,
</span><span class="cx"> configTemplate=None, workerID=None, workerCount=1):
</span><ins>+ if configTemplate == {}:
+ raise Exception('Got here!')
</ins><span class="cx"> if reactor is None:
</span><span class="cx"> from twisted.internet import reactor
</span><span class="cx"> self.server = server
</span><span class="lines">@@ -224,7 +246,7 @@
</span><span class="cx"> except UsageError, e:
</span><span class="cx"> raise SystemExit(str(e))
</span><span class="cx">
</span><del>- return cls.fromConfig(options.config, options['runtime'], output)
</del><ins>+ return cls.fromConfigObject(options.configObj, options['runtime'], output)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="lines">@@ -338,7 +360,6 @@
</span><span class="cx"> observers=observers,
</span><span class="cx"> records=records,
</span><span class="cx"> runtime=runtime,
</span><del>- reactor=reactor,
</del><span class="cx"> workers=workers,
</span><span class="cx"> configTemplate=configTemplate,
</span><span class="cx"> workerID=workerID,
</span><span class="lines">@@ -348,138 +369,47 @@
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="cx"> def fromConfigObject(cls, config, runtime=None, output=stdout):
</span><del>- workers = config['workers']
- if workers is None:
- # Client / place where the simulator actually runs configuration
- workerID = config.get("workerID", 0)
- workerCount = config.get("workerCount", 1)
- configTemplate = None
- server = config.get('server', 'http://127.0.0.1:8008')
- serializationPath = None
</del><ins>+ # if config.isManaging:
+ # observers =
</ins><span class="cx">
</span><del>- serializationPath = config['serializationPath']
</del><ins>+ # if config.isWorking:
+ # simulator = CalendarClientSimulator(
+ # config.records,
+ # config.parameters,
+ # config.reactor,
+ # config.server,
+ # config.serializationPath,
+ # config.workerID,
+ # config.workerCount,
+ # )
+ # arrival = config.arrival
</ins><span class="cx">
</span><del>- if 'arrival' in config:
- arrival = Arrival(
- namedAny(config['arrival']['factory']),
- config['arrival']['params'])
- else:
- arrival = Arrival(
- SmoothRampUp, dict(groups=10, groupSize=1, interval=3))
-
- parameters = PopulationParameters()
- if 'clients' in config:
- for clientConfig in config['clients']:
- parameters.addClient(
- clientConfig["weight"],
- ClientType(
- clientConfig["software"],
- clientConfig["params"],
- clientConfig["profiles"]
- )
- )
- # ClientType(
- # namedAny(clientConfig["software"]),
- # cls._convertParams(clientConfig["params"]),
- # [
- # ProfileType(
- # namedAny(profile["class"]),
- # cls._convertParams(profile["params"])
- # ) for profile in clientConfig["profiles"]
- # ]))
- if not parameters.clients:
- parameters.addClient(1,
- ClientType(OS_X_10_6, {},
- [Eventer, Inviter, Accepter]))
- else:
- # Manager / observer process.
- server = ''
- serializationPath = None
- arrival = None
- parameters = None
- workerID = 0
- configTemplate = config
- workerCount = 1
-
- # webadminPort =
- webadminPort = None
- if 'webadmin' in config:
- if config['webadmin']['enabled']:
- webadminPort = config['webadmin']['HTTPPort']
-
- serverStats = None
- if 'serverStats' in config:
- if config['serverStats']['enabled']:
- serverStats = config['serverStats']
- serverStats['server'] = config['server'] if 'server' in config else ''
-
- observers = []
- if 'observers' in config:
- for observer in config['observers']:
- observerName = observer["type"]
- observerParams = observer["params"]
- observers.append(namedAny(observerName)(**observerParams))
-
- records = []
- if 'accounts' in config:
- loader = config['accounts']['loader']
- params = config['accounts']['params']
- records.extend(namedAny(loader)(**params))
- output.write("Loaded {0} accounts.\n".format(len(records)))
-
</del><span class="cx"> return cls(
</span><del>- server,
- webadminPort,
- serverStats,
- serializationPath,
- arrival,
- parameters,
- observers=observers,
- records=records,
</del><ins>+ config.server,
+ config.webadminPort,
+ config.serverStats,
+ config.serializationPath,
+ config.arrival,
+ config.parameters,
+ observers=config.observers,
+ records=config.records,
</ins><span class="cx"> runtime=runtime,
</span><del>- reactor=reactor,
- workers=workers,
- configTemplate=configTemplate,
- workerID=workerID,
- workerCount=workerCount,
</del><ins>+ configTemplate=config.serializeForWorker,
+ workers=config.workers,
+ workerID=config.workerID,
+ workerCount=config.workerCount,
</ins><span class="cx"> )
</span><span class="cx">
</span><del>- @classmethod
- def _convertParams(cls, params):
- """
- Find parameter values which should be more structured than plistlib is
- capable of constructing and replace them with the more structured form.
</del><span class="cx">
</span><del>- Specifically, find keys that end with C{"Distribution"} and convert
- them into some kind of distribution object using the associated
- dictionary of keyword arguments.
- """
- for k, v in params.iteritems():
- if k.endswith('Distribution'):
- params[k] = cls._convertDistribution(v)
- return params
-
-
</del><span class="cx"> @classmethod
</span><del>- def _convertDistribution(cls, value):
- """
- Construct and return a new distribution object using the type and
- params specified by C{value}.
- """
- return namedAny(value['type'])(**value['params'])
-
-
- @classmethod
</del><span class="cx"> def main(cls, args=None):
</span><span class="cx"> simulator = cls.fromCommandLine(args)
</span><span class="cx"> raise SystemExit(simulator.run())
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def createSimulator(self):
</span><del>- populator = Populator(Random())
</del><span class="cx"> return CalendarClientSimulator(
</span><span class="cx"> self.records,
</span><del>- populator,
</del><span class="cx"> self.parameters,
</span><span class="cx"> self.reactor,
</span><span class="cx"> self.server,
</span><span class="lines">@@ -491,7 +421,8 @@
</span><span class="cx">
</span><span class="cx"> def createArrivalPolicy(self):
</span><span class="cx"> # print(self.arrival.parameters)
</span><del>- return self.arrival.factory(self.reactor, **self.arrival.parameters)
</del><ins>+ # return zory(self.reactor, **self.arrival.parameters)
+ return self.arrival
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def serviceClasses(self):
</span><span class="lines">@@ -503,6 +434,7 @@
</span><span class="cx"> PrimaryService = WorkerSpawnerService
</span><span class="cx"> else:
</span><span class="cx"> PrimaryService = SimulatorService
</span><ins>+
</ins><span class="cx"> return [
</span><span class="cx"> ObserverService,
</span><span class="cx"> ReporterService,
</span><span class="lines">@@ -575,6 +507,7 @@
</span><span class="cx"> data = ""
</span><span class="cx"> while not data.endswith("\n"):
</span><span class="cx"> d = s.recv(1024)
</span><ins>+ print("Received data from stats socket: ", d)
</ins><span class="cx"> if d:
</span><span class="cx"> data += d
</span><span class="cx"> else:
</span><span class="lines">@@ -622,6 +555,9 @@
</span><span class="cx"> """
</span><span class="cx"> super(ObserverService, self).startService()
</span><span class="cx"> for obs in self.loadsim.observers:
</span><ins>+ import random
+ val = random.random()
+ msg(type='log', text='Adding observer: ' + obs.__class__.__name__, val=str(val))
</ins><span class="cx"> addObserver(obs.observe)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -639,16 +575,19 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> def startService(self):
</span><ins>+ print("Starting simulator service")
</ins><span class="cx"> super(SimulatorService, self).startService()
</span><span class="cx"> self.clientsim = self.loadsim.createSimulator()
</span><span class="cx"> arrivalPolicy = self.loadsim.createArrivalPolicy()
</span><del>- arrivalPolicy.run(self.clientsim)
</del><ins>+ arrivalPolicy.run(self.loadsim.reactor, self.clientsim)
+ # self.loadsim.arrival.run(self.loadsim.reactor, self.loadsim.simulator)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def stopService(self):
</span><span class="cx"> yield super(SimulatorService, self).stopService()
</span><span class="cx"> yield self.clientsim.stop()
</span><ins>+ # yield self.loadsim.simulator.stop()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -662,6 +601,7 @@
</span><span class="cx"> """
</span><span class="cx"> Start observing.
</span><span class="cx"> """
</span><ins>+ print("Starting reporter service")
</ins><span class="cx"> super(ReporterService, self).startService()
</span><span class="cx"> self.loadsim.reporter = self
</span><span class="cx">
</span><span class="lines">@@ -736,6 +676,7 @@
</span><span class="cx"> super(WorkerSpawnerService, self).startService()
</span><span class="cx"> self.bridges = []
</span><span class="cx"> for workerID, worker in enumerate(self.loadsim.workers):
</span><ins>+ print("Building bridge for #" + str(workerID))
</ins><span class="cx"> bridge = ProcessProtocolBridge(
</span><span class="cx"> self, Manager(self.loadsim, workerID, len(self.loadsim.workers),
</span><span class="cx"> self.output)
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_distributionspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_distributions.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_distributions.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_distributions.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,25 +1,91 @@
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><span class="cx">
</span><span class="cx"> from contrib.performance.loadtest.distributions import (
</span><del>- LogNormalDistribution, UniformDiscreteDistribution,
- UniformIntegerDistribution, WorkDistribution, RecurrenceDistribution
</del><ins>+ # Continuous distributions
+ LogNormalDistribution, NormalDistribution,
+ # Discrete distributions
+ UniformDiscreteDistribution, UniformIntegerDistribution,
+ BernoulliDistribution, BinomialDistribution, FixedDistribution,
+ # Calendar-specific distributions
+ WorkDistribution, RecurrenceDistribution,
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx"> from pycalendar.timezone import Timezone
</span><span class="cx">
</span><ins>+from scipy import stats
+from scipy.optimize import curve_fit
+import itertools
+
+"""
+Disclaimer: These tests are nondeterministic, so be careful
+"""
+
</ins><span class="cx"> class DistributionTestBase(TestCase):
</span><del>- def getSamples(self, n):
</del><ins>+ def get_n_samples(self, dist, n):
</ins><span class="cx"> samples = []
</span><span class="cx"> for _ignore_i in xrange(n):
</span><del>- samples.append()
</del><ins>+ samples.append(dist.sample())
+ return samples
</ins><span class="cx">
</span><del>- def close(self, n):
- pass
</del><ins>+class DiscreteDistributionTests(DistributionTestBase):
+ def test_bernoulli(self):
+ sample_count = 1000
+ proportions = [0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]
+ for prop in proportions:
+ dist = BernoulliDistribution(proportion=prop)
+ samples = self.get_n_samples(dist, sample_count)
+ successes = samples.count(True)
</ins><span class="cx">
</span><del>-class DistributionTests(TestCase):
</del><ins>+ # This representes the likelihood that we would see as many successes
+ # as we did given that the true proportion is prop
+ p_value = stats.binom_test(successes, n=sample_count, p=prop)
+ self.assertFalse(p_value <= 0.01, "%d/%d, expected %f" % (successes, sample_count, prop))
</ins><span class="cx">
</span><ins>+ def test_binomial(self):
+ sample_counts = [100, 1000, 10000]
+ proportions = [0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]
+ for sample_count, prop in itertools.product(sample_counts, proportions):
+ dist = BinomialDistribution(p=prop, n=sample_count)
+ successes = dist.sample()
</ins><span class="cx">
</span><ins>+ # This representes the likelihood that we would see as many successes
+ # as we did given that the true proportion is prop
+ p_value = stats.binom_test(successes, n=sample_count, p=prop)
+ self.assertFalse(p_value <= 0.01, "%d/%d, expected %f" % (successes, sample_count, prop))
+
+ def test_fixed(self):
+ dist = FixedDistribution(4) # https://xkcd.com/221/
+ for _ignore_i in xrange(100):
+ self.assertEqual(dist.sample(), 4)
+
+ def test_uniformdiscrete(self):
+ population = [82, 101, 100, 109, 111, 110, 100]
+ counts = dict.fromkeys(population, 0)
+ dist = UniformDiscreteDistribution(population)
+ for _ignore_i in range(len(population) * 10):
+ counts[dist.sample()] += 1
+ self.assertEqual(dict.fromkeys(population, 10), counts)
+ # Do some chi squared stuff
+
+ def test_uniform(self):
+ dist = UniformIntegerDistribution(-5, 10)
+ for _ignore_i in range(100):
+ value = dist.sample()
+ self.assertTrue(-5 <= value < 10)
+ self.assertIsInstance(value, int)
+
+class ContinuousDistributionTests(TestCase):
+ def is_fit(self, pdf, xdata, ydata, pexp):
+ """
+ expected parameters
+ """
+ popt, pcov = curve_fit(pdf, xdata, ydata)
+ print popt
+
+ def test_normal(self):
+ dist = NormalDistribution()
+
</ins><span class="cx"> def test_lognormal(self):
</span><span class="cx"> dist = LogNormalDistribution(mu=1, sigma=1)
</span><span class="cx"> for _ignore_i in range(100):
</span><span class="lines">@@ -48,16 +114,8 @@
</span><span class="cx"> self.assertRaises(ValueError, LogNormalDistribution, mean=1)
</span><span class="cx"> self.assertRaises(ValueError, LogNormalDistribution, median=1)
</span><span class="cx">
</span><ins>+class CalendarDistributionTests(TestCase):
</ins><span class="cx">
</span><del>- def test_uniformdiscrete(self):
- population = [1, 5, 6, 9]
- counts = dict.fromkeys(population, 0)
- dist = UniformDiscreteDistribution(population)
- for _ignore_i in range(len(population) * 10):
- counts[dist.sample()] += 1
- self.assertEqual(dict.fromkeys(population, 10), counts)
-
-
</del><span class="cx"> def test_workdistribution(self):
</span><span class="cx"> tzname = "US/Eastern"
</span><span class="cx"> dist = WorkDistribution(["mon", "wed", "thu", "sat"], 10, 20, tzname)
</span><span class="lines">@@ -91,7 +149,7 @@
</span><span class="cx"> self.assertTrue(value is None)
</span><span class="cx">
</span><span class="cx"> dist = RecurrenceDistribution(True, {"daily": 1, "none": 2, "weekly": 1})
</span><del>- dist._helperDistribution = UniformDiscreteDistribution([0, 3, 2, 1, 0], randomize=False)
</del><ins>+ dist._helperDistribution = UniformDiscreteDistribution([0, 3, 2, 1, 0])
</ins><span class="cx"> value = dist.sample()
</span><span class="cx"> self.assertTrue(value is not None)
</span><span class="cx"> value = dist.sample()
</span><span class="lines">@@ -102,11 +160,3 @@
</span><span class="cx"> self.assertTrue(value is not None)
</span><span class="cx"> value = dist.sample()
</span><span class="cx"> self.assertTrue(value is not None)
</span><del>-
-
- def test_uniform(self):
- dist = UniformIntegerDistribution(-5, 10)
- for _ignore_i in range(100):
- value = dist.sample()
- self.assertTrue(-5 <= value < 10)
- self.assertIsInstance(value, int)
</del></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_pushpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_push.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_push.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_push.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -1,15 +1,19 @@
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><ins>+from twisted.internet.defer import inlineCallbacks
</ins><span class="cx">
</span><del>-from calendarserver.push.amppush import AMPPushMaster
-
</del><span class="cx"> from contrib.performance.loadtest.push import PushMonitor
</span><span class="cx">
</span><span class="cx"> class PushMonitorTests(TestCase):
</span><del>- def fakePush(self, inboundID, dataChangedTimestamp, priority=5):
- self.monitor._receivedAMPPush(inboundID, dataChangedTimestamp, priority)
</del><ins>+ def sendFakePush(self, pushkey):
+ self.monitor._receivedAMPPush(inboundID=pushkey, dataChangedTimestamp=None, priority=None)
</ins><span class="cx">
</span><ins>+ def receivedPush(self, calendar_href):
+ self.history.append(calendar_href)
+
</ins><span class="cx"> def setUp(self):
</span><del>- self.pushMaster = AMPPushMaster()
</del><ins>+ """
+ Creates and begins a PushMonitor with a history-tracking callback
+ """
</ins><span class="cx">
</span><span class="cx"> self.monitor = PushMonitor(
</span><span class="cx"> None,
</span><span class="lines">@@ -17,16 +21,94 @@
</span><span class="cx"> 62311,
</span><span class="cx"> self.receivedPush
</span><span class="cx"> )
</span><del>- self.monitor.begin()
</del><ins>+ self.history = []
+ return self.monitor.begin()
</ins><span class="cx">
</span><del>- def sendNotification(self, href, pushkey):
- self.pushMaster.notify(href, pushkey, None, None)
</del><ins>+ def test_noPushkey(self):
+ """
+ Monitor will not react to a push if there are no registered pushkeys
+ """
+ pushkey = '/CalDAV/abc/def/'
+ calendar_home = '/foo/bar/'
+ self.assertFalse(self.monitor.isSubscribedTo(calendar_home))
+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [])
</ins><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def test_addPushkey(self):
</span><del>- pass
</del><ins>+ """
+ Adding a pushkey triggers a notification for the corresponding calendar home
+ """
+ pushkey = '/CalDAV/abc/def/'
+ calendar_home = '/foo/bar/'
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home))
+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [calendar_home])
</ins><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def test_removePushkey(self):
</span><del>- pass
</del><ins>+ """
+ Pushkeys can be unregistered
+ """
+ pushkey = '/CalDAV/abc/def/'
+ calendar_home = '/foo/bar/'
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home))
+ yield self.monitor.removePushkey(pushkey)
+ self.assertFalse(self.monitor.isSubscribedTo(calendar_home))
</ins><span class="cx">
</span><ins>+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [])
+
+ @inlineCallbacks
+ def test_addDuplicatePushkeys(self):
+ """
+ Adding the same pushkey twice only registers it once
+ """
+ pushkey = '/CalDAV/abc/def/'
+ calendar_home = '/foo/bar/'
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home))
+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [calendar_home])
+
+ @inlineCallbacks
+ def test_addOverridePushkeys(self):
+ """
+ Adding the same pushkey with a different calendar home
+ unregisters the original
+ """
+ pushkey = '/CalDAV/abc/def/'
+ calendar_home = '/foo/bar/'
+ calendar_home_2 = '/foo/baz/'
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home))
+ yield self.monitor.addPushkey(pushkey, calendar_home_2)
+ self.assertFalse(self.monitor.isSubscribedTo(calendar_home))
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home_2))
+
+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [calendar_home_2])
+
+ @inlineCallbacks
+ def test_multiplePushkeys(self):
+ """
+ Monitor supports registering multiple pushkeys
+ """
+ pushkey = '/CalDAV/abc/def/'
+ pushkey_2 = '/CalDAV/abc/xyz/'
+ calendar_home = '/foo/bar/'
+ calendar_home_2 = '/foo/baz/'
+ yield self.monitor.addPushkey(pushkey, calendar_home)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home))
+ yield self.monitor.addPushkey(pushkey_2, calendar_home_2)
+ self.assertTrue(self.monitor.isSubscribedTo(calendar_home_2))
+ self.sendFakePush(pushkey_2)
+ self.assertEqual(self.history, [calendar_home_2])
+ self.sendFakePush(pushkey)
+ self.assertEqual(self.history, [calendar_home_2, calendar_home])
+
</ins><span class="cx"> def tearDown(self):
</span><del>- self.monitor.end()
</del><ins>+ return self.monitor.end()
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestteststest_resourcespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_resources.py (15080 => 15081)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_resources.py 2015-08-31 20:06:29 UTC (rev 15080)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/tests/test_resources.py 2015-08-31 20:37:45 UTC (rev 15081)
</span><span class="lines">@@ -37,4 +37,3 @@
</span><span class="cx"> """
</span><span class="cx"> event = Event(None, u'/bar/baz', u'etag')
</span><span class="cx"> self.assertIdentical(event.getUID(), None)
</span><del>-
</del></span></pre>
</div>
</div>
</body>
</html>