[CalendarServer-changes] [8246] CalendarServer/branches/users/glyph/parallel-sim/contrib/performance /loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Tue Nov 1 17:51:44 PDT 2011
Revision: 8246
http://trac.macosforge.org/projects/calendarserver/changeset/8246
Author: glyph at apple.com
Date: 2011-11-01 17:51:44 -0700 (Tue, 01 Nov 2011)
Log Message:
-----------
transfer accounts to each simulator worker via the AMP protocol, don't relay the configuration, since we may be using a cluster of machines without access to the file containing the actual list of accounts.
Modified Paths:
--------------
CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/ampsim.py
CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/sim.py
Modified: CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/ampsim.py
===================================================================
--- CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/ampsim.py 2011-11-02 00:51:29 UTC (rev 8245)
+++ CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/ampsim.py 2011-11-02 00:51:44 UTC (rev 8246)
@@ -22,36 +22,38 @@
if __name__ == '__main__':
# When run as a script, this is the worker process, receiving commands over
# stdin.
- import traceback
- try:
- from twisted.python.log import startLogging
- from sys import stderr, exit
+ def runmain():
+ import traceback
+ try:
+ from twisted.python.log import startLogging
+ from sys import stderr, exit
- startLogging(stderr)
+ startLogging(stderr)
- from twisted.internet import reactor
- from twisted.internet.stdio import StandardIO
+ from twisted.internet import reactor
+ from twisted.internet.stdio import StandardIO
- from contrib.performance.loadtest.ampsim import Worker
- from contrib.performance.loadtest.sim import LagTrackingReactor
+ from contrib.performance.loadtest.ampsim import Worker
+ from contrib.performance.loadtest.sim import LagTrackingReactor
- StandardIO(Worker(LagTrackingReactor(reactor)))
- reactor.run()
- except:
- traceback.print_exc()
- exit(1)
- else:
- exit(0)
+ StandardIO(Worker(LagTrackingReactor(reactor)))
+ reactor.run()
+ except:
+ traceback.print_exc()
+ exit(1)
+ else:
+ exit(0)
+ runmain()
from copy import deepcopy
-from plistlib import writePlistToString
-from twisted.protocols.amp import AMP, Command, String
+from plistlib import writePlistToString, readPlistFromString
+from twisted.protocols.amp import AMP, Command, String, Unicode
from twext.enterprise.adbapi2 import Pickle
-
from twisted.python.log import msg, addObserver
+from contrib.performance.loadtest.sim import _DirectoryRecord, LoadSimulator
class Configure(Command):
"""
@@ -63,12 +65,27 @@
class LogMessage(Command):
"""
- A log message was received.
+ This message represents an observed log message being relayed from a worker
+ process to the manager process.
"""
arguments = [("event", Pickle())]
+class Account(Command):
+ """
+ This message represents a L{_DirectoryRecord} loaded by the manager process
+ being relayed to a worker.
+ """
+ arguments = [
+ ("uid", Unicode()),
+ ("password", Unicode()),
+ ("commonName", Unicode()),
+ ("email", Unicode()),
+ ]
+
+
+
class Worker(AMP):
"""
Protocol to be run in the worker process, to handle messages from its
@@ -78,16 +95,22 @@
def __init__(self, reactor):
super(Worker, self).__init__()
self.reactor = reactor
+ self.records = []
+ @Account.responder
+ def account(self, **kw):
+ self.records.append(_DirectoryRecord(**kw))
+ return {}
+
+
@Configure.responder
def config(self, plist):
- from plistlib import readPlistFromString
- from contrib.performance.loadtest.sim import LoadSimulator
from sys import stderr
cfg = readPlistFromString(plist)
addObserver(self.emit)
sim = LoadSimulator.fromConfig(cfg)
+ sim.records = self.records
sim.attachServices(stderr)
return {}
@@ -116,11 +139,29 @@
def connectionMade(self):
super(Manager, self).connectionMade()
+
+ for record in self.loadsim.records:
+ self.callRemote(Account,
+ uid=record.uid,
+ password=record.password,
+ commonName=record.commonName,
+ email=record.email)
+
workerConfig = deepcopy(self.loadsim.configTemplate)
+ # The list of workers is for the manager only; the workers themselves
+ # know they're workers because they _don't_ receive this list.
del workerConfig["workers"]
+ # The manager loads the accounts via the configured loader, then sends
+ # them out to the workers (right above), which look at the state at an
+ # instance level and therefore don't need a globally-named directory
+ # record loader.
+ del workerConfig["accounts"]
+
workerConfig["workerID"] = self.whichWorker
workerConfig["workerCount"] = self.numWorkers
workerConfig["observers"] = []
+ workerConfig.pop("accounts", None)
+
plist = writePlistToString(workerConfig)
self.output.write("Initiating worker configuration\n")
def completed(x):
Modified: CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/sim.py
===================================================================
--- CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/sim.py 2011-11-02 00:51:29 UTC (rev 8245)
+++ CalendarServer/branches/users/glyph/parallel-sim/contrib/performance/loadtest/sim.py 2011-11-02 00:51:44 UTC (rev 8246)
@@ -41,7 +41,6 @@
CalendarClientSimulator)
from twisted.internet.defer import Deferred
from twisted.internet.defer import gatherResults
-from contrib.performance.loadtest.ampsim import Manager
class _DirectoryRecord(object):
@@ -177,7 +176,7 @@
@type arrival: L{Arrival}
@type parameters: L{PopulationParameters}
- @ivar records: A C{list} of L{DirectoryRecord} instances giving
+ @ivar records: A C{list} of L{_DirectoryRecord} instances giving
user information about the accounts on the server being put
under load.
"""
@@ -489,6 +488,7 @@
class WorkerSpawnerService(SimService):
def startService(self):
+ from contrib.performance.loadtest.ampsim import Manager
super(WorkerSpawnerService, self).startService()
self.bridges = []
for workerID, worker in enumerate(self.loadsim.workers):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111101/2b60a785/attachment.html>
More information about the calendarserver-changes
mailing list