Revision: 873 http://trac.macosforge.org/projects/calendarserver/changeset/873 Author: dreid@apple.com Date: 2006-12-20 14:24:32 -0800 (Wed, 20 Dec 2006) Log Message: ----------- patch procutils to not cause a nasty keyerror every time our subprocesses start. Also rework the way we do thing clustering, add an argument for the different servicetypes instead of doing a seperate subcommand for each type. Modified Paths: -------------- CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist CalendarServer/branches/users/dreid/cluster/support/patchmaker.py CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py Added Paths: ----------- CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch Modified: CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist =================================================================== --- CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist 2006-12-20 22:24:32 UTC (rev 873) @@ -187,7 +187,7 @@ </dict> <key>pydirLocation</key> - <string>/usr/share/caldavd/bin/pydir++.py</string> + <string>../pydirector-1.0.0/pydir++.py</string> </dict> </dict> Added: CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch =================================================================== --- CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch (rev 0) +++ CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch 2006-12-20 22:24:32 UTC (rev 873) @@ -0,0 +1,13 @@ +Index: twisted/python/procutils.py +=================================================================== +--- twisted/python/procutils.py (revision 18545) ++++ twisted/python/procutils.py (working copy) +@@ -33,7 +33,7 @@ + """ + result = [] + exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep)) +- for p in os.environ['PATH'].split(os.pathsep): ++ for p in os.environ.get('PATH', '').split(os.pathsep): + p = os.path.join(p, name) + if os.access(p, flags): + result.append(p) Modified: CalendarServer/branches/users/dreid/cluster/support/patchmaker.py =================================================================== --- CalendarServer/branches/users/dreid/cluster/support/patchmaker.py 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/support/patchmaker.py 2006-12-20 22:24:32 UTC (rev 873) @@ -30,7 +30,7 @@ cwd = os.getcwd() libpatches = os.path.join(cwd, "lib-patches") -svn = "/usr/bin/svn" +svn = "/usr/local/bin/svn" # Stuff we have to manually ignore because our ignore logic cannot cope ignores = set(( Modified: CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py =================================================================== --- CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py 2006-12-20 22:24:32 UTC (rev 873) @@ -1,5 +1,5 @@ from twistedcaldav.tap import CaldavServiceMaker -from twistedcaldav.cluster import ClusterServiceMaker +#from twistedcaldav.cluster import ClusterServiceMaker TwistedCalDAV = CaldavServiceMaker() -TwistedCalDAVCluster = ClusterServiceMaker() +# TwistedCalDAVCluster = ClusterServiceMaker() Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py =================================================================== --- CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py 2006-12-20 22:24:32 UTC (rev 873) @@ -7,7 +7,7 @@ from twisted.python import usage from twistedcaldav.config import config -from twistedcaldav.tap import CaldavOptions, CaldavServiceMaker +# from twistedcaldav.tap import CaldavOptions, CaldavServiceMaker serviceTemplate = """ <service name="%(name)s"> @@ -31,8 +31,6 @@ hostTemplate = '<host name="%(name)s" ip="127.0.0.1:%(port)s" />' -twistdTemplate = ('%(twistd)s caldav -f %(configFile)s ' - '-o Port=%(port)s -o SSLPort=%(sslPort)s') class TwistdSlaveProcess(object): prefix = "caldav" @@ -58,6 +56,7 @@ def getCommandLine(self): return [self.twistd, '-n', 'caldav', '-f', self.configFile, + '-t', 'standalone', '-o', 'Port=%s' % (self.port,), '-o', 'SSLPort=%s' % (self.sslPort,), '-o', 'PIDFile=%s' % (self.pidFile,)] @@ -72,82 +71,70 @@ return hostTemplate % {'name': name, 'port': port} +def makeService(self, options): + if not config.ClusterEnable: + raise usage.UsageError( + ("Clustering is not enabled in the config " + "file, use -o ClusterEnable=True to " + "override")) -class ClusterServiceMaker(CaldavServiceMaker): - tapname = "caldavcluster" - - description = "A cluster of Calendar Servers" + service = procmon.ProcessMonitor() - def makeService(self, options): - _twistdTemplate = twistdTemplate % { - 'twistd': config.twistdLocation, - 'configFile': options['config'], - 'port': '%(port)s', - 'sslPort': '%(sslPort)s', - } - - if not config.ClusterEnable: - raise usage.UsageError( - ("Clustering is not enabled in the config " - "file, use -o ClusterEnable=True to " - "override, or use --degrade")) + hosts = [] + sslHosts = [] - service = procmon.ProcessMonitor() + port = config.Port + sslport = config.SSLPort - hosts = [] - sslHosts = [] + for p in xrange(0, config.Cluster['processes']): + port += 1 + sslport += 1 - port = config.Port - sslport = config.SSLPort + process = TwistdSlaveProcess(config.twistdLocation, + options['config'], + port, sslport) - for p in xrange(0, config.Cluster['processes']): - port += 1 - sslport += 1 + service.addProcess(process.getName(), + process.getCommandLine(), + uid=options.parent['uid'], + gid=options.parent['gid']) + + if not config.SSLOnly: + hosts.append(process.getHostLine()) - process = TwistdSlaveProcess(config.twistdLocation, - options['config'], - port, sslport) + if config.SSLEnable: + sslHosts.append(process.getHostLine(ssl=True)) - service.addProcess(process.getName(), - process.getCommandLine(), - uid=options.parent['uid'], - gid=options.parent['gid']) - - if not config.SSLOnly: - hosts.append(process.getHostLine()) + services = [] - if config.SSLEnable: - sslHosts.append(process.getHostLine(ssl=True)) + if not config.SSLOnly: - services = [] - services.append(serviceTemplate % { 'name': 'http', 'port': config.Port, 'scheduler': config.Cluster['scheduler'], 'hosts': '\n'.join(hosts) }) - - - if config.SSLEnable: - services.append(serviceTemplate % { - 'name': 'https', - 'port': config.SSLPort, - 'scheduler': config.Cluster['scheduler'], - 'hosts': '\n'.join(sslHosts), - }) + + if config.SSLEnable: + services.append(serviceTemplate % { + 'name': 'https', + 'port': config.SSLPort, + 'scheduler': config.Cluster['scheduler'], + 'hosts': '\n'.join(sslHosts), + }) - pdconfig = configTemplate % { - 'services': '\n'.join(services), - 'username': config.Cluster['admin']['username'], - 'password': config.Cluster['admin']['password'], - } - - fd, fname = tempfile.mkstemp(prefix='pydir') - os.write(fd, pdconfig) - os.close(fd) - - service.addProcess('pydir', [config.Cluster['pydirLocation'], - fname]) - - return service + pdconfig = configTemplate % { + 'services': '\n'.join(services), + 'username': config.Cluster['admin']['username'], + 'password': config.Cluster['admin']['password'], + } + + fd, fname = tempfile.mkstemp(prefix='pydir') + os.write(fd, pdconfig) + os.close(fd) + + service.addProcess('pydir', [config.Cluster['pydirLocation'], + fname]) + + return service Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py =================================================================== --- CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py 2006-12-20 22:24:32 UTC (rev 873) @@ -49,7 +49,7 @@ 'SACLEnable': False, 'AuthSchemes': ['Basic'], 'AdminPrincipals': ['/principals/user/admin'], - 'ClusterEnable': Trueb, + 'ClusterEnable': True, 'Cluster': { 'processes': 10, 'scheduler': 'leastconns', Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py =================================================================== --- CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py 2006-12-20 21:24:27 UTC (rev 872) +++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py 2006-12-20 22:24:32 UTC (rev 873) @@ -42,6 +42,7 @@ from twisted.web2.log import LogWrapperResource from twisted.web2.server import Site +from twistedcaldav.cluster import makeService as makeService_cluster from twistedcaldav.config import config, parseConfig, defaultConfig from twistedcaldav.logging import RotatingFileAccessLoggingObserver from twistedcaldav.root import RootResource @@ -53,6 +54,7 @@ optParameters = [ ["config", "f", "/etc/caldavd/caldavd.plist", "Path to configuration file."], + ["type", "t", "standalone", "Select the type of service to run"], ] zsh_actions = {"config" : "_files -g '*.plist'"} @@ -122,7 +124,7 @@ principalResourceClass = DirectoryPrincipalProvisioningResource calendarResourceClass = CalendarHomeProvisioningFile - def makeService(self, options): + def makeService_standalone(self, options): # # Setup the Directory # @@ -238,3 +240,22 @@ httpsService.setServiceParent(service) return service + + makeService_slave = makeService_standalone + + makeService_cluster = makeService_cluster + + def makeService(self, options): + serviceType = options['type'] + + serviceMethod = getattr(self, 'makeService_%s' % (serviceType,)) + + if not serviceMethod: + raise usage.UsageError( + ("Unknown service type %s, please choose: " % (serviceType,))) + + else: + return serviceMethod(options) + + +