[CalendarServer-changes] [873] CalendarServer/branches/users/dreid/cluster

20 Dec 2006

Revision: 873
          http://trac.macosforge.org/projects/calendarserver/changeset/873
Author:   dreid@apple.com
Date:     2006-12-20 14:24:32 -0800 (Wed, 20 Dec 2006)

Log Message:
-----------
patch procutils to not cause a nasty keyerror every time our subprocesses start.  Also rework the way we do thing clustering, add an argument for the different servicetypes instead of doing a seperate subcommand for each type.

Modified Paths:
--------------
    CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist
    CalendarServer/branches/users/dreid/cluster/support/patchmaker.py
    CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py

Added Paths:
-----------
    CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch

Modified: CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist
===================================================================

--- CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist	2006-12-20 22:24:32 UTC (rev 873)
@@ -187,7 +187,7 @@
     </dict>
     
     <key>pydirLocation</key>
-    <string>/usr/share/caldavd/bin/pydir++.py</string>
+    <string>../pydirector-1.0.0/pydir++.py</string>
 
   </dict>
 </dict>

Added: CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch
===================================================================
--- CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch	                        (rev 0)
+++ CalendarServer/branches/users/dreid/cluster/lib-patches/Twisted/twisted.python.procutils.patch	2006-12-20 22:24:32 UTC (rev 873)
@@ -0,0 +1,13 @@
+Index: twisted/python/procutils.py
+===================================================================
+--- twisted/python/procutils.py	(revision 18545)
++++ twisted/python/procutils.py	(working copy)
+@@ -33,7 +33,7 @@
+     """
+     result = []
+     exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
+-    for p in os.environ['PATH'].split(os.pathsep):
++    for p in os.environ.get('PATH', '').split(os.pathsep):
+         p = os.path.join(p, name)
+         if os.access(p, flags):
+             result.append(p)

Modified: CalendarServer/branches/users/dreid/cluster/support/patchmaker.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/support/patchmaker.py	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/support/patchmaker.py	2006-12-20 22:24:32 UTC (rev 873)
@@ -30,7 +30,7 @@
 cwd = os.getcwd()
 libpatches = os.path.join(cwd, "lib-patches")
 
-svn = "/usr/bin/svn"
+svn = "/usr/local/bin/svn"
 
 # Stuff we have to manually ignore because our ignore logic cannot cope
 ignores = set((

Modified: CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py	2006-12-20 22:24:32 UTC (rev 873)
@@ -1,5 +1,5 @@
 from twistedcaldav.tap import CaldavServiceMaker
-from twistedcaldav.cluster import ClusterServiceMaker
+#from twistedcaldav.cluster import ClusterServiceMaker
 
 TwistedCalDAV = CaldavServiceMaker()
-TwistedCalDAVCluster = ClusterServiceMaker()
+# TwistedCalDAVCluster = ClusterServiceMaker()

Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py	2006-12-20 22:24:32 UTC (rev 873)
@@ -7,7 +7,7 @@
 from twisted.python import usage
 
 from twistedcaldav.config import config
-from twistedcaldav.tap import CaldavOptions, CaldavServiceMaker
+# from twistedcaldav.tap import CaldavOptions, CaldavServiceMaker
 
 serviceTemplate = """
     <service name="%(name)s">
@@ -31,8 +31,6 @@
 
 hostTemplate = '<host name="%(name)s" ip="127.0.0.1:%(port)s" />'
 
-twistdTemplate = ('%(twistd)s caldav -f %(configFile)s '
-                  '-o Port=%(port)s -o SSLPort=%(sslPort)s')
 
 class TwistdSlaveProcess(object):
     prefix = "caldav"
@@ -58,6 +56,7 @@
     def getCommandLine(self):
         return [self.twistd, '-n', 'caldav', 
                 '-f', self.configFile,
+                '-t', 'standalone',
                 '-o', 'Port=%s' % (self.port,),
                 '-o', 'SSLPort=%s' % (self.sslPort,),
                 '-o', 'PIDFile=%s' % (self.pidFile,)]
@@ -72,82 +71,70 @@
         return hostTemplate % {'name': name,
                                'port': port}
 
+def makeService(self, options):
+    if not config.ClusterEnable:
+        raise usage.UsageError(
+            ("Clustering is not enabled in the config "
+             "file, use -o ClusterEnable=True to "
+             "override"))
 
-class ClusterServiceMaker(CaldavServiceMaker):
-    tapname = "caldavcluster"
-    
-    description = "A cluster of Calendar Servers"
+    service = procmon.ProcessMonitor()
 
-    def makeService(self, options):
-        _twistdTemplate = twistdTemplate % {
-            'twistd': config.twistdLocation,
-            'configFile': options['config'],
-            'port': '%(port)s',
-            'sslPort': '%(sslPort)s',
-            }
-        
-        if not config.ClusterEnable:
-            raise usage.UsageError(
-                ("Clustering is not enabled in the config "
-                 "file, use -o ClusterEnable=True to "
-                 "override, or use --degrade"))
+    hosts = []
+    sslHosts = []
 
-        service = procmon.ProcessMonitor()
+    port = config.Port
+    sslport = config.SSLPort
 
-        hosts = []
-        sslHosts = []
+    for p in xrange(0, config.Cluster['processes']):
+        port += 1
+        sslport += 1
 
-        port = config.Port
-        sslport = config.SSLPort
+        process = TwistdSlaveProcess(config.twistdLocation,
+                                     options['config'],
+                                     port, sslport)
 
-        for p in xrange(0, config.Cluster['processes']):
-            port += 1
-            sslport += 1
+        service.addProcess(process.getName(),
+                           process.getCommandLine(),
+                           uid=options.parent['uid'],
+                           gid=options.parent['gid'])
+        
+        if not config.SSLOnly:
+            hosts.append(process.getHostLine())
 
-            process = TwistdSlaveProcess(config.twistdLocation,
-                                         options['config'],
-                                         port, sslport)
+        if config.SSLEnable:
+            sslHosts.append(process.getHostLine(ssl=True))
 
-            service.addProcess(process.getName(),
-                               process.getCommandLine(),
-                               uid=options.parent['uid'],
-                               gid=options.parent['gid'])
-            
-            if not config.SSLOnly:
-                hosts.append(process.getHostLine())
+    services = []
 
-            if config.SSLEnable:
-                sslHosts.append(process.getHostLine(ssl=True))
+    if not config.SSLOnly:
 
-        services = []
-
         services.append(serviceTemplate % {
                 'name': 'http',
                 'port': config.Port,
                 'scheduler': config.Cluster['scheduler'],
                 'hosts': '\n'.join(hosts)
                 })
-                
-
-        if config.SSLEnable:
-            services.append(serviceTemplate % {
-                    'name': 'https',
-                    'port': config.SSLPort,
-                    'scheduler': config.Cluster['scheduler'],
-                    'hosts': '\n'.join(sslHosts),
-                    })
+        
+    if config.SSLEnable:
+        services.append(serviceTemplate % {
+                'name': 'https',
+                'port': config.SSLPort,
+                'scheduler': config.Cluster['scheduler'],
+                'hosts': '\n'.join(sslHosts),
+                })
                     
-        pdconfig = configTemplate % {
-            'services': '\n'.join(services),
-            'username': config.Cluster['admin']['username'],
-            'password': config.Cluster['admin']['password'],
-            }
-            
-        fd, fname = tempfile.mkstemp(prefix='pydir')
-        os.write(fd, pdconfig)
-        os.close(fd)
-
-        service.addProcess('pydir', [config.Cluster['pydirLocation'],
-                                     fname])
-
-        return service
+    pdconfig = configTemplate % {
+        'services': '\n'.join(services),
+        'username': config.Cluster['admin']['username'],
+        'password': config.Cluster['admin']['password'],
+        }
+                    
+    fd, fname = tempfile.mkstemp(prefix='pydir')
+    os.write(fd, pdconfig)
+    os.close(fd)
+    
+    service.addProcess('pydir', [config.Cluster['pydirLocation'],
+                                 fname])
+    
+    return service

Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py	2006-12-20 22:24:32 UTC (rev 873)
@@ -49,7 +49,7 @@
     'SACLEnable': False,
     'AuthSchemes': ['Basic'],
     'AdminPrincipals': ['/principals/user/admin'],
-    'ClusterEnable': Trueb,
+    'ClusterEnable': True,
     'Cluster': {
         'processes': 10,
         'scheduler': 'leastconns',

Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py	2006-12-20 21:24:27 UTC (rev 872)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py	2006-12-20 22:24:32 UTC (rev 873)
@@ -42,6 +42,7 @@
 from twisted.web2.log import LogWrapperResource
 from twisted.web2.server import Site
 
+from twistedcaldav.cluster import makeService as makeService_cluster
 from twistedcaldav.config import config, parseConfig, defaultConfig
 from twistedcaldav.logging import RotatingFileAccessLoggingObserver
 from twistedcaldav.root import RootResource
@@ -53,6 +54,7 @@
     optParameters = [
         ["config", "f", "/etc/caldavd/caldavd.plist",
          "Path to configuration file."],
+        ["type", "t", "standalone", "Select the type of service to run"],
         ]
 
     zsh_actions = {"config" : "_files -g '*.plist'"}
@@ -122,7 +124,7 @@
     principalResourceClass = DirectoryPrincipalProvisioningResource
     calendarResourceClass = CalendarHomeProvisioningFile
 
-    def makeService(self, options):
+    def makeService_standalone(self, options):
         #
         # Setup the Directory
         #
@@ -238,3 +240,22 @@
             httpsService.setServiceParent(service)
             
         return service
+
+    makeService_slave = makeService_standalone
+
+    makeService_cluster = makeService_cluster
+
+    def makeService(self, options):
+        serviceType = options['type']
+        
+        serviceMethod = getattr(self, 'makeService_%s' % (serviceType,))
+
+        if not serviceMethod:
+            raise usage.UsageError(
+                ("Unknown service type %s, please choose: " % (serviceType,)))
+
+        else:
+            return serviceMethod(options)
+            
+
+

    

[CalendarServer-changes] [873] CalendarServer/branches/users/dreid/cluster

source_changes＠macosforge.org