[CalendarServer-changes] [870] CalendarServer/branches/users/dreid/cluster

20 Dec 2006

Revision: 870
          http://trac.macosforge.org/projects/calendarserver/changeset/870
Author:   dreid@apple.com
Date:     2006-12-20 13:06:22 -0800 (Wed, 20 Dec 2006)

Log Message:
-----------
The simplest thing that could possibly work with regards to cluster support

Modified Paths:
--------------
    CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist
    CalendarServer/branches/users/dreid/cluster/conf/caldavd.plist
    CalendarServer/branches/users/dreid/cluster/run
    CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py

Added Paths:
-----------
    CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py

Modified: CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist
===================================================================

--- CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/conf/caldavd-test.plist	2006-12-20 21:06:22 UTC (rev 870)
@@ -166,5 +166,29 @@
     <string>/principals/user/admin</string>
   </array>
 
+  <key>ClusterEnable</key>
+  <false/>
+  
+  <key>Cluster</key>
+  <dict>
+    <key>processes</key>
+    <integer>0</integer>
+
+    <key>scheduler</key>
+    <string>leastconns</string>
+
+    <key>admin</key>
+    <dict>
+      <key>username</key>
+      <string></string>
+
+      <key>password</key>
+      <string></string>
+    </dict>
+    
+    <key>pydirLocation</key>
+    <string>/usr/share/caldavd/bin/pydir++.py</string>
+
+  </dict>
 </dict>
 </plist>

Modified: CalendarServer/branches/users/dreid/cluster/conf/caldavd.plist
===================================================================
--- CalendarServer/branches/users/dreid/cluster/conf/caldavd.plist	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/conf/caldavd.plist	2006-12-20 21:06:22 UTC (rev 870)
@@ -112,5 +112,28 @@
     <string>/principals/user/admin</string>
   </array>
 
+  <key>ClusterEnable</key>
+  <false/>
+  
+  <key>Cluster</key>
+  <dict>
+    <key>processes</key>
+    <integer>0</integer>
+
+    <key>scheduler</key>
+    <string>leastconns</string>
+
+    <key>admin</key>
+    <dict>
+      <key>username</key>
+      <string></string>
+
+      <key>password</key>
+      <string></string>
+    </dict>
+    
+    <key>pydirLocation</key>
+    <string>/usr/share/caldavd/bin/pydir++.py</string>
+  </dict>
 </dict>
 </plist>

Modified: CalendarServer/branches/users/dreid/cluster/run
===================================================================
--- CalendarServer/branches/users/dreid/cluster/run	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/run	2006-12-20 21:06:22 UTC (rev 870)
@@ -527,6 +527,20 @@
 fi;
 
 #
+# PyDirector
+#
+
+if ! py_have_module pydirector; then
+    pydirector="${top}/pydirector-1.0.0";
+    
+    www_get "PyDirector" "${pydirector}" http://easynews.dl.sourceforge.net/sourceforge/pythondirector/pydirector-1.0...;
+    py_build "PyDirector" "${pydirector}" false;
+    py_install "PyDirector" "${pydirector}";
+
+    export PYTHONPATH="${PYTHONPATH}:${pydirector}/build/${py_platform_libdir}";
+fi;
+
+#
 # TwistedCalDAV
 #
 

Modified: CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/twisted/plugins/caldav.py	2006-12-20 21:06:22 UTC (rev 870)
@@ -1,3 +1,5 @@
 from twistedcaldav.tap import CaldavServiceMaker
+from twistedcaldav.cluster import ClusterServiceMaker
 
 TwistedCalDAV = CaldavServiceMaker()
+TwistedCalDAVCluster = ClusterServiceMaker()

Added: CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py	                        (rev 0)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/cluster.py	2006-12-20 21:06:22 UTC (rev 870)
@@ -0,0 +1,152 @@
+import os
+import tempfile
+import socket
+
+from twisted.runner import procmon
+
+from twisted.python import usage
+
+from twistedcaldav.config import config
+from twistedcaldav.tap import CaldavOptions, CaldavServiceMaker
+
+serviceTemplate = """
+    <service name="%(name)s">
+	<listen ip="127.0.0.1:%(port)s" />
+	<group name="main" scheduler="%(scheduler)s">
+	  %(hosts)s
+        </group>
+        <enable group="main" />
+    </service>
+"""
+
+configTemplate = """
+<pdconfig>
+    %(services)s
+    <admin listen="localhost:7001">
+	<user name="%(username)s" password="%(password)s" access="full"/>
+    </admin>
+    <logging file="pydir.log"/>
+</pdconfig>
+"""
+
+hostTemplate = '<host name="%(name)s" ip="127.0.0.1:%(port)s" />'
+
+twistdTemplate = ('%(twistd)s caldav -f %(configFile)s '
+                  '-o Port=%(port)s -o SSLPort=%(sslPort)s')
+
+class TwistdSlaveProcess(object):
+    prefix = "caldav"
+
+    def __init__(self, twistdLocation, configFile, port, sslPort):
+        self.twistd = twistdLocation
+
+        self.configFile = configFile
+
+        self.port = port
+        self.sslPort = sslPort
+
+        self.pidFile = os.path.join(
+            os.path.dirname(config.PIDFile),
+            '%s.pid' % (self.getName(),))
+
+    def getName(self):
+        return '%s-%s' % (self.prefix, self.port)
+    
+    def getSSLName(self):
+        return '%s-%s' % (self.prefix, self.sslPort)
+
+    def getCommandLine(self):
+        return [self.twistd, '-n', 'caldav', 
+                '-f', self.configFile,
+                '-o', 'Port=%s' % (self.port,),
+                '-o', 'SSLPort=%s' % (self.sslPort,),
+                '-o', 'PIDFile=%s' % (self.pidFile,)]
+
+    def getHostLine(self, ssl=None):
+        name = self.getName()
+        port = self.port
+        if ssl:
+            name = self.getSSLName()
+            port = self.sslPort
+
+        return hostTemplate % {'name': name,
+                               'port': port}
+
+
+class ClusterServiceMaker(CaldavServiceMaker):
+    tapname = "caldavcluster"
+    
+    description = "A cluster of Calendar Servers"
+
+    def makeService(self, options):
+        _twistdTemplate = twistdTemplate % {
+            'twistd': config.twistdLocation,
+            'configFile': options['config'],
+            'port': '%(port)s',
+            'sslPort': '%(sslPort)s',
+            }
+        
+        if not config.ClusterEnable:
+            raise usage.UsageError(
+                ("Clustering is not enabled in the config "
+                 "file, use -o ClusterEnable=True to "
+                 "override, or use --degrade"))
+
+        service = procmon.ProcessMonitor()
+
+        hosts = []
+        sslHosts = []
+
+        port = config.Port
+        sslport = config.SSLPort
+
+        for p in xrange(0, config.Cluster['processes']):
+            port += 1
+            sslport += 1
+
+            process = TwistdSlaveProcess(config.twistdLocation,
+                                         options['config'],
+                                         port, sslport)
+            
+            service.addProcess(process.getName(),
+                               process.getCommandLine(),
+                               uid=options.parent['uid'],
+                               gid=options.parent['gid'])
+
+            hosts.append(process.getHostLine())
+
+            if config.SSLEnable:
+                sslHosts.append(process.getHostLine(ssl=True))
+
+        services = []
+
+        services.append(serviceTemplate % {
+                'name': 'http',
+                'port': config.Port,
+                'scheduler': config.Cluster['scheduler'],
+                'hosts': '\n'.join(hosts)
+                })
+                
+
+        if config.SSLEnable:
+            services.append(serviceTemplate % {
+                    'name': 'https',
+                    'port': config.SSLPort,
+                    'scheduler': config.Cluster['scheduler'],
+                    'hosts': '\n'.join(sslHosts),
+                    })
+                    
+        pdconfig = configTemplate % {
+            'services': '\n'.join(services),
+            'username': config.Cluster['admin']['username'],
+            'password': config.Cluster['admin']['password'],
+            }
+            
+        fd, fname = tempfile.mkstemp(prefix='pydir')
+        os.write(fd, pdconfig)
+        os.close(fd)
+
+        service.addProcess('pydir', [config.Cluster['pydirLocation'],
+                                     fname])
+
+        return service

Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/config.py	2006-12-20 21:06:22 UTC (rev 870)
@@ -48,8 +48,18 @@
     'twistdLocation': '/usr/share/caldavd/bin/twistd',
     'SACLEnable': False,
     'AuthSchemes': ['Basic'],
-    'AdminPrincipals': ['/principals/user/admin']
-}
+    'AdminPrincipals': ['/principals/user/admin'],
+    'ClusterEnable': Trueb,
+    'Cluster': {
+        'processes': 10,
+        'scheduler': 'leastconns',
+        'admin': {
+            'username': None,
+            'password': None,
+            },
+        'pydirLocation': '/usr/share/caldavd/bin/pydir++.py',
+        },
+    }
 
 class Config (object):
     def __init__(self, defaults):

Modified: CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py	2006-12-20 19:59:24 UTC (rev 869)
+++ CalendarServer/branches/users/dreid/cluster/twistedcaldav/tap.py	2006-12-20 21:06:22 UTC (rev 870)
@@ -80,7 +80,7 @@
                 elif isinstance(defaultConfig[key], (int, float, long)):
                     value = type(defaultConfig[key])(value)
                 
-                elif isinstance(defaultConfig[key], (list, tuples)):
+                elif isinstance(defaultConfig[key], (list, tuple)):
                     value = value.split(',')
 
                 elif isinstance(defaultConfig[key], dict):
@@ -104,6 +104,7 @@
         self.parent['logfile'] = config.ErrorLogFile
         self.parent['pidfile'] = config.PIDFile
 
+
 class CaldavServiceMaker(object):
     implements(IPlugin, service.IServiceMaker)

    

[CalendarServer-changes] [870] CalendarServer/branches/users/dreid/cluster

source_changes＠macosforge.org