[CalendarServer-changes] [4642] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Oct 26 22:00:52 PDT 2009
Revision: 4642
http://trac.macosforge.org/projects/calendarserver/changeset/4642
Author: sagen at apple.com
Date: 2009-10-26 22:00:50 -0700 (Mon, 26 Oct 2009)
Log Message:
-----------
Instead of using the software loadbalancer, the master process binds the required ports and the child processes inherit the file descriptors.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/memcacheclient.py
CalendarServer/trunk/support/Makefile.Apple
CalendarServer/trunk/support/build.sh
CalendarServer/trunk/twext/web2/channel/http.py
CalendarServer/trunk/twistedcaldav/accesslog.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/test/test_config.py
Added Paths:
-----------
CalendarServer/trunk/twext/internet/tcp.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -24,7 +24,7 @@
import socket
import stat
import sys
-from time import sleep
+from time import sleep, time
from tempfile import mkstemp
from subprocess import Popen, PIPE
@@ -39,7 +39,7 @@
from twisted.python.usage import Options, UsageError
from twisted.python.reflect import namedClass
from twisted.plugin import IPlugin
-from twisted.internet.reactor import callLater
+from twisted.internet.reactor import callLater, spawnProcess
from twisted.internet.process import ProcessExitedAlready
from twisted.internet.protocol import Protocol, Factory
from twisted.internet.address import IPv4Address
@@ -56,7 +56,8 @@
from twisted.web2.http import Request, RedirectResponse
from twext.internet.ssl import ChainingOpenSSLContextFactory
-from twext.web2.channel.http import HTTP503LoggingFactory
+from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
+from twext.web2.channel.http import HTTP503LoggingFactory, LimitingHTTPFactory, SSLRedirectRequest
try:
from twistedcaldav.version import version
@@ -86,7 +87,6 @@
from twistedcaldav.mail import IMIPReplyInboxResource
from twistedcaldav.timezones import TimezoneCache
from twistedcaldav.upgrade import upgradeData
-from twistedcaldav.pdmonster import PDClientAddressWrapper
from twistedcaldav import memcachepool
from twistedcaldav.notify import installNotificationClient
from twistedcaldav.util import getNCPU
@@ -372,7 +372,7 @@
if not serviceMethod:
raise UsageError(
"Unknown server type %s. "
- "Please choose: Master, Slave, Single or Combined"
+ "Please choose: Slave, Single or Combined"
% (config.ProcessType,)
)
else:
@@ -713,17 +713,7 @@
self.log_info("Setting up service")
if config.ProcessType == "Slave":
- if (
- config.MultiProcess.ProcessCount > 1 and
- config.MultiProcess.LoadBalancer.Enabled
- ):
- realRoot = PDClientAddressWrapper(
- logWrapper,
- config.PythonDirector.ControlSocket,
- directory,
- )
- else:
- realRoot = logWrapper
+ realRoot = logWrapper
if config.ControlSocket:
mode = "AF_UNIX"
@@ -741,7 +731,7 @@
self.deleteStaleSocketFiles()
realRoot = logWrapper
-
+
logObserver = RotatingFileAccessLoggingObserver(
config.AccessLogFile,
)
@@ -752,44 +742,36 @@
site = Site(realRoot)
- channel = HTTP503LoggingFactory(
+ httpFactory = LimitingHTTPFactory(
site,
- maxRequests = config.MaxRequests,
- retryAfter = config.HTTPRetryAfter,
- betweenRequestsTimeOut = config.IdleConnectionTimeOut,
- vary = True,
+ maxRequests=config.MaxRequests,
+ maxAccepts=config.MaxAccepts,
+ betweenRequestsTimeOut=config.IdleConnectionTimeOut,
+ vary=True,
)
+ if config.RedirectHTTPToHTTPS:
+ redirectFactory = LimitingHTTPFactory(
+ SSLRedirectRequest,
+ maxRequests=config.MaxRequests,
+ maxAccepts=config.MaxAccepts,
+ betweenRequestsTimeOut=config.IdleConnectionTimeOut,
+ vary=True,
+ )
- def updateChannel(configDict):
- channel.maxRequests = configDict.MaxRequests
- channel.retryAfter = configDict.HTTPRetryAfter
+ def updateFactory(configDict):
+ httpFactory.maxRequests = configDict.MaxRequests
+ httpFactory.maxAccepts = configDict.MaxAccepts
+ if config.RedirectHTTPToHTTPS:
+ redirectFactory.maxRequests = configDict.MaxRequests
+ redirectFactory.maxAccepts = configDict.MaxAccepts
- config.addPostUpdateHook(updateChannel)
+ config.addPostUpdateHook(updateFactory)
- if not config.BindAddresses:
- config.BindAddresses = [""]
+ if config.InheritFDs or config.InheritSSLFDs:
- for bindAddress in config.BindAddresses:
- if config.BindHTTPPorts:
- if config.HTTPPort == 0:
- raise UsageError(
- "HTTPPort required if BindHTTPPorts is not empty"
- )
- elif config.HTTPPort != 0:
- config.BindHTTPPorts = [config.HTTPPort]
+ for fd in config.InheritSSLFDs:
+ fd = int(fd)
- if config.BindSSLPorts:
- if config.SSLPort == 0:
- raise UsageError(
- "SSLPort required if BindSSLPorts is not empty"
- )
- elif config.SSLPort != 0:
- config.BindSSLPorts = [config.SSLPort]
-
- for port in config.BindSSLPorts:
- self.log_info("Adding SSL server at %s:%s"
- % (bindAddress, port))
-
try:
contextFactory = ChainingOpenSSLContextFactory(
config.SSLPrivateKey,
@@ -799,53 +781,100 @@
sslmethod=getattr(OpenSSL.SSL, config.SSLMethod),
)
except SSLError, e:
- self.log_error("Unable to set up SSL context factory: %s"
- % (e,))
- self.log_error("Disabling SSL port: %s" % (port,))
+ log.error("Unable to set up SSL context factory: %s" % (e,))
else:
- httpsService = SSLServer(
- int(port), channel,
- contextFactory, interface=bindAddress,
+ MaxAcceptSSLServer(
+ fd, httpFactory,
+ contextFactory,
backlog=config.ListenBacklog,
- )
- httpsService.setServiceParent(service)
+ inherit=True
+ ).setServiceParent(service)
- for port in config.BindHTTPPorts:
+ for fd in config.InheritFDs:
+ fd = int(fd)
if config.RedirectHTTPToHTTPS:
- #
- # Redirect non-SSL ports to the configured SSL port.
- #
- class SSLRedirectRequest(Request):
- def process(self):
- if config.SSLPort == 443:
- location = (
- "https://%s%s"
- % (config.ServerHostName, self.uri)
- )
- else:
- location = (
- "https://%s:%d%s"
- % (config.ServerHostName, config.SSLPort, self.uri)
- )
- self.writeResponse(RedirectResponse(location))
+ self.log_info("Redirecting to HTTPS port %s" % (config.SSLPort,))
+ useFactory = redirectFactory
+ else:
+ useFactory = httpFactory
- self.log_info(
- "Redirecting HTTP port %s:%s to HTTPS port %s:%s"
- % (bindAddress, port, bindAddress, config.SSLPort)
- )
- TCPServer(int(port), HTTPFactory(SSLRedirectRequest),
- interface=bindAddress, backlog=config.ListenBacklog,
- ).setServiceParent(service)
+ MaxAcceptTCPServer(
+ fd, useFactory,
+ backlog=config.ListenBacklog,
+ inherit=True
+ ).setServiceParent(service)
- else:
- # Set up non-SSL port
- self.log_info(
- "Adding server at %s:%s"
- % (bindAddress, port)
- )
- TCPServer(int(port), channel,
- interface=bindAddress, backlog=config.ListenBacklog,
+
+ else: # Not inheriting, therefore we open our own:
+
+ if not config.BindAddresses:
+ config.BindAddresses = [""]
+
+ for bindAddress in config.BindAddresses:
+ if config.BindHTTPPorts:
+ if config.HTTPPort == 0:
+ raise UsageError(
+ "HTTPPort required if BindHTTPPorts is not empty"
+ )
+ elif config.HTTPPort != 0:
+ config.BindHTTPPorts = [config.HTTPPort]
+
+ if config.BindSSLPorts:
+ if config.SSLPort == 0:
+ raise UsageError(
+ "SSLPort required if BindSSLPorts is not empty"
+ )
+ elif config.SSLPort != 0:
+ config.BindSSLPorts = [config.SSLPort]
+
+ for port in config.BindSSLPorts:
+ self.log_info("Adding SSL server at %s:%s"
+ % (bindAddress, port))
+
+ try:
+ contextFactory = ChainingOpenSSLContextFactory(
+ config.SSLPrivateKey,
+ config.SSLCertificate,
+ certificateChainFile=config.SSLAuthorityChain,
+ passwdCallback=getSSLPassphrase,
+ sslmethod=getattr(OpenSSL.SSL, config.SSLMethod),
+ )
+ except SSLError, e:
+ self.log_error("Unable to set up SSL context factory: %s"
+ % (e,))
+ self.log_error("Disabling SSL port: %s" % (port,))
+ else:
+ httpsService = MaxAcceptSSLServer(
+ int(port), httpFactory,
+ contextFactory, interface=bindAddress,
+ backlog=config.ListenBacklog,
+ inherit=False
+ )
+ httpsService.setServiceParent(service)
+
+ for port in config.BindHTTPPorts:
+
+ if config.RedirectHTTPToHTTPS:
+ #
+ # Redirect non-SSL ports to the configured SSL port.
+ #
+ self.log_info("Redirecting HTTP port %s to HTTPS port %s"
+ % (port, config.SSLPort)
+ )
+ useFactory = redirectFactory
+ else:
+ self.log_info(
+ "Adding server at %s:%s"
+ % (bindAddress, port)
+ )
+ useFactory = httpFactory
+
+ MaxAcceptTCPServer(
+ int(port), useFactory,
+ interface=bindAddress,
+ backlog=config.ListenBacklog,
+ inherit=False
).setServiceParent(service)
@@ -861,7 +890,7 @@
# Make sure no old socket files are lying around.
self.deleteStaleSocketFiles()
-
+
# The logger service must come before the monitor service, otherwise
# we won't know which logging port to pass to the slaves' command lines
@@ -888,14 +917,6 @@
if "KRB5_KTNAME" in os.environ:
parentEnv["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
- hosts = []
- sslHosts = []
-
- port = [config.HTTPPort,]
- sslPort = [config.SSLPort,]
-
- bindAddress = ["127.0.0.1"]
-
#
# Attempt to calculate the number of processes to use 1 per processor
#
@@ -922,148 +943,68 @@
config.MultiProcess.ProcessCount = processCount
- if config.MultiProcess.ProcessCount > 1:
- if config.BindHTTPPorts:
- port = [list(reversed(config.BindHTTPPorts))[0]]
- if config.BindSSLPorts:
- sslPort = [list(reversed(config.BindSSLPorts))[0]]
+ # Open the socket(s) to be inherited by the slaves
- elif config.MultiProcess.ProcessCount == 1:
+ if not config.BindAddresses:
+ config.BindAddresses = [""]
+
+ inheritFDs = []
+ inheritSSLFDs = []
+
+ s._inheritedSockets = [] # keep a reference to these so they don't close
+
+ for bindAddress in config.BindAddresses:
if config.BindHTTPPorts:
- port = config.BindHTTPPorts
+ if config.HTTPPort == 0:
+ raise UsageError(
+ "HTTPPort required if BindHTTPPorts is not empty"
+ )
+ elif config.HTTPPort != 0:
+ config.BindHTTPPorts = [config.HTTPPort]
if config.BindSSLPorts:
- sslPort = config.BindSSLPorts
+ if config.SSLPort == 0:
+ raise UsageError(
+ "SSLPort required if BindSSLPorts is not empty"
+ )
+ elif config.SSLPort != 0:
+ config.BindSSLPorts = [config.SSLPort]
- if port[0] == 0:
- port = None
+ def _openSocket(addr, port):
+ log.info("Opening socket for inheritance at %s:%d" % (addr, port))
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.setblocking(0)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.bind((addr, port))
+ sock.listen(config.ListenBacklog)
+ s._inheritedSockets.append(sock)
+ return sock
- if sslPort[0] == 0:
- sslPort = None
+ for portNum in config.BindHTTPPorts:
+ sock = _openSocket(bindAddress, int(portNum))
+ inheritFDs.append(sock.fileno())
- # If the load balancer isn't enabled, or if we only have one process
- # We listen directly on the interfaces.
+ for portNum in config.BindSSLPorts:
+ sock = _openSocket(bindAddress, int(portNum))
+ inheritSSLFDs.append(sock.fileno())
- if (
- not config.MultiProcess.LoadBalancer.Enabled or
- config.MultiProcess.ProcessCount == 1
- ):
- bindAddress = config.BindAddresses
for p in xrange(0, config.MultiProcess.ProcessCount):
- if config.MultiProcess.ProcessCount > 1:
- if port is not None:
- port = [port[0] + 1]
-
- if sslPort is not None:
- sslPort = [sslPort[0] + 1]
-
process = TwistdSlaveProcess(
config.Twisted.twistd,
self.tapname,
options["config"],
- bindAddress,
- port,
- sslPort
+ p,
+ config.BindAddresses,
+ inheritFDs=inheritFDs,
+ inheritSSLFDs=inheritSSLFDs
)
monitor.addProcessObject(process, parentEnv)
- if config.HTTPPort:
- hosts.append(process.getHostLine())
- if config.SSLPort:
- sslHosts.append(process.getHostLine(ssl=True))
- #
- # Set up pydirector config file.
- #
- if (config.MultiProcess.LoadBalancer.Enabled and
- config.MultiProcess.ProcessCount > 1):
- services = []
-
- if not config.BindAddresses:
- config.BindAddresses = [""]
-
- scheduler_map = {
- "LeastConnections": "leastconns",
- "RoundRobin": "roundrobin",
- "LeastConnectionsAndRoundRobin": "leastconnsrr",
- }
-
- for bindAddress in config.BindAddresses:
- httpListeners = []
- sslListeners = []
-
- httpPorts = config.BindHTTPPorts
- if not httpPorts:
- if config.HTTPPort != 0:
- httpPorts = (config.HTTPPort,)
-
- sslPorts = config.BindSSLPorts
- if not sslPorts:
- if config.SSLPort != 0:
- sslPorts = (config.SSLPort,)
-
- for ports, listeners in (
- (httpPorts, httpListeners),
- (sslPorts, sslListeners)
- ):
- for port in ports:
- listeners.append(
- """<listen ip="%s:%s" />""" % (bindAddress, port)
- )
-
- scheduler = config.MultiProcess.LoadBalancer.Scheduler
-
- pydirServiceTemplate = (
- """<service name="%(name)s">"""
- """%(listeningInterfaces)s"""
- """<group name="main" scheduler="%(scheduler)s">"""
- """%(hosts)s"""
- """</group>"""
- """<enable group="main" />"""
- """</service>"""
- )
-
- if httpPorts:
- services.append(
- pydirServiceTemplate % {
- "name": "http",
- "listeningInterfaces": "\n".join(httpListeners),
- "scheduler": scheduler_map[scheduler],
- "hosts": "\n".join(hosts)
- }
- )
-
- if sslPorts:
- services.append(
- pydirServiceTemplate % {
- "name": "https",
- "listeningInterfaces": "\n".join(sslListeners),
- "scheduler": scheduler_map[scheduler],
- "hosts": "\n".join(sslHosts),
- }
- )
-
- pdconfig = """<pdconfig>%s<control socket="%s" /></pdconfig>""" % (
- "\n".join(services), config.PythonDirector.ControlSocket,
- )
-
- fd, fname = mkstemp(prefix="pydir")
- os.write(fd, pdconfig)
- os.close(fd)
-
- self.log_info("Adding pydirector service with configuration: %s"
- % (fname,))
-
- monitor.addProcess(
- "pydir",
- [sys.executable, config.PythonDirector.pydir, fname],
- env=parentEnv,
- )
-
if config.Memcached.ServerEnabled:
self.log_info("Adding memcached service")
@@ -1152,26 +1093,7 @@
return s
- def makeService_Master(self, options):
- service = procmon.ProcessMonitor()
- parentEnv = {"PYTHONPATH": os.environ.get("PYTHONPATH", "")}
-
- self.log_info("Adding pydirector service with configuration: %s"
- % (config.PythonDirector.ConfigFile,))
-
- service.addProcess(
- "pydir",
- [
- sys.executable,
- config.PythonDirector.pydir,
- config.PythonDirector.ConfigFile
- ],
- env=parentEnv
- )
-
- return service
-
def deleteStaleSocketFiles(self):
# Check all socket files we use.
@@ -1205,28 +1127,25 @@
class TwistdSlaveProcess(object):
prefix = "caldav"
- def __init__(self, twistd, tapname, configFile, interfaces, port, sslPort):
+ def __init__(self, twistd, tapname, configFile, id, interfaces,
+ inheritFDs=None, inheritSSLFDs=None):
+
self.twistd = twistd
self.tapname = tapname
self.configFile = configFile
- self.ports = port
- self.sslPorts = sslPort
+ self.id = id
+ self.inheritFDs = inheritFDs
+ self.inheritSSLFDs = inheritSSLFDs
+
self.interfaces = interfaces
def getName(self):
- if self.ports is not None:
- return "%s-%s" % (self.prefix, self.ports[0])
- elif self.sslPorts is not None:
- return "%s-%s" % (self.prefix, self.sslPorts[0])
+ return '%s-%s' % (self.prefix, self.id)
- raise ConfigurationError(
- "Can't create TwistdSlaveProcess without a TCP Port"
- )
-
def getCommandLine(self):
args = [sys.executable, self.twistd]
@@ -1251,6 +1170,7 @@
"-o", "BindAddresses=%s" % (",".join(self.interfaces),),
"-o", "PIDFile=None",
"-o", "ErrorLogFile=None",
+ "-o", "LogID=%s" % (self.id,),
"-o", "MultiProcess/ProcessCount=%d"
% (config.MultiProcess.ProcessCount,),
"-o", "ControlPort=%d"
@@ -1260,34 +1180,19 @@
if config.Memcached.ServerEnabled:
args.extend(["-o", "Memcached/ClientEnabled=True"])
- if self.ports:
+ if self.inheritFDs:
args.extend([
- "-o", "BindHTTPPorts=%s" % (",".join(map(str, self.ports)),)
+ "-o", "InheritFDs=%s" % (",".join(map(str, self.inheritFDs)),)
])
- if self.sslPorts:
+ if self.inheritSSLFDs:
args.extend([
- "-o", "BindSSLPorts=%s" % (",".join(map(str, self.sslPorts)),)
+ "-o", "InheritSSLFDs=%s" % (",".join(map(str, self.inheritSSLFDs)),)
])
return args
- def getHostLine(self, ssl=False):
- name = self.getName()
- port = None
- if self.ports is not None:
- port = self.ports
-
- if ssl and self.sslPorts is not None:
- port = self.sslPorts
-
- if port is None:
- raise ConfigurationError("Can not add a host without a port")
-
- return """<host name="%s" ip="127.0.0.1:%s" />""" % (name, port[0])
-
-
class ControlPortTCPServer(TCPServer):
""" This TCPServer retrieves the port number that was actually assigned
when the service was started, and stores that into config.ControlPort
@@ -1376,7 +1281,29 @@
except ProcessExitedAlready:
pass
+ def startProcess(self, name):
+ if self.protocols.has_key(name):
+ return
+ p = self.protocols[name] = procmon.LoggingProtocol()
+ p.service = self
+ p.name = name
+ args, uid, gid, env = self.processes[name]
+ self.timeStarted[name] = time()
+ childFDs = { 0 : "w", 1 : "r", 2 : "r" }
+
+ # Examine args for -o InheritFDs= and -o InheritSSLFDs=
+ # Add any file descriptors listed in those args to the childFDs
+ # dictionary so those don't get closed across the spawn.
+ for i in xrange(len(args)-1):
+ if args[i] == "-o" and args[i+1].startswith("Inherit"):
+ for fd in map(int, args[i+1].split("=")[1].split(",")):
+ childFDs[fd] = fd
+
+ spawnProcess(p, args[0], args, uid=uid, gid=gid, env=env,
+ childFDs=childFDs)
+
+
def getSSLPassphrase(*ignored):
if not config.SSLPrivateKey:
Modified: CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -26,6 +26,7 @@
from twisted.web2.log import LogWrapperResource
from twext.python.plistlib import writePlist
+from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
from twistedcaldav.config import config, ConfigDict, _mergeData
from twistedcaldav.stdconfig import DEFAULT_CONFIG
@@ -127,7 +128,6 @@
myConfig = ConfigDict(DEFAULT_CONFIG)
myConfig.Authentication.Basic.Enabled = False
- myConfig.MultiProcess.LoadBalancer.Enabled = False
myConfig.HTTPPort = 80
myConfig.ServerHostName = "calendar.calenderserver.org"
@@ -139,10 +139,6 @@
self.config.parseOptions(args)
self.assertEquals(config.ServerHostName, myConfig["ServerHostName"])
- self.assertEquals(
- config.MultiProcess.LoadBalancer.Enabled,
- myConfig.MultiProcess.LoadBalancer.Enabled
- )
self.assertEquals(config.HTTPPort, myConfig.HTTPPort)
self.assertEquals(
config.Authentication.Basic.Enabled,
@@ -242,9 +238,9 @@
"""
Test the default options of the dispatching makeService
"""
- validServices = ["Slave", "Master", "Combined"]
+ validServices = ["Slave", "Combined"]
- self.config["HTTPPort"] = 80
+ self.config["HTTPPort"] = 8008
for service in validServices:
self.config["ProcessType"] = service
@@ -294,8 +290,8 @@
service = self.makeService()
expectedSubServices = (
- (internet.TCPServer, self.config["HTTPPort"]),
- (internet.SSLServer, self.config["SSLPort"]),
+ (MaxAcceptTCPServer, self.config["HTTPPort"]),
+ (MaxAcceptSSLServer, self.config["SSLPort"]),
)
configuredSubServices = [(s.__class__, s.args) for s in service.services]
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2009-10-27 05:00:50 UTC (rev 4642)
@@ -413,19 +413,6 @@
<dict>
<key>ProcessCount</key>
<integer>2</integer> <!-- 0 = larger of: 4 or (2 * CPU count) -->
-
- <key>LoadBalancer</key>
- <dict>
- <!--
- Valid values are:
- - LeastConnections
- - RoundRobin
- - LeastConnectionsAndRoundRobin
- -->
- <key>Scheduler</key>
- <string>LeastConnections</string>
- </dict>
-
</dict>
Modified: CalendarServer/trunk/memcacheclient.py
===================================================================
--- CalendarServer/trunk/memcacheclient.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/memcacheclient.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -1280,6 +1280,9 @@
Is not a string (Raises MemcachedKeyError)
Is None (Raises MemcachedKeyError)
"""
+
+ return # Short-circuit this expensive method
+
if type(key) == types.TupleType: key = key[1]
if not key:
raise Client.MemcachedKeyNoneError, ("Key is None")
Modified: CalendarServer/trunk/support/Makefile.Apple
===================================================================
--- CalendarServer/trunk/support/Makefile.Apple 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/support/Makefile.Apple 2009-10-27 05:00:50 UTC (rev 4642)
@@ -47,17 +47,16 @@
PyXML-0.8.4:: $(BuildDirectory)/PyXML-0.8.4
vobject:: $(BuildDirectory)/vobject
Twisted:: $(BuildDirectory)/Twisted
-pydirector-1.0.0:: $(BuildDirectory)/pydirector-1.0.0
$(Project):: $(BuildDirectory)/$(Project)
-build:: PyKerberos PyOpenDirectory PyXML-0.8.4 pydirector-1.0.0 vobject Twisted $(Project)
+build:: PyKerberos PyOpenDirectory PyXML-0.8.4 vobject Twisted $(Project)
setup:
$(_v) ./run -g
-prep:: setup CalDAVTester.tgz PyKerberos.tgz PyOpenDirectory.tgz PyXML-0.8.4.tgz pydirector-1.0.0.tgz vobject.tgz Twisted.tgz
+prep:: setup CalDAVTester.tgz PyKerberos.tgz PyOpenDirectory.tgz PyXML-0.8.4.tgz vobject.tgz Twisted.tgz
-PyKerberos PyOpenDirectory PyXML-0.8.4 pydirector-1.0.0 vobject $(Project)::
+PyKerberos PyOpenDirectory PyXML-0.8.4 vobject $(Project)::
@echo "Building $@..."
$(_v) cd $(BuildDirectory)/$@ && $(Environment) $(PYTHON) setup.py build
@@ -79,7 +78,6 @@
--install-data="$(ETCDIR)"
$(_v) cd $(BuildDirectory)/PyKerberos && $(Environment) $(PYTHON) setup.py install $(PY_INSTALL_FLAGS)
$(_v) cd $(BuildDirectory)/PyOpenDirectory && $(Environment) $(PYTHON) setup.py install $(PY_INSTALL_FLAGS)
- $(_v) cd $(BuildDirectory)/pydirector-1.0.0 && $(Environment) $(PYTHON) setup.py install $(PY_INSTALL_FLAGS)
$(_v) cd $(BuildDirectory)/PyXML-0.8.4 && $(Environment) $(PYTHON) setup.py install $(PY_INSTALL_FLAGS)
$(_v) cd $(BuildDirectory)/vobject && $(Environment) $(PYTHON) setup.py install $(PY_INSTALL_FLAGS)
$(_v) cd $(BuildDirectory)/Twisted && $(TwistedSubEnvironment) $(PYTHON) twisted/runner/topfiles/setup.py install $(PY_INSTALL_FLAGS)
Modified: CalendarServer/trunk/support/build.sh
===================================================================
--- CalendarServer/trunk/support/build.sh 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/support/build.sh 2009-10-27 05:00:50 UTC (rev 4642)
@@ -495,10 +495,6 @@
"svn" "${base}/vobject/trunk" \
false true true true 212;
- py_dependency "PyDirector" "pydirector" "pydirector-1.0.0" \
- "www" http://internap.dl.sourceforge.net/sourceforge/pythondirector/pydirector-1.0.0.tar.gz \
- false false false false 0;
-
# Tool dependencies. The code itself doesn't depend on these, but you probably want them.
svn_get "CalDAVTester" "${top}/CalDAVTester" "${svn_uri_base}/CalDAVTester/trunk" 4517;
svn_get "Pyflakes" "${top}/Pyflakes" http://divmod.org/svn/Divmod/trunk/Pyflakes 17198;
Added: CalendarServer/trunk/twext/internet/tcp.py
===================================================================
--- CalendarServer/trunk/twext/internet/tcp.py (rev 0)
+++ CalendarServer/trunk/twext/internet/tcp.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -0,0 +1,137 @@
+##
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Extentions to twisted.internet.tcp.
+"""
+
+__all__ = [
+ "InheritedPort",
+ "InheritedSSLPort",
+ "InheritTCPServer",
+ "InheritSSLServer",
+]
+
+from OpenSSL import SSL
+from twisted.application import internet
+from twisted.internet import tcp, ssl
+from twistedcaldav.log import Logger, logLevelForNamespace, setLogLevelForNamespace
+import socket
+
+log = Logger()
+
+
+class MaxAcceptPortMixin(object):
+ """ Mixin for resetting maxAccepts """
+
+ def doRead(self):
+ self.numberAccepts = min(self.factory.maxRequests - self.factory.outstandingRequests, self.factory.maxAccepts)
+ tcp.Port.doRead(self)
+
+class MaxAcceptTCPPort(tcp.Port, MaxAcceptPortMixin):
+ """ Use for non-inheriting tcp ports """
+ pass
+
+class MaxAcceptSSLPort(ssl.Port, MaxAcceptPortMixin):
+ """ Use for non-inheriting SSL ports """
+ pass
+
+class InheritedTCPPort(MaxAcceptTCPPort):
+ """ A tcp port which uses an inherited file descriptor """
+
+ def __init__(self, fd, factory, reactor):
+ tcp.Port.__init__(self, 0, factory, reactor=reactor)
+ # MOR: careful because fromfd dup()'s the socket, so we need to
+ # make sure we don't leak file descriptors
+ self.socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
+ self._realPortNumber = self.port = self.socket.getsockname()[1]
+
+ def createInternetSocket(self):
+ return self.socket
+
+ def startListening(self):
+ log.msg("%s starting on %s" % (self.factory.__class__, self._realPortNumber))
+ self.factory.doStart()
+ self.connected = 1
+ self.fileno = self.socket.fileno
+ self.numberAccepts = self.factory.maxRequests
+ self.startReading()
+
+
+class InheritedSSLPort(InheritedTCPPort):
+ """ An SSL port which uses an inherited file descriptor """
+
+ _socketShutdownMethod = 'sock_shutdown'
+
+ transport = ssl.Server
+
+ def __init__(self, fd, factory, ctxFactory, reactor):
+ InheritedTCPPort.__init__(self, fd, factory, reactor)
+ self.ctxFactory = ctxFactory
+ self.socket = SSL.Connection(self.ctxFactory.getContext(), self.socket)
+
+ def _preMakeConnection(self, transport):
+ transport._startTLS()
+ return tcp.Port._preMakeConnection(self, transport)
+
+class MaxAcceptTCPServer(internet.TCPServer):
+ """ TCP server which will uses MaxAcceptTCPPorts (and optionally,
+ inherited ports)
+ """
+
+ def __init__(self, *args, **kwargs):
+ internet.TCPServer.__init__(self, *args, **kwargs)
+ self.args[1].myServer = self
+ self.inherit = self.kwargs.get("inherit", False)
+ self.backlog = self.kwargs.get("backlog", None)
+ self.interface = self.kwargs.get("interface", None)
+
+ def _getPort(self):
+ from twisted.internet import reactor
+
+ if self.inherit:
+ port = InheritedTCPPort(self.args[0], self.args[1], reactor)
+ else:
+ port = MaxAcceptTCPPort(self.args[0], self.args[1], self.backlog, self.interface, reactor)
+
+ port.startListening()
+ self.myPort = port
+ return port
+
+class MaxAcceptSSLServer(internet.SSLServer):
+ """ SSL server which will uses MaxAcceptSSLPorts (and optionally,
+ inherited ports)
+ """
+
+ def __init__(self, *args, **kwargs):
+ internet.SSLServer.__init__(self, *args, **kwargs)
+ self.args[1].myServer = self
+ self.inherit = self.kwargs.get("inherit", False)
+ self.backlog = self.kwargs.get("backlog", None)
+ self.interface = self.kwargs.get("interface", None)
+
+ def _getPort(self):
+ from twisted.internet import reactor
+
+ if self.inherit:
+ port = InheritedSSLPort(self.args[0], self.args[1], self.args[2], reactor)
+ else:
+ port = MaxAcceptSSLPort(self.args[0], self.args[1], self.args[2], self.backlog, self.interface, self.reactor)
+
+ port.startListening()
+ self.myPort = port
+ return port
+
Modified: CalendarServer/trunk/twext/web2/channel/http.py
===================================================================
--- CalendarServer/trunk/twext/web2/channel/http.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/twext/web2/channel/http.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -20,11 +20,16 @@
from twisted.python import log
from twisted.web2.channel.http import HTTPFactory, HTTPChannelRequest,\
HTTPChannel
+from twisted.web2.http import Request, RedirectResponse
+from twistedcaldav.config import config
+
from twistedcaldav import accounting
import time
__all__ = [
"HTTP503LoggingFactory",
+ "LimitingHTTPFactory",
+ "SSLRedirectRequest",
]
class OverloadedLoggingServerProtocol (protocol.Protocol):
@@ -163,3 +168,58 @@
accounting.emitAccounting("HTTP", "all", "".join(self.logData.request) + "".join(self.logData.response))
HTTPChannel.chanRequestFactory = HTTPLoggingChannelRequest
+
+
+
+class LimitingHTTPChannel(HTTPChannel):
+ """ HTTPChannel that takes itself out of the reactor once it has enough
+ requests in flight.
+ """
+
+ def connectionMade(self):
+ HTTPChannel.connectionMade(self)
+ if self.factory.outstandingRequests >= self.factory.maxRequests:
+ self.factory.myServer.myPort.stopReading()
+
+ def connectionLost(self, reason):
+ HTTPChannel.connectionLost(self, reason)
+ if self.factory.outstandingRequests < self.factory.maxRequests:
+ self.factory.myServer.myPort.startReading()
+
+class LimitingHTTPFactory(HTTPFactory):
+ """ HTTPFactory which stores maxAccepts on behalf of the MaxAcceptPortMixin
+ """
+
+ protocol = LimitingHTTPChannel
+
+ def __init__(self, requestFactory, maxRequests=600, maxAccepts=100,
+ **kwargs):
+ HTTPFactory.__init__(self, requestFactory, maxRequests, **kwargs)
+ self.maxAccepts = maxAccepts
+
+ def buildProtocol(self, addr):
+
+ p = protocol.ServerFactory.buildProtocol(self, addr)
+ for arg, value in self.protocolArgs.iteritems():
+ setattr(p, arg, value)
+ return p
+
+
+
+class SSLRedirectRequest(Request):
+ """ For redirecting HTTP to HTTPS port """
+
+ def process(self):
+ if config.SSLPort == 443:
+ location = (
+ "https://%s%s"
+ % (config.ServerHostName, self.uri)
+ )
+ else:
+ location = (
+ "https://%s:%d%s"
+ % (config.ServerHostName, config.SSLPort, self.uri)
+ )
+ self.writeResponse(RedirectResponse(location))
+
+
Modified: CalendarServer/trunk/twistedcaldav/accesslog.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/accesslog.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/twistedcaldav/accesslog.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -146,7 +146,7 @@
"bytesSent" : loginfo.bytesSent,
"referer" : request.headers.getHeader("referer", "-"),
"userAgent" : request.headers.getHeader("user-agent", "-"),
- "serverInstance" : request.serverInstance,
+ "serverInstance" : config.LogID,
"timeSpent" : (time.time() - request.initTime) * 1000,
"outstandingRequests" : request.chanRequest.channel.factory.outstandingRequests,
}
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -72,6 +72,8 @@
"BindAddresses": [], # List of IP addresses to bind to [empty = all]
"BindHTTPPorts": [], # List of port numbers to bind to for HTTP [empty = same as "Port"]
"BindSSLPorts" : [], # List of port numbers to bind to for SSL [empty = same as "SSLPort"]
+ "InheritFDs": [], # File descriptors to inherit for HTTP requests (empty = don't inherit)
+ "InheritSSLFDs": [], # File descriptors to inherit for HTTPS requests (empty = don't inherit)
#
# Data store
@@ -152,6 +154,7 @@
"EnableExtendedAccessLog": True,
"DefaultLogLevel" : "",
"LogLevels" : {},
+ "LogID" : "",
"AccountingCategories": {
"iTIP": False,
@@ -185,10 +188,6 @@
"MultiProcess": {
"ProcessCount": 0,
"MinProcessCount": 4,
- "LoadBalancer": {
- "Enabled": True,
- "Scheduler": "LeastConnections",
- },
"StaggeredStartup": {
"Enabled": False,
"Interval": 15,
@@ -315,9 +314,10 @@
#
# Set the maximum number of outstanding requests to this server.
- "MaxRequests": 600,
+ "MaxRequests": 1,
+ "MaxAccepts": 1,
- "ListenBacklog": 50,
+ "ListenBacklog": 2024,
"IdleConnectionTimeOut": 15,
"UIDReservationTimeOut": 30 * 60,
Modified: CalendarServer/trunk/twistedcaldav/test/test_config.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_config.py 2009-10-26 23:50:15 UTC (rev 4641)
+++ CalendarServer/trunk/twistedcaldav/test/test_config.py 2009-10-27 05:00:50 UTC (rev 4642)
@@ -125,11 +125,11 @@
self.assertEquals(config.SSLPort, 8443)
def testMerge(self):
- self.assertEquals(config.MultiProcess.LoadBalancer.Enabled, True)
+ self.assertEquals(config.MultiProcess.StaggeredStartup.Enabled, False)
config.update({"MultiProcess": {}})
- self.assertEquals(config.MultiProcess.LoadBalancer.Enabled, True)
+ self.assertEquals(config.MultiProcess.StaggeredStartup.Enabled, False)
def testDirectoryService_noChange(self):
self.assertEquals(config.DirectoryService.type, "twistedcaldav.directory.xmlfile.XMLDirectoryService")
@@ -206,7 +206,7 @@
def testMergeDefaults(self):
config.updateDefaults({"MultiProcess": {}})
- self.assertEquals(config._provider.getDefaults().MultiProcess.LoadBalancer.Enabled, True)
+ self.assertEquals(config._provider.getDefaults().MultiProcess.StaggeredStartup.Enabled, False)
def testSetDefaults(self):
config.updateDefaults({"SSLPort": 8443})
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20091026/20ecd74d/attachment-0001.html>
More information about the calendarserver-changes
mailing list