<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[12843] CalendarServer/branches/release/CalendarServer-5.2-dev</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/12843">12843</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2014-03-07 12:38:57 -0800 (Fri, 07 Mar 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Fix master-child slot counting issues.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devcalendarservertapcaldavpy">CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devcalendarservertaptesttest_caldavpy">CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/test/test_caldav.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtwextinternetsendfdportpy">CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/sendfdport.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtwextinternettesttest_sendfdportpy">CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/test/test_sendfdport.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtwextweb2metafdpy">CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/metafd.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtwextweb2testtest_metafdpy">CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/test/test_metafd.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesreleaseCalendarServer52devcalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/caldav.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/caldav.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/caldav.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -500,7 +500,9 @@
</span><span class="cx"> def startService(self):
</span><span class="cx"> for slaveNumber in xrange(0, config.MultiProcess.ProcessCount):
</span><span class="cx"> if config.UseMetaFD:
</span><del>- extraArgs = dict(metaSocket=self.dispatcher.addSocket())
</del><ins>+ extraArgs = dict(
+ metaSocket=self.dispatcher.addSocket(slaveNumber)
+ )
</ins><span class="cx"> else:
</span><span class="cx"> extraArgs = dict(inheritFDs=self.inheritFDs,
</span><span class="cx"> inheritSSLFDs=self.inheritSSLFDs)
</span><span class="lines">@@ -1802,7 +1804,6 @@
</span><span class="cx"> @ivar metaSocket: an AF_UNIX/SOCK_DGRAM socket (initialized from the
</span><span class="cx"> dispatcher passed to C{__init__}) that is to be inherited by the
</span><span class="cx"> subprocess and used to accept incoming connections.
</span><del>-
</del><span class="cx"> @type metaSocket: L{socket.socket}
</span><span class="cx">
</span><span class="cx"> @ivar ampSQLDispenser: a factory for AF_UNIX/SOCK_STREAM sockets that are
</span><span class="lines">@@ -1835,6 +1836,30 @@
</span><span class="cx"> self.ampDBSocket = None
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def starting(self):
+ """
+ Called when the process is being started (or restarted). Allows for various initialization
+ operations to be done. The child process will itself signal back to the master when it is ready
+ to accept sockets - until then the master socket is marked as "starting" which means it is not
+ active and won't be dispatched to.
+ """
+
+ # Always tell any metafd socket that we have started, so it can re-initialize state.
+ if self.metaSocket is not None:
+ self.metaSocket.start()
+
+
+ def stopped(self):
+ """
+ Called when the process has stopped (died). The socket is marked as "stopped" which means it is
+ not active and won't be dispatched to.
+ """
+
+ # Always tell any metafd socket that we have started, so it can re-initialize state.
+ if self.metaSocket is not None:
+ self.metaSocket.stop()
+
+
</ins><span class="cx"> def getName(self):
</span><span class="cx"> return '%s-%s' % (self.prefix, self.id)
</span><span class="cx">
</span><span class="lines">@@ -1863,7 +1888,7 @@
</span><span class="cx"> fds = {}
</span><span class="cx"> extraFDs = []
</span><span class="cx"> if self.metaSocket is not None:
</span><del>- extraFDs.append(self.metaSocket.fileno())
</del><ins>+ extraFDs.append(self.metaSocket.childSocket().fileno())
</ins><span class="cx"> if self.ampSQLDispenser is not None:
</span><span class="cx"> self.ampDBSocket = self.ampSQLDispenser.dispense()
</span><span class="cx"> extraFDs.append(self.ampDBSocket.fileno())
</span><span class="lines">@@ -1922,7 +1947,7 @@
</span><span class="cx">
</span><span class="cx"> if self.metaSocket is not None:
</span><span class="cx"> args.extend([
</span><del>- "-o", "MetaFD=%s" % (self.metaSocket.fileno(),)
</del><ins>+ "-o", "MetaFD=%s" % (self.metaSocket.childSocket().fileno(),)
</ins><span class="cx"> ])
</span><span class="cx"> if self.ampDBSocket is not None:
</span><span class="cx"> args.extend([
</span><span class="lines">@@ -2015,6 +2040,10 @@
</span><span class="cx"> exists
</span><span class="cx"> """
</span><span class="cx"> class SimpleProcessObject(object):
</span><ins>+ def starting(self):
+ pass
+ def stopped(self):
+ pass
</ins><span class="cx"> def getName(self):
</span><span class="cx"> return name
</span><span class="cx"> def getCommandLine(self):
</span><span class="lines">@@ -2103,7 +2132,10 @@
</span><span class="cx"> def processEnded(self, name):
</span><span class="cx"> """
</span><span class="cx"> When a child process has ended it calls me so I can fire the
</span><del>- appropriate deferred which was created in stopService
</del><ins>+ appropriate deferred which was created in stopService.
+
+ Also make sure to signal the dispatcher so that the socket is
+ marked as inactive.
</ins><span class="cx"> """
</span><span class="cx"> # Cancel the scheduled _forceStopProcess function if the process
</span><span class="cx"> # dies naturally
</span><span class="lines">@@ -2112,6 +2144,8 @@
</span><span class="cx"> self.murder[name].cancel()
</span><span class="cx"> del self.murder[name]
</span><span class="cx">
</span><ins>+ self.processes[name][0].stopped()
+
</ins><span class="cx"> del self.protocols[name]
</span><span class="cx">
</span><span class="cx"> if self._reactor.seconds() - self.timeStarted[name] < self.threshold:
</span><span class="lines">@@ -2198,6 +2232,8 @@
</span><span class="cx">
</span><span class="cx"> childFDs.update(procObj.getFileDescriptors())
</span><span class="cx">
</span><ins>+ procObj.starting()
+
</ins><span class="cx"> args = procObj.getCommandLine()
</span><span class="cx">
</span><span class="cx"> self._reactor.spawnProcess(
</span></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devcalendarservertaptesttest_caldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/test/test_caldav.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/test/test_caldav.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/calendarserver/tap/test/test_caldav.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -954,6 +954,14 @@
</span><span class="cx"> self.args = args
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def starting(self):
+ pass
+
+
+ def stopped(self):
+ pass
+
+
</ins><span class="cx"> def getCommandLine(self):
</span><span class="cx"> """
</span><span class="cx"> Simple command line.
</span><span class="lines">@@ -1204,12 +1212,35 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+class FakeSubsocket(object):
+
+ def __init__(self, fakefd):
+ self.fakefd = fakefd
+
+
+ def childSocket(self):
+ return self.fakefd
+
+
+ def start(self):
+ pass
+
+
+ def restarted(self):
+ pass
+
+
+ def stop(self):
+ pass
+
+
+
</ins><span class="cx"> class FakeDispatcher(object):
</span><span class="cx"> n = 3
</span><span class="cx">
</span><span class="cx"> def addSocket(self):
</span><span class="cx"> self.n += 1
</span><del>- return FakeFD(self.n)
</del><ins>+ return FakeSubsocket(FakeFD(self.n))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtwextinternetsendfdportpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/sendfdport.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/sendfdport.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/sendfdport.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -94,8 +94,8 @@
</span><span class="cx"> A socket in the master process pointing at a file descriptor that can be
</span><span class="cx"> used to transmit sockets to a subprocess.
</span><span class="cx">
</span><del>- @ivar skt: the UNIX socket used as the sendmsg() transport.
- @type skt: L{socket.socket}
</del><ins>+ @ivar outSocket: the UNIX socket used as the sendmsg() transport.
+ @type outSocket: L{socket.socket}
</ins><span class="cx">
</span><span class="cx"> @ivar outgoingSocketQueue: an outgoing queue of sockets to send to the
</span><span class="cx"> subprocess, along with their descriptions (strings describing their
</span><span class="lines">@@ -115,16 +115,61 @@
</span><span class="cx"> @type dispatcher: L{InheritedSocketDispatcher}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, dispatcher, skt, status):
</del><ins>+ def __init__(self, dispatcher, inSocket, outSocket, status, slavenum):
</ins><span class="cx"> FileDescriptor.__init__(self, dispatcher.reactor)
</span><span class="cx"> self.status = status
</span><ins>+ self.slavenum = slavenum
</ins><span class="cx"> self.dispatcher = dispatcher
</span><del>- self.skt = skt # XXX needs to be set non-blocking by somebody
- self.fileno = skt.fileno
</del><ins>+ self.inSocket = inSocket
+ self.outSocket = outSocket # XXX needs to be set non-blocking by somebody
+ self.fileno = outSocket.fileno
</ins><span class="cx"> self.outgoingSocketQueue = []
</span><span class="cx"> self.pendingCloseSocketQueue = []
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def childSocket(self):
+ """
+ Return the socket that the child process will use to communicate with the master.
+ """
+ return self.inSocket
+
+
+ def start(self):
+ """
+ The master process monitor is about to start the child process associated with this socket.
+ Update status to ensure dispatcher know what is going on.
+ """
+ self.status.start()
+ self.dispatcher.statusChanged()
+
+
+ def restarted(self):
+ """
+ The child process associated with this socket has signaled it is ready.
+ Update status to ensure dispatcher know what is going on.
+ """
+ self.status.restarted()
+ self.dispatcher.statusChanged()
+
+
+ def stop(self):
+ """
+ The master process monitor has determined the child process associated with this socket
+ has died. Update status to ensure dispatcher know what is going on.
+ """
+ self.status.stop()
+ self.dispatcher.statusChanged()
+
+
+ def remove(self):
+ """
+ Remove this socket.
+ """
+ self.status.stop()
+ self.dispatcher.statusChanged()
+ self.dispatcher.removeSocket()
+
+
</ins><span class="cx"> def sendSocketToPeer(self, skt, description):
</span><span class="cx"> """
</span><span class="cx"> Enqueue a socket to send to the subprocess.
</span><span class="lines">@@ -138,7 +183,7 @@
</span><span class="cx"> Receive a status / health message and record it.
</span><span class="cx"> """
</span><span class="cx"> try:
</span><del>- data, _ignore_flags, _ignore_ancillary = recvmsg(self.skt.fileno())
</del><ins>+ data, _ignore_flags, _ignore_ancillary = recvmsg(self.outSocket.fileno())
</ins><span class="cx"> except SocketError, se:
</span><span class="cx"> if se.errno not in (EAGAIN, ENOBUFS):
</span><span class="cx"> raise
</span><span class="lines">@@ -155,7 +200,7 @@
</span><span class="cx"> while self.outgoingSocketQueue:
</span><span class="cx"> skt, desc = self.outgoingSocketQueue.pop(0)
</span><span class="cx"> try:
</span><del>- sendfd(self.skt.fileno(), skt.fileno(), desc)
</del><ins>+ sendfd(self.outSocket.fileno(), skt.fileno(), desc)
</ins><span class="cx"> except SocketError, se:
</span><span class="cx"> if se.errno in (EAGAIN, ENOBUFS):
</span><span class="cx"> self.outgoingSocketQueue.insert(0, (skt, desc))
</span><span class="lines">@@ -170,6 +215,51 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+class IStatus(Interface):
+ """
+ Defines the status of a socket. This keeps track of active connections etc.
+ """
+
+ def effective():
+ """
+ The current effective load.
+
+ @return: The current effective load.
+ @rtype: L{int}
+ """
+
+ def active():
+ """
+ Whether the socket should be active (able to be dispatched to).
+
+ @return: Active state.
+ @rtype: L{bool}
+ """
+
+ def start():
+ """
+ Worker process is starting. Mark status accordingly but do not make
+ it active.
+
+ @return: C{self}
+ """
+
+ def restarted():
+ """
+ Worker process has signaled it is ready so make this active.
+
+ @return: C{self}
+ """
+
+ def stop():
+ """
+ Worker process has stopped so make this inactive.
+
+ @return: C{self}
+ """
+
+
+
</ins><span class="cx"> class IStatusWatcher(Interface):
</span><span class="cx"> """
</span><span class="cx"> A provider of L{IStatusWatcher} tracks the I{status messages} reported by
</span><span class="lines">@@ -197,7 +287,7 @@
</span><span class="cx"> than the somewhat more abstract language that would be accurate.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def initialStatus(): #@NoSelf
</del><ins>+ def initialStatus():
</ins><span class="cx"> """
</span><span class="cx"> A new socket was created and added to the dispatcher. Compute an
</span><span class="cx"> initial value for its status.
</span><span class="lines">@@ -205,8 +295,7 @@
</span><span class="cx"> @return: the new status.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-
- def newConnectionStatus(previousStatus): #@NoSelf
</del><ins>+ def newConnectionStatus(previousStatus):
</ins><span class="cx"> """
</span><span class="cx"> A new connection was sent to a given socket. Compute its status based
</span><span class="cx"> on the previous status of that socket.
</span><span class="lines">@@ -217,8 +306,7 @@
</span><span class="cx"> @return: the socket's status after incrementing its outstanding work.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-
- def statusFromMessage(previousStatus, message): #@NoSelf
</del><ins>+ def statusFromMessage(previousStatus, message):
</ins><span class="cx"> """
</span><span class="cx"> A status message was received by a worker. Convert the previous status
</span><span class="cx"> value (returned from L{newConnectionStatus}, L{initialStatus}, or
</span><span class="lines">@@ -231,8 +319,7 @@
</span><span class="cx"> account.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-
- def closeCountFromStatus(previousStatus): #@NoSelf
</del><ins>+ def closeCountFromStatus(previousStatus):
</ins><span class="cx"> """
</span><span class="cx"> Based on a status previously returned from a method on this
</span><span class="cx"> L{IStatusWatcher}, determine how many sockets may be closed.
</span><span class="lines">@@ -250,7 +337,7 @@
</span><span class="cx"> list of available sockets that connect to I{worker process}es and sends
</span><span class="cx"> inbound connections to be inherited over those sockets, by those processes.
</span><span class="cx">
</span><del>- L{InheritedSocketDispatcher} is therefore insantiated in the I{master
</del><ins>+ L{InheritedSocketDispatcher} is therefore instantiated in the I{master
</ins><span class="cx"> process}.
</span><span class="cx">
</span><span class="cx"> @ivar statusWatcher: The object which will handle status messages and
</span><span class="lines">@@ -272,27 +359,43 @@
</span><span class="cx"> @property
</span><span class="cx"> def statuses(self):
</span><span class="cx"> """
</span><del>- Yield the current status of all subprocess sockets.
</del><ins>+ Yield the current status of all subprocess sockets in the current priority order.
</ins><span class="cx"> """
</span><span class="cx"> for subsocket in self._subprocessSockets:
</span><span class="cx"> yield subsocket.status
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @property
+ def slavestates(self):
+ """
+ Yield the current status of all subprocess sockets, ordered by slave number.
+ """
+ for subsocket in sorted(self._subprocessSockets, key=lambda x: x.slavenum):
+ yield (subsocket.slavenum, subsocket.status,)
+
+
+ def statusChanged(self):
+ """
+ Someone is telling us a child socket status changed.
+ """
+ self.statusWatcher.statusesChanged(self.statuses)
+
+
</ins><span class="cx"> def statusMessage(self, subsocket, message):
</span><span class="cx"> """
</span><span class="cx"> The status of a connection has changed; update all registered status
</span><span class="cx"> change listeners.
</span><span class="cx"> """
</span><del>- watcher = self.statusWatcher
- status = watcher.statusFromMessage(subsocket.status, message)
- closeCount, subsocket.status = watcher.closeCountFromStatus(status)
- watcher.statusesChanged(self.statuses)
</del><ins>+ status = self.statusWatcher.statusFromMessage(subsocket.status, message)
+ closeCount, subsocket.status = self.statusWatcher.closeCountFromStatus(status)
+ self.statusChanged()
</ins><span class="cx"> return closeCount
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def sendFileDescriptor(self, skt, description):
</span><span class="cx"> """
</span><del>- A connection has been received. Dispatch it.
</del><ins>+ A connection has been received. Dispatch it to active sockets, sorted by
+ how much work they have.
</ins><span class="cx">
</span><span class="cx"> @param skt: the I{connection socket} (i.e.: not the listening socket)
</span><span class="cx"> @type skt: L{socket.socket}
</span><span class="lines">@@ -301,23 +404,15 @@
</span><span class="cx"> L{InheritedPort} what type of transport to create for this socket.
</span><span class="cx"> @type description: C{bytes}
</span><span class="cx"> """
</span><del>- # We want None to sort after 0 and before 1, so coerce to 0.5 - this
- # allows the master to first schedule all child process that are up but
- # not yet busy ahead of those that are still starting up.
- def sortKey(conn):
- if conn.status is None:
- return 0.5
- else:
- return conn.status
- self._subprocessSockets.sort(key=sortKey)
- selectedSocket = self._subprocessSockets[0]
</del><ins>+ self._subprocessSockets.sort(key=lambda x: x.status.effective())
+ selectedSocket = filter(lambda x: x.status.active(), self._subprocessSockets)[0]
</ins><span class="cx"> selectedSocket.sendSocketToPeer(skt, description)
</span><span class="cx"> # XXX Maybe want to send along 'description' or 'skt' or some
</span><span class="cx"> # properties thereof? -glyph
</span><span class="cx"> selectedSocket.status = self.statusWatcher.newConnectionStatus(
</span><span class="cx"> selectedSocket.status
</span><span class="cx"> )
</span><del>- self.statusWatcher.statusesChanged(self.statuses)
</del><ins>+ self.statusChanged()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def startDispatching(self):
</span><span class="lines">@@ -329,7 +424,7 @@
</span><span class="cx"> subSocket.startReading()
</span><span class="cx">
</span><span class="cx">
</span><del>- def addSocket(self, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
</del><ins>+ def addSocket(self, slavenum=0, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
</ins><span class="cx"> """
</span><span class="cx"> Add a C{sendmsg()}-oriented AF_UNIX socket to the pool of sockets being
</span><span class="cx"> used for transmitting file descriptors to child processes.
</span><span class="lines">@@ -341,14 +436,22 @@
</span><span class="cx"> i, o = socketpair()
</span><span class="cx"> i.setblocking(False)
</span><span class="cx"> o.setblocking(False)
</span><del>- a = _SubprocessSocket(self, o, self.statusWatcher.initialStatus())
</del><ins>+ a = _SubprocessSocket(self, i, o, self.statusWatcher.initialStatus(), slavenum)
</ins><span class="cx"> self._subprocessSockets.append(a)
</span><span class="cx"> if self._isDispatching:
</span><span class="cx"> a.startReading()
</span><del>- return i
</del><ins>+ return a
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ def removeSocket(self, skt):
+ """
+ Removes a previously added socket from the pool of sockets being used
+ for transmitting file descriptors to child processes.
+ """
+ self._subprocessSockets.remove(skt)
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class InheritedPort(FileDescriptor, object):
</span><span class="cx"> """
</span><span class="cx"> An L{InheritedPort} is an L{IReadDescriptor}/L{IWriteDescriptor} created in
</span></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtwextinternettesttest_sendfdportpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/test/test_sendfdport.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/test/test_sendfdport.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/twext/internet/test/test_sendfdport.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -1,4 +1,3 @@
</span><del>-from twext.internet.sendfdport import IStatusWatcher
</del><span class="cx"> # -*- test-case-name: twext.internet.test.test_sendfdport -*-
</span><span class="cx"> ##
</span><span class="cx"> # Copyright (c) 2010-2014 Apple Inc. All rights reserved.
</span><span class="lines">@@ -27,6 +26,7 @@
</span><span class="cx"> from zope.interface import implementer
</span><span class="cx">
</span><span class="cx"> from twext.internet.sendfdport import InheritedSocketDispatcher
</span><ins>+from twext.internet.sendfdport import IStatusWatcher, IStatus
</ins><span class="cx">
</span><span class="cx"> from twext.web2.metafd import ConnectionLimiter
</span><span class="cx"> from twisted.internet.interfaces import IReactorFDSet
</span><span class="lines">@@ -93,6 +93,38 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+@verifiedImplementer(IStatus)
+class Status(object):
+ def __init__(self):
+ self.count = 0
+ self.available = False
+
+
+ def effective(self):
+ return self.count
+
+
+ def active(self):
+ return self.available
+
+
+ def start(self):
+ self.available = False
+ return self
+
+
+ def restarted(self):
+ self.available = True
+ return self
+
+
+ def stop(self):
+ self.count = 0
+ self.available = False
+ return self
+
+
+
</ins><span class="cx"> @verifiedImplementer(IStatusWatcher)
</span><span class="cx"> class Watcher(object):
</span><span class="cx"> def __init__(self, q):
</span><span class="lines">@@ -101,19 +133,21 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def newConnectionStatus(self, previous):
</span><del>- return previous + 1
</del><ins>+ previous.count += 1
+ return previous
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def statusFromMessage(self, previous, message):
</span><del>- return previous - 1
</del><ins>+ previous.count -= 1
+ return previous
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def statusesChanged(self, statuses):
</span><del>- self.q.append(list(statuses))
</del><ins>+ self.q.append([(status.count, status.available) for status in statuses])
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def initialStatus(self):
</span><del>- return 0
</del><ins>+ return Status()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def closeCountFromStatus(self, status):
</span><span class="lines">@@ -152,9 +186,10 @@
</span><span class="cx"> two = SocketForClosing()
</span><span class="cx"> three = SocketForClosing()
</span><span class="cx">
</span><del>- self.dispatcher.addSocket(
</del><ins>+ skt = self.dispatcher.addSocket(
</ins><span class="cx"> lambda: (SocketForClosing(), SocketForClosing())
</span><span class="cx"> )
</span><ins>+ skt.restarted()
</ins><span class="cx">
</span><span class="cx"> self.dispatcher.sendFileDescriptor(one, "one")
</span><span class="cx"> self.dispatcher.sendFileDescriptor(two, "two")
</span><span class="lines">@@ -214,10 +249,11 @@
</span><span class="cx"> dispatcher.statusWatcher = Watcher(q)
</span><span class="cx"> description = "whatever"
</span><span class="cx"> # Need to have a socket that will accept the descriptors.
</span><del>- dispatcher.addSocket()
</del><ins>+ skt = dispatcher.addSocket()
+ skt.restarted()
</ins><span class="cx"> dispatcher.sendFileDescriptor(object(), description)
</span><span class="cx"> dispatcher.sendFileDescriptor(object(), description)
</span><del>- self.assertEquals(q, [[1], [2]])
</del><ins>+ self.assertEquals(q, [[(0, True)], [(1, True)], [(2, True)]])
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def test_statusesChangedOnStatusMessage(self):
</span><span class="lines">@@ -235,4 +271,33 @@
</span><span class="cx"> subskt = dispatcher._subprocessSockets[0]
</span><span class="cx"> dispatcher.statusMessage(subskt, message)
</span><span class="cx"> dispatcher.statusMessage(subskt, message)
</span><del>- self.assertEquals(q, [[-1], [-2]])
</del><ins>+ self.assertEquals(q, [[(-1, False)], [(-2, False)]])
+
+
+ def test_statusesChangedOnStartRestartStop(self):
+ """
+ L{_SubprocessSocket} will update its C{status} when state change.
+ """
+ q = []
+ dispatcher = self.dispatcher
+ dispatcher.statusWatcher = Watcher(q)
+ message = "whatever"
+ # Need to have a socket that will accept the descriptors.
+ subskt = dispatcher.addSocket()
+ subskt.start()
+ subskt.restarted()
+ dispatcher.sendFileDescriptor(subskt, message)
+ subskt.stop()
+ subskt.start()
+ subskt.restarted()
+ self.assertEquals(
+ q,
+ [
+ [(0, False)],
+ [(0, True)],
+ [(1, True)],
+ [(0, False)],
+ [(0, False)],
+ [(0, True)],
+ ]
+ )
</ins></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtwextweb2metafdpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/metafd.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/metafd.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/metafd.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -21,12 +21,11 @@
</span><span class="cx"> """
</span><span class="cx"> from __future__ import print_function
</span><span class="cx">
</span><del>-from functools import total_ordering
-
</del><span class="cx"> from zope.interface import implementer
</span><span class="cx">
</span><span class="cx"> from twext.internet.sendfdport import (
</span><del>- InheritedPort, InheritedSocketDispatcher, InheritingProtocolFactory)
</del><ins>+ InheritedPort, InheritedSocketDispatcher, InheritingProtocolFactory,
+ IStatus)
</ins><span class="cx"> from twext.internet.tcp import MaxAcceptTCPServer
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twext.web2.channel.http import HTTPFactory
</span><span class="lines">@@ -164,17 +163,26 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-@total_ordering
</del><ins>+@implementer(IStatus)
</ins><span class="cx"> class WorkerStatus(FancyStrMixin, object):
</span><span class="cx"> """
</span><span class="cx"> The status of a worker process.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- showAttributes = ("acknowledged unacknowledged started abandoned unclosed"
</del><ins>+ showAttributes = ("acknowledged unacknowledged total started abandoned unclosed starting stopped"
</ins><span class="cx"> .split())
</span><span class="cx">
</span><del>- def __init__(self, acknowledged=0, unacknowledged=0, started=0,
- abandoned=0, unclosed=0):
</del><ins>+ def __init__(
+ self,
+ acknowledged=0,
+ unacknowledged=0,
+ total=0,
+ started=0,
+ abandoned=0,
+ unclosed=0,
+ starting=1,
+ stopped=0
+ ):
</ins><span class="cx"> """
</span><span class="cx"> Create a L{ConnectionStatus} with a number of sent connections and a
</span><span class="cx"> number of un-acknowledged connections.
</span><span class="lines">@@ -187,20 +195,32 @@
</span><span class="cx"> the subprocess which have never received a status response (a
</span><span class="cx"> "C{+}" status message).
</span><span class="cx">
</span><ins>+ @param total: The total number of acknowledged connections over
+ the lifetime of this socket.
+
+ @param started: The number of times this worker has been started.
+
</ins><span class="cx"> @param abandoned: The number of connections which have been sent to
</span><span class="cx"> this worker, but were not acknowledged at the moment that the
</span><del>- worker restarted.
</del><ins>+ worker was stopped.
</ins><span class="cx">
</span><del>- @param started: The number of times this worker has been started.
-
</del><span class="cx"> @param unclosed: The number of sockets which have been sent to the
</span><span class="cx"> subprocess but not yet closed.
</span><ins>+
+ @param starting: The process that owns this socket is starting. Do not
+ dispatch to it until we receive the started message.
+
+ @param stopped: The process that owns this socket has stopped. Do not
+ dispatch to it.
</ins><span class="cx"> """
</span><span class="cx"> self.acknowledged = acknowledged
</span><span class="cx"> self.unacknowledged = unacknowledged
</span><ins>+ self.total = total
</ins><span class="cx"> self.started = started
</span><span class="cx"> self.abandoned = abandoned
</span><span class="cx"> self.unclosed = unclosed
</span><ins>+ self.starting = starting
+ self.stopped = stopped
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def effective(self):
</span><span class="lines">@@ -210,43 +230,67 @@
</span><span class="cx"> return self.acknowledged + self.unacknowledged
</span><span class="cx">
</span><span class="cx">
</span><del>- def restarted(self):
</del><ins>+ def active(self):
</ins><span class="cx"> """
</span><del>- The L{WorkerStatus} derived from the current status of a process and
- the fact that it just restarted.
</del><ins>+ Is the subprocess associated with this socket available to dispatch to.
+ i.e, this socket is neither stopped nor starting
</ins><span class="cx"> """
</span><del>- return self.__class__(0, 0, self.started + 1, self.unacknowledged)
</del><ins>+ return self.starting == 0 and self.stopped == 0
</ins><span class="cx">
</span><span class="cx">
</span><del>- def _tuplify(self):
- return tuple(getattr(self, attr) for attr in self.showAttributes)
</del><ins>+ def start(self):
+ """
+ The child process for this L{WorkerStatus} is about to (re)start. Reset the status to indicate it
+ is starting - that should prevent any new connections being dispatched.
+ """
+ return self.reset(
+ starting=1,
+ stopped=0,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __lt__(self, other):
- if not isinstance(other, WorkerStatus):
- return NotImplemented
- return self.effective() < other.effective()
</del><ins>+ def restarted(self):
+ """
+ The child process for this L{WorkerStatus} has indicated it is now available to accept
+ connections, so reset the starting status so this socket will be available for dispatch.
+ """
+ return self.reset(
+ started=self.started + 1,
+ starting=0,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __eq__(self, other):
- if not isinstance(other, WorkerStatus):
- return NotImplemented
- return self._tuplify() == other._tuplify()
</del><ins>+ def stop(self):
+ """
+ The child process for this L{WorkerStatus} has stopped. Stop the socket and clear out
+ existing counters, but track abandoned connections.
+ """
+ return self.reset(
+ acknowledged=0,
+ unacknowledged=0,
+ abandoned=self.abandoned + self.unacknowledged,
+ starting=0,
+ stopped=1,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __add__(self, other):
- if not isinstance(other, WorkerStatus):
- return NotImplemented
- a = self._tuplify()
- b = other._tuplify()
- c = [a1 + b1 for (a1, b1) in zip(a, b)]
- return self.__class__(*c)
</del><ins>+ def adjust(self, **kwargs):
+ """
+ Update the L{WorkerStatus} by adding the supplied values to the specified attributes.
+ """
+ for k, v in kwargs.items():
+ newval = getattr(self, k) + v
+ setattr(self, k, max(newval, 0))
+ return self
</ins><span class="cx">
</span><span class="cx">
</span><del>- def __sub__(self, other):
- if not isinstance(other, WorkerStatus):
- return NotImplemented
- return self + self.__class__(*[-x for x in other._tuplify()])
</del><ins>+ def reset(self, **kwargs):
+ """
+ Reset the L{WorkerStatus} by setting the supplied values in the specified attributes.
+ """
+ for k, v in kwargs.items():
+ setattr(self, k, v)
+ return self
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -272,6 +316,7 @@
</span><span class="cx"> self.dispatcher = InheritedSocketDispatcher(self)
</span><span class="cx"> self.maxAccepts = maxAccepts
</span><span class="cx"> self.maxRequests = maxRequests
</span><ins>+ self.overloaded = False
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def startService(self):
</span><span class="lines">@@ -314,20 +359,20 @@
</span><span class="cx"> # A connection has gone away in a subprocess; we should start
</span><span class="cx"> # accepting connections again if we paused (see
</span><span class="cx"> # newConnectionStatus)
</span><del>- return previousStatus - WorkerStatus(acknowledged=1)
</del><ins>+ return previousStatus.adjust(acknowledged=-1)
+
</ins><span class="cx"> elif message == '0':
</span><del>- # A new process just started accepting new connections. It might
- # still have some unacknowledged connections, but any connections
- # that it acknowledged working on are now completed. (We have no
- # way of knowing whether the acknowledged connections were acted
- # upon or dropped, so we have to treat that number with a healthy
- # amount of skepticism.)
</del><ins>+ # A new process just started accepting new connections.
</ins><span class="cx"> return previousStatus.restarted()
</span><ins>+
</ins><span class="cx"> else:
</span><span class="cx"> # '+' acknowledges that the subprocess has taken on the work.
</span><del>- return previousStatus + WorkerStatus(acknowledged=1,
- unacknowledged=-1,
- unclosed=1)
</del><ins>+ return previousStatus.adjust(
+ acknowledged=1,
+ unacknowledged=-1,
+ total=1,
+ unclosed=1,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def closeCountFromStatus(self, status):
</span><span class="lines">@@ -335,21 +380,22 @@
</span><span class="cx"> Determine the number of sockets to close from the current status.
</span><span class="cx"> """
</span><span class="cx"> toClose = status.unclosed
</span><del>- return (toClose, status - WorkerStatus(unclosed=toClose))
</del><ins>+ return (toClose, status.adjust(unclosed=-toClose))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def newConnectionStatus(self, previousStatus):
</span><span class="cx"> """
</span><del>- Determine the effect of a new connection being sent on a subprocess
- socket.
</del><ins>+ A connection was just sent to the process, but not yet acknowledged.
</ins><span class="cx"> """
</span><del>- return previousStatus + WorkerStatus(unacknowledged=1)
</del><ins>+ return previousStatus.adjust(unacknowledged=1)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def statusesChanged(self, statuses):
</span><span class="cx"> """
</span><span class="cx"> The L{InheritedSocketDispatcher} is reporting that the list of
</span><del>- connection-statuses have changed.
</del><ins>+ connection-statuses have changed. Check to see if we are overloaded
+ or if there are no active processes left. If so, stop the protocol
+ factory from processing more requests until capacity is back.
</ins><span class="cx">
</span><span class="cx"> (The argument to this function is currently duplicated by the
</span><span class="cx"> C{self.dispatcher.statuses} attribute, which is what
</span><span class="lines">@@ -360,8 +406,10 @@
</span><span class="cx"> self._outstandingRequests = current # preserve for or= field in log
</span><span class="cx"> maximum = self.maxRequests
</span><span class="cx"> overloaded = (current >= maximum)
</span><ins>+ available = len(filter(lambda x: x.active(), self.dispatcher.statuses))
+ self.overloaded = (overloaded or available == 0)
</ins><span class="cx"> for f in self.factories:
</span><del>- if overloaded:
</del><ins>+ if self.overloaded:
</ins><span class="cx"> f.loadAboveMaximum()
</span><span class="cx"> else:
</span><span class="cx"> f.loadNominal()
</span></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtwextweb2testtest_metafdpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/test/test_metafd.py (12842 => 12843)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/test/test_metafd.py 2014-03-07 20:24:02 UTC (rev 12842)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/twext/web2/test/test_metafd.py 2014-03-07 20:38:57 UTC (rev 12843)
</span><span class="lines">@@ -71,15 +71,12 @@
</span><span class="cx"> def startReading(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def stopReading(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def startWriting(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def stopWriting(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><span class="lines">@@ -93,19 +90,15 @@
</span><span class="cx"> def startReading(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def stopReading(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def startWriting(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def stopWriting(self):
</span><span class="cx"> "Do nothing."
</span><span class="cx">
</span><del>-
</del><span class="cx"> def __init__(self, *a, **kw):
</span><span class="cx"> super(ServerTransportForTesting, self).__init__(*a, **kw)
</span><span class="cx"> self.reactor = None
</span><span class="lines">@@ -225,12 +218,28 @@
</span><span class="cx"> L{WorkerStatus.__repr__} will show all the values associated with the
</span><span class="cx"> status of the worker.
</span><span class="cx"> """
</span><del>- self.assertEquals(repr(WorkerStatus(1, 2, 3, 4, 5)),
- "<WorkerStatus acknowledged=1 unacknowledged=2 "
- "started=3 abandoned=4 unclosed=5>")
</del><ins>+ self.assertEquals(repr(WorkerStatus(1, 2, 3, 4, 5, 6, 7, 8)),
+ "<WorkerStatus acknowledged=1 unacknowledged=2 total=3 "
+ "started=4 abandoned=5 unclosed=6 starting=7 stopped=8>")
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ def test_workerStatusNonNegative(self):
+ """
+ L{WorkerStatus.__repr__} will show all the values associated with the
+ status of the worker.
+ """
+ w = WorkerStatus()
+ w.adjust(
+ acknowledged=1,
+ unacknowledged=-1,
+ total=1,
+ )
+ self.assertEquals(w.acknowledged, 1)
+ self.assertEquals(w.unacknowledged, 0)
+ self.assertEquals(w.total, 1)
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class LimiterBuilder(object):
</span><span class="cx"> """
</span><span class="cx"> A L{LimiterBuilder} can build a L{ConnectionLimiter} and associated objects
</span><span class="lines">@@ -251,7 +260,9 @@
</span><span class="cx"> self.limiter.addPortService("TCP", 4321, "127.0.0.1", 5,
</span><span class="cx"> self.serverServiceMakerMaker(self.service))
</span><span class="cx"> for ignored in xrange(socketCount):
</span><del>- self.dispatcher.addSocket()
</del><ins>+ subskt = self.dispatcher.addSocket()
+ subskt.start()
+ subskt.restarted()
</ins><span class="cx"> # Has to be running in order to add stuff.
</span><span class="cx"> self.limiter.startService()
</span><span class="cx"> self.port = self.service.myPort
</span><span class="lines">@@ -296,7 +307,7 @@
</span><span class="cx"> @param count: Amount of load to add; default to the maximum that the
</span><span class="cx"> limiter.
</span><span class="cx"> """
</span><del>- for x in range(count or self.limiter.maxRequests):
</del><ins>+ for _ignore_x in range(count or self.limiter.maxRequests):
</ins><span class="cx"> self.dispatcher.sendFileDescriptor(None, "SSL")
</span><span class="cx"> if acknowledged:
</span><span class="cx"> self.dispatcher.statusMessage(
</span><span class="lines">@@ -305,6 +316,8 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def processRestart(self):
</span><ins>+ self.dispatcher._subprocessSockets[0].stop()
+ self.dispatcher._subprocessSockets[0].start()
</ins><span class="cx"> self.dispatcher.statusMessage(
</span><span class="cx"> self.dispatcher._subprocessSockets[0], "0"
</span><span class="cx"> )
</span></span></pre>
</div>
</div>
</body>
</html>