[CalendarServer-changes] [12844] twext/trunk/twext
source_changes at macosforge.org
source_changes at macosforge.org
Fri Mar 7 12:48:27 PST 2014
Revision: 12844
http://trac.calendarserver.org//changeset/12844
Author: cdaboo at apple.com
Date: 2014-03-07 12:48:27 -0800 (Fri, 07 Mar 2014)
Log Message:
-----------
Fix master-child slot counting issues.
Modified Paths:
--------------
twext/trunk/twext/application/masterchild.py
twext/trunk/twext/internet/sendfdport.py
twext/trunk/twext/internet/test/test_sendfdport.py
Modified: twext/trunk/twext/application/masterchild.py
===================================================================
--- twext/trunk/twext/application/masterchild.py 2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/application/masterchild.py 2014-03-07 20:48:27 UTC (rev 12844)
@@ -37,7 +37,6 @@
import sys
from os import close, unlink
from tempfile import mkstemp
-from functools import total_ordering
from zope.interface import implementer
@@ -54,7 +53,7 @@
from twisted.internet.protocol import ProcessProtocol
from twext.python.log import Logger
-from twext.internet.sendfdport import InheritingProtocolFactory
+from twext.internet.sendfdport import InheritingProtocolFactory, IStatus
from twext.internet.sendfdport import InheritedSocketDispatcher
from twext.internet.sendfdport import IStatusWatcher
from twext.internet.sendfdport import InheritedPort
@@ -205,7 +204,10 @@
@staticmethod
def newConnectionStatus(previousStatus):
- return previousStatus + ChildStatus(unacknowledged=1)
+ """
+ A connection was just sent to the process, but not yet acknowledged.
+ """
+ return previousStatus.adjust(unacknowledged=1)
@staticmethod
@@ -214,40 +216,19 @@
# A connection has gone away in a subprocess; we should start
# accepting connections again if we paused (see
# newConnectionStatus)
- return previousStatus - ChildStatus(acknowledged=1)
+ return previousStatus.adjust(acknowledged=-1)
elif message == "0":
- # A new process just started accepting new connections. It might
- # still have some unacknowledged connections, but any connections
- # that it acknowledged working on are now completed. (We have no
- # way of knowing whether the acknowledged connections were acted
- # upon or dropped, so we have to treat that number with a healthy
- # amount of skepticism.)
+ # A new process just started accepting new connections.
+ return previousStatus.restarted()
- # Do some sanity checks... no attempt to fix, but log critically
- # if there are unexpected connection counts, as that means we
- # don't know what's going on with our connection management.
-
- def checkForWeirdness(what, expected):
- n = getattr(previousStatus, what)
- if n != expected:
- MasterService.log.critical(
- "New process has {count} {type} connections, "
- "expected {expected}."
- .format(count=n, type=what, expected=expected)
- )
-
- checkForWeirdness("acknowledged", 0)
- checkForWeirdness("unacknowledged", 1)
- checkForWeirdness("unclosed", 1)
-
- return previousStatus
-
elif message == "+":
# Acknowledges that the subprocess has taken on the work.
- return (
- previousStatus +
- ChildStatus(acknowledged=1, unacknowledged=-1, unclosed=1)
+ return previousStatus.adjust(
+ acknowledged=1,
+ unacknowledged=-1,
+ total=1,
+ unclosed=1,
)
else:
@@ -257,7 +238,7 @@
@staticmethod
def closeCountFromStatus(previousStatus):
toClose = previousStatus.unclosed
- return (toClose, previousStatus - ChildStatus(unclosed=toClose))
+ return (toClose, previousStatus.adjust(unclosed=-toClose))
def statusesChanged(self, statuses):
@@ -266,22 +247,21 @@
self.log.info("Status changed: {0}".format(tuple(statuses)))
- # current = sum(
- # status.effective()
- # for status in self.dispatcher.statuses
- # )
+# current = sum(status.effective()
+# for status in self.dispatcher.statuses)
+# self._outstandingRequests = current # preserve for or= field in log
+# maximum = self.maxRequests
+# overloaded = (current >= maximum)
+# available = len(filter(lambda x: x.active(), self.dispatcher.statuses))
+# self.overloaded = (overloaded or available == 0)
+# for f in self.factories:
+# if self.overloaded:
+# f.loadAboveMaximum()
+# else:
+# f.loadNominal()
- # maximum = self.maxRequests
- # overloaded = (current >= maximum)
- # for f in self.factories:
- # if overloaded:
- # f.loadAboveMaximum()
- # else:
- # f.loadNominal()
-
-
@implementer(IServiceMaker)
class MasterServiceMaker(object):
"""
@@ -402,7 +382,7 @@
from twisted.internet import reactor
inheritedSocket = self.dispatcher.addSocket()
- inheritedFD = inheritedSocket.fileno()
+ inheritedFD = inheritedSocket.childSocket().fileno()
processProtocol = ChildProcessProtocol(self, inheritedSocket)
@@ -491,12 +471,14 @@
# FIXME: deserialize log events from child
# log = Logger()
-
def __init__(self, service, inheritedSocket):
self.service = service
self.inheritedSocket = inheritedSocket
+ # Always tell any metafd socket that we have started, so it can re-initialize state.
+ self.inheritedSocket.start()
+
def outReceived(self, data):
# self.log.info(u"{data}", data=data)
sys.stdout.write(data)
@@ -509,6 +491,8 @@
def processExited(self, reason):
+ # Always tell any metafd socket that we have started, so it can re-initialize state.
+ self.inheritedSocket.stop()
self.service.childDidExit(self, reason)
@@ -684,7 +668,7 @@
- at total_ordering
+ at implementer(IStatus)
class ChildStatus(FancyStrMixin, object):
"""
The status of a child process.
@@ -693,11 +677,26 @@
showAttributes = (
"acknowledged",
"unacknowledged",
+ "total",
+ "started",
+ "abandoned",
"unclosed",
+ "starting",
+ "stopped",
)
- def __init__(self, acknowledged=0, unacknowledged=0, unclosed=0):
+ def __init__(
+ self,
+ acknowledged=0,
+ unacknowledged=0,
+ total=0,
+ started=0,
+ abandoned=0,
+ unclosed=0,
+ starting=0,
+ stopped=0
+ ):
"""
Create a L{ConnectionStatus} with a number of sent connections and a
number of un-acknowledged connections.
@@ -710,12 +709,32 @@
the subprocess which have never received a status response (a
"C{+}" status message).
+ @param total: The total number of acknowledged connections over
+ the lifetime of this socket.
+
+ @param started: The number of times this worker has been started.
+
+ @param abandoned: The number of connections which have been sent to
+ this worker, but were not acknowledged at the moment that the
+ worker was stopped.
+
@param unclosed: The number of sockets which have been sent to the
subprocess but not yet closed.
+
+ @param starting: The process that owns this socket is starting. Do not
+ dispatch to it until we receive the started message.
+
+ @param stopped: The process that owns this socket has stopped. Do not
+ dispatch to it.
"""
self.acknowledged = acknowledged
self.unacknowledged = unacknowledged
+ self.total = total
+ self.started = started
+ self.abandoned = abandoned
self.unclosed = unclosed
+ self.starting = starting
+ self.stopped = stopped
def effectiveLoad(self):
@@ -725,37 +744,64 @@
return self.acknowledged + self.unacknowledged
- def _tuplify(self):
- return tuple(getattr(self, attr) for attr in self.showAttributes)
+ def active(self):
+ """
+ Is the subprocess associated with this socket available to dispatch to.
+ i.e, this socket is neither stopped nor starting
+ """
+ return self.starting == 0 and self.stopped == 0
- def __lt__(self, other):
- if not isinstance(other, ChildStatus):
- return NotImplemented
+ def start(self):
+ """
+ The child process for this L{WorkerStatus} is about to (re)start. Reset the status to indicate it
+ is starting - that should prevent any new connections being dispatched.
+ """
+ return self.reset(
+ starting=1,
+ stopped=0,
+ )
- return self.effectiveLoad() < other.effectiveLoad()
+ def restarted(self):
+ """
+ The child process for this L{WorkerStatus} has indicated it is now available to accept
+ connections, so reset the starting status so this socket will be available for dispatch.
+ """
+ return self.reset(
+ started=self.started + 1,
+ starting=0,
+ )
- def __eq__(self, other):
- if not isinstance(other, ChildStatus):
- return NotImplemented
- return self._tuplify() == other._tuplify()
+ def stop(self):
+ """
+ The child process for this L{WorkerStatus} has stopped. Stop the socket and clear out
+ existing counters, but track abandoned connections.
+ """
+ return self.reset(
+ acknowledged=0,
+ unacknowledged=0,
+ abandoned=self.abandoned + self.unacknowledged,
+ starting=0,
+ stopped=1,
+ )
- def __add__(self, other):
- if not isinstance(other, ChildStatus):
- return NotImplemented
+ def adjust(self, **kwargs):
+ """
+ Update the L{WorkerStatus} by adding the supplied values to the specified attributes.
+ """
+ for k, v in kwargs.items():
+ newval = getattr(self, k) + v
+ setattr(self, k, max(newval, 0))
+ return self
- a = self._tuplify()
- b = other._tuplify()
- sum = [a1 + b1 for (a1, b1) in zip(a, b)]
- return self.__class__(*sum)
-
-
- def __sub__(self, other):
- if not isinstance(other, ChildStatus):
- return NotImplemented
-
- return self + self.__class__(*[-x for x in other._tuplify()])
+ def reset(self, **kwargs):
+ """
+ Reset the L{WorkerStatus} by setting the supplied values in the specified attributes.
+ """
+ for k, v in kwargs.items():
+ setattr(self, k, v)
+ return self
Modified: twext/trunk/twext/internet/sendfdport.py
===================================================================
--- twext/trunk/twext/internet/sendfdport.py 2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/internet/sendfdport.py 2014-03-07 20:48:27 UTC (rev 12844)
@@ -116,9 +116,10 @@
@type dispatcher: L{InheritedSocketDispatcher}
"""
- def __init__(self, dispatcher, inSocket, outSocket, status):
+ def __init__(self, dispatcher, inSocket, outSocket, status, slavenum):
FileDescriptor.__init__(self, dispatcher.reactor)
self.status = status
+ self.slavenum = slavenum
self.dispatcher = dispatcher
self.inSocket = inSocket
self.outSocket = outSocket # XXX needs to be set non-blocking by somebody
@@ -127,6 +128,49 @@
self.pendingCloseSocketQueue = []
+ def childSocket(self):
+ """
+ Return the socket that the child process will use to communicate with the master.
+ """
+ return self.inSocket
+
+
+ def start(self):
+ """
+ The master process monitor is about to start the child process associated with this socket.
+ Update status to ensure dispatcher know what is going on.
+ """
+ self.status.start()
+ self.dispatcher.statusChanged()
+
+
+ def restarted(self):
+ """
+ The child process associated with this socket has signaled it is ready.
+ Update status to ensure dispatcher know what is going on.
+ """
+ self.status.restarted()
+ self.dispatcher.statusChanged()
+
+
+ def stop(self):
+ """
+ The master process monitor has determined the child process associated with this socket
+ has died. Update status to ensure dispatcher know what is going on.
+ """
+ self.status.stop()
+ self.dispatcher.statusChanged()
+
+
+ def remove(self):
+ """
+ Remove this socket.
+ """
+ self.status.stop()
+ self.dispatcher.statusChanged()
+ self.dispatcher.removeSocket()
+
+
def sendSocketToPeer(self, skt, description):
"""
Enqueue a socket to send to the subprocess.
@@ -174,6 +218,51 @@
+class IStatus(Interface):
+ """
+ Defines the status of a socket. This keeps track of active connections etc.
+ """
+
+ def effective():
+ """
+ The current effective load.
+
+ @return: The current effective load.
+ @rtype: L{int}
+ """
+
+ def active():
+ """
+ Whether the socket should be active (able to be dispatched to).
+
+ @return: Active state.
+ @rtype: L{bool}
+ """
+
+ def start():
+ """
+ Worker process is starting. Mark status accordingly but do not make
+ it active.
+
+ @return: C{self}
+ """
+
+ def restarted():
+ """
+ Worker process has signaled it is ready so make this active.
+
+ @return: C{self}
+ """
+
+ def stop():
+ """
+ Worker process has stopped so make this inactive.
+
+ @return: C{self}
+ """
+
+
+
class IStatusWatcher(Interface):
"""
A provider of L{IStatusWatcher} tracks the I{status messages} reported by
@@ -209,7 +298,6 @@
@return: the new status.
"""
-
def newConnectionStatus(previousStatus):
"""
A new connection was sent to a given socket. Compute its status based
@@ -221,7 +309,6 @@
@return: the socket's status after incrementing its outstanding work.
"""
-
def statusFromMessage(previousStatus, message):
"""
A status message was received by a worker. Convert the previous status
@@ -235,7 +322,6 @@
account.
"""
-
def closeCountFromStatus(previousStatus):
"""
Based on a status previously returned from a method on this
@@ -254,7 +340,7 @@
list of available sockets that connect to I{worker process}es and sends
inbound connections to be inherited over those sockets, by those processes.
- L{InheritedSocketDispatcher} is therefore insantiated in the I{master
+ L{InheritedSocketDispatcher} is therefore instantiated in the I{master
process}.
@ivar statusWatcher: The object which will handle status messages and
@@ -276,27 +362,43 @@
@property
def statuses(self):
"""
- Yield the current status of all subprocess sockets.
+ Yield the current status of all subprocess sockets in the current priority order.
"""
for subsocket in self._subprocessSockets:
yield subsocket.status
+ @property
+ def slavestates(self):
+ """
+ Yield the current status of all subprocess sockets, ordered by slave number.
+ """
+ for subsocket in sorted(self._subprocessSockets, key=lambda x: x.slavenum):
+ yield (subsocket.slavenum, subsocket.status,)
+
+
+ def statusChanged(self):
+ """
+ Someone is telling us a child socket status changed.
+ """
+ self.statusWatcher.statusesChanged(self.statuses)
+
+
def statusMessage(self, subsocket, message):
"""
The status of a connection has changed; update all registered status
change listeners.
"""
- watcher = self.statusWatcher
- status = watcher.statusFromMessage(subsocket.status, message)
- closeCount, subsocket.status = watcher.closeCountFromStatus(status)
- watcher.statusesChanged(self.statuses)
+ status = self.statusWatcher.statusFromMessage(subsocket.status, message)
+ closeCount, subsocket.status = self.statusWatcher.closeCountFromStatus(status)
+ self.statusChanged()
return closeCount
def sendFileDescriptor(self, skt, description):
"""
- A connection has been received. Dispatch it.
+ A connection has been received. Dispatch it to active sockets, sorted by
+ how much work they have.
@param skt: the I{connection socket} (i.e.: not the listening socket)
@type skt: L{socket.socket}
@@ -305,23 +407,15 @@
L{InheritedPort} what type of transport to create for this socket.
@type description: C{bytes}
"""
- # We want None to sort after 0 and before 1, so coerce to 0.5 - this
- # allows the master to first schedule all child process that are up but
- # not yet busy ahead of those that are still starting up.
- def sortKey(conn):
- if conn.status is None:
- return 0.5
- else:
- return conn.status
- self._subprocessSockets.sort(key=sortKey)
- selectedSocket = self._subprocessSockets[0]
+ self._subprocessSockets.sort(key=lambda x: x.status.effective())
+ selectedSocket = filter(lambda x: x.status.active(), self._subprocessSockets)[0]
selectedSocket.sendSocketToPeer(skt, description)
# XXX Maybe want to send along 'description' or 'skt' or some
# properties thereof? -glyph
selectedSocket.status = self.statusWatcher.newConnectionStatus(
selectedSocket.status
)
- self.statusWatcher.statusesChanged(self.statuses)
+ self.statusChanged()
def startDispatching(self):
@@ -333,7 +427,7 @@
subSocket.startReading()
- def addSocket(self, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
+ def addSocket(self, slavenum=0, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
"""
Add a L{send1msg}-oriented AF_UNIX socket to the pool of sockets being
used for transmitting file descriptors to child processes.
@@ -345,11 +439,11 @@
i, o = socketpair()
i.setblocking(False)
o.setblocking(False)
- a = _SubprocessSocket(self, i, o, self.statusWatcher.initialStatus())
+ a = _SubprocessSocket(self, i, o, self.statusWatcher.initialStatus(), slavenum)
self._subprocessSockets.append(a)
if self._isDispatching:
a.startReading()
- return i
+ return a
def removeSocket(self, skt):
@@ -357,12 +451,7 @@
Removes a previously added socket from the pool of sockets being used
for transmitting file descriptors to child processes.
"""
- for a in self._subprocessSockets:
- if a.inSocket == skt:
- self._subprocessSockets.remove(a)
- break
- else:
- raise ValueError("Unknown socket: {0}".format(skt))
+ self._subprocessSockets.remove(skt)
Modified: twext/trunk/twext/internet/test/test_sendfdport.py
===================================================================
--- twext/trunk/twext/internet/test/test_sendfdport.py 2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/internet/test/test_sendfdport.py 2014-03-07 20:48:27 UTC (rev 12844)
@@ -1,4 +1,3 @@
-from twext.internet.sendfdport import IStatusWatcher
# -*- test-case-name: twext.internet.test.test_sendfdport -*-
##
# Copyright (c) 2010-2014 Apple Inc. All rights reserved.
@@ -27,6 +26,7 @@
from zope.interface import implementer
from twext.internet.sendfdport import InheritedSocketDispatcher
+from twext.internet.sendfdport import IStatusWatcher, IStatus
from twisted.internet.interfaces import IReactorFDSet
from twisted.trial.unittest import TestCase
@@ -101,6 +101,38 @@
+ at verifiedImplementer(IStatus)
+class Status(object):
+ def __init__(self):
+ self.count = 0
+ self.available = False
+
+
+ def effective(self):
+ return self.count
+
+
+ def active(self):
+ return self.available
+
+
+ def start(self):
+ self.available = False
+ return self
+
+
+ def restarted(self):
+ self.available = True
+ return self
+
+
+ def stop(self):
+ self.count = 0
+ self.available = False
+ return self
+
+
+
@verifiedImplementer(IStatusWatcher)
class Watcher(object):
def __init__(self, q):
@@ -109,19 +141,21 @@
def newConnectionStatus(self, previous):
- return previous + 1
+ previous.count += 1
+ return previous
def statusFromMessage(self, previous, message):
- return previous - 1
+ previous.count -= 1
+ return previous
def statusesChanged(self, statuses):
- self.q.append(list(statuses))
+ self.q.append([(status.count, status.available) for status in statuses])
def initialStatus(self):
- return 0
+ return Status()
def closeCountFromStatus(self, status):
@@ -164,9 +198,10 @@
two = SocketForClosing()
three = SocketForClosing()
- self.dispatcher.addSocket(
+ skt = self.dispatcher.addSocket(
lambda: (SocketForClosing(), SocketForClosing())
)
+ skt.restarted()
self.dispatcher.sendFileDescriptor(one, "one")
self.dispatcher.sendFileDescriptor(two, "two")
@@ -228,10 +263,11 @@
dispatcher.statusWatcher = Watcher(q)
description = "whatever"
# Need to have a socket that will accept the descriptors.
- dispatcher.addSocket()
+ skt = dispatcher.addSocket()
+ skt.restarted()
dispatcher.sendFileDescriptor(object(), description)
dispatcher.sendFileDescriptor(object(), description)
- self.assertEquals(q, [[1], [2]])
+ self.assertEquals(q, [[(0, True)], [(1, True)], [(2, True)]])
def test_statusesChangedOnStatusMessage(self):
@@ -249,4 +285,33 @@
subskt = dispatcher._subprocessSockets[0]
dispatcher.statusMessage(subskt, message)
dispatcher.statusMessage(subskt, message)
- self.assertEquals(q, [[-1], [-2]])
+ self.assertEquals(q, [[(-1, False)], [(-2, False)]])
+
+
+ def test_statusesChangedOnStartRestartStop(self):
+ """
+ L{_SubprocessSocket} will update its C{status} when state change.
+ """
+ q = []
+ dispatcher = self.dispatcher
+ dispatcher.statusWatcher = Watcher(q)
+ message = "whatever"
+ # Need to have a socket that will accept the descriptors.
+ subskt = dispatcher.addSocket()
+ subskt.start()
+ subskt.restarted()
+ dispatcher.sendFileDescriptor(subskt, message)
+ subskt.stop()
+ subskt.start()
+ subskt.restarted()
+ self.assertEquals(
+ q,
+ [
+ [(0, False)],
+ [(0, True)],
+ [(1, True)],
+ [(0, False)],
+ [(0, False)],
+ [(0, True)],
+ ]
+ )
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140307/10691e36/attachment-0001.html>
More information about the calendarserver-changes
mailing list