[CalendarServer-changes] [12844] twext/trunk/twext

source_changes at macosforge.org source_changes at macosforge.org
Fri Mar 7 12:48:27 PST 2014


Revision: 12844
          http://trac.calendarserver.org//changeset/12844
Author:   cdaboo at apple.com
Date:     2014-03-07 12:48:27 -0800 (Fri, 07 Mar 2014)
Log Message:
-----------
Fix master-child slot counting issues.

Modified Paths:
--------------
    twext/trunk/twext/application/masterchild.py
    twext/trunk/twext/internet/sendfdport.py
    twext/trunk/twext/internet/test/test_sendfdport.py

Modified: twext/trunk/twext/application/masterchild.py
===================================================================
--- twext/trunk/twext/application/masterchild.py	2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/application/masterchild.py	2014-03-07 20:48:27 UTC (rev 12844)
@@ -37,7 +37,6 @@
 import sys
 from os import close, unlink
 from tempfile import mkstemp
-from functools import total_ordering
 
 from zope.interface import implementer
 
@@ -54,7 +53,7 @@
 from twisted.internet.protocol import ProcessProtocol
 
 from twext.python.log import Logger
-from twext.internet.sendfdport import InheritingProtocolFactory
+from twext.internet.sendfdport import InheritingProtocolFactory, IStatus
 from twext.internet.sendfdport import InheritedSocketDispatcher
 from twext.internet.sendfdport import IStatusWatcher
 from twext.internet.sendfdport import InheritedPort
@@ -205,7 +204,10 @@
 
     @staticmethod
     def newConnectionStatus(previousStatus):
-        return previousStatus + ChildStatus(unacknowledged=1)
+        """
+        A connection was just sent to the process, but not yet acknowledged.
+        """
+        return previousStatus.adjust(unacknowledged=1)
 
 
     @staticmethod
@@ -214,40 +216,19 @@
             # A connection has gone away in a subprocess; we should start
             # accepting connections again if we paused (see
             # newConnectionStatus)
-            return previousStatus - ChildStatus(acknowledged=1)
+            return previousStatus.adjust(acknowledged=-1)
 
         elif message == "0":
-            # A new process just started accepting new connections.  It might
-            # still have some unacknowledged connections, but any connections
-            # that it acknowledged working on are now completed.  (We have no
-            # way of knowing whether the acknowledged connections were acted
-            # upon or dropped, so we have to treat that number with a healthy
-            # amount of skepticism.)
+            # A new process just started accepting new connections.
+            return previousStatus.restarted()
 
-            # Do some sanity checks... no attempt to fix, but log critically
-            # if there are unexpected connection counts, as that means we
-            # don't know what's going on with our connection management.
-
-            def checkForWeirdness(what, expected):
-                n = getattr(previousStatus, what)
-                if n != expected:
-                    MasterService.log.critical(
-                        "New process has {count} {type} connections, "
-                        "expected {expected}."
-                        .format(count=n, type=what, expected=expected)
-                    )
-
-            checkForWeirdness("acknowledged", 0)
-            checkForWeirdness("unacknowledged", 1)
-            checkForWeirdness("unclosed", 1)
-
-            return previousStatus
-
         elif message == "+":
             # Acknowledges that the subprocess has taken on the work.
-            return (
-                previousStatus +
-                ChildStatus(acknowledged=1, unacknowledged=-1, unclosed=1)
+            return previousStatus.adjust(
+                acknowledged=1,
+                unacknowledged=-1,
+                total=1,
+                unclosed=1,
             )
 
         else:
@@ -257,7 +238,7 @@
     @staticmethod
     def closeCountFromStatus(previousStatus):
         toClose = previousStatus.unclosed
-        return (toClose, previousStatus - ChildStatus(unclosed=toClose))
+        return (toClose, previousStatus.adjust(unclosed=-toClose))
 
 
     def statusesChanged(self, statuses):
@@ -266,22 +247,21 @@
 
         self.log.info("Status changed: {0}".format(tuple(statuses)))
 
-        # current = sum(
-        #     status.effective()
-        #     for status in self.dispatcher.statuses
-        # )
+#        current = sum(status.effective()
+#                      for status in self.dispatcher.statuses)
+#        self._outstandingRequests = current # preserve for or= field in log
+#        maximum = self.maxRequests
+#        overloaded = (current >= maximum)
+#        available = len(filter(lambda x: x.active(), self.dispatcher.statuses))
+#        self.overloaded = (overloaded or available == 0)
+#        for f in self.factories:
+#            if self.overloaded:
+#                f.loadAboveMaximum()
+#            else:
+#                f.loadNominal()
 
-        # maximum = self.maxRequests
-        # overloaded = (current >= maximum)
 
-        # for f in self.factories:
-        #     if overloaded:
-        #         f.loadAboveMaximum()
-        #     else:
-        #         f.loadNominal()
 
-
-
 @implementer(IServiceMaker)
 class MasterServiceMaker(object):
     """
@@ -402,7 +382,7 @@
         from twisted.internet import reactor
 
         inheritedSocket = self.dispatcher.addSocket()
-        inheritedFD = inheritedSocket.fileno()
+        inheritedFD = inheritedSocket.childSocket().fileno()
 
         processProtocol = ChildProcessProtocol(self, inheritedSocket)
 
@@ -491,12 +471,14 @@
     # FIXME: deserialize log events from child
     # log = Logger()
 
-
     def __init__(self, service, inheritedSocket):
         self.service = service
         self.inheritedSocket = inheritedSocket
 
+        # Always tell any metafd socket that we have started, so it can re-initialize state.
+        self.inheritedSocket.start()
 
+
     def outReceived(self, data):
         # self.log.info(u"{data}", data=data)
         sys.stdout.write(data)
@@ -509,6 +491,8 @@
 
 
     def processExited(self, reason):
+        # Always tell any metafd socket that we have started, so it can re-initialize state.
+        self.inheritedSocket.stop()
         self.service.childDidExit(self, reason)
 
 
@@ -684,7 +668,7 @@
 
 
 
- at total_ordering
+ at implementer(IStatus)
 class ChildStatus(FancyStrMixin, object):
     """
     The status of a child process.
@@ -693,11 +677,26 @@
     showAttributes = (
         "acknowledged",
         "unacknowledged",
+        "total",
+        "started",
+        "abandoned",
         "unclosed",
+        "starting",
+        "stopped",
     )
 
 
-    def __init__(self, acknowledged=0, unacknowledged=0, unclosed=0):
+    def __init__(
+        self,
+        acknowledged=0,
+        unacknowledged=0,
+        total=0,
+        started=0,
+        abandoned=0,
+        unclosed=0,
+        starting=0,
+        stopped=0
+    ):
         """
         Create a L{ConnectionStatus} with a number of sent connections and a
         number of un-acknowledged connections.
@@ -710,12 +709,32 @@
             the subprocess which have never received a status response (a
             "C{+}" status message).
 
+        @param total: The total number of acknowledged connections over
+            the lifetime of this socket.
+
+        @param started: The number of times this worker has been started.
+
+        @param abandoned: The number of connections which have been sent to
+            this worker, but were not acknowledged at the moment that the
+            worker was stopped.
+
         @param unclosed: The number of sockets which have been sent to the
             subprocess but not yet closed.
+
+        @param starting: The process that owns this socket is starting. Do not
+            dispatch to it until we receive the started message.
+
+        @param stopped: The process that owns this socket has stopped. Do not
+            dispatch to it.
         """
         self.acknowledged = acknowledged
         self.unacknowledged = unacknowledged
+        self.total = total
+        self.started = started
+        self.abandoned = abandoned
         self.unclosed = unclosed
+        self.starting = starting
+        self.stopped = stopped
 
 
     def effectiveLoad(self):
@@ -725,37 +744,64 @@
         return self.acknowledged + self.unacknowledged
 
 
-    def _tuplify(self):
-        return tuple(getattr(self, attr) for attr in self.showAttributes)
+    def active(self):
+        """
+        Is the subprocess associated with this socket available to dispatch to.
+        i.e, this socket is neither stopped nor starting
+        """
+        return self.starting == 0 and self.stopped == 0
 
 
-    def __lt__(self, other):
-        if not isinstance(other, ChildStatus):
-            return NotImplemented
+    def start(self):
+        """
+        The child process for this L{WorkerStatus} is about to (re)start. Reset the status to indicate it
+        is starting - that should prevent any new connections being dispatched.
+        """
+        return self.reset(
+            starting=1,
+            stopped=0,
+        )
 
-        return self.effectiveLoad() < other.effectiveLoad()
 
+    def restarted(self):
+        """
+        The child process for this L{WorkerStatus} has indicated it is now available to accept
+        connections, so reset the starting status so this socket will be available for dispatch.
+        """
+        return self.reset(
+            started=self.started + 1,
+            starting=0,
+        )
 
-    def __eq__(self, other):
-        if not isinstance(other, ChildStatus):
-            return NotImplemented
 
-        return self._tuplify() == other._tuplify()
+    def stop(self):
+        """
+        The child process for this L{WorkerStatus} has stopped. Stop the socket and clear out
+        existing counters, but track abandoned connections.
+        """
+        return self.reset(
+            acknowledged=0,
+            unacknowledged=0,
+            abandoned=self.abandoned + self.unacknowledged,
+            starting=0,
+            stopped=1,
+        )
 
 
-    def __add__(self, other):
-        if not isinstance(other, ChildStatus):
-            return NotImplemented
+    def adjust(self, **kwargs):
+        """
+        Update the L{WorkerStatus} by adding the supplied values to the specified attributes.
+        """
+        for k, v in kwargs.items():
+            newval = getattr(self, k) + v
+            setattr(self, k, max(newval, 0))
+        return self
 
-        a = self._tuplify()
-        b = other._tuplify()
-        sum = [a1 + b1 for (a1, b1) in zip(a, b)]
 
-        return self.__class__(*sum)
-
-
-    def __sub__(self, other):
-        if not isinstance(other, ChildStatus):
-            return NotImplemented
-
-        return self + self.__class__(*[-x for x in other._tuplify()])
+    def reset(self, **kwargs):
+        """
+        Reset the L{WorkerStatus} by setting the supplied values in the specified attributes.
+        """
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        return self

Modified: twext/trunk/twext/internet/sendfdport.py
===================================================================
--- twext/trunk/twext/internet/sendfdport.py	2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/internet/sendfdport.py	2014-03-07 20:48:27 UTC (rev 12844)
@@ -116,9 +116,10 @@
     @type dispatcher: L{InheritedSocketDispatcher}
     """
 
-    def __init__(self, dispatcher, inSocket, outSocket, status):
+    def __init__(self, dispatcher, inSocket, outSocket, status, slavenum):
         FileDescriptor.__init__(self, dispatcher.reactor)
         self.status = status
+        self.slavenum = slavenum
         self.dispatcher = dispatcher
         self.inSocket = inSocket
         self.outSocket = outSocket   # XXX needs to be set non-blocking by somebody
@@ -127,6 +128,49 @@
         self.pendingCloseSocketQueue = []
 
 
+    def childSocket(self):
+        """
+        Return the socket that the child process will use to communicate with the master.
+        """
+        return self.inSocket
+
+
+    def start(self):
+        """
+        The master process monitor is about to start the child process associated with this socket.
+        Update status to ensure dispatcher know what is going on.
+        """
+        self.status.start()
+        self.dispatcher.statusChanged()
+
+
+    def restarted(self):
+        """
+        The child process associated with this socket has signaled it is ready.
+        Update status to ensure dispatcher know what is going on.
+        """
+        self.status.restarted()
+        self.dispatcher.statusChanged()
+
+
+    def stop(self):
+        """
+        The master process monitor has determined the child process associated with this socket
+        has died. Update status to ensure dispatcher know what is going on.
+        """
+        self.status.stop()
+        self.dispatcher.statusChanged()
+
+
+    def remove(self):
+        """
+        Remove this socket.
+        """
+        self.status.stop()
+        self.dispatcher.statusChanged()
+        self.dispatcher.removeSocket()
+
+
     def sendSocketToPeer(self, skt, description):
         """
         Enqueue a socket to send to the subprocess.
@@ -174,6 +218,51 @@
 
 
 
+class IStatus(Interface):
+    """
+    Defines the status of a socket. This keeps track of active connections etc.
+    """
+
+    def effective():
+        """
+        The current effective load.
+
+        @return: The current effective load.
+        @rtype: L{int}
+        """
+
+    def active():
+        """
+        Whether the socket should be active (able to be dispatched to).
+
+        @return: Active state.
+        @rtype: L{bool}
+        """
+
+    def start():
+        """
+        Worker process is starting. Mark status accordingly but do not make
+        it active.
+
+        @return: C{self}
+        """
+
+    def restarted():
+        """
+        Worker process has signaled it is ready so make this active.
+
+        @return: C{self}
+        """
+
+    def stop():
+        """
+        Worker process has stopped so make this inactive.
+
+        @return: C{self}
+        """
+
+
+
 class IStatusWatcher(Interface):
     """
     A provider of L{IStatusWatcher} tracks the I{status messages} reported by
@@ -209,7 +298,6 @@
         @return: the new status.
         """
 
-
     def newConnectionStatus(previousStatus):
         """
         A new connection was sent to a given socket.  Compute its status based
@@ -221,7 +309,6 @@
         @return: the socket's status after incrementing its outstanding work.
         """
 
-
     def statusFromMessage(previousStatus, message):
         """
         A status message was received by a worker.  Convert the previous status
@@ -235,7 +322,6 @@
             account.
         """
 
-
     def closeCountFromStatus(previousStatus):
         """
         Based on a status previously returned from a method on this
@@ -254,7 +340,7 @@
     list of available sockets that connect to I{worker process}es and sends
     inbound connections to be inherited over those sockets, by those processes.
 
-    L{InheritedSocketDispatcher} is therefore insantiated in the I{master
+    L{InheritedSocketDispatcher} is therefore instantiated in the I{master
     process}.
 
     @ivar statusWatcher: The object which will handle status messages and
@@ -276,27 +362,43 @@
     @property
     def statuses(self):
         """
-        Yield the current status of all subprocess sockets.
+        Yield the current status of all subprocess sockets in the current priority order.
         """
         for subsocket in self._subprocessSockets:
             yield subsocket.status
 
 
+    @property
+    def slavestates(self):
+        """
+        Yield the current status of all subprocess sockets, ordered by slave number.
+        """
+        for subsocket in sorted(self._subprocessSockets, key=lambda x: x.slavenum):
+            yield (subsocket.slavenum, subsocket.status,)
+
+
+    def statusChanged(self):
+        """
+        Someone is telling us a child socket status changed.
+        """
+        self.statusWatcher.statusesChanged(self.statuses)
+
+
     def statusMessage(self, subsocket, message):
         """
         The status of a connection has changed; update all registered status
         change listeners.
         """
-        watcher = self.statusWatcher
-        status = watcher.statusFromMessage(subsocket.status, message)
-        closeCount, subsocket.status = watcher.closeCountFromStatus(status)
-        watcher.statusesChanged(self.statuses)
+        status = self.statusWatcher.statusFromMessage(subsocket.status, message)
+        closeCount, subsocket.status = self.statusWatcher.closeCountFromStatus(status)
+        self.statusChanged()
         return closeCount
 
 
     def sendFileDescriptor(self, skt, description):
         """
-        A connection has been received.  Dispatch it.
+        A connection has been received.  Dispatch it to active sockets, sorted by
+        how much work they have.
 
         @param skt: the I{connection socket} (i.e.: not the listening socket)
         @type skt: L{socket.socket}
@@ -305,23 +407,15 @@
             L{InheritedPort} what type of transport to create for this socket.
         @type description: C{bytes}
         """
-        # We want None to sort after 0 and before 1, so coerce to 0.5 - this
-        # allows the master to first schedule all child process that are up but
-        # not yet busy ahead of those that are still starting up.
-        def sortKey(conn):
-            if conn.status is None:
-                return 0.5
-            else:
-                return conn.status
-        self._subprocessSockets.sort(key=sortKey)
-        selectedSocket = self._subprocessSockets[0]
+        self._subprocessSockets.sort(key=lambda x: x.status.effective())
+        selectedSocket = filter(lambda x: x.status.active(), self._subprocessSockets)[0]
         selectedSocket.sendSocketToPeer(skt, description)
         # XXX Maybe want to send along 'description' or 'skt' or some
         # properties thereof? -glyph
         selectedSocket.status = self.statusWatcher.newConnectionStatus(
             selectedSocket.status
         )
-        self.statusWatcher.statusesChanged(self.statuses)
+        self.statusChanged()
 
 
     def startDispatching(self):
@@ -333,7 +427,7 @@
             subSocket.startReading()
 
 
-    def addSocket(self, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
+    def addSocket(self, slavenum=0, socketpair=lambda: socketpair(AF_UNIX, SOCK_DGRAM)):
         """
         Add a L{send1msg}-oriented AF_UNIX socket to the pool of sockets being
         used for transmitting file descriptors to child processes.
@@ -345,11 +439,11 @@
         i, o = socketpair()
         i.setblocking(False)
         o.setblocking(False)
-        a = _SubprocessSocket(self, i, o, self.statusWatcher.initialStatus())
+        a = _SubprocessSocket(self, i, o, self.statusWatcher.initialStatus(), slavenum)
         self._subprocessSockets.append(a)
         if self._isDispatching:
             a.startReading()
-        return i
+        return a
 
 
     def removeSocket(self, skt):
@@ -357,12 +451,7 @@
         Removes a previously added socket from the pool of sockets being used
         for transmitting file descriptors to child processes.
         """
-        for a in self._subprocessSockets:
-            if a.inSocket == skt:
-                self._subprocessSockets.remove(a)
-                break
-        else:
-            raise ValueError("Unknown socket: {0}".format(skt))
+        self._subprocessSockets.remove(skt)
 
 
 

Modified: twext/trunk/twext/internet/test/test_sendfdport.py
===================================================================
--- twext/trunk/twext/internet/test/test_sendfdport.py	2014-03-07 20:38:57 UTC (rev 12843)
+++ twext/trunk/twext/internet/test/test_sendfdport.py	2014-03-07 20:48:27 UTC (rev 12844)
@@ -1,4 +1,3 @@
-from twext.internet.sendfdport import IStatusWatcher
 # -*- test-case-name: twext.internet.test.test_sendfdport -*-
 ##
 # Copyright (c) 2010-2014 Apple Inc. All rights reserved.
@@ -27,6 +26,7 @@
 from zope.interface import implementer
 
 from twext.internet.sendfdport import InheritedSocketDispatcher
+from twext.internet.sendfdport import IStatusWatcher, IStatus
 
 from twisted.internet.interfaces import IReactorFDSet
 from twisted.trial.unittest import TestCase
@@ -101,6 +101,38 @@
 
 
 
+ at verifiedImplementer(IStatus)
+class Status(object):
+    def __init__(self):
+        self.count = 0
+        self.available = False
+
+
+    def effective(self):
+        return self.count
+
+
+    def active(self):
+        return self.available
+
+
+    def start(self):
+        self.available = False
+        return self
+
+
+    def restarted(self):
+        self.available = True
+        return self
+
+
+    def stop(self):
+        self.count = 0
+        self.available = False
+        return self
+
+
+
 @verifiedImplementer(IStatusWatcher)
 class Watcher(object):
     def __init__(self, q):
@@ -109,19 +141,21 @@
 
 
     def newConnectionStatus(self, previous):
-        return previous + 1
+        previous.count += 1
+        return previous
 
 
     def statusFromMessage(self, previous, message):
-        return previous - 1
+        previous.count -= 1
+        return previous
 
 
     def statusesChanged(self, statuses):
-        self.q.append(list(statuses))
+        self.q.append([(status.count, status.available) for status in statuses])
 
 
     def initialStatus(self):
-        return 0
+        return Status()
 
 
     def closeCountFromStatus(self, status):
@@ -164,9 +198,10 @@
         two = SocketForClosing()
         three = SocketForClosing()
 
-        self.dispatcher.addSocket(
+        skt = self.dispatcher.addSocket(
             lambda: (SocketForClosing(), SocketForClosing())
         )
+        skt.restarted()
 
         self.dispatcher.sendFileDescriptor(one, "one")
         self.dispatcher.sendFileDescriptor(two, "two")
@@ -228,10 +263,11 @@
         dispatcher.statusWatcher = Watcher(q)
         description = "whatever"
         # Need to have a socket that will accept the descriptors.
-        dispatcher.addSocket()
+        skt = dispatcher.addSocket()
+        skt.restarted()
         dispatcher.sendFileDescriptor(object(), description)
         dispatcher.sendFileDescriptor(object(), description)
-        self.assertEquals(q, [[1], [2]])
+        self.assertEquals(q, [[(0, True)], [(1, True)], [(2, True)]])
 
 
     def test_statusesChangedOnStatusMessage(self):
@@ -249,4 +285,33 @@
         subskt = dispatcher._subprocessSockets[0]
         dispatcher.statusMessage(subskt, message)
         dispatcher.statusMessage(subskt, message)
-        self.assertEquals(q, [[-1], [-2]])
+        self.assertEquals(q, [[(-1, False)], [(-2, False)]])
+
+
+    def test_statusesChangedOnStartRestartStop(self):
+        """
+        L{_SubprocessSocket} will update its C{status} when state change.
+        """
+        q = []
+        dispatcher = self.dispatcher
+        dispatcher.statusWatcher = Watcher(q)
+        message = "whatever"
+        # Need to have a socket that will accept the descriptors.
+        subskt = dispatcher.addSocket()
+        subskt.start()
+        subskt.restarted()
+        dispatcher.sendFileDescriptor(subskt, message)
+        subskt.stop()
+        subskt.start()
+        subskt.restarted()
+        self.assertEquals(
+            q,
+            [
+                [(0, False)],
+                [(0, True)],
+                [(1, True)],
+                [(0, False)],
+                [(0, False)],
+                [(0, True)],
+            ]
+        )
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140307/10691e36/attachment-0001.html>


More information about the calendarserver-changes mailing list