[CalendarServer-changes] [2533] CalendarServer/branches/memcache-reconnect-2/twistedcaldav/ memcachepool.py

[source_changes at macosforge.org]

Revision: 2533
          http://trac.macosforge.org/projects/calendarserver/changeset/2533
Author:   dreid at apple.com
Date:     2008-06-03 19:21:56 -0700 (Tue, 03 Jun 2008)

Log Message:
-----------
Add some more debug logging, and fix the races that caused too many clients to be used.

Modified Paths:
--------------
    CalendarServer/branches/memcache-reconnect-2/twistedcaldav/memcachepool.py

Modified: CalendarServer/branches/memcache-reconnect-2/twistedcaldav/memcachepool.py
===================================================================
--- CalendarServer/branches/memcache-reconnect-2/twistedcaldav/memcachepool.py	2008-06-03 23:42:34 UTC (rev 2532)
+++ CalendarServer/branches/memcache-reconnect-2/twistedcaldav/memcachepool.py	2008-06-04 02:21:56 UTC (rev 2533)
@@ -119,6 +119,8 @@
 
     @ivar _busyClients: A C{set} that contains all currently busy clients.
     @ivar _freeClients: A C{set} that contains all currently free clients.
+    @ivar _pendingConnects: A C{int} indicating how many connections are in
+        progress.
     """
     clientFactory = MemCacheClientFactory
 
@@ -140,6 +142,7 @@
 
         self._busyClients = set([])
         self._freeClients = set([])
+        self._pendingConnects = 0
         self._commands = []
 
 
@@ -153,6 +156,13 @@
                 self._serverAddress,))
         self._logClientStats()
 
+        self._pendingConnects += 1
+
+        def _connected(client):
+            self._pendingConnects -= 1
+
+            return client
+
         factory = self.clientFactory()
 
         factory.connectionPool = self
@@ -160,9 +170,12 @@
         self._reactor.connectTCP(self._serverAddress.host,
                                  self._serverAddress.port,
                                  factory)
-        return factory.deferred
+        d = factory.deferred
 
+        d.addCallback(_connected)
+        return d
 
+
     def _performRequestOnClient(self, client, command, *args, **kwargs):
         """
         Perform the given request on the given client.
@@ -184,6 +197,7 @@
             self.clientFree(client)
             return result
 
+        self.clientBusy(client)
         method = getattr(client, command, None)
         if method is not None:
             d = method(*args, **kwargs)
@@ -210,14 +224,16 @@
         """
         if len(self._freeClients) > 0:
             client = self._freeClients.pop()
-            self.clientBusy(client)
 
             d = self._performRequestOnClient(
                 client, command, *args, **kwargs)
 
-        elif len(self._busyClients) >= self._maxClients:
+        elif len(self._busyClients) + self._pendingConnects >= self._maxClients:
             d = Deferred()
             self._commands.append((d, command, args, kwargs))
+            self.log_debug("Command queued: %s, %r, %r" % (
+                    command, args, kwargs))
+            self._logClientStats()
 
         else:
             d = self._newClientConnection()
@@ -228,9 +244,12 @@
 
 
     def _logClientStats(self):
-        self.log_debug("Clients #free: %d, #busy: %d" % (
+        self.log_debug("Clients #free: %d, #busy: %d, "
+                       "#pending: %d, #queued: %d" % (
                 len(self._freeClients),
-                len(self._busyClients)))
+                len(self._busyClients),
+                self._pendingConnects,
+                len(self._commands)))
 
 
     def clientGone(self, client):
@@ -277,6 +296,11 @@
 
         if len(self._commands) > 0:
             d, command, args, kwargs = self._commands.pop(0)
+
+            self.log_debug("Performing Queued Command: %s, %r, %r" % (
+                    command, args, kwargs))
+            self._logClientStats()
+
             _ign_d = self.performRequest(
                 command, *args, **kwargs)
 

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080603/17248785/attachment.htm