[CalendarServer-changes] [4578] CalendarServer/branches/users/cdaboo/deployment-partition-4524/ twistedcaldav

source_changes at macosforge.org source_changes at macosforge.org
Fri Oct 9 17:25:52 PDT 2009


Revision: 4578
          http://trac.macosforge.org/projects/calendarserver/changeset/4578
Author:   cdaboo at apple.com
Date:     2009-10-09 17:25:51 -0700 (Fri, 09 Oct 2009)
Log Message:
-----------
Backport from SL branch: introduce a split memcached option so that different caches can exist on different memcached
servers.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/cache.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcachepool.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacheprops.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacher.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/notify.py
    CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/cache.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/cache.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/cache.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -65,11 +65,11 @@
 
 class MemcacheChangeNotifier(LoggingMixIn, CachePoolUserMixIn):
 
-    def __init__(self, resource, cachePool=None):
+    def __init__(self, resource, cachePool=None, cacheHandle="Default"):
         self._resource = resource
         self._cachePool = cachePool
+        self._cachePoolHandle = cacheHandle
 
-
     def _newCacheToken(self):
         return str(uuid.uuid4())
 

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/config.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -396,8 +396,24 @@
         "MaxClients": 5,
         "ClientEnabled": True,
         "ServerEnabled": True,
-        "BindAddress": "127.0.0.1",
-        "Port": 11211,
+        "Pools": {
+            "Default": {
+                "Enabled": True,
+                "BindAddress": "127.0.0.1",
+                "Port": 11211,
+                "HandleCacheTypes": [
+                    "Default",
+                ]
+            },
+            "ProxyDB": {
+                "Enabled": True,
+                "BindAddress": "127.0.0.1",
+                "Port": 11211,
+                "HandleCacheTypes": [
+                    "ProxyDB", "PrincipalToken",
+                ]
+            },
+        },
         "memcached": "memcached", # Find in PATH
         "MaxMemory": 0, # Megabytes
         "Options": [],

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/calendaruserproxy.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -330,7 +330,7 @@
                 found.append(p)
                 # Make sure any outstanding deletion timer entries for
                 # existing principals are removed
-                yield self._index()._memcacher.clearDeletionTimer(uid)
+                yield self._index().refreshPrincipal(uid)
             else:
                 missing.append(uid)
 
@@ -434,7 +434,7 @@
     def __init__(self, dbID, dbapiName, dbapiArgs, **kwargs):
         AbstractADBAPIDatabase.__init__(self, dbID, dbapiName, dbapiArgs, True, **kwargs)
         
-        self._memcacher = ProxyDB.ProxyDBMemcacher("proxyDB")
+        self._memcacher = ProxyDB.ProxyDBMemcacher("ProxyDB")
 
     @inlineCallbacks
     def setGroupMembers(self, principalUID, members):
@@ -560,6 +560,16 @@
         yield self._memcacher.deleteMembership(principalUID)
         yield self._memcacher.clearDeletionTimer(principalUID)
 
+    def refreshPrincipal(self, principalUID):
+        """
+        Bring back to life a principal that was previously deleted.
+
+        @param principalUID:
+        @type principalUID:
+        """
+        
+        return self._memcacher.clearDeletionTimer(principalUID)
+
     @inlineCallbacks
     def getMembers(self, principalUID):
         """

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/directory/principal.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -370,7 +370,7 @@
         """
         super(DirectoryPrincipalResource, self).__init__(NotFilePath(isdir=True))
 
-        self.cacheNotifier = self.cacheNotifierFactory(self)
+        self.cacheNotifier = self.cacheNotifierFactory(self, cacheHandle="PrincipalToken")
 
         if self.isCollection():
             slash = "/"

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcachepool.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcachepool.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcachepool.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -15,6 +15,7 @@
 ##
 
 from twisted.python.failure import Failure
+from twisted.internet.address import IPv4Address
 from twisted.internet.defer import Deferred, fail
 from twisted.internet.protocol import ReconnectingClientFactory
 
@@ -343,22 +344,44 @@
     @ivar _cachePool: A saved cachePool.
     """
     _cachePool = None
+    _cachePoolHandle = "Default"
 
     def getCachePool(self):
         if self._cachePool is None:
-            return defaultCachePool()
+            return defaultCachePool(self._cachePoolHandle)
 
         return self._cachePool
 
 
 
-_memCachePool = None
+_memCachePools = {}         # Maps a name to a pool object
+_memCachePoolHandler = {}   # Maps a handler id to a named pool
 
-def installPool(serverAddress, maxClients=5, reactor=None):
-    global _memCachePool
-    _memCachePool = MemCachePool(serverAddress,
+def installPools(pools, maxClients=5, reactor=None):
+    
+    for name, pool in pools.items():
+        if pool["Enabled"]:
+            _installPool(
+                name,
+                pool["HandleCacheTypes"],
+                IPv4Address(
+                    "TCP",
+                    pool["BindAddress"],
+                    pool["Port"],
+                ),
+                maxClients,
+                reactor,
+            )
+
+def _installPool(name, handleTypes, serverAddress, maxClients=5, reactor=None):
+
+    pool = MemCachePool(serverAddress,
                                  maxClients=maxClients,
                                  reactor=None)
+    _memCachePools[name] = pool
 
-def defaultCachePool():
-    return _memCachePool
+    for handle in handleTypes:
+        _memCachePoolHandler[handle] = pool
+
+def defaultCachePool(name):
+    return _memCachePoolHandler[name]

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacheprops.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacheprops.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -60,7 +60,9 @@
 
             log.info("Instantiating memcache connection for MemcachePropertyCollection")
 
-            MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient(["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
+            MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient([
+                    "%s:%s" % (config.Memcached.Pools.Default.BindAddress, config.Memcached.Pools.Default.Port)
+                ],
                 debug=0,
                 pickleProtocol=2,
             )

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacher.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/memcacher.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -82,6 +82,7 @@
         @type no_invalidation: C{bool}
         """
         self._memcacheProtocol = None
+        self._cachePoolHandle = namespace
         self._namespace = namespace
         self._pickle = pickle
         self._noInvalidation = no_invalidation

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/notify.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/notify.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/notify.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -1296,12 +1296,10 @@
         # Configure Memcached Client Pool
         #
         if config.Memcached.ClientEnabled:
-            memcachepool.installPool(
-                IPv4Address(
-                    'TCP',
-                    config.Memcached.BindAddress,
-                    config.Memcached.Port),
-                config.Memcached.MaxClients)
+            memcachepool.installPools(
+                config.Memcached.Pools,
+                config.Memcached.MaxClients,
+            )
 
         multiService = service.MultiService()
 

Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py	2009-10-09 23:30:17 UTC (rev 4577)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4524/twistedcaldav/tap.py	2009-10-10 00:25:51 UTC (rev 4578)
@@ -525,12 +525,10 @@
         # Configure Memcached Client Pool
         #
         if config.Memcached["ClientEnabled"]:
-            memcachepool.installPool(
-                IPv4Address(
-                    'TCP',
-                    config.Memcached["BindAddress"],
-                    config.Memcached["Port"]),
-                config.Memcached["MaxClients"])
+            memcachepool.installPools(
+                config.Memcached.Pools,
+                config.Memcached.MaxClients,
+            )
 
         #
         # Configure NotificationClient
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20091009/692eb20c/attachment-0001.html>


More information about the calendarserver-changes mailing list