[CalendarServer-changes] [4577] CalendarServer/branches/users/cdaboo/partition-4464
source_changes at macosforge.org
source_changes at macosforge.org
Fri Oct 9 16:30:18 PDT 2009
Revision: 4577
http://trac.macosforge.org/projects/calendarserver/changeset/4577
Author: cdaboo at apple.com
Date: 2009-10-09 16:30:17 -0700 (Fri, 09 Oct 2009)
Log Message:
-----------
Introduce a split memcached option so that different caches can exist on different memcached
servers.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/sidecar/task.py
CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/principals.py
CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/util.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/cache.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/cachingdirectory.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/mail.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcachepool.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacheprops.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacher.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/notify.py
CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py
Modified: CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/sidecar/task.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/sidecar/task.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/sidecar/task.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -380,12 +380,8 @@
# Configure Memcached Client Pool
#
if config.Memcached.ClientEnabled:
- memcachepool.installPool(
- IPv4Address(
- "TCP",
- config.Memcached.BindAddress,
- config.Memcached.Port,
- ),
+ memcachepool.installPools(
+ config.Memcached.Pools,
config.Memcached.MaxClients,
)
Modified: CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tap/caldav.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -535,12 +535,8 @@
# Configure Memcached Client Pool
#
if config.Memcached.ClientEnabled:
- memcachepool.installPool(
- IPv4Address(
- "TCP",
- config.Memcached.BindAddress,
- config.Memcached.Port,
- ),
+ memcachepool.installPools(
+ config.Memcached.Pools,
config.Memcached.MaxClients,
)
Modified: CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/principals.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/principals.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/principals.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -218,12 +218,8 @@
# Connect to memcached, notifications
#
if config.Memcached.ClientEnabled:
- memcachepool.installPool(
- IPv4Address(
- "TCP",
- config.Memcached.BindAddress,
- config.Memcached.Port,
- ),
+ memcachepool.installPools(
+ config.Memcached.Pools,
config.Memcached.MaxClients
)
if config.Notifications.Enabled:
Modified: CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/util.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/calendarserver/tools/util.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -142,7 +142,7 @@
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
- s.connect((config.Memcached.BindAddress, config.Memcached.Port))
+ s.connect((config.Memcached.Pools.Default.BindAddress, config.Memcached.Pools.Default.Port))
s.close()
except socket.error:
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/cache.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/cache.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/cache.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -61,9 +61,10 @@
class MemcacheChangeNotifier(LoggingMixIn, CachePoolUserMixIn):
- def __init__(self, resource, cachePool=None):
+ def __init__(self, resource, cachePool=None, cacheHandle="Default"):
self._resource = resource
self._cachePool = cachePool
+ self._cachePoolHandle = cacheHandle
def _newCacheToken(self):
return str(uuid.uuid4())
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/cachingdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/cachingdirectory.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/cachingdirectory.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -154,7 +154,7 @@
def _getMemcacheClient(self, refresh=False):
if refresh or not hasattr(self, "memcacheClient"):
self.memcacheClient = memcacheclient.ClientFactory.getClient(['%s:%s' %
- (config.Memcached.BindAddress, config.Memcached.Port)],
+ (config.Memcached.Pools.Default.BindAddress, config.Memcached.Pools.Default.Port)],
debug=0, pickleProtocol=2)
return self.memcacheClient
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/calendaruserproxy.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -334,7 +334,7 @@
found.append(p)
# Make sure any outstanding deletion timer entries for
# existing principals are removed
- yield self._index()._memcacher.clearDeletionTimer(uid)
+ yield self._index().refreshPrincipal(uid)
else:
missing.append(uid)
@@ -442,7 +442,7 @@
def __init__(self, dbID, dbapiName, dbapiArgs, **kwargs):
AbstractADBAPIDatabase.__init__(self, dbID, dbapiName, dbapiArgs, True, **kwargs)
- self._memcacher = ProxyDB.ProxyDBMemcacher("proxyDB")
+ self._memcacher = ProxyDB.ProxyDBMemcacher("ProxyDB")
@inlineCallbacks
def setGroupMembers(self, principalUID, members):
@@ -543,7 +543,7 @@
# No timer was previously set
self.log_debug("Delaying removal of missing proxy principal '%s'" %
(principalUID,))
- self._memcacher.setDeletionTimer(principalUID, delay=delay)
+ yield self._memcacher.setDeletionTimer(principalUID, delay=delay)
returnValue(None)
self.log_warn("Removing missing proxy principal for '%s'" %
@@ -566,8 +566,18 @@
yield self._delete_from_db_member(principalUID)
yield self._memcacher.deleteMembership(principalUID)
- self._memcacher.clearDeletionTimer(principalUID)
+ yield self._memcacher.clearDeletionTimer(principalUID)
+ def refreshPrincipal(self, principalUID):
+ """
+ Bring back to life a principal that was previously deleted.
+
+ @param principalUID:
+ @type principalUID:
+ """
+
+ return self._memcacher.clearDeletionTimer(principalUID)
+
@inlineCallbacks
def getMembers(self, principalUID):
"""
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/principal.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -508,7 +508,7 @@
"""
super(DirectoryPrincipalResource, self).__init__(NotFilePath(isdir=True))
- self.cacheNotifier = self.cacheNotifierFactory(self)
+ self.cacheNotifier = self.cacheNotifierFactory(self, cacheHandle="PrincipalToken")
if self.isCollection():
slash = "/"
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -252,6 +252,7 @@
d.addCallback(check)
return d
+ @inlineCallbacks
def test_setGroupMemberSet(self):
class StubMemberDB(object):
def __init__(self):
@@ -279,14 +280,14 @@
davxml.HRef.fromString(
"/XMLDirectoryService/__uids__/5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1/"))
- proxyGroup.setGroupMemberSet(new_members, None)
+ yield proxyGroup.setGroupMemberSet(new_members, None)
self.assertEquals(
set([str(p) for p in memberdb.members]),
set(["5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1",
"8B4288F6-CC82-491D-8EF9-642EF4F3E7D0"]))
-
+ @inlineCallbacks
def test_setGroupMemberSetNotifiesPrincipalCaches(self):
class StubCacheNotifier(object):
changedCount = 0
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/mail.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/mail.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/mail.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -545,12 +545,8 @@
def makeService(self, options):
if config.Memcached.ClientEnabled:
- memcachepool.installPool(
- IPv4Address(
- "TCP",
- config.Memcached.BindAddress,
- config.Memcached.Port,
- ),
+ memcachepool.installPools(
+ config.Memcached.Pools,
config.Memcached.MaxClients,
)
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcachepool.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcachepool.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcachepool.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -15,6 +15,7 @@
##
from twisted.python.failure import Failure
+from twisted.internet.address import IPv4Address
from twisted.internet.defer import Deferred, fail
from twisted.internet.protocol import ReconnectingClientFactory
@@ -370,22 +371,44 @@
@ivar _cachePool: A saved cachePool.
"""
_cachePool = None
+ _cachePoolHandle = "Default"
def getCachePool(self):
if self._cachePool is None:
- return defaultCachePool()
+ return defaultCachePool(self._cachePoolHandle)
return self._cachePool
-_memCachePool = None
+_memCachePools = {} # Maps a name to a pool object
+_memCachePoolHandler = {} # Maps a handler id to a named pool
-def installPool(serverAddress, maxClients=5, reactor=None):
- global _memCachePool
- _memCachePool = MemCachePool(serverAddress,
+def installPools(pools, maxClients=5, reactor=None):
+
+ for name, pool in pools.items():
+ if pool["Enabled"]:
+ _installPool(
+ name,
+ pool["HandleCacheTypes"],
+ IPv4Address(
+ "TCP",
+ pool["BindAddress"],
+ pool["Port"],
+ ),
+ maxClients,
+ reactor,
+ )
+
+def _installPool(name, handleTypes, serverAddress, maxClients=5, reactor=None):
+
+ pool = MemCachePool(serverAddress,
maxClients=maxClients,
reactor=None)
+ _memCachePools[name] = pool
-def defaultCachePool():
- return _memCachePool
+ for handle in handleTypes:
+ _memCachePoolHandler[handle] = pool
+
+def defaultCachePool(name):
+ return _memCachePoolHandler[name]
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacheprops.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacheprops.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -60,7 +60,9 @@
log.info("Instantiating memcache connection for MemcachePropertyCollection")
- MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient(["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
+ MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient([
+ "%s:%s" % (config.Memcached.Pools.Default.BindAddress, config.Memcached.Pools.Default.Port)
+ ],
debug=0,
pickleProtocol=2,
)
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacher.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/memcacher.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -105,6 +105,7 @@
assert len(namespace) <= Memcacher.NAMESPACE_MAX_LENGTH, "Memcacher namespace must be less than or equal to %s characters long" % (Memcacher.NAMESPACE_MAX_LENGTH,)
self._memcacheProtocol = None
+ self._cachePoolHandle = namespace
self._namespace = namespace
self._pickle = pickle
self._noInvalidation = no_invalidation
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/notify.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/notify.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/notify.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -1297,12 +1297,10 @@
# Configure Memcached Client Pool
#
if config.Memcached.ClientEnabled:
- memcachepool.installPool(
- IPv4Address(
- 'TCP',
- config.Memcached.BindAddress,
- config.Memcached.Port),
- config.Memcached.MaxClients)
+ memcachepool.installPools(
+ config.Memcached.Pools,
+ config.Memcached.MaxClients,
+ )
multiService = service.MultiService()
Modified: CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py 2009-10-09 21:02:38 UTC (rev 4576)
+++ CalendarServer/branches/users/cdaboo/partition-4464/twistedcaldav/stdconfig.py 2009-10-09 23:30:17 UTC (rev 4577)
@@ -429,8 +429,24 @@
"MaxClients": 5,
"ClientEnabled": True,
"ServerEnabled": True,
- "BindAddress": "127.0.0.1",
- "Port": 11211,
+ "Pools": {
+ "Default": {
+ "Enabled": True,
+ "BindAddress": "127.0.0.1",
+ "Port": 11211,
+ "HandleCacheTypes": [
+ "Default",
+ ]
+ },
+ "ProxyDB": {
+ "Enabled": True,
+ "BindAddress": "127.0.0.1",
+ "Port": 11211,
+ "HandleCacheTypes": [
+ "ProxyDB", "PrincipalToken",
+ ]
+ },
+ },
"memcached": "memcached", # Find in PATH
"MaxMemory": 0, # Megabytes
"Options": [],
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20091009/c7555bda/attachment-0001.html>
More information about the calendarserver-changes
mailing list