[CalendarServer-changes] [11521] CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jul 17 10:27:32 PDT 2013
Revision: 11521
http://trac.calendarserver.org//changeset/11521
Author: sagen at apple.com
Date: 2013-07-17 10:27:32 -0700 (Wed, 17 Jul 2013)
Log Message:
-----------
Backport of group cacher changes from trunk: Don't wait for group cacher to finish before scheduling the next one, and use deferToThread so we don't block.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_directory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/upgrade.py
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/calendarserver/tap/caldav.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/calendarserver/tap/caldav.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -705,6 +705,7 @@
directory,
config.GroupCaching.UpdateSeconds,
config.GroupCaching.ExpireSeconds,
+ config.GroupCaching.LockSeconds,
namespace=config.GroupCaching.MemcachedPool,
useExternalProxies=config.GroupCaching.UseExternalProxies
)
@@ -1019,6 +1020,7 @@
directory,
config.GroupCaching.UpdateSeconds,
config.GroupCaching.ExpireSeconds,
+ config.GroupCaching.LockSeconds,
namespace=config.GroupCaching.MemcachedPool,
useExternalProxies=config.GroupCaching.UseExternalProxies
)
@@ -1402,6 +1404,7 @@
directory,
config.GroupCaching.UpdateSeconds,
config.GroupCaching.ExpireSeconds,
+ config.GroupCaching.LockSeconds,
namespace=config.GroupCaching.MemcachedPool,
useExternalProxies=config.GroupCaching.UseExternalProxies
)
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -569,13 +569,14 @@
"""
def __init__(self, namespace, pickle=True, no_invalidation=False,
- key_normalization=True, expireSeconds=0):
+ key_normalization=True, expireSeconds=0, lockSeconds=60):
super(GroupMembershipCache, self).__init__(namespace, pickle=pickle,
no_invalidation=no_invalidation,
key_normalization=key_normalization)
self.expireSeconds = expireSeconds
+ self.lockSeconds = lockSeconds
def setGroupsFor(self, guid, memberships):
@@ -614,7 +615,35 @@
returnValue(value is not None)
+ def acquireLock(self):
+ """
+ Acquire a memcached lock named group-cacher-lock
+ return: Deferred firing True if successful, False if someone already has
+ the lock
+ """
+ self.log_debug("add group-cacher-lock")
+ return self.add("group-cacher-lock", "1", expireTime=self.lockSeconds)
+
+
+ def extendLock(self):
+ """
+ Update the expiration time of the memcached lock
+ Return: Deferred firing True if successful, False otherwise
+ """
+ self.log_debug("extend group-cacher-lock")
+ return self.set("group-cacher-lock", "1", expireTime=self.lockSeconds)
+
+
+ def releaseLock(self):
+ """
+ Release the memcached lock
+ Return: Deferred firing True if successful, False otherwise
+ """
+ self.log_debug("delete group-cacher-lock")
+ return self.delete("group-cacher-lock")
+
+
class GroupMembershipCacheUpdater(LoggingMixIn):
"""
Responsible for updating memcached with group memberships. This will run
@@ -623,7 +652,7 @@
"""
def __init__(self, proxyDB, directory, updateSeconds, expireSeconds,
- cache=None, namespace=None, useExternalProxies=False,
+ lockSeconds, cache=None, namespace=None, useExternalProxies=False,
externalProxiesSource=None):
self.proxyDB = proxyDB
self.directory = directory
@@ -635,7 +664,8 @@
if cache is None:
assert namespace is not None, "namespace must be specified if GroupMembershipCache is not provided"
- cache = GroupMembershipCache(namespace, expireSeconds=expireSeconds)
+ cache = GroupMembershipCache(namespace, expireSeconds=expireSeconds,
+ lockSeconds=lockSeconds)
self.cache = cache
@@ -723,6 +753,8 @@
# TODO: add memcached eviction protection
+ useLock = True
+
# See if anyone has completely populated the group membership cache
isPopulated = (yield self.cache.isPopulated())
@@ -733,6 +765,9 @@
self.log_info("Group membership cache is already populated")
returnValue((fast, 0))
+ # We don't care what others are doing right now, we need to update
+ useLock = False
+
self.log_info("Updating group membership cache")
dataRoot = FilePath(config.DataRoot)
@@ -750,6 +785,14 @@
previousMembers = pickle.loads(membershipsCacheFile.getContent())
callGroupsChanged = True
+ if useLock:
+ self.log_info("Attempting to acquire group membership cache lock")
+ acquiredLock = (yield self.cache.acquireLock())
+ if not acquiredLock:
+ self.log_info("Group membership cache lock held by another process")
+ returnValue((fast, 0))
+ self.log_info("Acquired lock")
+
if not fast and self.useExternalProxies:
# Load in cached copy of external proxies so we can diff against them
@@ -759,11 +802,17 @@
(extProxyCacheFile.path,))
previousAssignments = pickle.loads(extProxyCacheFile.getContent())
+ if useLock:
+ yield self.cache.extendLock()
+
self.log_info("Retrieving proxy assignments from directory")
assignments = self.externalProxiesSource()
self.log_info("%d proxy assignments retrieved from directory" %
(len(assignments),))
+ if useLock:
+ yield self.cache.extendLock()
+
changed, removed = diffAssignments(previousAssignments, assignments)
# changed is the list of proxy assignments (either new or updates).
# removed is the list of principals who used to have an external
@@ -919,6 +968,10 @@
yield self.cache.setPopulatedMarker()
+ if useLock:
+ self.log_info("Releasing lock")
+ yield self.cache.releaseLock()
+
self.log_info("Group memberships cache updated")
returnValue((fast, len(members), len(changedMembers)))
@@ -936,16 +989,19 @@
groupCacher = getattr(self.transaction, "_groupCacher", None)
if groupCacher is not None:
+
+ # Schedule next update
+ notBefore = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=groupCacher.updateSeconds))
+ log.debug("Scheduling next group cacher update: %s" % (notBefore,))
+ yield self.transaction.enqueue(GroupCacherPollingWork,
+ notBefore=notBefore)
+
try:
- yield groupCacher.updateCache()
+ groupCacher.updateCache()
except Exception, e:
log.error("Failed to update group membership cache (%s)" % (e,))
- finally:
- notBefore = (datetime.datetime.utcnow() +
- datetime.timedelta(seconds=groupCacher.updateSeconds))
- log.debug("Scheduling next group cacher update: %s" % (notBefore,))
- yield self.transaction.enqueue(GroupCacherPollingWork,
- notBefore=notBefore)
+
else:
notBefore = (datetime.datetime.utcnow() +
datetime.timedelta(seconds=10))
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -58,6 +58,7 @@
from twistedcaldav.directory.augment import AugmentRecord
from twistedcaldav.directory.util import splitIntoBatches
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
+from twisted.internet.threads import deferToThread
from twext.web2.http import HTTPError, StatusResponse
from twext.web2 import responsecode
@@ -1141,6 +1142,7 @@
return succeed(records)
+ @inlineCallbacks
def recordsMatchingFields(self, fields, operand="or", recordType=None):
"""
Carries out the work of a principal-property-search against LDAP
@@ -1187,10 +1189,10 @@
# Query the LDAP server
self.log_debug("LDAP search %s %s %s" %
(ldap.dn.dn2str(base), scope, filterstr))
- results = self.timedSearch(ldap.dn.dn2str(base), scope,
+ results = (yield deferToThread(self.timedSearch, ldap.dn.dn2str(base), scope,
filterstr=filterstr, attrlist=self.attrlist,
timeoutSeconds=self.requestTimeoutSeconds,
- resultLimit=self.requestResultsLimit)
+ resultLimit=self.requestResultsLimit))
self.log_debug("LDAP search returned %d results" % (len(results),))
numMissingGuids = 0
numMissingRecordNames = 0
@@ -1228,7 +1230,7 @@
(numMissingRecordNames, recordType))
self.log_debug("Principal property search matched %d records" % (len(records),))
- return succeed(records)
+ returnValue(records)
@inlineCallbacks
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_directory.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_directory.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -172,7 +172,7 @@
self.directoryService.groupMembershipCache = cache
updater = GroupMembershipCacheUpdater(
- calendaruserproxy.ProxyDBService, self.directoryService, 30, 30,
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
cache=cache, useExternalProxies=False)
# Exercise getGroups()
@@ -371,7 +371,7 @@
]
updater = GroupMembershipCacheUpdater(
- calendaruserproxy.ProxyDBService, self.directoryService, 30, 30,
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
cache=cache, useExternalProxies=True,
externalProxiesSource=fakeExternalProxies)
@@ -455,7 +455,7 @@
]
updater = GroupMembershipCacheUpdater(
- calendaruserproxy.ProxyDBService, self.directoryService, 30, 30,
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
cache=cache, useExternalProxies=True,
externalProxiesSource=fakeExternalProxiesRemoved)
@@ -622,7 +622,7 @@
self.directoryService.groupMembershipCache = cache
updater = GroupMembershipCacheUpdater(
- calendaruserproxy.ProxyDBService, self.directoryService, 30, 30,
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
cache=cache)
dataRoot = FilePath(config.DataRoot)
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -1593,7 +1593,7 @@
cache = GroupMembershipCache("ProxyDB", expireSeconds=60)
self.service.groupMembershipCache = cache
updater = GroupMembershipCacheUpdater(calendaruserproxy.ProxyDBService,
- self.service, 30, 15, cache=cache, useExternalProxies=False)
+ self.service, 30, 15, 30, cache=cache, useExternalProxies=False)
self.assertEquals((False, 8, 8), (yield updater.updateCache()))
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/stdconfig.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/stdconfig.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -938,7 +938,8 @@
"Enabled": True,
"MemcachedPool" : "Default",
"UpdateSeconds" : 300,
- "ExpireSeconds" : 3600,
+ "ExpireSeconds" : 86400,
+ "LockSeconds" : 600,
"EnableUpdater" : True,
"UseExternalProxies" : False,
},
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/upgrade.py 2013-07-16 20:13:19 UTC (rev 11520)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/upgrade.py 2013-07-17 17:27:32 UTC (rev 11521)
@@ -1075,6 +1075,7 @@
directory,
self.config.GroupCaching.UpdateSeconds,
self.config.GroupCaching.ExpireSeconds,
+ self.config.GroupCaching.LockSeconds,
namespace=self.config.GroupCaching.MemcachedPool,
useExternalProxies=self.config.GroupCaching.UseExternalProxies)
yield updater.updateCache(fast=True)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130717/10166c37/attachment-0001.html>
More information about the calendarserver-changes
mailing list