[CalendarServer-changes] [3815] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 11 10:45:47 PDT 2009
Revision: 3815
http://trac.macosforge.org/projects/calendarserver/changeset/3815
Author: sagen at apple.com
Date: 2009-03-11 10:45:45 -0700 (Wed, 11 Mar 2009)
Log Message:
-----------
Rather than immediately remove "missing" principals from the proxy DB, defend against transient directory problems by only removing them after they have been missing for longer than the directory cache refresh period.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/trunk/twistedcaldav/notify.py
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2009-03-11 17:14:04 UTC (rev 3814)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2009-03-11 17:45:45 UTC (rev 3815)
@@ -42,6 +42,7 @@
import itertools
import os
+import time
class PermissionsMixIn (ReadOnlyWritePropertiesResourceMixIn):
def defaultAccessControlList(self):
@@ -338,14 +339,20 @@
p = self.pcollection.principalForUID(uid)
if p:
found.append(p)
+ # Make sure any outstanding deletion timer entries for
+ # existing principals are removed
+ yield self._index()._memcacher.clearDeletionTimer(uid)
else:
missing.append(uid)
# Clean-up ones that are missing
for uid in missing:
self.log_debug("Removing missing proxy principal for '%s' from %s" % (uid, self,))
- yield self._index().removePrincipal(uid)
+ cacheTimeout = config.DirectoryService.params.get("cacheTimeout", 30) * 60 # in seconds
+ yield self._index().removePrincipal(uid,
+ delay=cacheTimeout*2)
+
returnValue(found)
else:
# Fixed proxies
@@ -424,6 +431,33 @@
def deleteMembership(self, guid):
return self.delete("memberships:%s" % (str(guid),))
+ def setDeletionTimer(self, guid, delay):
+ return self.set("del:%s" % (str(guid),), str(self.getTime()+delay))
+
+ def checkDeletionTimer(self, guid):
+ # True means it's overdue, False means it's not, None means no timer
+ def _value(value):
+ if value:
+ if int(value) <= self.getTime():
+ return True
+ else:
+ return False
+ else:
+ return None
+ d = self.get("del:%s" % (str(guid),))
+ d.addCallback(_value)
+ return d
+
+ def clearDeletionTimer(self, guid):
+ return self.delete("del:%s" % (str(guid),))
+
+ def getTime(self):
+ if hasattr(self, 'theTime'):
+ theTime = self.theTime
+ else:
+ theTime = int(time.time())
+ return theTime
+
def __init__(self, path):
path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
super(CalendarUserProxyDatabase, self).__init__(path, True)
@@ -480,13 +514,32 @@
yield self._memcacher.deleteMember(principalUID)
@inlineCallbacks
- def removePrincipal(self, principalUID):
+ def removePrincipal(self, principalUID, delay=None):
"""
Remove a group membership record.
@param principalUID: the UID of the principal to remove.
"""
-
+
+ if delay:
+ # We are going to remove the principal only after <delay> seconds
+ # has passed since we first chose to remove it, to protect against
+ # transient directory problems.
+ # If <delay> is specified, first see if there was a timer set
+ # previously. If the timer is more than delay seconds old, we
+ # go ahead and remove the principal. Otherwise, do nothing.
+
+ overdue = yield self._memcacher.checkDeletionTimer(principalUID)
+
+ if overdue == False:
+ # Do nothing
+ returnValue(None)
+
+ elif overdue is None:
+ # No timer was previously set
+ self._memcacher.setDeletionTimer(principalUID, delay=delay)
+ returnValue(None)
+
for suffix in ("calendar-proxy-read", "calendar-proxy-write",):
groupUID = "%s#%s" % (principalUID, suffix,)
self._delete_from_db(groupUID)
@@ -505,6 +558,7 @@
self._delete_from_db_member(principalUID)
yield self._memcacher.deleteMembership(principalUID)
self._db_commit()
+ self._memcacher.clearDeletionTimer(principalUID)
@inlineCallbacks
def getMembers(self, principalUID):
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-03-11 17:14:04 UTC (rev 3814)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2009-03-11 17:45:45 UTC (rev 3815)
@@ -14,7 +14,7 @@
# limitations under the License.
##
-from twisted.internet.defer import DeferredList, inlineCallbacks
+from twisted.internet.defer import DeferredList, inlineCallbacks, returnValue
from twisted.web2.dav import davxml
from twistedcaldav.directory.directory import DirectoryService
@@ -23,8 +23,10 @@
from twistedcaldav.directory.principal import DirectoryPrincipalResource
from twistedcaldav.directory.xmlaccountsparser import XMLAccountsParser
from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
import twistedcaldav.test.util
+from twistedcaldav.config import config
directoryService = XMLDirectoryService(xmlFile)
@@ -362,8 +364,21 @@
@inlineCallbacks
def test_InvalidUserProxy(self):
+
+ # Set up the in-memory (non-null) memcacher:
+ config.ProcessType = "Single"
+ principal = self._getPrincipalByShortName(
+ DirectoryService.recordType_users, "wsanchez")
+ db = principal._calendar_user_proxy_index()
+
+ # Set the clock to the epoch:
+ theTime = 0
+ db._memcacher.theTime = theTime
+
+
for doMembershipFirst in (True, False,):
for proxyType in ("calendar-proxy-read", "calendar-proxy-write"):
+
principal = self._getPrincipalByShortName(DirectoryService.recordType_users, "wsanchez")
proxyGroup = principal.getChild(proxyType)
@@ -377,6 +392,7 @@
proxyType,
testPrincipal,
)
+
members = yield proxyGroup._index().getMembers(proxyGroup.uid)
self.assertEquals(len(members), 1)
@@ -414,20 +430,66 @@
self.assertEquals(len(memberships), 1)
@inlineCallbacks
- def _membersTest():
+ def _membersTest(theTime):
yield self._groupMembersTest(
DirectoryService.recordType_users, "wsanchez",
proxyType,
("Cyrus Daboo",),
)
+
+ # Trigger the proxy DB clean up, which won't actually
+ # remove anything because we haven't exceeded the timeout
+ yield proxyGroup.groupMembers()
+
+ # Advance 10 seconds
+ theTime += 10
+ db._memcacher.theTime = theTime
+
+ # When we first examine the members, we have not exceeded
+ # the clean-up timeout, so we'll still have 2:
members = yield proxyGroup._index().getMembers(proxyGroup.uid)
+ self.assertEquals(len(members), 2)
+
+ # Restore removed user
+ parser = XMLAccountsParser(directoryService.xmlFile)
+ directoryService._parsedAccounts = parser.items
+
+ # Trigger the proxy DB clean up, which will actually
+ # remove the deletion timer because the principal has been
+ # restored
+ yield proxyGroup.groupMembers()
+
+ # Verify the deletion timer has been removed
+ result = yield db._memcacher.checkDeletionTimer("5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1")
+ self.assertEquals(result, None)
+
+ # Remove the dreid user from the directory service
+ del directoryService._accounts()[DirectoryService.recordType_users]["dreid"]
+
+ # Trigger the proxy DB clean up, which won't actually
+ # remove anything because we haven't exceeded the timeout
+ yield proxyGroup.groupMembers()
+
+ cacheTimeout = config.DirectoryService.params.get("cacheTimeout", 30) * 60 * 2
+ # Advance beyond the timeout
+ theTime += cacheTimeout
+ db._memcacher.theTime = theTime
+
+ # Trigger the proxy DB clean up
+ yield proxyGroup.groupMembers()
+
+ # The missing principal has now been cleaned out of the
+ # proxy DB
+ members = yield proxyGroup._index().getMembers(proxyGroup.uid)
self.assertEquals(len(members), 1)
+ returnValue(theTime)
+
if doMembershipFirst:
yield _membershipTest()
- yield _membersTest()
+ theTime = yield _membersTest(theTime)
else:
- yield _membersTest()
+ theTime = yield _membersTest(theTime)
yield _membershipTest()
# Restore removed user
Modified: CalendarServer/trunk/twistedcaldav/notify.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/notify.py 2009-03-11 17:14:04 UTC (rev 3814)
+++ CalendarServer/trunk/twistedcaldav/notify.py 2009-03-11 17:45:45 UTC (rev 3815)
@@ -368,16 +368,16 @@
# reschedule for delaySeconds in the future
delayed.reset(self.delaySeconds)
self.uris[uri][1] = count
- self.log_info("Delaying: %s" % (uri,))
+ self.log_debug("Delaying: %s" % (uri,))
else:
- self.log_info("Not delaying to avoid starvation: %s" % (uri,))
+ self.log_debug("Not delaying to avoid starvation: %s" % (uri,))
else:
- self.log_info("Scheduling: %s" % (uri,))
+ self.log_debug("Scheduling: %s" % (uri,))
self.uris[uri] = [self.reactor.callLater(self.delaySeconds,
self.delayedEnqueue, op, uri), 0]
def delayedEnqueue(self, op, uri):
- self.log_info("Time to send: %s" % (uri,))
+ self.log_debug("Time to send: %s" % (uri,))
self.uris[uri][1] = 0
for notifier in self.notifiers:
notifier.enqueue(op, uri)
@@ -907,14 +907,14 @@
for child in iq.children[0].children:
jid = child['jid']
if self.allowedInRoster(jid):
- self.log_info("In roster: %s" % (jid,))
+ self.log_debug("In roster: %s" % (jid,))
if not self.roster.has_key(jid):
self.roster[jid] = { 'debug' : False, 'available' : False }
else:
self.log_info("JID not allowed in roster: %s" % (jid,))
def handlePresence(self, iq):
- self.log_info("Presence IQ: %s" %
+ self.log_debug("Presence IQ: %s" %
(iq.toXml().encode('ascii', 'replace')),)
presenceType = iq.getAttribute('type')
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090311/af24f7db/attachment-0001.html>
More information about the calendarserver-changes
mailing list