[CalendarServer-changes] [13815] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jul 31 21:41:47 PDT 2014
Revision: 13815
http://trac.calendarserver.org//changeset/13815
Author: gaya at apple.com
Date: 2014-07-31 21:41:47 -0700 (Thu, 31 Jul 2014)
Log Message:
-----------
delete cached groups after they go missing from the directory for config.AutomaticPurging.GroupPurgeIntervalSeconds
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/who/groups.py
CalendarServer/trunk/txdav/who/test/test_groups.py
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2014-08-01 02:18:10 UTC (rev 13814)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2014-08-01 04:41:47 UTC (rev 13815)
@@ -1084,6 +1084,7 @@
"CheckStaggerSeconds": 0, # No staggering
"PurgeIntervalSeconds": 7 * 24 * 60 * 60, # 7 days
"HomePurgeDelaySeconds": 60,
+ "GroupPurgeIntervalSeconds": 7 * 24 * 60 * 60, # 7 days
},
"Manhole": {
Modified: CalendarServer/trunk/txdav/who/groups.py
===================================================================
--- CalendarServer/trunk/txdav/who/groups.py 2014-08-01 02:18:10 UTC (rev 13814)
+++ CalendarServer/trunk/txdav/who/groups.py 2014-08-01 04:41:47 UTC (rev 13815)
@@ -104,18 +104,12 @@
)
else:
- notBefore = (
- datetime.datetime.utcnow() +
- datetime.timedelta(seconds=10)
- )
log.debug(
"Rescheduling group refresh for {group}: {when}",
- group=self.groupUid, when=notBefore
+ group=self.groupUid,
+ when=datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
)
- yield self.transaction.enqueue(
- GroupRefreshWork,
- groupUID=self.groupUid, notBefore=notBefore
- )
+ yield self.reschedule(self.transaction, 10, groupUID=self.groupUid)
@@ -317,30 +311,46 @@
# Figure out which groups matter
groupUIDs = yield self.groupsToRefresh(txn)
self.log.debug(
- "Number of groups to refresh: {num}", num=len(groupUIDs)
+ "Groups to refresh: {g}", g=groupUIDs
)
+
+ gr = schema.GROUPS
+ if config.AutomaticPurging.Enabled and groupUIDs:
+ # remove unused groups and groups that have not been seen in a while
+ dateLimit = (
+ datetime.datetime.utcnow() -
+ datetime.timedelta(seconds=float(config.AutomaticPurging.GroupPurgeIntervalSeconds))
+ )
+ rows = yield Delete(
+ From=gr,
+ Where=(
+ (gr.EXTANT == 0).And(gr.MODIFIED < dateLimit)
+ ).Or(
+ gr.GROUP_UID.NotIn(
+ Parameter("groupUIDs", len(groupUIDs))
+ )
+ ) if groupUIDs else None,
+ Return=[gr.GROUP_UID]
+ ).on(txn, groupUIDs=groupUIDs)
+ else:
+ # remove unused groups
+ rows = yield Delete(
+ From=gr,
+ Where=gr.GROUP_UID.NotIn(
+ Parameter("groupUIDs", len(groupUIDs))
+ ) if groupUIDs else None,
+ Return=[gr.GROUP_UID]
+ ).on(txn, groupUIDs=groupUIDs)
+ deletedGroupUIDs = [row[0] for row in rows]
+ if deletedGroupUIDs:
+ self.log.debug("Deleted old or unused groups {d}", d=deletedGroupUIDs)
+
# For each of those groups, create a per-group refresh work item
- for groupUID in groupUIDs:
- notBefore = (
- datetime.datetime.utcnow() +
- datetime.timedelta(seconds=1)
- )
+ for groupUID in set(groupUIDs) - set(deletedGroupUIDs):
self.log.debug("Enqueuing group refresh for {u}", u=groupUID)
- yield txn.enqueue(
- GroupRefreshWork, groupUid=groupUID, notBefore=notBefore
- )
- self.log.debug("Enqueued group refresh for {u}", u=groupUID)
+ yield GroupRefreshWork.reschedule(txn, 0, groupUid=groupUID)
- # remove unused groups
- gr = schema.GROUPS
- yield Delete(
- From=gr,
- Where=gr.GROUP_UID.NotIn(
- Parameter("groupUIDs", len(groupUIDs))
- ) if groupUIDs else None
- ).on(txn, groupUIDs=groupUIDs)
-
@inlineCallbacks
def applyExternalAssignments(self, txn, newAssignments):
Modified: CalendarServer/trunk/txdav/who/test/test_groups.py
===================================================================
--- CalendarServer/trunk/txdav/who/test/test_groups.py 2014-08-01 02:18:10 UTC (rev 13814)
+++ CalendarServer/trunk/txdav/who/test/test_groups.py 2014-08-01 04:41:47 UTC (rev 13815)
@@ -18,11 +18,14 @@
Group membership caching implementation tests
"""
+from twext.enterprise.jobqueue import JobItem
from twext.who.idirectory import RecordType
+from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
+from twistedcaldav.stdconfig import config
from twistedcaldav.test.util import StoreTestCase
from txdav.common.icommondatastore import NotFoundError
-from txdav.who.groups import GroupCacher, diffAssignments
+from txdav.who.groups import GroupCacher, diffAssignments, GroupRefreshWork
from txdav.who.test.support import TestRecord, CalendarInMemoryDirectoryService
@@ -498,6 +501,17 @@
yield group.setMembers(members)
+ def doWork(self):
+ self.transaction._groupCacher = groupCacher
+ return unpatchedDoWork(self)
+
+ groupCacher = self.groupCacher
+ unpatchedDoWork = GroupRefreshWork.doWork
+ self.patch(GroupRefreshWork, "doWork", doWork)
+
+ config.AutomaticPurging.Enabled = True
+
+
@inlineCallbacks
def test_extant(self):
"""
@@ -577,7 +591,7 @@
@inlineCallbacks
- def test_update_delete(self):
+ def test_update_delete_unused(self):
"""
Verify that unused groups are deleted from group cache
"""
@@ -596,6 +610,7 @@
txn = store.newTransaction()
yield self.groupCacher.update(txn)
groupID = (yield txn.groupByUID(uid, create=False))[0]
+ yield txn.commit()
self.assertEqual(groupID, None)
@@ -611,8 +626,13 @@
txn = store.newTransaction()
yield self.groupCacher.update(txn)
+ yield txn.commit()
+ yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
+
+ txn = store.newTransaction()
groupID = (yield txn.groupByUID(uid, create=False))[0]
yield txn.commit()
+
self.assertNotEqual(groupID, None)
# delegate group is deleted. unused group is deleted
@@ -628,9 +648,67 @@
txn = store.newTransaction()
yield self.groupCacher.update(txn)
+ yield txn.commit()
+ yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
+
+ txn = store.newTransaction()
testGroupID = (yield txn.groupByUID(u"testgroup", create=False))[0]
emptyGroupID = (yield txn.groupByUID(u"emptygroup", create=False))[0]
yield txn.commit()
self.assertEqual(testGroupID, None)
self.assertNotEqual(emptyGroupID, None)
+
+
+ @inlineCallbacks
+ def test_update_delete_old_nonextant(self):
+ """
+ Verify that old missing groups are deleted from group cache
+ """
+
+ oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds
+ store = self.storeUnderTest()
+
+ for uid in (u"testgroup", u"emptygroup",):
+
+ config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
+ txn = store.newTransaction()
+ groupID = (yield txn.groupByUID(uid))[0]
+ yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=groupID, readWrite=True)
+ (
+ groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
+ extant
+ ) = yield txn.groupByUID(uid, create=False)
+ yield txn.commit()
+
+ self.assertTrue(extant)
+ self.assertNotEqual(groupID, None)
+
+ # Remove the group, still cached
+ yield self.directory.removeRecords([uid])
+ txn = store.newTransaction()
+ yield self.groupCacher.update(txn)
+ (
+ groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
+ extant
+ ) = yield txn.groupByUID(uid, create=False)
+ yield txn.commit()
+ yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
+
+ txn = store.newTransaction()
+ (
+ groupID, _ignore_name, _ignore_membershipHash, _ignore_modified,
+ extant
+ ) = yield txn.groupByUID(uid, create=False)
+ yield txn.commit()
+ self.assertNotEqual(groupID, None)
+ self.assertFalse(extant)
+
+ # delete the group
+ config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
+
+ txn = store.newTransaction()
+ yield self.groupCacher.update(txn)
+ groupID = (yield txn.groupByUID(uid, create=False))[0]
+ yield txn.commit()
+ self.assertEqual(groupID, None)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140731/e9cacce8/attachment-0001.html>
More information about the calendarserver-changes
mailing list