[CalendarServer-changes] [9777] CalendarServer/trunk/twistedcaldav/directory
source_changes at macosforge.org
source_changes at macosforge.org
Wed Sep 5 13:07:54 PDT 2012
Revision: 9777
http://trac.macosforge.org/projects/calendarserver/changeset/9777
Author: sagen at apple.com
Date: 2012-09-05 13:07:52 -0700 (Wed, 05 Sep 2012)
Log Message:
-----------
Clean out proxy assignments that originally had an external assignment but no longer do.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2012-09-05 03:36:05 UTC (rev 9776)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2012-09-05 20:07:52 UTC (rev 9777)
@@ -721,17 +721,18 @@
self.log_info("Updating group membership cache")
dataRoot = FilePath(config.DataRoot)
- snapshotFile = dataRoot.child("memberships_cache")
+ membershipsCacheFile = dataRoot.child("memberships_cache")
+ extProxyCacheFile = dataRoot.child("external_proxy_cache")
- if not snapshotFile.exists():
+ if not membershipsCacheFile.exists():
self.log_info("Group membership snapshot file does not yet exist")
fast = False
previousMembers = {}
callGroupsChanged = False
else:
self.log_info("Group membership snapshot file exists: %s" %
- (snapshotFile.path,))
- previousMembers = pickle.loads(snapshotFile.getContent())
+ (membershipsCacheFile.path,))
+ previousMembers = pickle.loads(membershipsCacheFile.getContent())
callGroupsChanged = True
if useLock:
@@ -743,37 +744,75 @@
self.log_info("Acquired lock")
if not fast and self.useExternalProxies:
+
+ # Load in cached copy of external proxies so we can diff against them
+ previousAssignments = []
+ if extProxyCacheFile.exists():
+ self.log_info("External proxies snapshot file exists: %s" %
+ (extProxyCacheFile.path,))
+ previousAssignments = pickle.loads(extProxyCacheFile.getContent())
+
self.log_info("Retrieving proxy assignments from directory")
assignments = self.externalProxiesSource()
self.log_info("%d proxy assignments retrieved from directory" %
(len(assignments),))
+
+ changed, removed = diffAssignments(previousAssignments, assignments)
+ # changed is the list of proxy assignments (either new or updates).
+ # removed is the list of principals who used to have an external
+ # delegate but don't anymore.
+
# populate proxy DB from external resource info
- self.log_info("Applying proxy assignment changes")
- assignmentCount = 0
- totalNumAssignments = len(assignments)
- currentAssignmentNum = 0
- for principalUID, members in assignments:
- currentAssignmentNum += 1
- if currentAssignmentNum % 1000 == 0:
- self.log_info("...proxy assignment %d of %d" % (currentAssignmentNum,
- totalNumAssignments))
- try:
- current = (yield self.proxyDB.getMembers(principalUID))
- if members != current:
+ if changed:
+ self.log_info("Updating proxy assignments")
+ assignmentCount = 0
+ totalNumAssignments = len(changed)
+ currentAssignmentNum = 0
+ for principalUID, members in changed:
+ currentAssignmentNum += 1
+ if currentAssignmentNum % 1000 == 0:
+ self.log_info("...proxy assignment %d of %d" % (currentAssignmentNum,
+ totalNumAssignments))
+ try:
+ current = (yield self.proxyDB.getMembers(principalUID))
+ if members != current:
+ assignmentCount += 1
+ yield self.proxyDB.setGroupMembers(principalUID, members)
+ except Exception, e:
+ self.log_error("Unable to update proxy assignment: principal=%s, members=%s, error=%s" % (principalUID, members, e))
+ self.log_info("Updated %d assignment%s in proxy database" %
+ (assignmentCount, "" if assignmentCount == 1 else "s"))
+
+ if removed:
+ self.log_info("Deleting proxy assignments")
+ assignmentCount = 0
+ totalNumAssignments = len(removed)
+ currentAssignmentNum = 0
+ for principalUID in removed:
+ currentAssignmentNum += 1
+ if currentAssignmentNum % 1000 == 0:
+ self.log_info("...proxy assignment %d of %d" % (currentAssignmentNum,
+ totalNumAssignments))
+ try:
assignmentCount += 1
- yield self.proxyDB.setGroupMembers(principalUID, members)
- except Exception, e:
- self.log_error("Unable to apply proxy assignment: principal=%s, members=%s, error=%s" % (principalUID, members, e))
- self.log_info("Applied %d assignment%s to proxy database" %
- (assignmentCount, "" if assignmentCount == 1 else "s"))
+ yield self.proxyDB.setGroupMembers(principalUID, [])
+ except Exception, e:
+ self.log_error("Unable to remove proxy assignment: principal=%s, members=%s, error=%s" % (principalUID, members, e))
+ self.log_info("Removed %d assignment%s from proxy database" %
+ (assignmentCount, "" if assignmentCount == 1 else "s"))
+ # Store external proxy snapshot
+ self.log_info("Taking snapshot of external proxies to %s" %
+ (extProxyCacheFile.path,))
+ extProxyCacheFile.setContent(pickle.dumps(assignments))
+
if fast:
# If there is an on-disk snapshot of the membership information,
# load that and put into memcached, bypassing the faulting in of
# any records, so that the server can start up quickly.
self.log_info("Loading group memberships from snapshot")
- members = pickle.loads(snapshotFile.getContent())
+ members = pickle.loads(membershipsCacheFile.getContent())
else:
# Fetch the group hierarchy from the directory, fetch the list
@@ -813,8 +852,8 @@
# Store snapshot
self.log_info("Taking snapshot of group memberships to %s" %
- (snapshotFile.path,))
- snapshotFile.setContent(pickle.dumps(members))
+ (membershipsCacheFile.path,))
+ membershipsCacheFile.setContent(pickle.dumps(members))
# Update ownership
uid = gid = -1
@@ -822,7 +861,7 @@
uid = pwd.getpwnam(config.UserName).pw_uid
if config.GroupName:
gid = grp.getgrnam(config.GroupName).gr_gid
- os.chown(snapshotFile.path, uid, gid)
+ os.chown(membershipsCacheFile.path, uid, gid)
self.log_info("Storing %d group memberships in memcached" %
(len(members),))
@@ -1115,7 +1154,35 @@
return cacherService
+def diffAssignments(old, new):
+ """
+ Compare two proxy assignment lists and return their differences in the form of
+ two lists -- one for added/updated assignments, and one for removed assignments.
+ @param old: list of (group, set(members)) tuples
+ @type old: C{list}
+ @param new: list of (group, set(members)) tuples
+ @type new: C{list}
+ @return: Tuple of two lists; the first list contains tuples of (proxy-principal,
+ set(members)), and represents all the new or updated assignments. The
+ second list contains all the proxy-principals which used to have a delegate
+ but don't anymore.
+ """
+ old = dict(old)
+ new = dict(new)
+ changed = []
+ removed = []
+ for key in old.iterkeys():
+ if key not in new:
+ removed.append(key)
+ else:
+ if old[key] != new[key]:
+ changed.append((key, new[key]))
+ for key in new.iterkeys():
+ if key not in old:
+ changed.append((key, new[key]))
+ return changed, removed
+
class DirectoryRecord(LoggingMixIn):
implements(IDirectoryRecord)
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2012-09-05 03:36:05 UTC (rev 9776)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2012-09-05 20:07:52 UTC (rev 9777)
@@ -21,7 +21,7 @@
from twistedcaldav.test.util import TestCase
from twistedcaldav.test.util import xmlFile, augmentsFile, proxiesFile, dirTest
from twistedcaldav.config import config
-from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord, GroupMembershipCacherService, GroupMembershipCache, GroupMembershipCacheUpdater
+from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord, GroupMembershipCacherService, GroupMembershipCache, GroupMembershipCacheUpdater, diffAssignments
from twistedcaldav.directory.xmlfile import XMLDirectoryService
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
from twistedcaldav.directory import augment, calendaruserproxy
@@ -485,7 +485,172 @@
groups,
)
+ #
+ # Now remove two external assignments, and those should take effect.
+ #
+ def fakeExternalProxiesRemoved():
+ return [
+ (
+ "transporter#calendar-proxy-write",
+ set(["8B4288F6-CC82-491D-8EF9-642EF4F3E7D0"])
+ ),
+ ]
+ updater = GroupMembershipCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30,
+ cache=cache, useExternalProxies=True,
+ externalProxiesSource=fakeExternalProxiesRemoved)
+
+ yield updater.updateCache()
+
+ delegates = (
+
+ # record name
+ # read-write delegators
+ # read-only delegators
+ # groups delegate is in (restricted to only those groups
+ # participating in delegation)
+
+ # Note: "transporter" is now gone for wsanchez and cdaboo
+
+ ("wsanchez",
+ set(["mercury", "apollo", "orion", "gemini"]),
+ set(["non_calendar_proxy"]),
+ set(['left_coast',
+ 'both_coasts',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ 'gemini#calendar-proxy-write',
+ ]),
+ ),
+ ("cdaboo",
+ set(["apollo", "orion", "non_calendar_proxy"]),
+ set(["non_calendar_proxy"]),
+ set(['both_coasts',
+ 'non_calendar_group',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ ]),
+ ),
+ ("lecroy",
+ set(["apollo", "mercury", "non_calendar_proxy", "transporter"]),
+ set(),
+ set(['both_coasts',
+ 'left_coast',
+ 'non_calendar_group',
+ 'transporter#calendar-proxy-write',
+ ]),
+ ),
+ )
+
+ for name, write, read, groups in delegates:
+ delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
+
+ proxyFor = (yield delegate.proxyFor(True))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ write,
+ )
+ proxyFor = (yield delegate.proxyFor(False))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ read,
+ )
+ groupsIn = (yield delegate.groupMemberships())
+ uids = set()
+ for group in groupsIn:
+ try:
+ uid = group.uid # a sub-principal
+ except AttributeError:
+ uid = group.record.guid # a regular group
+ uids.add(uid)
+ self.assertEquals(
+ set(uids),
+ groups,
+ )
+
+ def test_diffAssignments(self):
+ """
+ Ensure external proxy assignment diffing works
+ """
+
+ self.assertEquals(
+ (
+ # changed
+ [ ],
+ # removed
+ [ ],
+ ),
+ diffAssignments(
+ # old
+ [ ],
+ # new
+ [ ],
+ )
+ )
+
+ self.assertEquals(
+ (
+ # changed
+ [ ],
+ # removed
+ [ ],
+ ),
+ diffAssignments(
+ # old
+ [ ("B", set(["3"])), ("A", set(["1", "2"])), ],
+ # new
+ [ ("A", set(["1", "2"])), ("B", set(["3"])), ],
+ )
+ )
+
+ self.assertEquals(
+ (
+ # changed
+ [ ("A", set(["1", "2"])), ("B", set(["3"])), ],
+ # removed
+ [ ],
+ ),
+ diffAssignments(
+ # old
+ [ ],
+ # new
+ [ ("A", set(["1", "2"])), ("B", set(["3"])), ],
+ )
+ )
+
+ self.assertEquals(
+ (
+ # changed
+ [ ],
+ # removed
+ [ "A", "B" ],
+ ),
+ diffAssignments(
+ # old
+ [ ("A", set(["1", "2"])), ("B", set(["3"])), ],
+ # new
+ [ ],
+ )
+ )
+
+ self.assertEquals(
+ (
+ # changed
+ [ ("A", set(["2"])), ("C", set(["4", "5"])), ("D", set(["6"])), ],
+ # removed
+ [ "B" ],
+ ),
+ diffAssignments(
+ # old
+ [ ("A", set(["1", "2"])), ("B", set(["3"])), ("C", set(["4"])), ],
+ # new
+ [ ("D", set(["6"])), ("C", set(["4", "5"])), ("A", set(["2"])), ],
+ )
+ )
+
+
+
@inlineCallbacks
def test_groupMembershipCacheSnapshot(self):
"""
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120905/5d2ce813/attachment-0001.html>
More information about the calendarserver-changes
mailing list