[CalendarServer-changes] [7751] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jul 8 16:27:52 PDT 2011
Revision: 7751
http://trac.macosforge.org/projects/calendarserver/changeset/7751
Author: sagen at apple.com
Date: 2011-07-08 16:27:50 -0700 (Fri, 08 Jul 2011)
Log Message:
-----------
Removing proxy cacher, replacing with group membership cacher. Now the members
of groups participating in delegation are cached, but proxy assignment
expansions are not, therefore assignment changes update in real time. Still
turned off by default, enabled in GroupCaching plist section.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/calendarserver/tools/util.py
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/twisted/plugins/caldav.py
CalendarServer/trunk/twistedcaldav/directory/aggregate.py
CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
CalendarServer/trunk/twistedcaldav/directory/principal.py
CalendarServer/trunk/twistedcaldav/directory/test/test_aggregate.py
CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -506,24 +506,24 @@
self.monitor.addProcess("mailgateway", mailGatewayArgv,
env=PARENT_ENVIRONMENT)
- if config.ProxyCaching.Enabled and config.ProxyCaching.EnableUpdater:
- self.maker.log_info("Adding proxy caching service")
+ if config.GroupCaching.Enabled and config.GroupCaching.EnableUpdater:
+ self.maker.log_info("Adding group caching service")
- proxyCacherArgv = [
+ groupMembershipCacherArgv = [
sys.executable,
sys.argv[0],
]
if config.UserName:
- proxyCacherArgv.extend(("-u", config.UserName))
+ groupMembershipCacherArgv.extend(("-u", config.UserName))
if config.GroupName:
- proxyCacherArgv.extend(("-g", config.GroupName))
- proxyCacherArgv.extend((
+ groupMembershipCacherArgv.extend(("-g", config.GroupName))
+ groupMembershipCacherArgv.extend((
"--reactor=%s" % (config.Twisted.reactor,),
- "-n", self.maker.proxyCacherTapName,
+ "-n", self.maker.groupMembershipCacherTapName,
"-f", self.configPath,
))
- self.monitor.addProcess("proxycache", proxyCacherArgv,
+ self.monitor.addProcess("groupcacher", groupMembershipCacherArgv,
env=PARENT_ENVIRONMENT)
@@ -540,7 +540,7 @@
#
mailGatewayTapName = "caldav_mailgateway"
notifierTapName = "caldav_notifier"
- proxyCacherTapName = "caldav_proxycacher"
+ groupMembershipCacherTapName = "caldav_groupcacher"
def makeService(self, options):
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -50,6 +50,7 @@
from twistedcaldav.directory.aggregate import AggregateDirectoryService
from twistedcaldav.directory.calendar import DirectoryCalendarHomeProvisioningResource
from twistedcaldav.directory.digest import QopDigestCredentialFactory
+from twistedcaldav.directory.directory import GroupMembershipCache
from twistedcaldav.directory.internal import InternalDirectoryService
from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
from twistedcaldav.directory.sudo import SudoDirectoryService
@@ -225,12 +226,14 @@
raise
#
- # Setup the proxy cacher
+ # Setup the group membership cacher
#
- if config.ProxyCaching.Enabled:
- proxyCache = calendaruserproxy.ProxyMemberCache(config.ProxyCaching.MemcachedPool)
+ if config.GroupCaching.Enabled:
+ groupMembershipCache = GroupMembershipCache(
+ config.GroupCaching.MemcachedPool,
+ expireSeconds=config.GroupCaching.ExpireSeconds)
else:
- proxyCache = None
+ groupMembershipCache = None
#
# Setup the Directory
@@ -244,7 +247,7 @@
% (config.DirectoryService.type,))
config.DirectoryService.params.augmentService = augmentService
- config.DirectoryService.params.proxyCache = proxyCache
+ config.DirectoryService.params.groupMembershipCache = groupMembershipCache
baseDirectory = directoryClass(config.DirectoryService.params)
# Wait for the directory to become available
@@ -262,7 +265,7 @@
log.info("Configuring resource service of type: %s" % (resourceClass,))
config.ResourceService.params.augmentService = augmentService
- config.ResourceService.params.proxyCache = proxyCache
+ config.ResourceService.params.groupMembershipCache = groupMembershipCache
resourceDirectory = resourceClass(config.ResourceService.params)
resourceDirectory.realmName = baseDirectory.realmName
directories.append(resourceDirectory)
@@ -302,7 +305,7 @@
internalDirectory = InternalDirectoryService(baseDirectory.realmName)
directories.append(internalDirectory)
- directory = AggregateDirectoryService(directories)
+ directory = AggregateDirectoryService(directories, groupMembershipCache)
if sudoDirectory:
directory.userRecordTypes.insert(0,
Modified: CalendarServer/trunk/calendarserver/tools/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/util.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/calendarserver/tools/util.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -151,7 +151,7 @@
resourceDirectory.realmName = directory.realmName
directories.append(resourceDirectory)
- aggregate = MyDirectoryService(directories)
+ aggregate = MyDirectoryService(directories, None)
#
# Wire up the resource hierarchy
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2011-07-08 23:27:50 UTC (rev 7751)
@@ -898,8 +898,8 @@
</array>
</dict>
- <!-- Proxy Membership Caching -->
- <key>ProxyCaching</key>
+ <!-- Group Membership Caching -->
+ <key>GroupCaching</key>
<dict>
<key>Enabled</key>
<false/>
Modified: CalendarServer/trunk/twisted/plugins/caldav.py
===================================================================
--- CalendarServer/trunk/twisted/plugins/caldav.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twisted/plugins/caldav.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -52,4 +52,4 @@
TwistedCalDAV = TAP("calendarserver.tap.caldav.CalDAVServiceMaker")
CalDAVNotifier = TAP("twistedcaldav.notify.NotificationServiceMaker")
CalDAVMailGateway = TAP("twistedcaldav.mail.MailGatewayServiceMaker")
-CalDAVProxyCacher = TAP("twistedcaldav.directory.calendaruserproxy.ProxyCacherServiceMaker")
+CalDAVGroupCacher = TAP("twistedcaldav.directory.directory.GroupMembershipCacherServiceMaker")
Modified: CalendarServer/trunk/twistedcaldav/directory/aggregate.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/aggregate.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/aggregate.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -38,11 +38,12 @@
"""
baseGUID = "06FB225F-39E7-4D34-B1D1-29925F5E619B"
- def __init__(self, services):
+ def __init__(self, services, groupMembershipCache):
super(AggregateDirectoryService, self).__init__()
realmName = None
recordTypes = {}
+ self.groupMembershipCache = groupMembershipCache
for service in services:
service = IDirectoryService(service)
Modified: CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -77,7 +77,7 @@
self.recordType_groups,
),
'augmentService' : None,
- 'proxyCache' : None,
+ 'groupMembershipCache' : None,
}
ignored = ('requireComputerRecord',)
params = self.getParams(params, defaults, ignored)
@@ -96,7 +96,7 @@
raise
self.augmentService = params['augmentService']
- self.proxyCache = params['proxyCache']
+ self.groupMembershipCache = params['groupMembershipCache']
self.realmName = params['node']
self.directory = directory
self.node = params['node']
@@ -307,24 +307,9 @@
return guids
- def proxiesForGUID(self, recordType, guid):
-
- # Lookup in index
- try:
- # TODO:
- return ()
- except KeyError:
- return ()
- def readOnlyProxiesForGUID(self, recordType, guid):
-
- # Lookup in index
- try:
- # TODO:
- return ()
- except KeyError:
- return ()
+
_ODFields = {
'fullName' : {
'odField' : dsattributes.kDS1AttrDistinguishedName,
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -46,15 +46,8 @@
from twistedcaldav.extensions import DAVPrincipalResource,\
DAVResourceWithChildrenMixin
from twistedcaldav.extensions import ReadOnlyWritePropertiesResourceMixIn
-from twistedcaldav import memcachepool
from twistedcaldav.memcacher import Memcacher
from twistedcaldav.resource import CalDAVComplianceMixIn
-from twisted.python.reflect import namedClass
-from twisted.python.usage import Options, UsageError
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
-from twisted.application import service
-from twisted.plugin import IPlugin
-from zope.interface import implements
log = Logger()
@@ -360,38 +353,15 @@
returnValue(found)
def groupMembers(self):
- cache = getattr(self.parent.record.service, "proxyCache", None)
- if cache is not None:
- return self.expandedGroupMembers()
- else:
- return self._expandMemberUIDs()
+ return self._expandMemberUIDs()
@inlineCallbacks
def expandedGroupMembers(self):
"""
Return the complete, flattened set of principals belonging to this
group.
-
- If the directory service is using a ProxyMemberCache, take advantage
- of it and use the set of cached uids. Otherwise go through
- _expandMemberUIDs.
"""
- cache = getattr(self.parent.record.service, "proxyCache", None)
- if cache is not None:
- log.debug("expandedGroupMembers using proxyCache")
- if not (yield cache.checkMarker()):
- raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
- "Proxy membership cache not yet populated"))
- principals = set()
- memberUIDs = (yield cache.getMembers(self.uid))
- if memberUIDs:
- for uid in memberUIDs:
- principal = self.pcollection.principalForUID(uid)
- if principal is not None:
- principals.add(principal)
- returnValue(principals)
- else:
- returnValue((yield self._expandMemberUIDs(infinity=True)))
+ returnValue((yield self._expandMemberUIDs(infinity=True)))
def groupMemberships(self):
# Get membership UIDs and map to principal resources
@@ -857,375 +827,3 @@
davxml.Protected(),
),
)
-
-
-class ProxyMemberCache(Memcacher, LoggingMixIn):
- """
- Caches expanded proxy membership information
-
- This cache is periodically updated by a side car so that worker processes
- never have to ask the directory service directly for group membership
- information.
-
- Keys in this cache are:
-
- "proxy-members:<GUID>" : comma-separated list of member guids (flattened)
- (where <GUID> will be that of a sub-principal, e.g.
- 5A985493-EE2C-4665-94CF-4DFEA3A89500#calendar-proxy-write)
-
- "read-proxy-for:<GUID>" : comma-separated list of principals
- who have granted read access to GUID
-
- "write-proxy-for:<GUID>" : comma-separated list of principals
- who have granted read-write access to GUID
-
- "proxy-cache-populated" : gets set to "true" after the cache is populated,
- so clients know they can now use it. Note, this needs to be made robust
- in the face of memcached evictions.
-
- """
-
-
- def setMembers(self, guid, members, expireSeconds=0):
- self.log_debug("set proxy-members %s : %s" % (guid, members))
- return self.set("proxy-members:%s" % (str(guid),), str(",".join(members)), expire_time=expireSeconds)
-
- def getMembers(self, guid):
- self.log_debug("get proxy-members %s" % (guid,))
- def _value(value):
- if value:
- return set(value.split(","))
- elif value is None:
- return None
- else:
- return set()
- d = self.get("proxy-members:%s" % (str(guid),))
- d.addCallback(_value)
- return d
-
- def deleteMembers(self, guid):
- return self.delete("proxy-members:%s" % (str(guid),))
-
- def setProxyFor(self, guid, proxyType, memberships, expireSeconds=0):
- self.log_debug("set %s-proxy-for %s : %s" %
- (proxyType, guid, memberships))
- return self.set("%s-proxy-for:%s" %
- (proxyType, str(guid)), str(",".join(memberships)),
- expire_time=expireSeconds)
-
- def getProxyFor(self, guid, proxyType):
- self.log_debug("get %s-proxy-for %s" % (proxyType, guid))
- def _value(value):
- if value:
- return set(value.split(","))
- elif value is None:
- return None
- else:
- return set()
- d = self.get("%s-proxy-for:%s" % (proxyType, str(guid),))
- d.addCallback(_value)
- return d
-
- def deleteProxyFor(self, guid, proxyType):
- return self.delete("%s-proxy-for:%s" % (proxyType, str(guid),))
-
- def createMarker(self, expireSeconds=0):
- return self.set("proxy-cache-populated", "true",
- expire_time=expireSeconds)
-
- def checkMarker(self):
- def _value(value):
- return value == "true"
- d = self.get("proxy-cache-populated")
- d.addCallback(_value)
- return d
-
-
-class ProxyMemberCacheUpdater(LoggingMixIn):
- """
- Responsible for updating memcached with proxy assignments. This will run
- in a sidecar. There are two sources of proxy data to pull from: the local
- proxy database, and the location/resource info in the directory system.
-
- TODO: Implement location/resource
- """
-
- def __init__(self, proxyDB, directory, expireSeconds, cache=None,
- namespace=None):
- self.proxyDB = proxyDB
- self.directory = directory
- if cache is None:
- assert namespace is not None, "namespace must be specified if ProxyMemberCache is not provided"
- cache = ProxyMemberCache(namespace)
- self.cache = cache
- self.previousProxyGroups = set()
- self.previousProxyFor = {
- "read" : { },
- "write" : { },
- }
- self.expireSeconds = expireSeconds
-
- @inlineCallbacks
- def updateCache(self):
- """
- Iterate the proxy database to retrieve all the principals who have been
- delegated to. Fault these principals in. For any of these principals
- that are groups, expand the members of that group and store those in
- the cache
- """
- # TODO: add memcached eviction protection
-
- self.log_debug("Updating proxy membership cache")
-
- numProxyGroupsUpdated = 0
- numProxiesUpdated = {
- "read" : 0,
- "write" : 0,
- }
-
- currentProxyGroups = set()
- currentProxyFor = {
- "read" : { },
- "write" : { },
- }
- proxyGroups = (yield self.proxyDB.getAllGroups())
- for proxyGroup in proxyGroups:
-
- # Protect against bogus entries in proxy db:
- if "#" not in proxyGroup:
- continue
-
- # Populate delegator -> delegate cache
- combinedGUIDs = set()
- for proxyGUID in (yield self.proxyDB.getMembers(proxyGroup)):
- record = self.directory.recordWithGUID(proxyGUID)
- if record:
- # TODO: What if this guid is not in the directory?
- combinedGUIDs.add(record.guid)
- if record.recordType == self.directory.recordType_groups:
- members = record.expandedMembers()
- guids = set([r.guid for r in members])
- combinedGUIDs.update(guids)
-
- self.cache.setMembers(proxyGroup, combinedGUIDs,
- expireSeconds=self.expireSeconds)
- numProxyGroupsUpdated += 1
-
- if proxyGroup in self.previousProxyGroups:
- # whatever remains in previousProxyGroups needs to be deleted
- # from memcached
- self.previousProxyGroups.remove(proxyGroup)
- currentProxyGroups.add(proxyGroup)
-
- # Populate delegate -> delegator mapping
- delegator, suffix = proxyGroup.split("#")
- proxyType = "read" if suffix.endswith("read") else "write"
- for guid in combinedGUIDs:
- proxyFor = currentProxyFor[proxyType].setdefault(guid, set())
- proxyFor.add(delegator)
-
- for proxyType in ("read", "write"):
- for guid, memberships in currentProxyFor[proxyType].iteritems():
- self.cache.setProxyFor(guid, proxyType, memberships,
- expireSeconds=self.expireSeconds)
- numProxiesUpdated[proxyType] += 1
- if self.previousProxyFor[proxyType].has_key(guid):
- # whatever remains in previousProxyFor needs to be deleted
- # from memcached
- del self.previousProxyFor[proxyType][guid]
-
- self.log_debug("%d proxyGroups updated" % (numProxyGroupsUpdated,))
- for proxyType in ("read", "write"):
- self.log_debug("%d %s proxies updated" %
- (numProxiesUpdated[proxyType], proxyType))
-
- # Delete obsolete memcached keys
- for proxyGroup in self.previousProxyGroups:
- self.log_debug("Deleting proxyGroup members for %s" %
- (proxyGroup,))
- self.cache.deleteMembers(proxyGroup)
- for proxyType in ("read", "write"):
- for guid in self.previousProxyFor[proxyType].iterkeys():
- self.log_debug("Deleting %s proxyFor members for %s" %
- (proxyType, guid,))
- self.cache.deleteProxyFor(guid, proxyType)
-
- self.previousProxyGroups = currentProxyGroups
- self.previousProxyFor = currentProxyFor
-
- # Put a special key into memcached to let workers know proxyCache is
- # populated
- self.cache.createMarker(expireSeconds=self.expireSeconds)
-
-
-class ProxyCacherOptions(Options):
- optParameters = [[
- "config", "f", DEFAULT_CONFIG_FILE, "Path to configuration file."
- ]]
-
- def __init__(self, *args, **kwargs):
- super(ProxyCacherOptions, self).__init__(*args, **kwargs)
-
- self.overrides = {}
-
- def _coerceOption(self, configDict, key, value):
- """
- Coerce the given C{val} to type of C{configDict[key]}
- """
- if key in configDict:
- if isinstance(configDict[key], bool):
- value = value == "True"
-
- elif isinstance(configDict[key], (int, float, long)):
- value = type(configDict[key])(value)
-
- elif isinstance(configDict[key], (list, tuple)):
- value = value.split(',')
-
- elif isinstance(configDict[key], dict):
- raise UsageError(
- "Dict options not supported on the command line"
- )
-
- elif value == 'None':
- value = None
-
- return value
-
- def _setOverride(self, configDict, path, value, overrideDict):
- """
- Set the value at path in configDict
- """
- key = path[0]
-
- if len(path) == 1:
- overrideDict[key] = self._coerceOption(configDict, key, value)
- return
-
- if key in configDict:
- if not isinstance(configDict[key], dict):
- raise UsageError(
- "Found intermediate path element that is not a dictionary"
- )
-
- if key not in overrideDict:
- overrideDict[key] = {}
-
- self._setOverride(
- configDict[key], path[1:],
- value, overrideDict[key]
- )
-
-
- def opt_option(self, option):
- """
- Set an option to override a value in the config file. True, False, int,
- and float options are supported, as well as comma seperated lists. Only
- one option may be given for each --option flag, however multiple
- --option flags may be specified.
- """
-
- if "=" in option:
- path, value = option.split('=')
- self._setOverride(
- DEFAULT_CONFIG,
- path.split('/'),
- value,
- self.overrides
- )
- else:
- self.opt_option('%s=True' % (option,))
-
- opt_o = opt_option
-
- def postOptions(self):
- config.load(self['config'])
- config.updateDefaults(self.overrides)
- self.parent['pidfile'] = None
-
-
-
-class ProxyCacherService(service.Service, LoggingMixIn):
- """
- Service to update the proxy cache at a configured interval
- """
-
- def __init__(self, proxyDB, directory, namespace, updateSeconds,
- expireSeconds, reactor=None, updateMethod=None):
-
- self.updater = ProxyMemberCacheUpdater(proxyDB, directory,
- expireSeconds, namespace=namespace)
-
- if reactor is None:
- from twisted.internet import reactor
- self.reactor = reactor
- self.updateSeconds = updateSeconds
- self.nextUpdate = None
- if updateMethod:
- self.updateMethod = updateMethod
- else:
- self.updateMethod = self.updater.updateCache
-
- def startService(self):
- self.log_warn("Starting proxy cacher service")
- service.Service.startService(self)
- return self.update()
-
- @inlineCallbacks
- def update(self):
- self.nextUpdate = None
- try:
- yield self.updateMethod()
- finally:
- self.log_debug("Scheduling next proxy cacher update")
- self.nextUpdate = self.reactor.callLater(self.updateSeconds,
- self.update)
-
- def stopService(self):
- self.log_warn("Stopping proxy cacher service")
- service.Service.stopService(self)
- if self.nextUpdate is not None:
- self.nextUpdate.cancel()
-
-
-class ProxyCacherServiceMaker(LoggingMixIn):
- """
- Configures and returns a ProxyCacherService
- """
- implements(IPlugin, service.IServiceMaker)
-
- tapname = "caldav_proxycacher"
- description = "Proxy Cacher"
- options = ProxyCacherOptions
-
- def makeService(self, options):
-
- # Setup the directory
- from calendarserver.tap.util import directoryFromConfig
- directory = directoryFromConfig(config)
-
- # Setup the ProxyDB Service
- proxydbClass = namedClass(config.ProxyDBService.type)
-
- self.log_warn("Configuring proxydb service of type: %s" % (proxydbClass,))
-
- try:
- proxyDB = proxydbClass(**config.ProxyDBService.params)
- except IOError:
- self.log_error("Could not start proxydb service")
- raise
-
- # Setup memcached pools
- memcachepool.installPools(
- config.Memcached.Pools,
- config.Memcached.MaxClients,
- )
-
- proxyCacherService = ProxyCacherService(proxyDB, directory,
- config.ProxyCaching.MemcachedPool,
- config.ProxyCaching.UpdateSeconds,
- config.ProxyCaching.ExpireSeconds
- )
-
- return proxyCacherService
-
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -36,7 +36,7 @@
from twisted.cred.error import UnauthorizedLogin
from twisted.cred.checkers import ICredentialsChecker
from twext.web2.dav.auth import IPrincipalCredentials
-from twisted.internet.defer import succeed
+from twisted.internet.defer import succeed, inlineCallbacks
from twext.python.log import LoggingMixIn
@@ -45,6 +45,14 @@
from twistedcaldav.directory.util import uuidFromName
from twistedcaldav.scheduling.cuaddress import normalizeCUAddr
from twistedcaldav import servers
+from twistedcaldav.memcacher import Memcacher
+from twistedcaldav import memcachepool
+from twisted.python.reflect import namedClass
+from twisted.python.usage import Options, UsageError
+from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
+from twisted.application import service
+from twisted.plugin import IPlugin
+from zope.interface import implements
class DirectoryService(LoggingMixIn):
implements(IDirectoryService, ICredentialsChecker)
@@ -340,6 +348,317 @@
"""
raise NotImplementedError("Subclass must implement createRecords")
+ @inlineCallbacks
+ def cacheGroupMembership(self, guids):
+ """
+ Update the "which groups is each principal in" cache. The only groups
+ that the server needs to worry about are the ones which have been
+ delegated to. So instead of faulting in all groups a principal is in,
+ we pre-fault in all the delgated-to groups and build an index to
+ quickly look up principal membership.
+
+ guids is the set of every guid that's been directly delegated to, and
+ can be a mixture of users and groups.
+ """
+ groups = set()
+ for guid in guids:
+ record = self.recordWithGUID(guid)
+ if record is not None and record.recordType == self.recordType_groups:
+ groups.add(record)
+
+ members = { }
+ for group in groups:
+ groupMembers = group.expandedMembers()
+ for member in groupMembers:
+ if member.recordType == self.recordType_users:
+ memberships = members.setdefault(member.guid, set())
+ memberships.add(group.guid)
+
+ for member, groups in members.iteritems():
+ yield self.groupMembershipCache.setGroupsFor(member, groups)
+
+ self.groupMembershipCache.createMarker()
+
+
+class GroupMembershipCache(Memcacher, LoggingMixIn):
+ """
+ Caches group membership information
+
+ This cache is periodically updated by a side car so that worker processes
+ never have to ask the directory service directly for group membership
+ information.
+
+ Keys in this cache are:
+
+ "group-membership-cache-populated" : gets set to "true" after the cache
+ is populated, so clients know they can now use it. Note, this needs to
+ be made robust in the face of memcached evictions.
+
+ "groups-for:<GUID>" : comma-separated list of groups that GUID is a member
+ of
+
+ """
+
+ def __init__(self, namespace, pickle=False, no_invalidation=False,
+ key_normalization=True, expireSeconds=0):
+
+ super(GroupMembershipCache, self).__init__(namespace, pickle=pickle,
+ no_invalidation=no_invalidation,
+ key_normalization=key_normalization)
+
+ self.expireSeconds = expireSeconds
+
+ def setGroupsFor(self, guid, memberships):
+ self.log_debug("set groups-for %s : %s" % (guid, memberships))
+ return self.set("groups-for:%s" %
+ (str(guid)), str(",".join(memberships)),
+ expire_time=self.expireSeconds)
+
+ def getGroupsFor(self, guid):
+ self.log_debug("get groups-for %s" % (guid,))
+ def _value(value):
+ if value:
+ return set(value.split(","))
+ else:
+ return set()
+ d = self.get("groups-for:%s" % (str(guid),))
+ d.addCallback(_value)
+ return d
+
+ def deleteGroupsFor(self, guid, proxyType):
+ return self.delete("groups-for:%s" % (str(guid),))
+
+ def createMarker(self):
+ return self.set("proxy-cache-populated", "true",
+ expire_time=self.expireSeconds)
+
+ def checkMarker(self):
+ def _value(value):
+ return value == "true"
+ d = self.get("proxy-cache-populated")
+ d.addCallback(_value)
+ return d
+
+
+class GroupMembershipCacheUpdater(LoggingMixIn):
+ """
+ Responsible for updating memcached with group memberships. This will run
+ in a sidecar. There are two sources of proxy data to pull from: the local
+ proxy database, and the location/resource info in the directory system.
+
+ TODO: Implement location/resource
+ """
+
+ def __init__(self, proxyDB, directory, expireSeconds, cache=None,
+ namespace=None):
+ self.proxyDB = proxyDB
+ self.directory = directory
+ if cache is None:
+ assert namespace is not None, "namespace must be specified if GroupMembershipCache is not provided"
+ cache = GroupMembershipCache(namespace, expireSeconds=expireSeconds)
+ self.cache = cache
+
+ @inlineCallbacks
+ def updateCache(self):
+ """
+ Iterate the proxy database to retrieve all the principals who have been
+ delegated to. Fault these principals in. For any of these principals
+ that are groups, expand the members of that group and store those in
+ the cache
+ """
+ # TODO: add memcached eviction protection
+
+ self.log_debug("Updating group membership cache")
+
+ guids = set()
+
+ proxyGroups = (yield self.proxyDB.getAllGroups())
+ for proxyGroup in proxyGroups:
+
+ # Protect against bogus entries in proxy db:
+ if "#" not in proxyGroup:
+ continue
+
+ for guid in (yield self.proxyDB.getMembers(proxyGroup)):
+ guids.add(guid)
+
+
+ yield self.directory.cacheGroupMembership(guids)
+
+
+class GroupMembershipCacherOptions(Options):
+ optParameters = [[
+ "config", "f", DEFAULT_CONFIG_FILE, "Path to configuration file."
+ ]]
+
+ def __init__(self, *args, **kwargs):
+ super(GroupMembershipCacherOptions, self).__init__(*args, **kwargs)
+
+ self.overrides = {}
+
+ def _coerceOption(self, configDict, key, value):
+ """
+ Coerce the given C{val} to type of C{configDict[key]}
+ """
+ if key in configDict:
+ if isinstance(configDict[key], bool):
+ value = value == "True"
+
+ elif isinstance(configDict[key], (int, float, long)):
+ value = type(configDict[key])(value)
+
+ elif isinstance(configDict[key], (list, tuple)):
+ value = value.split(',')
+
+ elif isinstance(configDict[key], dict):
+ raise UsageError(
+ "Dict options not supported on the command line"
+ )
+
+ elif value == 'None':
+ value = None
+
+ return value
+
+ def _setOverride(self, configDict, path, value, overrideDict):
+ """
+ Set the value at path in configDict
+ """
+ key = path[0]
+
+ if len(path) == 1:
+ overrideDict[key] = self._coerceOption(configDict, key, value)
+ return
+
+ if key in configDict:
+ if not isinstance(configDict[key], dict):
+ raise UsageError(
+ "Found intermediate path element that is not a dictionary"
+ )
+
+ if key not in overrideDict:
+ overrideDict[key] = {}
+
+ self._setOverride(
+ configDict[key], path[1:],
+ value, overrideDict[key]
+ )
+
+
+ def opt_option(self, option):
+ """
+ Set an option to override a value in the config file. True, False, int,
+ and float options are supported, as well as comma seperated lists. Only
+ one option may be given for each --option flag, however multiple
+ --option flags may be specified.
+ """
+
+ if "=" in option:
+ path, value = option.split('=')
+ self._setOverride(
+ DEFAULT_CONFIG,
+ path.split('/'),
+ value,
+ self.overrides
+ )
+ else:
+ self.opt_option('%s=True' % (option,))
+
+ opt_o = opt_option
+
+ def postOptions(self):
+ config.load(self['config'])
+ config.updateDefaults(self.overrides)
+ self.parent['pidfile'] = None
+
+
+
+class GroupMembershipCacherService(service.Service, LoggingMixIn):
+ """
+ Service to update the group membership cache at a configured interval
+ """
+
+ def __init__(self, proxyDB, directory, namespace, updateSeconds,
+ expireSeconds, reactor=None, updateMethod=None):
+
+ self.updater = GroupMembershipCacheUpdater(proxyDB, directory,
+ expireSeconds, namespace=namespace)
+
+ if reactor is None:
+ from twisted.internet import reactor
+ self.reactor = reactor
+ self.updateSeconds = updateSeconds
+ self.nextUpdate = None
+ if updateMethod:
+ self.updateMethod = updateMethod
+ else:
+ self.updateMethod = self.updater.updateCache
+
+ def startService(self):
+ self.log_warn("Starting group membership cacher service")
+ service.Service.startService(self)
+ return self.update()
+
+ @inlineCallbacks
+ def update(self):
+ self.nextUpdate = None
+ try:
+ yield self.updateMethod()
+ finally:
+ self.log_debug("Scheduling next group membership update")
+ self.nextUpdate = self.reactor.callLater(self.updateSeconds,
+ self.update)
+
+ def stopService(self):
+ self.log_warn("Stopping group membership cacher service")
+ service.Service.stopService(self)
+ if self.nextUpdate is not None:
+ self.nextUpdate.cancel()
+
+
+class GroupMembershipCacherServiceMaker(LoggingMixIn):
+ """
+ Configures and returns a GroupMembershipCacherService
+ """
+ implements(IPlugin, service.IServiceMaker)
+
+ tapname = "caldav_groupcacher"
+ description = "Group Membership Cacher"
+ options = GroupMembershipCacherOptions
+
+ def makeService(self, options):
+
+ # Setup the directory
+ from calendarserver.tap.util import directoryFromConfig
+ directory = directoryFromConfig(config)
+
+ # Setup the ProxyDB Service
+ proxydbClass = namedClass(config.ProxyDBService.type)
+
+ self.log_warn("Configuring proxydb service of type: %s" % (proxydbClass,))
+
+ try:
+ proxyDB = proxydbClass(**config.ProxyDBService.params)
+ except IOError:
+ self.log_error("Could not start proxydb service")
+ raise
+
+ # Setup memcached pools
+ memcachepool.installPools(
+ config.Memcached.Pools,
+ config.Memcached.MaxClients,
+ )
+
+ cacherService = GroupMembershipCacherService(proxyDB, directory,
+ config.GroupCaching.MemcachedPool,
+ config.GroupCaching.UpdateSeconds,
+ config.GroupCaching.ExpireSeconds
+ )
+
+ return cacherService
+
+
+
class DirectoryRecord(LoggingMixIn):
implements(IDirectoryRecord)
@@ -515,6 +834,15 @@
def groups(self):
return ()
+
+ def cachedGroups(self):
+ """
+ Return the set of groups (guids) this record is a member of, based on
+ the data cached by cacheGroupMembership( )
+ """
+ return self.service.groupMembershipCache.getGroupsFor(self.guid)
+
+
def verifyCredentials(self, credentials):
return False
Modified: CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -75,7 +75,7 @@
defaults = {
"augmentService" : None,
- "proxyCache" : None,
+ "groupMembershipCache" : None,
"cacheTimeout": 1, # Minutes
"negativeCaching": False,
"restrictEnabledRecords": False,
@@ -146,7 +146,7 @@
params["negativeCaching"])
self.augmentService = params["augmentService"]
- self.proxyCache = params["proxyCache"]
+ self.groupMembershipCache = params["groupMembershipCache"]
self.realmName = params["uri"]
self.uri = params["uri"]
self.tls = params["tls"]
Modified: CalendarServer/trunk/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -664,28 +664,24 @@
@inlineCallbacks
def proxyFor(self, read_write, resolve_memberships=True):
- cache = getattr(self.record.service, "proxyCache", None)
- if cache is not None:
- log.debug("proxyFor is using proxyCache")
- if not (yield cache.checkMarker()):
- raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
- "Proxy membership cache not yet populated"))
-
- principals = set()
- proxyType = "write" if read_write else "read"
- delegatorUIDs = (yield cache.getProxyFor(self.record.guid, proxyType))
- if delegatorUIDs:
- for uid in delegatorUIDs:
- principal = self.parent.principalForUID(uid)
- if principal is not None:
- principals.add(principal)
- returnValue(principals)
-
- # Slower, non cached method:
proxyFors = set()
if resolve_memberships:
- memberships = self._getRelatives("groups", infinity=True)
+ cache = getattr(self.record.service, "groupMembershipCache", None)
+ if cache:
+ log.debug("proxyFor is using groupMembershipCache")
+ if not (yield cache.checkMarker()):
+ raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
+ "Group membership cache not yet populated"))
+ guids = (yield self.record.cachedGroups())
+ memberships = set()
+ for guid in guids:
+ principal = self.parent.principalForUID(guid)
+ if principal:
+ memberships.add(principal)
+ else:
+ memberships = self._getRelatives("groups", infinity=True)
+
for membership in memberships:
results = (yield membership.proxyFor(read_write, False))
proxyFors.update(results)
@@ -751,29 +747,21 @@
@inlineCallbacks
def groupMemberships(self, infinity=False):
- cache = getattr(self.record.service, "proxyCache", None)
- if cache is not None:
- # We only need to worry about groups participating in delegation
- log.debug("groupMemberships is using proxyCache")
+ cache = getattr(self.record.service, "groupMembershipCache", None)
+ if cache:
+ log.debug("groupMemberships is using groupMembershipCache")
if not (yield cache.checkMarker()):
raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
- "Proxy membership cache not yet populated"))
+ "Group membership cache not yet populated"))
+ guids = (yield self.record.cachedGroups())
groups = set()
- for proxyType in ("read", "write"):
- delegatorUIDs = (yield cache.getProxyFor(self.record.guid,
- proxyType))
- if delegatorUIDs:
- for uid in delegatorUIDs:
- principal = self.parent.principalForUID(uid)
- if principal is not None:
- group = principal.getChild("calendar-proxy-%s" %
- (proxyType,))
- groups.add(group)
- returnValue(groups)
+ for guid in guids:
+ principal = self.parent.principalForUID(guid)
+ if principal:
+ groups.add(principal)
+ else:
+ groups = self._getRelatives("groups", infinity=infinity)
- # Slower, fetching-many-groups method:
- groups = self._getRelatives("groups", infinity=infinity)
-
if config.EnableProxyPrincipals:
# Get proxy group UIDs and map to principal resources
proxies = []
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_aggregate.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_aggregate.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_aggregate.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -72,7 +72,7 @@
xmlService.recordTypePrefix = xml_prefix
- return AggregateDirectoryService((xmlService,))
+ return AggregateDirectoryService((xmlService,), None)
def test_setRealm(self):
"""
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_directory.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -14,9 +14,19 @@
# limitations under the License.
##
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.task import Clock
+from twext.web2.http import HTTPError
+
from twistedcaldav.test.util import TestCase
+from twistedcaldav.test.util import xmlFile, augmentsFile, proxiesFile
from twistedcaldav.config import config
-from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
+from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord, GroupMembershipCacherService, GroupMembershipCache, GroupMembershipCacheUpdater
+from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
+from twistedcaldav.directory import augment, calendaruserproxy
+from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
+
def StubCheckSACL(cls, username, service):
services = {
@@ -57,3 +67,234 @@
record.applySACLs()
self.assertEquals(record.enabledForCalendaring, cal)
self.assertEquals(record.enabledForAddressBooks, ab)
+
+class GroupMembershipTests (TestCase):
+
+ @inlineCallbacks
+ def setUp(self):
+ super(GroupMembershipTests, self).setUp()
+
+ self.directoryService = XMLDirectoryService(
+ {
+ 'xmlFile' : xmlFile,
+ 'augmentService' :
+ augment.AugmentXMLDB(xmlFiles=(augmentsFile.path,)),
+ }
+ )
+ calendaruserproxy.ProxyDBService = calendaruserproxy.ProxySqliteDB("proxies.sqlite")
+
+ # Set up a principals hierarchy for each service we're testing with
+ self.principalRootResources = {}
+ name = self.directoryService.__class__.__name__
+ url = "/" + name + "/"
+
+ provisioningResource = DirectoryPrincipalProvisioningResource(url, self.directoryService)
+
+ self.site.resource.putChild(name, provisioningResource)
+
+ self.principalRootResources[self.directoryService.__class__.__name__] = provisioningResource
+
+ yield XMLCalendarUserProxyLoader(proxiesFile.path).updateProxyDB()
+
+ def tearDown(self):
+ """ Empty the proxy db between tests """
+ return calendaruserproxy.ProxyDBService.clean()
+
+ def _getPrincipalByShortName(self, type, name):
+ provisioningResource = self.principalRootResources[self.directoryService.__class__.__name__]
+ return provisioningResource.principalForShortName(type, name)
+
+ def _updateMethod(self):
+ """
+ Update a counter in the following test
+ """
+ self.count += 1
+
+ @inlineCallbacks
+ def test_groupMembershipCacherService(self):
+ """
+ Instantiate a GroupMembershipCacherService and make sure its update
+ method fires at the right interval, in this case 30 seconds. The
+ updateMethod keyword arg is purely for testing purposes, so we can
+ directly detect it getting called in this test.
+ """
+ clock = Clock()
+ self.count = 0
+ service = GroupMembershipCacherService(
+ None, None, "Testing", 30, 60, reactor=clock,
+ updateMethod=self._updateMethod)
+
+ yield service.startService()
+
+ self.assertEquals(self.count, 1)
+ clock.advance(29)
+ self.assertEquals(self.count, 1)
+ clock.advance(1)
+ self.assertEquals(self.count, 2)
+
+ service.stopService()
+
+
+
+ @inlineCallbacks
+ def test_groupMembershipCacheMarker(self):
+ """
+ If the group member cache is not populated (as noted by the existence
+ of a special memcached key), a 503 should be raised
+ """
+ cache = GroupMembershipCache("ProxyDB")
+ # Having a groupMembershipCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.groupMembershipCache = cache
+
+ userc = self._getPrincipalByShortName(DirectoryService.recordType_users, "userc")
+
+ try:
+ yield userc.proxyFor(True)
+ except HTTPError:
+ pass
+ else:
+ self.fail("HTTPError was unexpectedly not raised")
+
+ try:
+ yield userc.groupMemberships(True)
+ except HTTPError:
+ pass
+ else:
+ self.fail("HTTPError was unexpectedly not raised")
+
+
+ def test_expandedMembers(self):
+ """
+ Make sure expandedMembers( ) returns a complete, flattened set of
+ members of a group, including all sub-groups.
+ """
+ bothCoasts = self.directoryService.recordWithShortName(
+ DirectoryService.recordType_groups, "both_coasts")
+ self.assertEquals(
+ set([r.guid for r in bothCoasts.expandedMembers()]),
+ set(['8B4288F6-CC82-491D-8EF9-642EF4F3E7D0',
+ '6423F94A-6B76-4A3A-815B-D52CFD77935D',
+ '5A985493-EE2C-4665-94CF-4DFEA3A89500',
+ '5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1',
+ 'left_coast',
+ 'right_coast'])
+ )
+
+
+ @inlineCallbacks
+ def test_groupMembershipCache(self):
+ """
+ Ensure we get back what we put in
+ """
+ cache = GroupMembershipCache("ProxyDB", expireSeconds=10)
+
+ yield cache.setGroupsFor("a", ["b", "c", "d"]) # a is in b, c, d
+ members = (yield cache.getGroupsFor("a"))
+ self.assertEquals(members, set(["b", "c", "d"]))
+
+ yield cache.setGroupsFor("b", []) # b not in any groups
+ members = (yield cache.getGroupsFor("b"))
+ self.assertEquals(members, set())
+
+ cache._memcacheProtocol.advanceClock(10)
+
+ members = (yield cache.getGroupsFor("a")) # has expired
+ self.assertEquals(members, set())
+
+
+
+
+ @inlineCallbacks
+ def test_groupMembershipCacheUpdater(self):
+ """
+ Let the GroupMembershipCacheUpdater populate the cache, then make
+ sure proxyFor( ) and groupMemberships( ) work from the cache
+ """
+ cache = GroupMembershipCache("ProxyDB", 60)
+ # Having a groupMembershipCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.groupMembershipCache = cache
+
+ updater = GroupMembershipCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService, 30,
+ cache=cache)
+ yield updater.updateCache()
+
+ delegates = (
+
+ # record name
+ # read-write delegators
+ # read-only delegators
+ # groups delegate is in (restricted to only those groups
+ # participating in delegation)
+
+ ("wsanchez",
+ set(["mercury", "apollo", "orion", "gemini"]),
+ set(["non_calendar_proxy"]),
+ set(['left_coast',
+ 'both_coasts',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ 'gemini#calendar-proxy-write',
+ ]),
+ ),
+ ("cdaboo",
+ set(["apollo", "orion", "non_calendar_proxy"]),
+ set(["non_calendar_proxy"]),
+ set(['both_coasts',
+ 'non_calendar_group',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ ]),
+ ),
+ ("lecroy",
+ set(["apollo", "mercury", "non_calendar_proxy"]),
+ set(),
+ set(['both_coasts',
+ 'left_coast',
+ 'non_calendar_group',
+ ]),
+ ),
+ ("usera",
+ set(),
+ set(),
+ set(),
+ ),
+ ("userb",
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
+ set(),
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
+ ),
+ ("userc",
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
+ set(),
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
+ ),
+ )
+
+ for name, write, read, groups in delegates:
+ delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
+
+ proxyFor = (yield delegate.proxyFor(True))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ write,
+ )
+ proxyFor = (yield delegate.proxyFor(False))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ read,
+ )
+ groupsIn = (yield delegate.groupMemberships())
+ uids = set()
+ for group in groupsIn:
+ try:
+ uid = group.uid # a sub-principal
+ except AttributeError:
+ uid = group.record.guid # a regular group
+ uids.add(uid)
+ self.assertEquals(
+ set(uids),
+ groups,
+ )
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -17,7 +17,6 @@
from twisted.internet.defer import DeferredList, inlineCallbacks, returnValue,\
succeed
from twext.web2.dav import davxml
-from twext.web2.http import HTTPError
from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.test.util import xmlFile, augmentsFile, proxiesFile
@@ -29,7 +28,6 @@
from twistedcaldav.config import config
from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
-from twisted.internet.task import Clock
class ProxyPrincipals (twistedcaldav.test.util.TestCase):
@@ -579,325 +577,3 @@
provisioningResource.principalForUID(uid)
- @inlineCallbacks
- def test_proxyMemberCache(self):
- """
- Ensure we get back what we put in
- """
- cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
-
- yield cache.setMembers("a", ["b", "c", "d"]) # has members
- members = (yield cache.getMembers("a"))
- self.assertEquals(members, set(["b", "c", "d"]))
-
- yield cache.setMembers("b", []) # has no members
- members = (yield cache.getMembers("b"))
- self.assertEquals(members, set())
-
- members = (yield cache.getMembers("c")) # wasn't specified at all
- self.assertEquals(members, None)
-
-
- yield cache.setProxyFor("a", "read", ["b", "c", "d"]) # has members
- proxyFor = (yield cache.getProxyFor("a", "read"))
- self.assertEquals(proxyFor, set(["b", "c", "d"]))
-
- yield cache.setProxyFor("b", "read", []) # has no members
- proxyFor = (yield cache.getProxyFor("b", "read"))
- self.assertEquals(proxyFor, set())
-
- proxyFor = (yield cache.getProxyFor("c", "read"))
- # wasn't specified at all
- self.assertEquals(proxyFor, None)
-
-
- @inlineCallbacks
- def test_expandedGroupMembersFromCache(self):
- """
- Put proxy data directly into cache, then make sure
- CalendarUserProxyPrincipalResour.expandedGroupMembers( ) goes to the
- cache for that info.
- """
-
- cdaboo = "5A985493-EE2C-4665-94CF-4DFEA3A89500"
- lecroy = "8B4288F6-CC82-491D-8EF9-642EF4F3E7D0"
-
- cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
-
- delegator = self._getPrincipalByShortName(DirectoryService.recordType_users, "wsanchez")
-
- # Having a proxyCache assigned to the directory service is the
- # trigger to use such a cache:
- self.directoryService.proxyCache = cache
-
- proxyGroup = delegator.getChild("calendar-proxy-write")
- yield cache.setMembers(proxyGroup.uid, [cdaboo, lecroy],
- expireSeconds=10)
- yield cache.createMarker(expireSeconds=20)
-
- members = (yield proxyGroup.expandedGroupMembers())
- self.assertEquals(
- set([p.record.guid for p in members]),
- set([cdaboo, lecroy])
- )
-
- # Now go 10 seconds in the future and the key will be expired
- # Note the marker is set to expire later, otherwise the
- # expandedGroupMembers( ) call would get a 503
- cache._memcacheProtocol.advanceClock(10)
- members = (yield proxyGroup.expandedGroupMembers())
- self.assertEquals(members, set())
-
-
- @inlineCallbacks
- def test_proxyMemberCacheUpdater(self):
- """
- Let the ProxyMemberCacheUpdater populate the cache, then make
- sure CalendarUserProxyPrincipalResource.expandedGroupMembers( ) goes
- to the cache for that info.
- """
- cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
- updater = calendaruserproxy.ProxyMemberCacheUpdater(
- calendaruserproxy.ProxyDBService, self.directoryService, 30,
- cache=cache)
- yield updater.updateCache()
-
- delegator = self._getPrincipalByShortName(DirectoryService.recordType_locations, "apollo")
-
- # Having a proxyCache assigned to the directory service is the
- # trigger to use such a cache:
- self.directoryService.proxyCache = cache
-
- proxyGroup = delegator.getChild("calendar-proxy-write")
-
- members = (yield proxyGroup.expandedGroupMembers())
- self.assertEquals(
- set([p.record.guid for p in members]),
- set(['8B4288F6-CC82-491D-8EF9-642EF4F3E7D0',
- '6423F94A-6B76-4A3A-815B-D52CFD77935D',
- '5A985493-EE2C-4665-94CF-4DFEA3A89500',
- '5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1',
- 'both_coasts',
- 'left_coast',
- 'right_coast'])
- )
-
- delegates = (
-
- # record name
- # read-write delegators
- # read-only delegators
- # groups delegate is in (which is now just the "sub principals")
-
- ("wsanchez",
- set(["mercury", "apollo", "orion", "gemini"]),
- set(["non_calendar_proxy"]),
- set(['apollo#calendar-proxy-write',
- 'gemini#calendar-proxy-write',
- 'mercury#calendar-proxy-write',
- 'non_calendar_proxy#calendar-proxy-read',
- 'orion#calendar-proxy-write']),
- ),
- ("cdaboo",
- set(["apollo", "orion", "non_calendar_proxy"]),
- set(["non_calendar_proxy"]),
- set(['apollo#calendar-proxy-write',
- 'non_calendar_proxy#calendar-proxy-read',
- 'non_calendar_proxy#calendar-proxy-write',
- 'orion#calendar-proxy-write']),
- ),
- ("lecroy",
- set(["apollo", "mercury", "non_calendar_proxy"]),
- set(),
- set(['apollo#calendar-proxy-write',
- 'mercury#calendar-proxy-write',
- 'non_calendar_proxy#calendar-proxy-write']),
- ),
- ("usera",
- set(),
- set(),
- set(),
- ),
- ("userb",
- set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
- set(),
- set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
- ),
- ("userc",
- set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
- set(),
- set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
- ),
- )
-
- for name, write, read, groups in delegates:
- delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
-
- proxyFor = (yield delegate.proxyFor(True))
- self.assertEquals(
- set([p.record.guid for p in proxyFor]),
- write,
- )
- proxyFor = (yield delegate.proxyFor(False))
- self.assertEquals(
- set([p.record.guid for p in proxyFor]),
- read,
- )
- groupsIn = (yield delegate.groupMemberships())
- self.assertEquals(
- set([p.uid for p in groupsIn]),
- groups,
- )
-
- #
- # Remove proxy assignments and see that the appropriate memcached
- # keys are updated/deleted
- #
- usera = self._getPrincipalByShortName(DirectoryService.recordType_users,
- "usera")
- userb = self._getPrincipalByShortName(DirectoryService.recordType_users,
- "userb")
- userc = self._getPrincipalByShortName(DirectoryService.recordType_users,
- "userc")
- useraProxyGroup = usera.getChild("calendar-proxy-write")
-
- # First, make sure there are two in the usera write proxy group
- members = (yield cache.getMembers(useraProxyGroup.uid))
- self.assertEquals(members, set([userb.record.guid, userc.record.guid]))
- members = (yield useraProxyGroup.expandedGroupMembers())
- self.assertEquals(
- set([p.record.shortNames[0] for p in members]),
- set(["userb", "userc"])
- )
- # ...and that userc is a write proxy for usera, talking directly to
- # the cache, and by going through principal.proxyFor( )
- proxyFor = (yield cache.getProxyFor(userc.record.guid, "write"))
- self.assertEquals(proxyFor, set([usera.record.guid]))
- proxyFor = (yield userc.proxyFor(True))
- self.assertEquals(set([p.record.shortNames[0] for p in proxyFor]),
- set(["usera"]))
-
- # Remove userb as a proxy
- yield self._removeProxy(
- DirectoryService.recordType_users, "usera",
- "calendar-proxy-write",
- DirectoryService.recordType_users, "userb",
- )
- yield updater.updateCache()
-
- # Next, there should only be one in the group
- members = (yield cache.getMembers(useraProxyGroup.uid))
- self.assertEquals(
- members,
- set([userc.record.guid])
- )
- members = (yield useraProxyGroup.expandedGroupMembers())
- self.assertEquals(
- set([p.record.shortNames[0] for p in members]),
- set(["userc"])
- )
- yield self._removeProxy(
- DirectoryService.recordType_users, "usera",
- "calendar-proxy-write",
- DirectoryService.recordType_users, "userc",
- )
- yield updater.updateCache()
-
- # Finally the group is empty and the key should be deleted
- members = (yield cache.getMembers(useraProxyGroup.uid))
- self.assertEquals(members, None)
- members = (yield useraProxyGroup.expandedGroupMembers())
- self.assertEquals(members, set())
-
- # ...and userc is not a write proxy for usera
- proxyFor = (yield cache.getProxyFor(userc.record.guid, "write"))
- self.assertEquals(proxyFor, None)
- proxyFor = (yield userc.proxyFor(True))
- self.assertEquals(proxyFor, set())
-
-
- def test_expandedMembers(self):
- """
- Make sure expandedMembers( ) returns a complete, flattened set of
- members of a group, including all sub-groups.
- """
- bothCoasts = self.directoryService.recordWithShortName(
- DirectoryService.recordType_groups, "both_coasts")
- self.assertEquals(
- set([r.guid for r in bothCoasts.expandedMembers()]),
- set(['8B4288F6-CC82-491D-8EF9-642EF4F3E7D0',
- '6423F94A-6B76-4A3A-815B-D52CFD77935D',
- '5A985493-EE2C-4665-94CF-4DFEA3A89500',
- '5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1',
- 'left_coast',
- 'right_coast'])
- )
-
- @inlineCallbacks
- def test_proxyCacheMarker(self):
- """
- If the proxy member cache is not populated (as noted by the existence
- of a special memcached key), a 503 should be raised
- """
- cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
- # Having a proxyCache assigned to the directory service is the
- # trigger to use such a cache:
- self.directoryService.proxyCache = cache
-
- userc = self._getPrincipalByShortName(DirectoryService.recordType_users, "userc")
-
- try:
- yield userc.proxyFor(True)
- except HTTPError:
- pass
- else:
- self.fail("HTTPError was unexpectedly not raised")
-
- try:
- yield userc.groupMemberships(True)
- except HTTPError:
- pass
- else:
- self.fail("HTTPError was unexpectedly not raised")
-
- usercProxyGroup = userc.getChild("calendar-proxy-write")
- try:
- yield usercProxyGroup.expandedGroupMembers()
- except HTTPError:
- pass
- else:
- self.fail("HTTPError was unexpectedly not raised")
-
-
- def _updateMethod(self):
- """
- Update a counter in the following test
- """
- self.count += 1
-
- @inlineCallbacks
- def test_proxyCacherService(self):
- """
- Instantiate a ProxyCacherService and make sure its update method
- fires at the right interval, in this case 30 seconds. The updateMethod
- keyword arg is purely for testing purposes, so we can directly detect
- it getting called in this test.
- """
- clock = Clock()
- self.count = 0
- service = calendaruserproxy.ProxyCacherService(
- calendaruserproxy.ProxyDBService,
- self.directoryService, "Testing", 30, 60, reactor=clock,
- updateMethod=self._updateMethod)
-
- yield service.startService()
-
- self.assertEquals(self.count, 1)
- clock.advance(29)
- self.assertEquals(self.count, 1)
- clock.advance(1)
- self.assertEquals(self.count, 2)
-
- service.stopService()
-
-
Modified: CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -70,7 +70,7 @@
'realmName' : '/Search',
'statSeconds' : 15,
'augmentService' : None,
- 'proxyCache' : None,
+ 'groupMembershipCache' : None,
}
ignored = None
params = self.getParams(params, defaults, ignored)
@@ -79,7 +79,7 @@
self.realmName = params['realmName']
self.statSeconds = params['statSeconds']
self.augmentService = params['augmentService']
- self.proxyCache = params['proxyCache']
+ self.groupMembershipCache = params['groupMembershipCache']
super(XMLDirectoryService, self).__init__()
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-07-08 21:04:12 UTC (rev 7750)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-07-08 23:27:50 UTC (rev 7751)
@@ -736,7 +736,7 @@
],
},
- "ProxyCaching" : {
+ "GroupCaching" : {
"Enabled": False,
"MemcachedPool" : "ProxyDB",
"UpdateSeconds" : 300,
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110708/71ed3404/attachment-0001.html>
More information about the calendarserver-changes
mailing list