[CalendarServer-changes] [7716] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jul 6 16:32:56 PDT 2011
Revision: 7716
http://trac.macosforge.org/projects/calendarserver/changeset/7716
Author: sagen at apple.com
Date: 2011-07-06 16:32:55 -0700 (Wed, 06 Jul 2011)
Log Message:
-----------
Adds a proxy-membership cacher which eliminates the need to fault in all the
groups (recursively!) a principal is in to see who they are a proxy for.
Instead, it turns the problem upside down and faults in the groups that have
been delegated to, and expands their membership. This is done in a sidecar
which updates the proxy-membership cache at a configurable interval. Changes
to proxy assignments will not take affect until that interval passes.
Note: this feature is off by default, but can be enabled in caldavd.plist
via the ProxyCaching key.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/twisted/plugins/caldav.py
CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/directory.py
CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
CalendarServer/trunk/twistedcaldav/directory/principal.py
CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -506,8 +506,28 @@
self.monitor.addProcess("mailgateway", mailGatewayArgv,
env=PARENT_ENVIRONMENT)
+ if config.ProxyCaching.Enabled:
+ self.maker.log_info("Adding proxy caching service")
+ proxyCacherArgv = [
+ sys.executable,
+ sys.argv[0],
+ ]
+ if config.UserName:
+ proxyCacherArgv.extend(("-u", config.UserName))
+ if config.GroupName:
+ proxyCacherArgv.extend(("-g", config.GroupName))
+ proxyCacherArgv.extend((
+ "--reactor=%s" % (config.Twisted.reactor,),
+ "-n", self.maker.proxyCacherTapName,
+ "-f", self.configPath,
+ ))
+ self.monitor.addProcess("proxycache", proxyCacherArgv,
+ env=PARENT_ENVIRONMENT)
+
+
+
class CalDAVServiceMaker (LoggingMixIn):
implements(IPlugin, IServiceMaker)
@@ -520,6 +540,7 @@
#
mailGatewayTapName = "caldav_mailgateway"
notifierTapName = "caldav_notifier"
+ proxyCacherTapName = "caldav_proxycacher"
def makeService(self, options):
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -222,6 +222,14 @@
raise
#
+ # Setup the proxy cacher
+ #
+ if config.ProxyCaching.Enabled:
+ proxyCache = calendaruserproxy.ProxyMemberCache(config.ProxyCaching.MemcachedPool)
+ else:
+ proxyCache = None
+
+ #
# Setup the Directory
#
directories = []
@@ -233,6 +241,7 @@
% (config.DirectoryService.type,))
config.DirectoryService.params.augmentService = augmentService
+ config.DirectoryService.params.proxyCache = proxyCache
baseDirectory = directoryClass(config.DirectoryService.params)
# Wait for the directory to become available
@@ -250,6 +259,7 @@
log.info("Configuring resource service of type: %s" % (resourceClass,))
config.ResourceService.params.augmentService = augmentService
+ config.ResourceService.params.proxyCache = proxyCache
resourceDirectory = resourceClass(config.ResourceService.params)
resourceDirectory.realmName = baseDirectory.realmName
directories.append(resourceDirectory)
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2011-07-06 23:32:55 UTC (rev 7716)
@@ -260,7 +260,7 @@
<key>recordName</key>
<string>uid</string>
<key>loginEnabledAttr</key>
- <string>loginEnabled</string>
+ <string></string>
<key>loginEnabledValue</key>
<string>yes</string>
</dict>
@@ -898,6 +898,17 @@
</array>
</dict>
+ <!-- Proxy Membership Caching -->
+ <key>ProxyCaching</key>
+ <dict>
+ <key>Enabled</key>
+ <false/>
+ <key>MemcachedPool</key>
+ <string>ProxyDB</string>
+ <key>UpdateSeconds</key>
+ <integer>300</integer>
+ </dict>
+
<!--
Twisted
-->
Modified: CalendarServer/trunk/twisted/plugins/caldav.py
===================================================================
--- CalendarServer/trunk/twisted/plugins/caldav.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twisted/plugins/caldav.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -52,3 +52,4 @@
TwistedCalDAV = TAP("calendarserver.tap.caldav.CalDAVServiceMaker")
CalDAVNotifier = TAP("twistedcaldav.notify.NotificationServiceMaker")
CalDAVMailGateway = TAP("twistedcaldav.mail.MailGatewayServiceMaker")
+CalDAVProxyCacher = TAP("twistedcaldav.directory.calendaruserproxy.ProxyCacherServiceMaker")
Modified: CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -70,13 +70,14 @@
'node' : '/Search',
'restrictEnabledRecords' : False,
'restrictToGroup' : '',
- 'cacheTimeout' : 1,
+ 'cacheTimeout' : 1, # Minutes
'negativeCaching' : False,
'recordTypes' : (
self.recordType_users,
self.recordType_groups,
),
'augmentService' : None,
+ 'proxyCache' : None,
}
ignored = ('requireComputerRecord',)
params = self.getParams(params, defaults, ignored)
@@ -95,6 +96,7 @@
raise
self.augmentService = params['augmentService']
+ self.proxyCache = params['proxyCache']
self.realmName = params['node']
self.directory = directory
self.node = params['node']
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -37,17 +37,27 @@
from twext.web2.dav.util import joinURL
from twext.web2.dav.noneprops import NonePropertyStore
-from twext.python.log import LoggingMixIn
+from twext.python.log import Logger, LoggingMixIn
+
from twistedcaldav.config import config, fullServerPath
from twistedcaldav.database import AbstractADBAPIDatabase, ADBAPISqliteMixin,\
ADBAPIPostgreSQLMixin
from twistedcaldav.extensions import DAVPrincipalResource,\
DAVResourceWithChildrenMixin
from twistedcaldav.extensions import ReadOnlyWritePropertiesResourceMixIn
+from twistedcaldav import memcachepool
from twistedcaldav.memcacher import Memcacher
from twistedcaldav.resource import CalDAVComplianceMixIn
+from twisted.python.reflect import namedClass
+from twisted.python.usage import Options, UsageError
+from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
+from twisted.application import service
+from twisted.plugin import IPlugin
+from zope.interface import implements
+log = Logger()
+
class PermissionsMixIn (ReadOnlyWritePropertiesResourceMixIn):
def defaultAccessControlList(self):
aces = (
@@ -350,11 +360,39 @@
returnValue(found)
def groupMembers(self):
- return self._expandMemberUIDs()
+ cache = getattr(self.parent.record.service, "proxyCache", None)
+ if cache is not None:
+ return self.expandedGroupMembers()
+ else:
+ return self._expandMemberUIDs()
+ @inlineCallbacks
def expandedGroupMembers(self):
- return self._expandMemberUIDs(infinity=True)
+ """
+ Return the complete, flattened set of principals belonging to this
+ group.
+ If the directory service is using a ProxyMemberCache, take advantage
+ of it and use the set of cached uids. Otherwise go through
+ _expandMemberUIDs.
+ """
+ cache = getattr(self.parent.record.service, "proxyCache", None)
+ if cache is not None:
+ log.debug("expandedGroupMembers using proxyCache")
+ if not (yield cache.checkMarker()):
+ raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
+ "Proxy membership cache not yet populated"))
+ principals = set()
+ memberUIDs = (yield cache.getMembers(self.uid))
+ if memberUIDs:
+ for uid in memberUIDs:
+ principal = self.pcollection.principalForUID(uid)
+ if principal is not None:
+ principals.add(principal)
+ returnValue(principals)
+ else:
+ returnValue((yield self._expandMemberUIDs(infinity=True)))
+
def groupMemberships(self):
# Get membership UIDs and map to principal resources
d = self._index().getMemberships(self.uid)
@@ -778,6 +816,9 @@
yield super(ProxyDB, self).clean()
+ @inlineCallbacks
+ def getAllGroups(self):
+ returnValue([row[0] for row in (yield self.query("select DISTINCT GROUPNAME from GROUPS"))])
ProxyDBService = None # Global proxyDB service
@@ -816,3 +857,346 @@
davxml.Protected(),
),
)
+
+
+class ProxyMemberCache(Memcacher, LoggingMixIn):
+ """
+ Caches expanded proxy membership information
+
+ This cache is periodically updated by a side car so that worker processes
+ never have to ask the directory service directly for group membership
+ information.
+
+ Keys in this cache are:
+
+ "proxy-members:<GUID>" : comma-separated list of member guids (flattened)
+ (where <GUID> will be that of a sub-principal, e.g.
+ 5A985493-EE2C-4665-94CF-4DFEA3A89500#calendar-proxy-write)
+
+ "proxy-read-memberships:<GUID>" : comma-separate list of principals
+ who have granted read access to GUID
+
+ "proxy-write-memberships:<GUID>" : comma-separate list of principals
+ who have granted read-write access to GUID
+
+ """
+
+
+ def setMembers(self, guid, members):
+ self.log_debug("set proxy-members %s : %s" % (guid, members))
+ return self.set("proxy-members:%s" % (str(guid),), str(",".join(members)))
+ def getMembers(self, guid):
+ self.log_debug("get proxy-members %s" % (guid,))
+ def _value(value):
+ if value:
+ return set(value.split(","))
+ elif value is None:
+ return None
+ else:
+ return set()
+ d = self.get("proxy-members:%s" % (str(guid),))
+ d.addCallback(_value)
+ return d
+
+ def deleteMembers(self, guid):
+ return self.delete("proxy-members:%s" % (str(guid),))
+
+ def setProxyFor(self, guid, proxyType, memberships):
+ self.log_debug("set %s-proxy-for %s : %s" %
+ (proxyType, guid, memberships))
+ return self.set("%s-proxy-for:%s" %
+ (proxyType, str(guid)), str(",".join(memberships)))
+
+ def getProxyFor(self, guid, proxyType):
+ self.log_debug("get %s-proxy-for %s" % (proxyType, guid))
+ def _value(value):
+ if value:
+ return set(value.split(","))
+ elif value is None:
+ return None
+ else:
+ return set()
+ d = self.get("%s-proxy-for:%s" % (proxyType, str(guid),))
+ d.addCallback(_value)
+ return d
+
+ def deleteProxyFor(self, guid, proxyType):
+ return self.delete("%s-proxy-for:%s" % (proxyType, str(guid),))
+
+ def createMarker(self):
+ return self.set("proxy-cache-populated", "true")
+
+ def checkMarker(self):
+ def _value(value):
+ return value == "true"
+ d = self.get("proxy-cache-populated")
+ d.addCallback(_value)
+ return d
+
+class ProxyMemberCacheUpdater(LoggingMixIn):
+ """
+ Responsible for updating memcached with proxy assignments. This will run
+ in a sidecar. There are two sources of proxy data to pull from: the local
+ proxy database, and the location/resource info in the directory system.
+
+ TODO: Implement location/resource
+ """
+
+ def __init__(self, proxyDB, directory, cache=None, namespace=None):
+ self.proxyDB = proxyDB
+ self.directory = directory
+ if cache is None:
+ assert namespace is not None, "namespace must be specified if ProxyMemberCache is not provided"
+ cache = ProxyMemberCache(namespace)
+ self.cache = cache
+ self.previousProxyGroups = set()
+ self.previousProxyFor = {
+ "read" : { },
+ "write" : { },
+ }
+
+ @inlineCallbacks
+ def updateCache(self):
+ """
+ Iterate the proxy database to retrieve all the principals who have been
+ delegated to. Fault these principals in. For any of these principals
+ that are groups, expand the members of that group and store those in
+ the cache
+ """
+ # TODO: add memcached eviction protection
+
+ self.log_debug("Updating proxy membership cache")
+
+ numProxyGroupsUpdated = 0
+ numProxiesUpdated = {
+ "read" : 0,
+ "write" : 0,
+ }
+
+ currentProxyGroups = set()
+ currentProxyFor = {
+ "read" : { },
+ "write" : { },
+ }
+ proxyGroups = (yield self.proxyDB.getAllGroups())
+ for proxyGroup in proxyGroups:
+
+ # Populate delegator -> delegate cache
+ combinedGUIDs = set()
+ for proxyGUID in (yield self.proxyDB.getMembers(proxyGroup)):
+ record = self.directory.recordWithGUID(proxyGUID)
+ if record:
+ # TODO: What if this guid is not in the directory?
+ combinedGUIDs.add(record.guid)
+ if record.recordType == self.directory.recordType_groups:
+ members = record.expandedMembers()
+ guids = set([r.guid for r in members])
+ combinedGUIDs.update(guids)
+
+ self.cache.setMembers(proxyGroup, combinedGUIDs)
+ numProxyGroupsUpdated += 1
+
+ if proxyGroup in self.previousProxyGroups:
+ # whatever remains in previousProxyGroups needs to be deleted
+ # from memcached
+ self.previousProxyGroups.remove(proxyGroup)
+ currentProxyGroups.add(proxyGroup)
+
+ # Populate delegate -> delegator mapping
+ delegator, suffix = proxyGroup.split("#")
+ proxyType = "read" if suffix.endswith("read") else "write"
+ for guid in combinedGUIDs:
+ proxyFor = currentProxyFor[proxyType].setdefault(guid, set())
+ proxyFor.add(delegator)
+
+ for proxyType in ("read", "write"):
+ for guid, memberships in currentProxyFor[proxyType].iteritems():
+ self.cache.setProxyFor(guid, proxyType, memberships)
+ numProxiesUpdated[proxyType] += 1
+ if self.previousProxyFor[proxyType].has_key(guid):
+ # whatever remains in previousProxyFor needs to be deleted
+ # from memcached
+ del self.previousProxyFor[proxyType][guid]
+
+ self.log_debug("%d proxyGroups updated" % (numProxyGroupsUpdated,))
+ for proxyType in ("read", "write"):
+ self.log_debug("%d %s proxies updated" % (numProxiesUpdated[proxyType], proxyType))
+
+ # Delete obsolete memcached keys
+ for proxyGroup in self.previousProxyGroups:
+ self.log_debug("Deleting proxyGroup members for %s" % (proxyGroup,))
+ self.cache.deleteMembers(proxyGroup)
+ for proxyType in ("read", "write"):
+ for guid in self.previousProxyFor[proxyType].iterkeys():
+ self.log_debug("Deleting %s proxyFor members for %s" % (proxyType, guid,))
+ self.cache.deleteProxyFor(guid, proxyType)
+
+ self.previousProxyGroups = currentProxyGroups
+ self.previousProxyFor = currentProxyFor
+
+ # Put a special key into memcached to let workers know proxyCache is
+ # populated
+ self.cache.createMarker()
+
+
+class ProxyCacherOptions(Options):
+ optParameters = [[
+ "config", "f", DEFAULT_CONFIG_FILE, "Path to configuration file."
+ ]]
+
+ def __init__(self, *args, **kwargs):
+ super(ProxyCacherOptions, self).__init__(*args, **kwargs)
+
+ self.overrides = {}
+
+ def _coerceOption(self, configDict, key, value):
+ """
+ Coerce the given C{val} to type of C{configDict[key]}
+ """
+ if key in configDict:
+ if isinstance(configDict[key], bool):
+ value = value == "True"
+
+ elif isinstance(configDict[key], (int, float, long)):
+ value = type(configDict[key])(value)
+
+ elif isinstance(configDict[key], (list, tuple)):
+ value = value.split(',')
+
+ elif isinstance(configDict[key], dict):
+ raise UsageError(
+ "Dict options not supported on the command line"
+ )
+
+ elif value == 'None':
+ value = None
+
+ return value
+
+ def _setOverride(self, configDict, path, value, overrideDict):
+ """
+ Set the value at path in configDict
+ """
+ key = path[0]
+
+ if len(path) == 1:
+ overrideDict[key] = self._coerceOption(configDict, key, value)
+ return
+
+ if key in configDict:
+ if not isinstance(configDict[key], dict):
+ raise UsageError(
+ "Found intermediate path element that is not a dictionary"
+ )
+
+ if key not in overrideDict:
+ overrideDict[key] = {}
+
+ self._setOverride(
+ configDict[key], path[1:],
+ value, overrideDict[key]
+ )
+
+
+ def opt_option(self, option):
+ """
+ Set an option to override a value in the config file. True, False, int,
+ and float options are supported, as well as comma seperated lists. Only
+ one option may be given for each --option flag, however multiple
+ --option flags may be specified.
+ """
+
+ if "=" in option:
+ path, value = option.split('=')
+ self._setOverride(
+ DEFAULT_CONFIG,
+ path.split('/'),
+ value,
+ self.overrides
+ )
+ else:
+ self.opt_option('%s=True' % (option,))
+
+ opt_o = opt_option
+
+ def postOptions(self):
+ config.load(self['config'])
+ config.updateDefaults(self.overrides)
+ self.parent['pidfile'] = None
+
+
+
+class ProxyCacherService(service.Service, LoggingMixIn):
+ """
+ Service to update the proxy cache at a configured interval
+ """
+
+ def __init__(self, proxyDB, directory, namespace, seconds, reactor=None):
+ self.updater = ProxyMemberCacheUpdater(proxyDB, directory,
+ namespace=namespace)
+ if reactor is None:
+ from twisted.internet import reactor
+ self.reactor = reactor
+ self.seconds = seconds
+ self.nextUpdate = None
+
+ def startService(self):
+ self.log_warn("Starting proxy cacher service")
+ service.Service.startService(self)
+ self.update()
+
+ @inlineCallbacks
+ def update(self):
+ self.nextUpdate = None
+ try:
+ yield self.updater.updateCache()
+ finally:
+ self.log_debug("Scheduling next proxy cacher update")
+ self.nextUpdate = self.reactor.callLater(self.seconds, self.update)
+
+ def stopService(self):
+ self.log_warn("Stopping proxy cacher service")
+ service.Service.stopService(self)
+ if self.nextUpdate is not None:
+ self.nextUpdate.cancel()
+
+
+class ProxyCacherServiceMaker(LoggingMixIn):
+ """
+ Configures and returns a ProxyCacherService
+ """
+ implements(IPlugin, service.IServiceMaker)
+
+ tapname = "caldav_proxycacher"
+ description = "Proxy Cacher"
+ options = ProxyCacherOptions
+
+ def makeService(self, options):
+
+ # Setup the directory
+ from calendarserver.tap.util import directoryFromConfig
+ directory = directoryFromConfig(config)
+
+ # Setup the ProxyDB Service
+ proxydbClass = namedClass(config.ProxyDBService.type)
+
+ self.log_warn("Configuring proxydb service of type: %s" % (proxydbClass,))
+
+ try:
+ proxyDB = proxydbClass(**config.ProxyDBService.params)
+ except IOError:
+ self.log_error("Could not start proxydb service")
+ raise
+
+ # Setup memcached pools
+ memcachepool.installPools(
+ config.Memcached.Pools,
+ config.Memcached.MaxClients,
+ )
+
+ proxyCacherService = ProxyCacherService(proxyDB, directory,
+ config.ProxyCaching.MemcachedPool,
+ config.ProxyCaching.UpdateSeconds)
+
+ return proxyCacherService
+
Modified: CalendarServer/trunk/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/directory.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -492,6 +492,26 @@
def members(self):
return ()
+ def expandedMembers(self, members=None, seen=None):
+ """
+ Return the complete, flattened set of members of a group, including
+ all sub-groups.
+ """
+ if members is None:
+ members = set()
+ if seen is None:
+ seen = set()
+
+ if self not in seen:
+ seen.add(self)
+ for member in self.members():
+ members.add(member)
+ if member.recordType == self.service.recordType_groups:
+ member.expandedMembers(members=members, seen=seen)
+
+ return members
+
+
def groups(self):
return ()
Modified: CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -75,6 +75,7 @@
defaults = {
"augmentService" : None,
+ "proxyCache" : None,
"cacheTimeout": 1, # Minutes
"negativeCaching": False,
"restrictEnabledRecords": False,
@@ -145,6 +146,7 @@
params["negativeCaching"])
self.augmentService = params["augmentService"]
+ self.proxyCache = params["proxyCache"]
self.realmName = params["uri"]
self.uri = params["uri"]
self.tls = params["tls"]
Modified: CalendarServer/trunk/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/principal.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -41,7 +41,7 @@
from twisted.internet.defer import succeed
from twext.web2.auth.digest import DigestedCredentials
from twext.web2 import responsecode
-from twext.web2.http import HTTPError
+from twext.web2.http import HTTPError, StatusResponse
from twext.web2.dav import davxml
from twext.web2.dav.util import joinURL
from twext.web2.dav.noneprops import NonePropertyStore
@@ -663,6 +663,25 @@
@inlineCallbacks
def proxyFor(self, read_write, resolve_memberships=True):
+
+ cache = getattr(self.record.service, "proxyCache", None)
+ if cache is not None:
+ log.debug("proxyFor is using proxyCache")
+ if not (yield cache.checkMarker()):
+ raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
+ "Proxy membership cache not yet populated"))
+
+ principals = set()
+ proxyType = "write" if read_write else "read"
+ delegatorUIDs = (yield cache.getProxyFor(self.record.guid, proxyType))
+ if delegatorUIDs:
+ for uid in delegatorUIDs:
+ principal = self.parent.principalForUID(uid)
+ if principal is not None:
+ principals.add(principal)
+ returnValue(principals)
+
+ # Slower, non cached method:
proxyFors = set()
if resolve_memberships:
@@ -731,6 +750,28 @@
@inlineCallbacks
def groupMemberships(self, infinity=False):
+
+ cache = getattr(self.record.service, "proxyCache", None)
+ if cache is not None:
+ # We only need to worry about groups participating in delegation
+ log.debug("groupMemberships is using proxyCache")
+ if not (yield cache.checkMarker()):
+ raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,
+ "Proxy membership cache not yet populated"))
+ groups = set()
+ for proxyType in ("read", "write"):
+ delegatorUIDs = (yield cache.getProxyFor(self.record.guid,
+ proxyType))
+ if delegatorUIDs:
+ for uid in delegatorUIDs:
+ principal = self.parent.principalForUID(uid)
+ if principal is not None:
+ group = principal.getChild("calendar-proxy-%s" %
+ (proxyType,))
+ groups.add(group)
+ returnValue(groups)
+
+ # Slower, fetching-many-groups method:
groups = self._getRelatives("groups", infinity=infinity)
if config.EnableProxyPrincipals:
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -17,6 +17,7 @@
from twisted.internet.defer import DeferredList, inlineCallbacks, returnValue,\
succeed
from twext.web2.dav import davxml
+from twext.web2.http import HTTPError
from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.test.util import xmlFile, augmentsFile, proxiesFile
@@ -29,6 +30,7 @@
from twistedcaldav.directory import augment, calendaruserproxy
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
+
class ProxyPrincipals (twistedcaldav.test.util.TestCase):
"""
Directory service provisioned principals.
@@ -574,3 +576,285 @@
memberships = yield proxyPrincipal._calendar_user_proxy_index().getMemberships(proxyPrincipal.principalUID())
for uid in memberships:
provisioningResource.principalForUID(uid)
+
+
+ @inlineCallbacks
+ def test_proxyMemberCache(self):
+ """
+ Ensure we get back what we put in
+ """
+ cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
+
+ yield cache.setMembers("a", ["b", "c", "d"]) # has members
+ members = (yield cache.getMembers("a"))
+ self.assertEquals(members, set(["b", "c", "d"]))
+
+ yield cache.setMembers("b", []) # has no members
+ members = (yield cache.getMembers("b"))
+ self.assertEquals(members, set())
+
+ members = (yield cache.getMembers("c")) # wasn't specified at all
+ self.assertEquals(members, None)
+
+
+ yield cache.setProxyFor("a", "read", ["b", "c", "d"]) # has members
+ proxyFor = (yield cache.getProxyFor("a", "read"))
+ self.assertEquals(proxyFor, set(["b", "c", "d"]))
+
+ yield cache.setProxyFor("b", "read", []) # has no members
+ proxyFor = (yield cache.getProxyFor("b", "read"))
+ self.assertEquals(proxyFor, set())
+
+ proxyFor = (yield cache.getProxyFor("c", "read"))
+ # wasn't specified at all
+ self.assertEquals(proxyFor, None)
+
+
+ @inlineCallbacks
+ def test_expandedGroupMembersFromCache(self):
+ """
+ Put proxy data directly into cache, then make sure
+ CalendarUserProxyPrincipalResour.expandedGroupMembers( ) goes to the
+ cache for that info.
+ """
+
+ cdaboo = "5A985493-EE2C-4665-94CF-4DFEA3A89500"
+ lecroy = "8B4288F6-CC82-491D-8EF9-642EF4F3E7D0"
+
+ cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
+
+ delegator = self._getPrincipalByShortName(DirectoryService.recordType_users, "wsanchez")
+
+ # Having a proxyCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.proxyCache = cache
+
+ proxyGroup = delegator.getChild("calendar-proxy-write")
+ yield cache.setMembers(proxyGroup.uid, [cdaboo, lecroy])
+ yield cache.createMarker()
+
+ members = (yield proxyGroup.expandedGroupMembers())
+ self.assertEquals(
+ set([p.record.guid for p in members]),
+ set([cdaboo, lecroy])
+ )
+
+
+ @inlineCallbacks
+ def test_proxyMemberCacheUpdater(self):
+ """
+ Let the ProxyMemberCacheUpdater populate the cache, then make
+ sure CalendarUserProxyPrincipalResource.expandedGroupMembers( ) goes
+ to the cache for that info.
+ """
+ cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
+ updater = calendaruserproxy.ProxyMemberCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService,
+ cache=cache)
+ yield updater.updateCache()
+
+ delegator = self._getPrincipalByShortName(DirectoryService.recordType_locations, "apollo")
+
+ # Having a proxyCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.proxyCache = cache
+
+ proxyGroup = delegator.getChild("calendar-proxy-write")
+
+ members = (yield proxyGroup.expandedGroupMembers())
+ self.assertEquals(
+ set([p.record.guid for p in members]),
+ set(['8B4288F6-CC82-491D-8EF9-642EF4F3E7D0',
+ '6423F94A-6B76-4A3A-815B-D52CFD77935D',
+ '5A985493-EE2C-4665-94CF-4DFEA3A89500',
+ '5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1',
+ 'both_coasts',
+ 'left_coast',
+ 'right_coast'])
+ )
+
+ delegates = (
+
+ # record name
+ # read-write delegators
+ # read-only delegators
+ # groups delegate is in (which is now just the "sub principals")
+
+ ("wsanchez",
+ set(["mercury", "apollo", "orion", "gemini"]),
+ set(["non_calendar_proxy"]),
+ set(['apollo#calendar-proxy-write',
+ 'gemini#calendar-proxy-write',
+ 'mercury#calendar-proxy-write',
+ 'non_calendar_proxy#calendar-proxy-read',
+ 'orion#calendar-proxy-write']),
+ ),
+ ("cdaboo",
+ set(["apollo", "orion", "non_calendar_proxy"]),
+ set(["non_calendar_proxy"]),
+ set(['apollo#calendar-proxy-write',
+ 'non_calendar_proxy#calendar-proxy-read',
+ 'non_calendar_proxy#calendar-proxy-write',
+ 'orion#calendar-proxy-write']),
+ ),
+ ("lecroy",
+ set(["apollo", "mercury", "non_calendar_proxy"]),
+ set(),
+ set(['apollo#calendar-proxy-write',
+ 'mercury#calendar-proxy-write',
+ 'non_calendar_proxy#calendar-proxy-write']),
+ ),
+ ("usera",
+ set(),
+ set(),
+ set(),
+ ),
+ ("userb",
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
+ set(),
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
+ ),
+ ("userc",
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D']),
+ set(),
+ set(['7423F94A-6B76-4A3A-815B-D52CFD77935D#calendar-proxy-write']),
+ ),
+ )
+
+ for name, write, read, groups in delegates:
+ delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
+
+ proxyFor = (yield delegate.proxyFor(True))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ write,
+ )
+ proxyFor = (yield delegate.proxyFor(False))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ read,
+ )
+ groupsIn = (yield delegate.groupMemberships())
+ self.assertEquals(
+ set([p.uid for p in groupsIn]),
+ groups,
+ )
+
+ #
+ # Remove proxy assignments and see that the appropriate memcached
+ # keys are updated/deleted
+ #
+ usera = self._getPrincipalByShortName(DirectoryService.recordType_users,
+ "usera")
+ userb = self._getPrincipalByShortName(DirectoryService.recordType_users,
+ "userb")
+ userc = self._getPrincipalByShortName(DirectoryService.recordType_users,
+ "userc")
+ useraProxyGroup = usera.getChild("calendar-proxy-write")
+
+ # First, make sure there are two in the usera write proxy group
+ members = (yield cache.getMembers(useraProxyGroup.uid))
+ self.assertEquals(members, set([userb.record.guid, userc.record.guid]))
+ members = (yield useraProxyGroup.expandedGroupMembers())
+ self.assertEquals(
+ set([p.record.shortNames[0] for p in members]),
+ set(["userb", "userc"])
+ )
+ # ...and that userc is a write proxy for usera, talking directly to
+ # the cache, and by going through principal.proxyFor( )
+ proxyFor = (yield cache.getProxyFor(userc.record.guid, "write"))
+ self.assertEquals(proxyFor, set([usera.record.guid]))
+ proxyFor = (yield userc.proxyFor(True))
+ self.assertEquals(set([p.record.shortNames[0] for p in proxyFor]),
+ set(["usera"]))
+
+ # Remove userb as a proxy
+ yield self._removeProxy(
+ DirectoryService.recordType_users, "usera",
+ "calendar-proxy-write",
+ DirectoryService.recordType_users, "userb",
+ )
+ yield updater.updateCache()
+
+ # Next, there should only be one in the group
+ members = (yield cache.getMembers(useraProxyGroup.uid))
+ self.assertEquals(
+ members,
+ set([userc.record.guid])
+ )
+ members = (yield useraProxyGroup.expandedGroupMembers())
+ self.assertEquals(
+ set([p.record.shortNames[0] for p in members]),
+ set(["userc"])
+ )
+ yield self._removeProxy(
+ DirectoryService.recordType_users, "usera",
+ "calendar-proxy-write",
+ DirectoryService.recordType_users, "userc",
+ )
+ yield updater.updateCache()
+
+ # Finally the group is empty and the key should be deleted
+ members = (yield cache.getMembers(useraProxyGroup.uid))
+ self.assertEquals(members, None)
+ members = (yield useraProxyGroup.expandedGroupMembers())
+ self.assertEquals(members, set())
+
+ # ...and userc is not a write proxy for usera
+ proxyFor = (yield cache.getProxyFor(userc.record.guid, "write"))
+ self.assertEquals(proxyFor, None)
+ proxyFor = (yield userc.proxyFor(True))
+ self.assertEquals(proxyFor, set())
+
+
+ def test_expandedMembers(self):
+ """
+ Make sure expandedMembers( ) returns a complete, flattened set of
+ members of a group, including all sub-groups.
+ """
+ bothCoasts = self.directoryService.recordWithShortName(
+ DirectoryService.recordType_groups, "both_coasts")
+ self.assertEquals(
+ set([r.guid for r in bothCoasts.expandedMembers()]),
+ set(['8B4288F6-CC82-491D-8EF9-642EF4F3E7D0',
+ '6423F94A-6B76-4A3A-815B-D52CFD77935D',
+ '5A985493-EE2C-4665-94CF-4DFEA3A89500',
+ '5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1',
+ 'left_coast',
+ 'right_coast'])
+ )
+
+ @inlineCallbacks
+ def test_proxyCacheMarker(self):
+ """
+ If the proxy member cache is not populated (as noted by the existence
+ of a special memcached key), a 503 should be raised
+ """
+ cache = calendaruserproxy.ProxyMemberCache("ProxyDB")
+ # Having a proxyCache assigned to the directory service is the
+ # trigger to use such a cache:
+ self.directoryService.proxyCache = cache
+
+ userc = self._getPrincipalByShortName(DirectoryService.recordType_users, "userc")
+
+ try:
+ yield userc.proxyFor(True)
+ except HTTPError:
+ pass
+ else:
+ self.fail("HTTPError was unexpectedly not raised")
+
+ try:
+ yield userc.groupMemberships(True)
+ except HTTPError:
+ pass
+ else:
+ self.fail("HTTPError was unexpectedly not raised")
+
+ usercProxyGroup = userc.getChild("calendar-proxy-write")
+ try:
+ yield usercProxyGroup.expandedGroupMembers()
+ except HTTPError:
+ pass
+ else:
+ self.fail("HTTPError was unexpectedly not raised")
Modified: CalendarServer/trunk/twistedcaldav/directory/xmlfile.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/directory/xmlfile.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -70,6 +70,7 @@
'realmName' : '/Search',
'statSeconds' : 15,
'augmentService' : None,
+ 'proxyCache' : None,
}
ignored = None
params = self.getParams(params, defaults, ignored)
@@ -78,6 +79,7 @@
self.realmName = params['realmName']
self.statSeconds = params['statSeconds']
self.augmentService = params['augmentService']
+ self.proxyCache = params['proxyCache']
super(XMLDirectoryService, self).__init__()
@@ -589,6 +591,7 @@
for recordType, shortName in self._members:
yield self.service.recordWithShortName(recordType, shortName)
+
def groups(self):
for shortName in self._groups:
yield self.service.recordWithShortName(DirectoryService.recordType_groups, shortName)
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-07-06 18:25:40 UTC (rev 7715)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-07-06 23:32:55 UTC (rev 7716)
@@ -48,14 +48,14 @@
},
"twistedcaldav.directory.appleopendirectory.OpenDirectoryService": {
"node": "/Search",
- "cacheTimeout": 1,
+ "cacheTimeout": 1, # Minutes
"negativeCaching": False,
"restrictEnabledRecords": False,
"restrictToGroup": "",
"recordTypes": ("users", "groups"),
},
"twistedcaldav.directory.ldapdirectory.LdapDirectoryService": {
- "cacheTimeout": 1,
+ "cacheTimeout": 1, # Minutes
"negativeCaching": False,
"restrictEnabledRecords": False,
"restrictToGroup": "",
@@ -125,7 +125,7 @@
},
"twistedcaldav.directory.appleopendirectory.OpenDirectoryService": {
"node": "/Search",
- "cacheTimeout": 1,
+ "cacheTimeout": 1, # Minutes
"negativeCaching": False,
"restrictEnabledRecords": False,
"restrictToGroup": "",
@@ -736,6 +736,12 @@
],
},
+ "ProxyCaching" : {
+ "Enabled": False,
+ "MemcachedPool" : "ProxyDB",
+ "UpdateSeconds" : 300,
+ },
+
"EnableKeepAlive": True,
"EnableResponseCache": True,
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110706/4c0d6e83/attachment-0001.html>
More information about the calendarserver-changes
mailing list