[CalendarServer-changes] [2489] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Sat May 24 00:20:00 PDT 2008


Revision: 2489
          http://trac.macosforge.org/projects/calendarserver/changeset/2489
Author:   cdaboo at apple.com
Date:     2008-05-24 00:19:59 -0700 (Sat, 24 May 2008)

Log Message:
-----------
Merged branches/users/cdaboo/proxy-db-cached-2451 to trunk and converted to use memcached based cache.

Modified Paths:
--------------
    CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.idav.patch
    CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.method.report_principal_match.patch
    CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.resource.patch
    CalendarServer/trunk/twistedcaldav/__init__.py
    CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
    CalendarServer/trunk/twistedcaldav/directory/principal.py
    CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py
    CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py
    CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
    CalendarServer/trunk/twistedcaldav/resource.py

Added Paths:
-----------
    CalendarServer/trunk/twistedcaldav/memcacher.py

Modified: CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.idav.patch
===================================================================
--- CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.idav.patch	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.idav.patch	2008-05-24 07:19:59 UTC (rev 2489)
@@ -120,9 +120,20 @@
  class IDAVPrincipalResource (IDAVResource):
      """
      WebDAV principal resource.  (RFC 3744, section 2)
-@@ -212,3 +283,14 @@
+@@ -203,12 +274,23 @@
+         """
+         Provides the principal URLs of principals that are direct members of
+         this (group) principal.  (RFC 3744, section 4.3)
+-        @return: a iterable of principal URLs.
++        @return: a deferred returning an iterable of principal URLs.
+         """
+ 
+     def groupMemberships():
+         """
+         Provides the URLs of the group principals in which the principal is
          directly a member.  (RFC 3744, section 4.4)
-         @return: a iterable of group principal URLs.
+-        @return: a iterable of group principal URLs.
++        @return: a deferred containing an iterable of group principal URLs.
          """
 +
 +class IDAVPrincipalCollectionResource (IDAVResource):

Modified: CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.method.report_principal_match.patch
===================================================================
--- CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.method.report_principal_match.patch	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.method.report_principal_match.patch	2008-05-24 07:19:59 UTC (rev 2489)
@@ -2,7 +2,7 @@
 ===================================================================
 --- twisted/web2/dav/method/report_principal_match.py	(revision 19773)
 +++ twisted/web2/dav/method/report_principal_match.py	(working copy)
-@@ -89,40 +89,61 @@
+@@ -89,40 +89,64 @@
          responses = []
          matchcount = 0
  
@@ -38,7 +38,10 @@
 +            selfItems = [principal,]
 +            
 +            # Get group memberships for "self" and add each of those
-+            selfItems.extend(principal.groupMemberships())
++            d = waitForDeferred(principal.groupMemberships())
++            yield d
++            memberships = d.getResult()
++            selfItems.extend(memberships)
 +            
 +            # Now add each principal found to the response provided the principal resource is a child of
 +            # the current resource.
@@ -94,12 +97,45 @@
              for child, uri in children:
                  # Try to read the requested property from this resource
                  try:
-@@ -137,7 +158,7 @@
+@@ -137,22 +161,26 @@
                          yield principal
                          principal = principal.getResult()
  
 -                        if principal and isPrincipalResource(principal) and principal.principalMatch(selfPrincipal):
-+                        if principal and isPrincipalResource(principal) and principal.principalMatch(selfPrincipalURL):
-                             # Check size of results is within limit
-                             matchcount += 1
-                             if matchcount > max_number_of_matches:
+-                            # Check size of results is within limit
+-                            matchcount += 1
+-                            if matchcount > max_number_of_matches:
+-                                raise NumberOfMatchesWithinLimits
+-
+-                            d = waitForDeferred(prop_common.responseForHref(
+-                                request,
+-                                responses,
+-                                davxml.HRef.fromString(uri),
+-                                child,
+-                                propertiesForResource,
+-                                propElement
+-                            ))
++                        if principal and isPrincipalResource(principal):
++                            d = waitForDeferred(principal.principalMatch(selfPrincipalURL))
+                             yield d
+-                            d.getResult()
++                            matched = d.getResult()
++                            if matched:
++                                # Check size of results is within limit
++                                matchcount += 1
++                                if matchcount > max_number_of_matches:
++                                    raise NumberOfMatchesWithinLimits
++    
++                                d = waitForDeferred(prop_common.responseForHref(
++                                    request,
++                                    responses,
++                                    davxml.HRef.fromString(uri),
++                                    child,
++                                    propertiesForResource,
++                                    propElement
++                                ))
++                                yield d
++                                d.getResult()
+                 except HTTPError:
+                     # Just ignore a failure to access the property. We treat this like a property that does not exist
+                     # or does not match the principal.

Modified: CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.resource.patch
===================================================================
--- CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.resource.patch	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/lib-patches/Twisted/twisted.web2.dav.resource.patch	2008-05-24 07:19:59 UTC (rev 2489)
@@ -547,20 +547,48 @@
          # Compare two HRefs and do group membership test as well
          if principal1 == principal2:
              yield True
-@@ -1302,9 +1383,9 @@
-         def testGroup(group):
-             # Get principal resource for principal2
-             if group and isinstance(group, DAVPrincipalResource):
+@@ -1289,6 +1370,7 @@
+ 
+     matchPrincipal = deferredGenerator(matchPrincipal)
+ 
++    @deferredGenerator
+     def principalIsGroupMember(self, principal1, principal2, request):
+         """
+         Check whether one principal is a group member of another.
+@@ -1299,18 +1381,21 @@
+         @return: L{Deferred} with result C{True} if principal1 is a member of principal2, C{False} otherwise
+         """
+         
+-        def testGroup(group):
+-            # Get principal resource for principal2
+-            if group and isinstance(group, DAVPrincipalResource):
 -                members = group.groupMembers()
 -                if principal1 in members:
 -                    return True
-+                for member in group.groupMembers():
-+                    if member.principalURL() == principal1:
-+                        return True
-                 
-             return False
+-                
+-            return False
++        d = waitForDeferred(request.locateResource(principal2))
++        yield d
++        group = d.getResult()
  
-@@ -1351,11 +1432,16 @@
+-        d = request.locateResource(principal2)
+-        d.addCallback(testGroup)
+-        return d
++        # Get principal resource for principal2
++        if group and isinstance(group, DAVPrincipalResource):
++            d = waitForDeferred(group.groupMembers())
++            yield d
++            members = d.getResult()
++            for member in members:
++                if member.principalURL() == principal1:
++                    yield True
++                    return
++            
++        yield False
+         
+     def validPrincipal(self, ace_principal, request):
+         """
+@@ -1351,11 +1436,16 @@
          @return C{True} if C{href_principal} is valid, C{False} otherwise.
  
          This implementation tests for a href element that corresponds to
@@ -580,7 +608,7 @@
          return d
  
      def resolvePrincipal(self, principal, request):
-@@ -1432,7 +1518,7 @@
+@@ -1432,7 +1522,7 @@
                  log.err("DAV:self ACE is set on non-principal resource %r" % (self,))
                  yield None
                  return
@@ -589,7 +617,7 @@
  
          if isinstance(principal, davxml.HRef):
              yield principal
-@@ -1517,6 +1603,270 @@
+@@ -1517,6 +1607,270 @@
          return None
  
      ##
@@ -860,7 +888,7 @@
      # HTTP
      ##
  
-@@ -1525,10 +1875,6 @@
+@@ -1525,10 +1879,6 @@
          #litmus = request.headers.getRawHeaders("x-litmus")
          #if litmus: log.msg("*** Litmus test: %s ***" % (litmus,))
  
@@ -871,7 +899,7 @@
          #
          # If this is a collection and the URI doesn't end in "/", redirect.
          #
-@@ -1567,7 +1913,7 @@
+@@ -1567,7 +1917,7 @@
      def findChildren(self, depth, request, callback, privileges=None, inherited_aces=None):
          return succeed(None)
  
@@ -880,7 +908,7 @@
      """
      Resource representing a WebDAV principal.  (RFC 3744, section 2)
      """
-@@ -1577,7 +1923,7 @@
+@@ -1577,7 +1927,7 @@
      # WebDAV
      ##
  
@@ -889,7 +917,7 @@
          (dav_namespace, "alternate-URI-set"),
          (dav_namespace, "principal-URL"    ),
          (dav_namespace, "group-member-set" ),
-@@ -1585,14 +1931,11 @@
+@@ -1585,14 +1935,11 @@
      )
  
      def davComplianceClasses(self):
@@ -905,26 +933,60 @@
      def readProperty(self, property, request):
          def defer():
              if type(property) is tuple:
-@@ -1610,10 +1953,10 @@
+@@ -1610,10 +1957,20 @@
                      return davxml.PrincipalURL(davxml.HRef(self.principalURL()))
  
                  if name == "group-member-set":
 -                    return davxml.GroupMemberSet(*[davxml.HRef(p) for p in self.groupMembers()])
-+                    return davxml.GroupMemberSet(*[davxml.HRef(p.principalURL()) for p in self.groupMembers()])
++                    def callback(members):
++                        return davxml.GroupMemberSet(*[davxml.HRef(p.principalURL()) for p in members])
++                    
++                    d = self.groupMembers()
++                    d.addCallback(callback)
++                    return d
  
                  if name == "group-membership":
 -                    return davxml.GroupMembership(*[davxml.HRef(g) for g in self.groupMemberships()])
-+                    return davxml.GroupMembership(*[davxml.HRef(g.principalURL()) for g in self.groupMemberships()])
++                    def callback(memberships):
++                        return davxml.GroupMembership(*[davxml.HRef(g.principalURL()) for g in memberships])
++                    
++                    d = self.groupMemberships()
++                    d.addCallback(callback)
++                    return d
  
                  if name == "resourcetype":
                      if self.isCollection():
-@@ -1677,8 +2020,27 @@
+@@ -1655,7 +2012,7 @@
+         principals.  Subclasses should override this method to provide member
+         URLs for this resource if appropriate.
+         """
+-        return ()
++        return succeed(())
+ 
+     def groupMemberships(self):
+         """
+@@ -1666,6 +2023,7 @@
+         """
+         unimplemented(self)
+ 
++    @deferredGenerator
+     def principalMatch(self, href):
+         """
+         Check whether the supplied principal matches this principal or is a
+@@ -1675,10 +2033,33 @@
+         """
+         uri = str(href)
          if self.principalURL() == uri:
-             return True
+-            return True
++            yield True
++            return
          else:
 -            return uri in self.groupMembers()
-+            member_uris = [member.principalURL() for member in self.groupMembers()]
-+            return uri in member_uris
++            d = waitForDeferred(self.groupMembers())
++            yield d
++            members = d.getResult()
++            member_uris = [member.principalURL() for member in members]
++            yield uri in member_uris
  
 +class DAVPrincipalCollectionResource (DAVResource):
 +    """
@@ -947,7 +1009,7 @@
  class AccessDeniedError(Exception):
      def __init__(self, errors):
          """ 
-@@ -1718,6 +2080,37 @@
+@@ -1718,6 +2099,37 @@
  davxml.registerElement(TwistedACLInheritable)
  davxml.ACE.allowed_children[(twisted_dav_namespace, "inheritable")] = (0, 1)
  

Modified: CalendarServer/trunk/twistedcaldav/__init__.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/__init__.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/__init__.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -40,6 +40,8 @@
     "instance",
     "itip",
     "log",
+    "memcache",
+    "memcacher",
     "principalindex",
     "resource",
     "root",

Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
+from twistedcaldav.memcacher import Memcacher
+from twisted.internet.defer import returnValue
 
 """
 Implements a calendar user proxy principal.
@@ -23,6 +25,8 @@
 ]
 
 from twisted.internet.defer import succeed, inlineCallbacks
+from twisted.internet.defer import deferredGenerator
+from twisted.internet.defer import waitForDeferred
 from twisted.web2 import responsecode
 from twisted.web2.dav import davxml
 from twisted.web2.dav.element.base import dav_namespace
@@ -37,6 +41,7 @@
 from twistedcaldav.sql import db_prefix
 from twistedcaldav.static import AutoProvisioningFileMixIn
 
+import itertools
 import os
 
 class PermissionsMixIn (ReadOnlyWritePropertiesResourceMixIn):
@@ -184,7 +189,7 @@
         # Map the principals to UIDs.
         uids = [p.principalUID() for p in principals]
 
-        self._index().setGroupMembers(self.uid, uids)
+        _ignore = yield self._index().setGroupMembers(self.uid, uids)
         changed = yield self.parent.cacheNotifier.changed()
         yield True
         return
@@ -193,46 +198,54 @@
     # HTTP
     ##
 
+    @deferredGenerator
     def renderDirectoryBody(self, request):
         # FIXME: Too much code duplication here from principal.py
         from twistedcaldav.directory.principal import format_list, format_principals, format_link
 
-        def gotSuper(output):
-            return "".join((
-                """<div class="directory-listing">"""
-                """<h1>Principal Details</h1>"""
-                """<pre><blockquote>"""
-                """Directory Information\n"""
-                """---------------------\n"""
-                """Directory GUID: %s\n"""         % (self.parent.record.service.guid,),
-                """Realm: %s\n"""                  % (self.parent.record.service.realmName,),
-                """\n"""
-                """Parent Principal Information\n"""
-                """---------------------\n"""
-                """GUID: %s\n"""                   % (self.parent.record.guid,),
-                """Record type: %s\n"""            % (self.parent.record.recordType,),
-                """Short name: %s\n"""             % (self.parent.record.shortName,),
-                """Full name: %s\n"""              % (self.parent.record.fullName,),
-                """Principal UID: %s\n"""          % (self.parent.principalUID(),),
-                """Principal URL: %s\n"""          % (format_link(self.parent.principalURL()),),
-                """\n"""
-                """Proxy Principal Information\n"""
-                """---------------------\n"""
-               #"""GUID: %s\n"""                   % (self.guid,),
-                """Principal UID: %s\n"""          % (self.principalUID(),),
-                """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
-                """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
-                """\nGroup members (%s):\n""" % ({False:"Locked", True:"Editable"}[self.hasEditableMembership()])
-                                                   , format_principals(self.groupMembers()),
-                """\nGroup memberships:\n"""       , format_principals(self.groupMemberships()),
-                """</pre></blockquote></div>""",
-                output
-            ))
+        d = waitForDeferred(super(CalendarUserProxyPrincipalResource, self).renderDirectoryBody(request))
+        yield d
+        output = d.getResult()
+        
+        d = waitForDeferred(self.groupMembers())
+        yield d
+        members = d.getResult()
+        
+        d = waitForDeferred(self.groupMemberships())
+        yield d
+        memberships = d.getResult() 
+        
+        yield "".join((
+            """<div class="directory-listing">"""
+            """<h1>Principal Details</h1>"""
+            """<pre><blockquote>"""
+            """Directory Information\n"""
+            """---------------------\n"""
+            """Directory GUID: %s\n"""         % (self.parent.record.service.guid,),
+            """Realm: %s\n"""                  % (self.parent.record.service.realmName,),
+            """\n"""
+            """Parent Principal Information\n"""
+            """---------------------\n"""
+            """GUID: %s\n"""                   % (self.parent.record.guid,),
+            """Record type: %s\n"""            % (self.parent.record.recordType,),
+            """Short name: %s\n"""             % (self.parent.record.shortName,),
+            """Full name: %s\n"""              % (self.parent.record.fullName,),
+            """Principal UID: %s\n"""          % (self.parent.principalUID(),),
+            """Principal URL: %s\n"""          % (format_link(self.parent.principalURL()),),
+            """\n"""
+            """Proxy Principal Information\n"""
+            """---------------------\n"""
+           #"""GUID: %s\n"""                   % (self.guid,),
+            """Principal UID: %s\n"""          % (self.principalUID(),),
+            """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
+            """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
+            """\nGroup members (%s):\n""" % ({False:"Locked", True:"Editable"}[self.hasEditableMembership()])
+                                               , format_principals(members),
+            """\nGroup memberships:\n"""       , format_principals(memberships),
+            """</pre></blockquote></div>""",
+            output
+        ))
 
-        d = super(CalendarUserProxyPrincipalResource, self).renderDirectoryBody(request)
-        d.addCallback(gotSuper)
-        return d
-
     ##
     # DAV
     ##
@@ -257,6 +270,7 @@
     def principalCollections(self):
         return self.parent.principalCollections()
 
+    @deferredGenerator
     def _expandMemberUIDs(self, uid=None, relatives=None, uids=None):
         if uid is None:
             uid = self.principalUID()
@@ -270,35 +284,48 @@
             uids.add(uid)
             principal = self.parent.parent.principalForUID(uid)
             if isinstance(principal, CalendarUserProxyPrincipalResource):
-                for member in self._directGroupMembers():
+                d = waitForDeferred(self._directGroupMembers())
+                yield d
+                members = d.getResult()
+                for member in members:
                     if member.principalUID() not in uids:
                         relatives.add(member)
-                        self._expandMemberUIDs(member.principalUID(), relatives, uids)
+                        d = waitForDeferred(self._expandMemberUIDs(member.principalUID(), relatives, uids))
+                        yield d
+                        d.getResult()
             elif isinstance(principal, DirectoryPrincipalResource):
-                relatives.update(principal.groupMembers())
+                d = waitForDeferred(principal.groupMembers())
+                yield d
+                members = d.getResult()
+                relatives.update(members)
 
-        return relatives
+        yield relatives
 
+    @deferredGenerator
     def _directGroupMembers(self):
         if self.hasEditableMembership():
             # Get member UIDs from database and map to principal resources
-            members = self._index().getMembers(self.uid)
-            return [p for p in [self.pcollection.principalForUID(uid) for uid in members] if p]
+            d = waitForDeferred(self._index().getMembers(self.uid))
+            yield d
+            members = d.getResult()
+            yield [p for p in [self.pcollection.principalForUID(uid) for uid in members] if p]
         else:
             # Fixed proxies
             if self.proxyType == "calendar-proxy-write":
-                return self.parent.proxies()
+                yield self.parent.proxies()
             else:
-                return self.parent.readOnlyProxies()
+                yield self.parent.readOnlyProxies()
 
-
     def groupMembers(self):
         return self._expandMemberUIDs()
 
+    @deferredGenerator
     def groupMemberships(self):
         # Get membership UIDs and map to principal resources
-        memberships = self._index().getMemberships(self.uid)
-        return [p for p in [self.pcollection.principalForUID(uid) for uid in memberships] if p]
+        d = waitForDeferred(self._index().getMemberships(self.uid))
+        yield d
+        memberships = d.getResult()
+        yield [p for p in [self.pcollection.principalForUID(uid) for uid in memberships] if p]
 
     def hasEditableMembership(self):
         return self.parent.hasEditableProxyMembership()
@@ -319,10 +346,52 @@
     dbFilename = db_prefix + "calendaruserproxy"
     dbFormatVersion = "4"
 
+    class ProxyDBMemcacher(Memcacher):
+        
+        def setMembers(self, guid, members):
+            return self.set("members:%s" % (guid,), str(",".join(members)))
+
+        def setMemberships(self, guid, memberships):
+            return self.set("memberships:%s" % (guid,), str(",".join(memberships)))
+
+        def getMembers(self, guid):
+            def _value(value):
+                if value:
+                    return set(value.split(","))
+                elif value is None:
+                    return None
+                else:
+                    return set()
+            d = self.get("members:%s" % (guid,))
+            d.addCallback(_value)
+            return d
+
+        def getMemberships(self, guid):
+            def _value(value):
+                if value:
+                    return set(value.split(","))
+                elif value is None:
+                    return None
+                else:
+                    return set()
+            d = self.get("memberships:%s" % (guid,))
+            d.addCallback(_value)
+            return d
+
+        def deleteMember(self, guid):
+            return self.delete("members:%s" % (guid,))
+
+        def deleteMembership(self, guid):
+            return self.delete("memberships:%s" % (guid,))
+
     def __init__(self, path):
         path = os.path.join(path, CalendarUserProxyDatabase.dbFilename)
         super(CalendarUserProxyDatabase, self).__init__(path, True)
+        
+        if config.Memcached['ClientEnabled']:
+            self._memcacher = CalendarUserProxyDatabase.ProxyDBMemcacher("proxyDB")
 
+    @inlineCallbacks
     def setGroupMembers(self, principalUID, members):
         """
         Add a group membership record.
@@ -335,34 +404,88 @@
         self._delete_from_db(principalUID)
         self._add_to_db(principalUID, members)
         self._db_commit()
+        
+        # Update cache if present
+        if config.Memcached['ClientEnabled']:
+            current_members = yield self.getMembers(principalUID)
+            if current_members is None:
+                current_members = ()
+            current_members = set(current_members)
+            update_members = set(members)
+            
+            remove_members = current_members.difference(update_members)
+            add_members = update_members.difference(current_members)
+            for member in itertools.chain(remove_members, add_members,):
+                _ignore = yield self._memcacher.deleteMembership(member)
+            _ignore = yield self._memcacher.deleteMember(principalUID)
 
+    @inlineCallbacks
     def removeGroup(self, principalUID):
         """
         Remove a group membership record.
 
         @param principalUID: the UID of the group principal to remove.
         """
+
         self._delete_from_db(principalUID)
         self._db_commit()
+        
+        # Update cache if present
+        if config.Memcached['ClientEnabled']:
+            members = yield self.getMembers(principalUID)
+            if members:
+                for member in members:
+                    _ignore = yield self._memcacher.deleteMembership(member)
+                _ignore = yield self._memcacher.deleteMember(principalUID)
 
+    @inlineCallbacks
     def getMembers(self, principalUID):
         """
         Return the list of group member UIDs for the specified principal.
+        
+        @return: a deferred returning a C{set} of members.
         """
-        members = set()
-        for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID):
-            members.add(row[0])
-        return members
 
+        def _members():
+            members = set()
+            for row in self._db_execute("select MEMBER from GROUPS where GROUPNAME = :1", principalUID):
+                members.add(row[0])
+            return members
+
+        # Pull from cache if present
+        if config.Memcached['ClientEnabled']:
+            result = yield self._memcacher.getMembers(principalUID)
+            if result is None:
+                result = _members()
+                _ignore = yield self._memcacher.setMembers(principalUID, result)
+            returnValue(result)
+        else:
+            returnValue(_members())
+
+    @inlineCallbacks
     def getMemberships(self, principalUID):
         """
         Return the list of group principal UIDs the specified principal is a member of.
+        
+        @return: a deferred returning a C{set} of memberships.
         """
-        members = set()
-        for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID):
-            members.add(row[0])
-        return members
 
+        def _members():
+            members = set()
+            for row in self._db_execute("select GROUPNAME from GROUPS where MEMBER = :1", principalUID):
+                members.add(row[0])
+            return members
+
+        # Pull from cache if present
+        if config.Memcached['ClientEnabled']:
+            result = yield self._memcacher.getMemberships(principalUID)
+            if result is None:
+                result = _members()
+                _ignore = yield self._memcacher.setMemberships(principalUID, result)
+            returnValue(result)
+        else:
+            returnValue(_members())
+
     def _add_to_db(self, principalUID, members):
         """
         Insert the specified entry into the database.

Modified: CalendarServer/trunk/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/principal.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/directory/principal.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -32,7 +32,9 @@
 from urlparse import urlparse
 
 from twisted.python.failure import Failure
+from twisted.internet.defer import deferredGenerator
 from twisted.internet.defer import succeed
+from twisted.internet.defer import waitForDeferred
 from twisted.web2 import responsecode
 from twisted.web2.http import HTTPError
 from twisted.web2.dav import davxml
@@ -401,36 +403,45 @@
     # HTTP
     ##
 
+    @deferredGenerator
     def renderDirectoryBody(self, request):
-        def gotSuper(output):
-            return "".join((
-                """<div class="directory-listing">"""
-                """<h1>Principal Details</h1>"""
-                """<pre><blockquote>"""
-                """Directory Information\n"""
-                """---------------------\n"""
-                """Directory GUID: %s\n"""         % (self.record.service.guid,),
-                """Realm: %s\n"""                  % (self.record.service.realmName,),
-                """\n"""
-                """Principal Information\n"""
-                """---------------------\n"""
-                """GUID: %s\n"""                   % (self.record.guid,),
-                """Record type: %s\n"""            % (self.record.recordType,),
-                """Short name: %s\n"""             % (self.record.shortName,),
-                """Full name: %s\n"""              % (self.record.fullName,),
-                """Principal UID: %s\n"""          % (self.principalUID(),),
-                """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
-                """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
-                """\nGroup members:\n"""           , format_principals(self.groupMembers()),
-                """\nGroup memberships:\n"""       , format_principals(self.groupMemberships()),
-                """</pre></blockquote></div>""",
-                output
-            ))
 
-        d = super(DirectoryPrincipalResource, self).renderDirectoryBody(request)
-        d.addCallback(gotSuper)
-        return d
+        d = waitForDeferred(super(DirectoryPrincipalResource, self).renderDirectoryBody(request))
+        yield d
+        output = d.getResult()
+        
+        d = waitForDeferred(self.groupMembers())
+        yield d
+        members = d.getResult()
+        
+        d = waitForDeferred(self.groupMemberships())
+        yield d
+        memberships = d.getResult()
 
+        yield "".join((
+            """<div class="directory-listing">"""
+            """<h1>Principal Details</h1>"""
+            """<pre><blockquote>"""
+            """Directory Information\n"""
+            """---------------------\n"""
+            """Directory GUID: %s\n"""         % (self.record.service.guid,),
+            """Realm: %s\n"""                  % (self.record.service.realmName,),
+            """\n"""
+            """Principal Information\n"""
+            """---------------------\n"""
+            """GUID: %s\n"""                   % (self.record.guid,),
+            """Record type: %s\n"""            % (self.record.recordType,),
+            """Short name: %s\n"""             % (self.record.shortName,),
+            """Full name: %s\n"""              % (self.record.fullName,),
+            """Principal UID: %s\n"""          % (self.principalUID(),),
+            """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
+            """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
+            """\nGroup members:\n"""           , format_principals(members),
+            """\nGroup memberships:\n"""       , format_principals(memberships),
+            """</pre></blockquote></div>""",
+            output
+        ))
+
     ##
     # DAV
     ##
@@ -499,8 +510,9 @@
         return relatives
 
     def groupMembers(self):
-        return self._getRelatives("members")
+        return succeed(self._getRelatives("members"))
 
+    @deferredGenerator
     def groupMemberships(self):
         groups = self._getRelatives("groups")
 
@@ -511,14 +523,17 @@
 
             # Get proxy group UIDs and map to principal resources
             proxies = []
-            for uid in self._calendar_user_proxy_index().getMemberships(self.principalUID()):
+            d = waitForDeferred(self._calendar_user_proxy_index().getMemberships(self.principalUID()))
+            yield d
+            memberships = d.getResult()
+            for uid in memberships:
                 subprincipal = self.parent.principalForUID(uid)
                 if subprincipal:
                     proxies.append(subprincipal)
 
             groups.update(proxies)
 
-        return groups
+        yield groups
 
 
     def principalCollections(self):
@@ -551,37 +566,46 @@
     """
     Directory calendar principal resource.
     """
+    @deferredGenerator
     def renderDirectoryBody(self, request):
-        def gotSuper(output):
-            return "".join((
-                """<div class="directory-listing">"""
-                """<h1>Principal Details</h1>"""
-                """<pre><blockquote>"""
-                """Directory Information\n"""
-                """---------------------\n"""
-                """Directory GUID: %s\n"""         % (self.record.service.guid,),
-                """Realm: %s\n"""                  % (self.record.service.realmName,),
-                """\n"""
-                """Principal Information\n"""
-                """---------------------\n"""
-                """GUID: %s\n"""                   % (self.record.guid,),
-                """Record type: %s\n"""            % (self.record.recordType,),
-                """Short name: %s\n"""             % (self.record.shortName,),
-                """Full name: %s\n"""              % (self.record.fullName,),
-                """Principal UID: %s\n"""          % (self.principalUID(),),
-                """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
-                """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
-                """\nGroup members:\n"""           , format_principals(self.groupMembers()),
-                """\nGroup memberships:\n"""       , format_principals(self.groupMemberships()),
-                """\nCalendar homes:\n"""          , format_list(format_link(u) for u in self.calendarHomeURLs()),
-                """\nCalendar user addresses:\n""" , format_list(format_link(a) for a in self.calendarUserAddresses()),
-                """</pre></blockquote></div>""",
-                output
-            ))
 
-        d = super(DirectoryPrincipalResource, self).renderDirectoryBody(request)
-        d.addCallback(gotSuper)
-        return d
+        d = waitForDeferred(super(DirectoryPrincipalResource, self).renderDirectoryBody(request))
+        yield d
+        output = d.getResult()
+        
+        d = waitForDeferred(self.groupMembers())
+        yield d
+        members = d.getResult()
+        
+        d = waitForDeferred(self.groupMemberships())
+        yield d
+        memberships = d.getResult()
+        
+        yield "".join((
+            """<div class="directory-listing">"""
+            """<h1>Principal Details</h1>"""
+            """<pre><blockquote>"""
+            """Directory Information\n"""
+            """---------------------\n"""
+            """Directory GUID: %s\n"""         % (self.record.service.guid,),
+            """Realm: %s\n"""                  % (self.record.service.realmName,),
+            """\n"""
+            """Principal Information\n"""
+            """---------------------\n"""
+            """GUID: %s\n"""                   % (self.record.guid,),
+            """Record type: %s\n"""            % (self.record.recordType,),
+            """Short name: %s\n"""             % (self.record.shortName,),
+            """Full name: %s\n"""              % (self.record.fullName,),
+            """Principal UID: %s\n"""          % (self.principalUID(),),
+            """Principal URL: %s\n"""          % (format_link(self.principalURL()),),
+            """\nAlternate URIs:\n"""          , format_list(format_link(u) for u in self.alternateURIs()),
+            """\nGroup members:\n"""           , format_principals(members),
+            """\nGroup memberships:\n"""       , format_principals(memberships),
+            """\nCalendar homes:\n"""          , format_list(format_link(u) for u in self.calendarHomeURLs()),
+            """\nCalendar user addresses:\n""" , format_list(format_link(a) for a in self.calendarUserAddresses()),
+            """</pre></blockquote></div>""",
+            output
+        ))
 
     ##
     # CalDAV

Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -14,15 +14,10 @@
 # limitations under the License.
 ##
 
-#from twisted.web2 import responsecode
-#from twisted.web2.iweb import IResponse
-#from twisted.web2.dav import davxml
-#from twisted.web2.dav.util import davXMLFromStream
-#from twisted.web2.test.test_server import SimpleRequest
-#from twistedcaldav import caldavxml
-
 import os
 
+from twisted.internet.defer import deferredGenerator
+from twisted.internet.defer import waitForDeferred
 from twisted.web2.dav import davxml
 from twisted.web2.dav.fileop import rmdir
 from twisted.web2.dav.resource import AccessDeniedError
@@ -244,19 +239,27 @@
         for provisioningResource, recordType, recordResource, record in self._allRecords():
             self.failUnless(recordResource.displayName())
 
+    @deferredGenerator
     def test_groupMembers(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
         for provisioningResource, recordType, recordResource, record in self._allRecords():
-            self.failUnless(set(record.members()).issubset(set(r.record for r in recordResource.groupMembers())))
+            d = waitForDeferred(recordResource.groupMembers())
+            yield d
+            members = d.getResult()
+            self.failUnless(set(record.members()).issubset(set(r.record for r in members)))
 
+    @deferredGenerator
     def test_groupMemberships(self):
         """
         DirectoryPrincipalResource.groupMemberships()
         """
         for provisioningResource, recordType, recordResource, record in self._allRecords():
-            self.failUnless(set(record.groups()).issubset(set(r.record for r in recordResource.groupMemberships() if hasattr(r, "record"))))
+            d = waitForDeferred(recordResource.groupMemberships())
+            yield d
+            memberships = d.getResult()
+            self.failUnless(set(record.groups()).issubset(set(r.record for r in memberships if hasattr(r, "record"))))
 
     def test_proxies(self):
         """

Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipaldb.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -13,161 +13,353 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
-from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
 
 import os
 
+from twisted.internet.defer import deferredGenerator
+from twisted.internet.defer import waitForDeferred
+from twistedcaldav.directory.calendaruserproxy import CalendarUserProxyDatabase
 import twistedcaldav.test.util
 
-#class ProxyPrincipalDB (twistedcaldav.test.util.TestCase):
-#    """
-#    Directory service provisioned principals.
-#    """
-#    
-#    class old_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
-#        
-#        def _db_version(self):
-#            """
-#            @return: the schema version assigned to this index.
-#            """
-#            return "3"
-#            
-#        def _db_init_data_tables(self, q):
-#            """
-#            Initialise the underlying database tables.
-#            @param q:           a database cursor to use.
-#            """
-#    
-#            #
-#            # GROUPS table
-#            #
-#            q.execute(
-#                """
-#                create table GROUPS (
-#                    GROUPNAME   text,
-#                    MEMBER      text
-#                )
-#                """
-#            )
-#
-#    class new_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
-#        
-#        def _db_version(self):
-#            """
-#            @return: the schema version assigned to this index.
-#            """
-#            return "11"
-#            
-#    class newer_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
-#        
-#        def _db_version(self):
-#            """
-#            @return: the schema version assigned to this index.
-#            """
-#            return "51"
-#            
-#    def test_normalDB(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = CalendarUserProxyDatabase(db_path)
-#        db.setGroupMembers("A", ("B", "C", "D",))
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#
-#    def test_DBIndexed(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = CalendarUserProxyDatabase(db_path)
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
-#
-#    def test_OldDB(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = self.old_CalendarUserProxyDatabase(db_path)
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
-#
-#    def test_DBUpgrade(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = self.old_CalendarUserProxyDatabase(db_path)
-#        db.setGroupMembers("A", ("B", "C", "D",))
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
-#        db._db_close()
-#        db = None
-#        
-#        db = CalendarUserProxyDatabase(db_path)
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
-#        db._db_close()
-#        db = None
-#
-#    def test_DBUpgradeNewer(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = self.old_CalendarUserProxyDatabase(db_path)
-#        db.setGroupMembers("A", ("B", "C", "D",))
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
-#        db._db_close()
-#        db = None
-#        
-#        db = self.new_CalendarUserProxyDatabase(db_path)
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
-#        db._db_close()
-#        db = None
-#
-#    def test_DBNoUpgradeNewer(self):
-#        """
-#        DirectoryPrincipalResource.groupMembers()
-#        """
-#    
-#        # Get the DB
-#        db_path = self.mktemp()
-#        os.mkdir(db_path)
-#        db = self.new_CalendarUserProxyDatabase(db_path)
-#        db.setGroupMembers("A", ("B", "C", "D",))
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
-#        db._db_close()
-#        db = None
-#        
-#        db = self.newer_CalendarUserProxyDatabase(db_path)
-#        self.assertEqual(db.getMembers("A"), set(("B", "C", "D",)))
-#        self.assertEqual(db.getMemberships("B"), set(("A",)))
-#        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
-#        db._db_close()
-#        db = None
+class ProxyPrincipalDB (twistedcaldav.test.util.TestCase):
+    """
+    Directory service provisioned principals.
+    """
+    
+    class old_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+        
+        def _db_version(self):
+            """
+            @return: the schema version assigned to this index.
+            """
+            return "3"
+            
+        def _db_init_data_tables(self, q):
+            """
+            Initialise the underlying database tables.
+            @param q:           a database cursor to use.
+            """
+    
+            #
+            # GROUPS table
+            #
+            q.execute(
+                """
+                create table GROUPS (
+                    GROUPNAME   text,
+                    MEMBER      text
+                )
+                """
+            )
 
+    class new_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+        
+        def _db_version(self):
+            """
+            @return: the schema version assigned to this index.
+            """
+            return "11"
+            
+    class newer_CalendarUserProxyDatabase(CalendarUserProxyDatabase):
+        
+        def _db_version(self):
+            """
+            @return: the schema version assigned to this index.
+            """
+            return "51"
+    
+    @deferredGenerator
+    def test_normalDB(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = CalendarUserProxyDatabase(db_path)
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+        
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+        
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+        
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+
+    def test_DBIndexed(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = CalendarUserProxyDatabase(db_path)
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+
+    def test_OldDB(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = self.old_CalendarUserProxyDatabase(db_path)
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+
+    def test_DBUpgrade(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = self.old_CalendarUserProxyDatabase(db_path)
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+        db._db_close()
+        db = None
+        
+        db = CalendarUserProxyDatabase(db_path)
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+        db._db_close()
+        db = None
+
+    def test_DBUpgradeNewer(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = self.old_CalendarUserProxyDatabase(db_path)
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set())
+        db._db_close()
+        db = None
+        
+        db = self.new_CalendarUserProxyDatabase(db_path)
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+        db._db_close()
+        db = None
+
+    def test_DBNoUpgradeNewer(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = self.new_CalendarUserProxyDatabase(db_path)
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+        db._db_close()
+        db = None
+        
+        db = self.newer_CalendarUserProxyDatabase(db_path)
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(set([row[1] for row in db._db_execute("PRAGMA index_list(GROUPS)")]), set(("GROUPNAMES", "MEMBERS")))
+        db._db_close()
+        db = None
+
+    def test_cachingDBInsert(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = CalendarUserProxyDatabase(db_path)
+        
+        # Do one insert and check the result
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("C"))
+        yield d
+        membershipsC = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("D"))
+        yield d
+        membershipsD = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("E"))
+        yield d
+        membershipsE = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(membershipsC, set(("A",)))
+        self.assertEqual(membershipsD, set(("A",)))
+        self.assertEqual(membershipsE, set(()))
+        
+        # Change and check the result
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "E",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("C"))
+        yield d
+        membershipsC = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("D"))
+        yield d
+        membershipsD = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("E"))
+        yield d
+        membershipsE = d.getResult()
+
+        self.assertEqual(db.membersA, set(("B", "C", "E",)))
+        self.assertEqual(membershipsB, set(("A",)))
+        self.assertEqual(membershipsC, set(("A",)))
+        self.assertEqual(membershipsD, set())
+        self.assertEqual(membershipsE, set(("A",)))
+
+    def test_cachingDBRemove(self):
+    
+        # Get the DB
+        db_path = self.mktemp()
+        os.mkdir(db_path)
+        db = CalendarUserProxyDatabase(db_path)
+        
+        # Do one insert and check the result
+        d = waitForDeferred(db.setGroupMembers("A", ("B", "C", "D",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.setGroupMembers("X", ("B", "C",)))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMembers("X"))
+        yield d
+        membersX = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("C"))
+        yield d
+        membershipsC = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("D"))
+        yield d
+        membershipsD = d.getResult()
+
+        self.assertEqual(membersA, set(("B", "C", "D",)))
+        self.assertEqual(membersX, set(("B", "C",)))
+        self.assertEqual(membershipsB, set(("A", "X",)))
+        self.assertEqual(membershipsC, set(("A", "X",)))
+        self.assertEqual(membershipsD, set(("A",)))
+        
+        # Remove and check the result
+        d = waitForDeferred(db.removeGroup("A"))
+        yield d
+        d.getResult()
+
+        d = waitForDeferred(db.getMembers("A"))
+        yield d
+        membersA = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("B"))
+        yield d
+        membershipsB = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("C"))
+        yield d
+        membershipsC = d.getResult()
+
+        d = waitForDeferred(db.getMemberships("D"))
+        yield d
+        membershipsD = d.getResult()
+
+        self.assertEqual(membersA, set())
+        self.assertEqual(membershipsB, set("X",))
+        self.assertEqual(membershipsC, set("X",))
+        self.assertEqual(membershipsD, set())
         
\ No newline at end of file

Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_proxyprincipalmembers.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -16,6 +16,8 @@
 
 import os
 
+from twisted.internet.defer import deferredGenerator
+from twisted.internet.defer import waitForDeferred
 from twisted.web2.dav.fileop import rmdir
 from twisted.web2.dav import davxml
 
@@ -52,66 +54,90 @@
 
         self.principalRootResources[directoryService.__class__.__name__] = provisioningResource
 
+    @deferredGenerator
     def test_groupMembersRegular(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_groups, "both_coasts").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_groups, "both_coasts").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Chris Lecroy', 'David Reid', 'Wilfredo Sanchez', 'West Coast', 'East Coast', 'Cyrus Daboo',)))
 
+    @deferredGenerator
     def test_groupMembersRecursive(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_groups, "recursive1_coasts").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_groups, "recursive1_coasts").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Wilfredo Sanchez', 'Recursive2 Coasts', 'Cyrus Daboo',)))
 
+    @deferredGenerator
     def test_groupMembersProxySingleUser(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_locations, "gemini").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_locations, "gemini").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Wilfredo Sanchez',)))
 
+    @deferredGenerator
     def test_groupMembersProxySingleGroup(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_locations, "mercury").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_locations, "mercury").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Chris Lecroy', 'David Reid', 'Wilfredo Sanchez', 'West Coast',)))
 
+    @deferredGenerator
     def test_groupMembersProxySingleGroupWithNestedGroups(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_locations, "apollo").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_locations, "apollo").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Chris Lecroy', 'David Reid', 'Wilfredo Sanchez', 'West Coast', 'East Coast', 'Cyrus Daboo', 'Both Coasts',)))
 
+    @deferredGenerator
     def test_groupMembersProxySingleGroupWithNestedRecursiveGroups(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_locations, "orion").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_locations, "orion").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Wilfredo Sanchez', 'Cyrus Daboo', 'Recursive1 Coasts', 'Recursive2 Coasts',)))
 
+    @deferredGenerator
     def test_groupMembersProxySingleGroupWithNonCalendarGroup(self):
         """
         DirectoryPrincipalResource.groupMembers()
         """
-        members = self._getRecordByShortName(DirectoryService.recordType_resources, "non_calendar_proxy").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_resources, "non_calendar_proxy").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set(('Chris Lecroy', 'Cyrus Daboo', 'Non-calendar group')))
 
-        memberships = self._getRecordByShortName(DirectoryService.recordType_groups, "non_calendar_group").groupMemberships()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_groups, "non_calendar_group").groupMemberships())
+        yield d
+        memberships = d.getResult()
         memberships = set([p.principalUID() for p in memberships])
         self.assertEquals(memberships, set(('non_calendar_proxy#calendar-proxy-write',)))
 
+    @deferredGenerator
     def test_groupMembersProxyMissingUser(self):
         """
         DirectoryPrincipalResource.groupMembers()
@@ -120,15 +146,22 @@
         # Setup the fake entry in the DB
         proxy = self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo")
         proxy_group = proxy.getChild("calendar-proxy-write")
-        members = proxy_group._index().getMembers("%s#calendar-proxy-write" % (proxy.principalUID(),))
+        d = waitForDeferred(proxy_group._index().getMembers("%s#calendar-proxy-write" % (proxy.principalUID(),)))
+        yield d
+        members = d.getResult()
         members.add("12345")
-        proxy_group._index().setGroupMembers("%s#calendar-proxy-write" % (proxy.principalUID(),), members)
+        d = waitForDeferred(proxy_group._index().setGroupMembers("%s#calendar-proxy-write" % (proxy.principalUID(),), members))
+        yield d
+        d.getResult()
 
         # Do the failing lookup
-        members = self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo").getChild("calendar-proxy-write").groupMembers()
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo").getChild("calendar-proxy-write").groupMembers())
+        yield d
+        members = d.getResult()
         members = set([p.displayName() for p in members])
         self.assertEquals(members, set())
 
+    @deferredGenerator
     def test_groupMembershipsMissingUser(self):
         """
         DirectoryPrincipalResource.groupMembers()
@@ -138,14 +171,20 @@
         fake_uid = "12345"
         proxy = self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo")
         proxy_group = proxy.getChild("calendar-proxy-write")
-        members = proxy_group._index().getMembers("%s#calendar-proxy-write" % (fake_uid,))
+        d = waitForDeferred(proxy_group._index().getMembers("%s#calendar-proxy-write" % (fake_uid,)))
+        yield d
+        members = d.getResult()
         members.add("%s#calendar-proxy-write" % (proxy.principalUID(),))
-        proxy_group._index().setGroupMembers("%s#calendar-proxy-write" % (fake_uid,), members)
+        d = waitForDeferred(proxy_group._index().setGroupMembers("%s#calendar-proxy-write" % (fake_uid,), members))
+        yield d
+        d.getResult()
 
         # Do the failing lookup
-        members = self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo").getChild("calendar-proxy-write").groupMemberships()
-        members = set([p.displayName() for p in members])
-        self.assertEquals(members, set())
+        d = waitForDeferred(self._getRecordByShortName(DirectoryService.recordType_users, "cdaboo").getChild("calendar-proxy-write").groupMemberships())
+        yield d
+        memberships = d.getResult()
+        memberships = set([p.displayName() for p in memberships])
+        self.assertEquals(memberships, set())
 
     def _getRecordByShortName(self, type, name):
         """

Added: CalendarServer/trunk/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacher.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/memcacher.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -0,0 +1,83 @@
+##
+# Copyright (c) 2008 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import succeed
+from twisted.internet.protocol import ClientCreator
+
+from twistedcaldav.log import LoggingMixIn
+from twistedcaldav.memcache import MemCacheProtocol
+from twistedcaldav.config import config
+
+class Memcacher(LoggingMixIn):
+    _memcacheProtocol = None
+
+    def __init__(self, namespace):
+        self._namespace = namespace
+        self._host = config.Memcached['BindAddress']
+        self._port = config.Memcached['Port']
+
+        from twisted.internet import reactor
+        self._reactor = reactor
+
+    def _getMemcacheProtocol(self):
+        if Memcacher._memcacheProtocol is not None:
+            return succeed(self._memcacheProtocol)
+
+        d = ClientCreator(self._reactor, MemCacheProtocol).connectTCP(
+            self._host,
+            self._port)
+
+        def _cacheProtocol(proto):
+            Memcacher._memcacheProtocol = proto
+            return proto
+
+        return d.addCallback(_cacheProtocol)
+
+    def set(self, key, value):
+
+        def _set(proto):
+            return proto.set('%s:%s' % (self._namespace, key), value)
+
+        self.log_debug("Changing Cache Token for %s" % (key,))
+        d = self._getMemcacheProtocol()
+        d.addCallback(_set)
+        return d
+
+    def get(self, key):
+        
+        def _gotit(result):
+            _ignore_flags, value = result
+            return value
+
+        def _get(proto):
+            d1 = proto.get('%s:%s' % (self._namespace, key))
+            d1.addCallback(_gotit)
+            return d1
+
+        self.log_debug("Getting Cache Token for %r" % (key,))
+        d = self._getMemcacheProtocol()
+        d.addCallback(_get)
+        return d
+
+    def delete(self, key):
+        
+        def _delete(proto):
+            return proto.delete('%s:%s' % (self._namespace, key))
+
+        self.log_debug("Deleting Cache Token for %r" % (key,))
+        d = self._getMemcacheProtocol()
+        d.addCallback(_delete)
+        return d

Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py	2008-05-24 07:14:07 UTC (rev 2488)
+++ CalendarServer/trunk/twistedcaldav/resource.py	2008-05-24 07:19:59 UTC (rev 2489)
@@ -667,10 +667,10 @@
         return maybeDeferred(defer)
 
     def groupMembers(self):
-        return ()
+        return succeed(())
 
     def groupMemberships(self):
-        return ()
+        return succeed(())
 
     def calendarHomeURLs(self):
         if self.hasDeadProperty((caldav_namespace, "calendar-home-set")):

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080524/1b3636b0/attachment-0001.htm 


More information about the calendarserver-changes mailing list