[CalendarServer-changes] [3536] CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/ twistedcaldav/directory

source_changes at macosforge.org source_changes at macosforge.org
Tue Dec 16 08:26:48 PST 2008


Revision: 3536
          http://trac.macosforge.org/projects/calendarserver/changeset/3536
Author:   cdaboo at apple.com
Date:     2008-12-16 08:26:47 -0800 (Tue, 16 Dec 2008)
Log Message:
-----------
Use directory service cache for looking up/faulting calendar user addresses/email addresses. This
will provide the performance we need for mapping addresses.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/appleopendirectory.py
    CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/directory.py
    CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/test/test_opendirectoryrecords.py

Modified: CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/appleopendirectory.py	2008-12-15 23:22:16 UTC (rev 3535)
+++ CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/appleopendirectory.py	2008-12-16 16:26:47 UTC (rev 3536)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2008 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -274,17 +274,25 @@
 
             # Cache miss; try looking the record up, in case it is new
             # FIXME: This is a blocking call (hopefully it's a fast one)
-            self.reloadCache(recordType, shortName=shortName)
+            self.reloadCache(recordType, lookup=("shortName", shortName,))
             record = self.recordsForType(recordType).get(shortName, None)
             if record is None:
                 # Add to negative cache
                 self._storage(recordType)["disabled names"].add(shortName)
             return record
 
+    def recordWithEmailAddress(self, emailAddress):
+        return self._recordWithAttribute("emails", "disabled emails", "email", emailAddress)
+
     def recordWithGUID(self, guid):
+        return self._recordWithAttribute("guids", "disabled guids", "guid", guid)
+
+    recordWithUID = recordWithGUID
+
+    def _recordWithAttribute(self, cacheKey, disabledKey, lookupKey, value):
         def lookup():
             for recordType in self.recordTypes():
-                record = self._storage(recordType)["guids"].get(guid, None)
+                record = self._storage(recordType)[cacheKey].get(value, None)
                 if record:
                     return record
             else:
@@ -296,26 +304,24 @@
             # Cache miss; try looking the record up, in case it is new
             for recordType in self.recordTypes():
                 # Check negative cache
-                if guid in self._storage(recordType)["disabled guids"]:
+                if value in self._storage(recordType)[disabledKey]:
                     continue
 
-                self.reloadCache(recordType, guid=guid)
+                self.reloadCache(recordType, lookup=(lookupKey, value,))
                 record = lookup()
 
                 if record is None:
-                    self._storage(recordType)["disabled guids"].add(guid)
+                    self._storage(recordType)[disabledKey].add(value)
                 else:
-                    self.log_info("Faulted record with GUID %s into %s record cache"
-                                  % (guid, recordType))
+                    self.log_info("Faulted record with %s %s into %s record cache"
+                                  % (lookupKey, value, recordType))
                     break
             else:
                 # Nothing found; add to negative cache
-                self.log_info("Unable to find any record with GUID %s" % (guid,))
+                self.log_info("Unable to find any record with %s %s" % (lookupKey, value,))
 
         return record
 
-    recordWithUID = recordWithGUID
-
     def groupsForGUID(self, guid):
         
         # Lookup in index
@@ -415,19 +421,15 @@
         return deferred
 
 
-    def reloadCache(self, recordType, shortName=None, guid=None):
-        if shortName is not None:
-            self.log_info("Faulting record with shortName %s into %s record cache" % (shortName, recordType))
-        elif guid is not None:
-            self.log_info("Faulting record with guid %s into %s record cache" % (guid, recordType))
-        elif shortName is None and guid is None:
+    def reloadCache(self, recordType, lookup=None):
+        if lookup is not None:
+            self.log_info("Faulting record with %s %s into %s record cache" % (lookup[0], lookup[1], recordType))
+        else:
             self.log_info("Reloading %s record cache" % (recordType,))
-        else:
-            raise AssertionError("%r.reloadCache(%s, %s, %s)" % (self, recordType, shortName, guid))
 
-        results = self._queryDirectory(recordType, shortName=shortName, guid=guid)
+        results = self._queryDirectory(recordType, lookup=lookup)
 
-        if shortName is None and guid is None:
+        if lookup is None:
             records = {}
             guids   = {}
             emails  = {}
@@ -634,7 +636,7 @@
                     else:
                         emails[email] = record
 
-        if shortName is None and guid is None:
+        if lookup is None:
             #
             # Replace the entire cache
             #
@@ -682,7 +684,7 @@
                 % (len(self._records[recordType]["guids"]), enabled_count, recordType, cacheTimeout)
             )
 
-    def _queryDirectory(self, recordType, shortName=None, guid=None):
+    def _queryDirectory(self, recordType, lookup=None):
         attrs = [
             dsattributes.kDS1AttrGeneratedUID,
             dsattributes.kDS1AttrDistinguishedName,
@@ -750,10 +752,14 @@
             self.log_debug("Got %d restricted group members" % (len(self.restrictedGUIDs),))
 
         query = None
-        if shortName is not None:
-            query = dsquery.match(dsattributes.kDSNAttrRecordName, shortName, dsattributes.eDSExact)
-        elif guid is not None:
-            query = dsquery.match(dsattributes.kDS1AttrGeneratedUID, guid, dsattributes.eDSExact)
+        if lookup is not None:
+            queryattr = {
+                "shortName" : dsattributes.kDSNAttrRecordName,
+                "guid"      : dsattributes.kDS1AttrGeneratedUID,
+                "email"     : dsattributes.kDSNAttrEMailAddress,
+            }.get(lookup[0])
+            assert queryattr is not None, "Invalid type for record faulting query"
+            query = dsquery.match(queryattr, lookup[1], dsattributes.eDSExact)
 
         try:
             if query:

Modified: CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/directory.py	2008-12-15 23:22:16 UTC (rev 3535)
+++ CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/directory.py	2008-12-16 16:26:47 UTC (rev 3536)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2008 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -138,11 +138,28 @@
 
     def recordWithCalendarUserAddress(self, address):
         address = normalizeCUAddr(address)
+        if address.startswith("urn:uuid:"):
+            guid = address[9:]
+            return self.recordWithGUID(guid)
+        elif address.startswith("mailto:"):
+            email = address[7:]
+            result = self.recordWithEmailAddress(email)
+            if result:
+                return result
+
         for record in self.allRecords():
             if address in record.calendarUserAddresses:
                 return record
+                
         return None
 
+    def recordWithEmailAddress(self, email):
+        for record in self.allRecords():
+            if email in record.emailAddresses:
+                return record
+                
+        return None
+
     def allRecords(self):
         for recordType in self.recordTypes():
             for record in self.listRecords(recordType):

Modified: CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/test/test_opendirectoryrecords.py
===================================================================
--- CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/test/test_opendirectoryrecords.py	2008-12-15 23:22:16 UTC (rev 3535)
+++ CalendarServer/branches/users/cdaboo/normalize-cuaddr-3531/twistedcaldav/directory/test/test_opendirectoryrecords.py	2008-12-16 16:26:47 UTC (rev 3536)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2008 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,19 +27,26 @@
     from twistedcaldav.directory.util import uuidFromName
 
     class OpenDirectoryService (RealOpenDirectoryService):
-        def _queryDirectory(self, recordType, shortName=None, guid=None):
-            if shortName is None and guid is None:
+        def _queryDirectory(self, recordType, lookup=None):
+            self._didQuery = True
+            if lookup is None:
                 return self.fakerecords[recordType]
 
-            assert shortName is None or guid is None
-            if guid is not None:
-                guid = guid.lower()
+            if lookup[0] == "guid":
+                lookup = (lookup[0], lookup[1].lower(),)
 
             records = []
 
             for name, record in self.fakerecords[recordType]:
-                if name == shortName or record[dsattributes.kDS1AttrGeneratedUID] == guid:
-                    records.append((name, record))
+                if lookup[0] == "shortName":
+                    if name == lookup[1]:
+                        records.append((name, record))
+                elif lookup[0] == "guid":
+                    if record[dsattributes.kDS1AttrGeneratedUID] == lookup[1]:
+                        records.append((name, record))
+                elif lookup[0] == "email":
+                    if record[dsattributes.kDSNAttrEMailAddress] == lookup[1]:
+                        records.append((name, record))
 
             return tuple(records)
     
@@ -95,6 +102,22 @@
             check("disabled names", expectedNames)
             check("disabled guids", (guid.lower() for guid in expectedGUIDs))
 
+        def verifyQuery(self, f, *args):
+            try:
+                delattr(self.service, "_didQuery")
+            except AttributeError:
+                pass
+            self.assertFalse(f(*args))
+            self.assertTrue(hasattr(self.service, "_didQuery"))
+
+        def verifyNoQuery(self, f, *args):
+            try:
+                delattr(self.service, "_didQuery")
+            except AttributeError:
+                pass
+            self.assertFalse(f(*args))
+            self.assertFalse(hasattr(self.service, "_didQuery"))
+
         def test_restrictionGroupName(self):
             service = OpenDirectoryService(
                 node="/Search",
@@ -226,8 +249,8 @@
                 ],
             }
 
-            self.service.reloadCache(DirectoryService.recordType_users, shortName="user02")
-            self.service.reloadCache(DirectoryService.recordType_users, guid="D10F3EE0-5014-41D3-8488-3819D3EF3B2A")
+            self.service.reloadCache(DirectoryService.recordType_users, lookup=("shortName", "user02",))
+            self.service.reloadCache(DirectoryService.recordType_users, lookup=("guid", "D10F3EE0-5014-41D3-8488-3819D3EF3B2A",))
 
             self.verifyRecords(DirectoryService.recordType_users, ("user01", "user02", "user03"))
             self.verifyDisabledRecords(DirectoryService.recordType_users, (), ())
@@ -353,8 +376,8 @@
                 ],
             }
 
-            self.service.reloadCache(DirectoryService.recordType_users, shortName="user04")
-            self.service.reloadCache(DirectoryService.recordType_users, guid="62368DDF-0C62-4C97-9A58-DE9FD46131A0")
+            self.service.reloadCache(DirectoryService.recordType_users, lookup=("shortName", "user04",))
+            self.service.reloadCache(DirectoryService.recordType_users, lookup=("guid", "62368DDF-0C62-4C97-9A58-DE9FD46131A0",))
 
             self.verifyRecords(DirectoryService.recordType_users, ("user01",))
             self.verifyDisabledRecords(
@@ -434,7 +457,7 @@
                     guidForShortName("user02"),
                 ]),
             ]
-            self.service.reloadCache(DirectoryService.recordType_groups, guid=guidForShortName("group03"))
+            self.service.reloadCache(DirectoryService.recordType_groups, lookup=("guid", guidForShortName("group03"),))
 
             group1 = self.service.recordWithShortName(DirectoryService.recordType_groups, "group01")
             self.assertTrue(group1 is not None)
@@ -453,6 +476,138 @@
             self.assertTrue(user2 is not None)
             self.assertEqual(set((group2, group3)), user2.groups()) 
 
+        def test_negativeCacheShortname(self):
+            self.loadRecords({
+                DirectoryService.recordType_users: [
+                    fakeODRecord("User 01"),
+                    fakeODRecord("User 02"),
+                    fakeODRecord("User 03"),
+                    fakeODRecord("User 04"),
+                ],
+                DirectoryService.recordType_groups: [
+                    fakeODRecord("Group 01"),
+                    fakeODRecord("Group 02"),
+                    fakeODRecord("Group 03"),
+                    fakeODRecord("Group 04"),
+                ],
+                DirectoryService.recordType_resources: [
+                    fakeODRecord("Resource 01"),
+                    fakeODRecord("Resource 02"),
+                    fakeODRecord("Resource 03"),
+                    fakeODRecord("Resource 04"),
+                ],
+                DirectoryService.recordType_locations: [
+                    fakeODRecord("Location 01"),
+                    fakeODRecord("Location 02"),
+                    fakeODRecord("Location 03"),
+                    fakeODRecord("Location 04"),
+                ],
+            })
+
+            self.assertTrue(self.service.recordWithShortName(DirectoryService.recordType_users, "user01"))
+            self.verifyQuery(self.service.recordWithShortName, DirectoryService.recordType_users, "user05")
+            self.verifyNoQuery(self.service.recordWithShortName, DirectoryService.recordType_users, "user05")
+
+            self.assertTrue(self.service.recordWithShortName(DirectoryService.recordType_groups, "group01"))
+            self.verifyQuery(self.service.recordWithShortName, DirectoryService.recordType_groups, "group05")
+            self.verifyNoQuery(self.service.recordWithShortName, DirectoryService.recordType_groups, "group05")
+
+            self.assertTrue(self.service.recordWithShortName(DirectoryService.recordType_resources, "resource01"))
+            self.verifyQuery(self.service.recordWithShortName, DirectoryService.recordType_resources, "resource05")
+            self.verifyNoQuery(self.service.recordWithShortName, DirectoryService.recordType_resources, "resource05")
+
+            self.assertTrue(self.service.recordWithShortName(DirectoryService.recordType_locations, "location01"))
+            self.verifyQuery(self.service.recordWithShortName, DirectoryService.recordType_locations, "location05")
+            self.verifyNoQuery(self.service.recordWithShortName, DirectoryService.recordType_locations, "location05")
+
+        def test_negativeCacheGUID(self):
+            self.loadRecords({
+                DirectoryService.recordType_users: [
+                    fakeODRecord("User 01"),
+                    fakeODRecord("User 02"),
+                    fakeODRecord("User 03"),
+                    fakeODRecord("User 04"),
+                ],
+                DirectoryService.recordType_groups: [
+                    fakeODRecord("Group 01"),
+                    fakeODRecord("Group 02"),
+                    fakeODRecord("Group 03"),
+                    fakeODRecord("Group 04"),
+                ],
+                DirectoryService.recordType_resources: [
+                    fakeODRecord("Resource 01"),
+                    fakeODRecord("Resource 02"),
+                    fakeODRecord("Resource 03"),
+                    fakeODRecord("Resource 04"),
+                ],
+                DirectoryService.recordType_locations: [
+                    fakeODRecord("Location 01"),
+                    fakeODRecord("Location 02"),
+                    fakeODRecord("Location 03"),
+                    fakeODRecord("Location 04"),
+                ],
+            })
+
+            self.assertTrue(self.service.recordWithGUID(guidForShortName("user01")))
+            self.verifyQuery(self.service.recordWithGUID, guidForShortName("user05"))
+            self.verifyNoQuery(self.service.recordWithGUID, guidForShortName("user05"))
+
+            self.assertTrue(self.service.recordWithGUID(guidForShortName("group01")))
+            self.verifyQuery(self.service.recordWithGUID, guidForShortName("group05"))
+            self.verifyNoQuery(self.service.recordWithGUID, guidForShortName("group05"))
+
+            self.assertTrue(self.service.recordWithGUID(guidForShortName("resource01")))
+            self.verifyQuery(self.service.recordWithGUID, guidForShortName("resource05"))
+            self.verifyNoQuery(self.service.recordWithGUID, guidForShortName("resource05"))
+
+            self.assertTrue(self.service.recordWithGUID(guidForShortName("location01")))
+            self.verifyQuery(self.service.recordWithGUID, guidForShortName("location05"))
+            self.verifyNoQuery(self.service.recordWithGUID, guidForShortName("location05"))
+
+        def test_negativeCacheEmailAddress(self):
+            self.loadRecords({
+                DirectoryService.recordType_users: [
+                    fakeODRecord("User 01"),
+                    fakeODRecord("User 02"),
+                    fakeODRecord("User 03"),
+                    fakeODRecord("User 04"),
+                ],
+                DirectoryService.recordType_groups: [
+                    fakeODRecord("Group 01"),
+                    fakeODRecord("Group 02"),
+                    fakeODRecord("Group 03"),
+                    fakeODRecord("Group 04"),
+                ],
+                DirectoryService.recordType_resources: [
+                    fakeODRecord("Resource 01"),
+                    fakeODRecord("Resource 02"),
+                    fakeODRecord("Resource 03"),
+                    fakeODRecord("Resource 04"),
+                ],
+                DirectoryService.recordType_locations: [
+                    fakeODRecord("Location 01"),
+                    fakeODRecord("Location 02"),
+                    fakeODRecord("Location 03"),
+                    fakeODRecord("Location 04"),
+                ],
+            })
+
+            self.assertTrue(self.service.recordWithEmailAddress("user01 at example.com"))
+            self.verifyQuery(self.service.recordWithEmailAddress, "user05 at example.com")
+            self.verifyNoQuery(self.service.recordWithEmailAddress, "user05 at example.com")
+
+            self.assertTrue(self.service.recordWithEmailAddress("group01 at example.com"))
+            self.verifyQuery(self.service.recordWithEmailAddress, "group05 at example.com")
+            self.verifyNoQuery(self.service.recordWithEmailAddress, "group05 at example.com")
+
+            self.assertTrue(self.service.recordWithEmailAddress("resource01 at example.com"))
+            self.verifyQuery(self.service.recordWithEmailAddress, "resource05 at example.com")
+            self.verifyNoQuery(self.service.recordWithEmailAddress, "resource05 at example.com")
+
+            self.assertTrue(self.service.recordWithEmailAddress("location01 at example.com"))
+            self.verifyQuery(self.service.recordWithEmailAddress, "location05 at example.com")
+            self.verifyNoQuery(self.service.recordWithEmailAddress, "location05 at example.com")
+
 def fakeODRecord(fullName, shortName=None, guid=None, email=None, members=None):
     if shortName is None:
         shortName = shortNameForFullName(fullName)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20081216/10fbd69b/attachment-0001.html>


More information about the calendarserver-changes mailing list