[CalendarServer-changes] [11584] CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/ twistedcaldav/directory

source_changes at macosforge.org source_changes at macosforge.org
Tue Aug 6 12:02:14 PDT 2013


Revision: 11584
          http://trac.calendarserver.org//changeset/11584
Author:   sagen at apple.com
Date:     2013-08-06 12:02:14 -0700 (Tue, 06 Aug 2013)
Log Message:
-----------
- Put result-count limits on LDAP queries
- Add new record-type specific calendarserver-principal-search contexts
- Normalize guids within delegate assignments from the directory

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
    CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
    CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py

Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py	2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py	2013-08-06 19:02:14 UTC (rev 11584)
@@ -82,6 +82,9 @@
     recordType_resources = "resources"
 
     searchContext_location = "location"
+    searchContext_resource = "resource"
+    searchContext_user     = "user"
+    searchContext_group    = "group"
     searchContext_attendee = "attendee"
 
     aggregateService = None
@@ -261,13 +264,19 @@
         """
         Map calendarserver-principal-search REPORT context value to applicable record types
 
-        @param context: The context value to map (either "location" or "attendee")
+        @param context: The context value to map
         @type context: C{str}
         @returns: The list of record types the context maps to
         @rtype: C{list} of C{str}
         """
         if context == self.searchContext_location:
             recordTypes = [self.recordType_locations]
+        elif context == self.searchContext_resource:
+            recordTypes = [self.recordType_resources]
+        elif context == self.searchContext_user:
+            recordTypes = [self.recordType_users]
+        elif context == self.searchContext_group:
+            recordTypes = [self.recordType_groups]
         elif context == self.searchContext_attendee:
             recordTypes = [self.recordType_users, self.recordType_groups,
                 self.recordType_resources]

Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py	2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py	2013-08-06 19:02:14 UTC (rev 11584)
@@ -56,7 +56,7 @@
     CachingDirectoryRecord)
 from twistedcaldav.directory.directory import DirectoryConfigurationError
 from twistedcaldav.directory.augment import AugmentRecord
-from twistedcaldav.directory.util import splitIntoBatches
+from twistedcaldav.directory.util import splitIntoBatches, normalizeUUID
 from twisted.internet.defer import succeed, inlineCallbacks, returnValue
 from twisted.internet.threads import deferToThread
 from twext.web2.http import HTTPError, StatusResponse
@@ -399,12 +399,15 @@
             dn = normalizeDNstr(dn)
             guid = self._getUniqueLdapAttribute(attrs, guidAttr)
             if guid:
+                guid = normalizeUUID(guid)
                 readDelegate = self._getUniqueLdapAttribute(attrs, readAttr)
                 if readDelegate:
+                    readDelegate = normalizeUUID(readDelegate)
                     assignments.append(("%s#calendar-proxy-read" % (guid,),
                         [readDelegate]))
                 writeDelegate = self._getUniqueLdapAttribute(attrs, writeAttr)
                 if writeDelegate:
+                    writeDelegate = normalizeUUID(writeDelegate)
                     assignments.append(("%s#calendar-proxy-write" % (guid,),
                         [writeDelegate]))
 
@@ -777,6 +780,7 @@
             if not guid:
                 self.log_debug("LDAP data for %s is missing guid attribute %s" % (shortNames, guidAttr))
                 raise MissingGuidException()
+            guid = normalizeUUID(guid)
 
         # Find or build email
         # (The emailAddresses mapping is a list of ldap fields)
@@ -1062,7 +1066,7 @@
                         % (recordTypes, indexType, indexKey))
 
 
-    def recordsMatchingTokens(self, tokens, context=None):
+    def recordsMatchingTokens(self, tokens, context=None, limitResults=50, timeoutSeconds=10):
         """
         @param tokens: The tokens to search on
         @type tokens: C{list} of C{str} (utf-8 bytes)
@@ -1082,29 +1086,35 @@
         are considered.
         """
         self.log_debug("Peforming calendar user search for %s (%s)" % (tokens, context))
+        startTime = time.time()
 
         records = []
         recordTypes = self.recordTypesForSearchContext(context)
         recordTypes = [r for r in recordTypes if r in self.recordTypes()]
-        guidAttr = self.rdnSchema["guidAttr"]
 
+        typeCounts = {}
         for recordType in recordTypes:
+            if limitResults == 0:
+                self.log_debug("LDAP search aggregate limit reached")
+                break
+            typeCounts[recordType] = 0
             base = self.typeDNs[recordType]
             scope = ldap.SCOPE_SUBTREE
-            filterstr = buildFilterFromTokens(self.rdnSchema[recordType]["mapping"],
+            filterstr = buildFilterFromTokens(recordType, self.rdnSchema[recordType]["mapping"],
                 tokens)
 
             if filterstr is not None:
                 # Query the LDAP server
-                self.log_debug("LDAP search %s %s %s" %
-                    (ldap.dn.dn2str(base), scope, filterstr))
+                self.log_debug("LDAP search %s %s (limit=%d)" %
+                    (ldap.dn.dn2str(base), filterstr, limitResults))
                 results = self.timedSearch(ldap.dn.dn2str(base), scope,
                     filterstr=filterstr, attrlist=self.attrlist,
-                    timeoutSeconds=self.requestTimeoutSeconds,
-                    resultLimit=self.requestResultsLimit)
+                    timeoutSeconds=timeoutSeconds,
+                    resultLimit=limitResults)
                 self.log_debug("LDAP search returned %d results" % (len(results),))
                 numMissingGuids = 0
                 numMissingRecordNames = 0
+                numNotEnabled = 0
                 for dn, attrs in results:
                     dn = normalizeDNstr(dn)
                     # Skip if group restriction is in place and guid is not
@@ -1120,9 +1130,12 @@
                         # not include in principal property search results
                         if (recordType != self.recordType_groups):
                             if not record.enabledForCalendaring:
+                                numNotEnabled += 1
                                 continue
 
                         records.append(record)
+                        typeCounts[recordType] += 1
+                        limitResults -= 1
 
                     except MissingGuidException:
                         numMissingGuids += 1
@@ -1130,15 +1143,11 @@
                     except MissingRecordNameException:
                         numMissingRecordNames += 1
 
-                if numMissingGuids:
-                    self.log_warn("%d %s records are missing %s" %
-                        (numMissingGuids, recordType, guidAttr))
+                self.log_debug("LDAP search returned %d results, %d usable" % (len(results), typeCounts[recordType]))
 
-                if numMissingRecordNames:
-                    self.log_warn("%d %s records are missing record name" %
-                        (numMissingRecordNames, recordType))
-
-        self.log_debug("Calendar user search matched %d records" % (len(records),))
+        typeCountsStr = ", ".join(["%s:%d" % (rt, ct) for (rt, ct) in typeCounts.iteritems()])
+        totalTime = time.time() - startTime
+        self.log_info("Calendar user search for %s matched %d records (%s) in %.2f seconds" % (tokens, len(records), typeCountsStr, totalTime))
         return succeed(records)
 
 
@@ -1413,12 +1422,13 @@
     return filterstr
 
 
-def buildFilterFromTokens(mapping, tokens):
+def buildFilterFromTokens(recordType, mapping, tokens):
     """
     Create an LDAP filter string from a list of query tokens.  Each token is
     searched for in each LDAP attribute corresponding to "fullName" and
     "emailAddresses" (could be multiple LDAP fields for either).
 
+    @param recordType: The recordType to use to customize the filter
     @param mapping: A dict mapping internal directory attribute names to ldap names.
     @type mapping: C{dict}
     @param tokens: The list of tokens to search for
@@ -1432,25 +1442,30 @@
     if len(tokens) == 0:
         return None
 
-    attributes = ["fullName", "emailAddresses"]
+    attributes = [
+        ("fullName", "(%s=*%s*)"),
+        ("emailAddresses", "(%s=%s*)"),
+    ]
 
     ldapFields = []
-    for attribute in attributes:
+    for attribute, template in attributes:
         ldapField = mapping.get(attribute, None)
         if ldapField:
             if isinstance(ldapField, str):
-                ldapFields.append(ldapField)
+                ldapFields.append((ldapField, template))
             else:
-                ldapFields.extend(ldapField)
+                for lf in ldapField:
+                    ldapFields.append((lf, template))
 
     if len(ldapFields) == 0:
         return None
 
     tokenFragments = []
+
     for token in tokens:
         fragments = []
-        for ldapField in ldapFields:
-            fragments.append("(%s=*%s*)" % (ldapField, token))
+        for ldapField, template in ldapFields:
+            fragments.append(template % (ldapField, token))
         if len(fragments) == 1:
             tokenFragment = fragments[0]
         else:

Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py	2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py	2013-08-06 19:02:14 UTC (rev 11584)
@@ -207,7 +207,7 @@
                         "fullName" : "cn",
                         "emailAddresses" : "mail",
                     },
-                    "expected" : "(|(cn=*foo*)(mail=*foo*))",
+                    "expected" : "(|(cn=*foo*)(mail=foo*))",
                 },
                 {
                     "tokens" : ["foo"],
@@ -215,7 +215,7 @@
                         "fullName" : "cn",
                         "emailAddresses" : ["mail", "mailAliases"],
                     },
-                    "expected" : "(|(cn=*foo*)(mail=*foo*)(mailAliases=*foo*))",
+                    "expected" : "(|(cn=*foo*)(mail=foo*)(mailAliases=foo*))",
                 },
                 {
                     "tokens" : [],
@@ -235,7 +235,7 @@
                     "mapping" : {
                         "emailAddresses" : "mail",
                     },
-                    "expected" : "(&(mail=*foo*)(mail=*bar*))",
+                    "expected" : "(&(mail=foo*)(mail=bar*))",
                 },
                 {
                     "tokens" : ["foo", "bar"],
@@ -243,7 +243,7 @@
                         "fullName" : "cn",
                         "emailAddresses" : "mail",
                     },
-                    "expected" : "(&(|(cn=*foo*)(mail=*foo*))(|(cn=*bar*)(mail=*bar*)))",
+                    "expected" : "(&(|(cn=*foo*)(mail=foo*))(|(cn=*bar*)(mail=bar*)))",
                 },
                 {
                     "tokens" : ["foo", "bar"],
@@ -251,7 +251,7 @@
                         "fullName" : "cn",
                         "emailAddresses" : ["mail", "mailAliases"],
                     },
-                    "expected" : "(&(|(cn=*foo*)(mail=*foo*)(mailAliases=*foo*))(|(cn=*bar*)(mail=*bar*)(mailAliases=*bar*)))",
+                    "expected" : "(&(|(cn=*foo*)(mail=foo*)(mailAliases=foo*))(|(cn=*bar*)(mail=bar*)(mailAliases=bar*)))",
                 },
                 {
                     "tokens" : ["foo", "bar", "baz("],
@@ -259,12 +259,12 @@
                         "fullName" : "cn",
                         "emailAddresses" : "mail",
                     },
-                    "expected" : "(&(|(cn=*foo*)(mail=*foo*))(|(cn=*bar*)(mail=*bar*))(|(cn=*baz\\28*)(mail=*baz\\28*)))",
+                    "expected" : "(&(|(cn=*foo*)(mail=foo*))(|(cn=*bar*)(mail=bar*))(|(cn=*baz\\28*)(mail=baz\\28*)))",
                 },
             ]
             for entry in entries:
                 self.assertEquals(
-                    buildFilterFromTokens(entry["mapping"], entry["tokens"]),
+                    buildFilterFromTokens(None, entry["mapping"], entry["tokens"]),
                     entry["expected"]
                 )
 
@@ -330,6 +330,10 @@
                             key, value = fragment.split("=")
                             if value in attrs.get(key, []):
                                 results.append(("ignored", (dn, attrs)))
+                                break
+                            elif value == "*" and key in attrs:
+                                results.append(("ignored", (dn, attrs)))
+                                break
 
             return results
 
@@ -401,7 +405,8 @@
                     "uid=odtestamanda,cn=users,dc=example,dc=com",
                     {
                         'uid': ['odtestamanda'],
-                        'apple-generateduid': ['9DC04A70-E6DD-11DF-9492-0800200C9A66'],
+                        # purposely throw in an un-normalized GUID
+                        'apple-generateduid': ['9dc04a70-e6dd-11df-9492-0800200c9a66'],
                         'sn': ['Test'],
                         'mail': ['odtestamanda at example.com', 'alternate at example.com'],
                         'givenName': ['Amanda'],
@@ -452,6 +457,30 @@
                         'cn': ['Wilfredo Sanchez']
                     }
                 ),
+                (
+                    "uid=testresource  ,  cn=resources  , dc=example,dc=com",
+                    {
+                        'uid': ['testresource'],
+                        'apple-generateduid': ['D91B21B9-B856-495A-8E36-0E5AD54EFB3A'],
+                        'sn': ['Resource'],
+                        'givenName': ['Test'],
+                        'cn': ['Test Resource'],
+                        # purposely throw in an un-normalized GUID
+                        'read-write-proxy' : ['6423f94a-6b76-4a3a-815b-d52cfd77935d'],
+                        'read-only-proxy' : ['5A985493-EE2C-4665-94CF-4DFEA3A89500'],
+                    }
+                ),
+                (
+                    "uid=testresource2  ,  cn=resources  , dc=example,dc=com",
+                    {
+                        'uid': ['testresource2'],
+                        'apple-generateduid': ['753E5A60-AFFD-45E4-BF2C-31DAB459353F'],
+                        'sn': ['Resource2'],
+                        'givenName': ['Test'],
+                        'cn': ['Test Resource2'],
+                        'read-write-proxy' : ['6423F94A-6B76-4A3A-815B-D52CFD77935D'],
+                    }
+                ),
             ),
             {
                 "augmentService" : None,
@@ -546,8 +575,8 @@
                 "resourceSchema": {
                     "resourceInfoAttr": "apple-resource-info", # contains location/resource info
                     "autoScheduleAttr": None,
-                    "proxyAttr": None,
-                    "readOnlyProxyAttr": None,
+                    "proxyAttr": "read-write-proxy",
+                    "readOnlyProxyAttr": "read-only-proxy",
                     "autoAcceptGroupAttr": None,
                 },
                 "partitionSchema": {
@@ -1468,6 +1497,21 @@
             self.assertEquals(record.autoAcceptGroup,
                 '77A8EB52-AA2A-42ED-8843-B2BEE863AC70')
 
+            # Record with lowercase guid
+            dn = "uid=odtestamanda,cn=users,dc=example,dc=com"
+            guid = '9dc04a70-e6dd-11df-9492-0800200c9a66'
+            attrs = {
+                'uid': ['odtestamanda'],
+                'apple-generateduid': [guid],
+                'sn': ['Test'],
+                'mail': ['odtestamanda at example.com', 'alternate at example.com'],
+                'givenName': ['Amanda'],
+                'cn': ['Amanda Test']
+            }
+            record = self.service._ldapResultToRecord(dn, attrs,
+                self.service.recordType_users)
+            self.assertEquals(record.guid, guid.upper())
+
         def test_listRecords(self):
             """
             listRecords makes an LDAP query (with fake results in this test)
@@ -1576,7 +1620,7 @@
         @inlineCallbacks
         def test_groupMembershipAliases(self):
             """
-            Exercise a directory enviornment where group membership does not refer
+            Exercise a directory environment where group membership does not refer
             to guids but instead uses LDAP DNs.  This example uses the LDAP attribute
             "uniqueMember" to specify members of a group.  The value of this attribute
             is each members' DN.  Even though the proxy database deals strictly in
@@ -1608,6 +1652,26 @@
                 self.assertEquals(groups, (yield record.cachedGroups()))
 
 
+        def test_getExternalProxyAssignments(self):
+            """
+            Verify getExternalProxyAssignments can extract assignments from the
+            directory, and that guids are normalized.
+            """
+            self.setupService(self.nestedUsingDifferentAttributeUsingDN)
+            self.assertEquals(
+                self.service.getExternalProxyAssignments(),
+                [
+                    ('D91B21B9-B856-495A-8E36-0E5AD54EFB3A#calendar-proxy-read',
+                        ['5A985493-EE2C-4665-94CF-4DFEA3A89500']),
+                    ('D91B21B9-B856-495A-8E36-0E5AD54EFB3A#calendar-proxy-write',
+                        ['6423F94A-6B76-4A3A-815B-D52CFD77935D']),
+                    ('753E5A60-AFFD-45E4-BF2C-31DAB459353F#calendar-proxy-write',
+                        ['6423F94A-6B76-4A3A-815B-D52CFD77935D'])
+                ]
+            )
+
+
+
         def test_splitIntoBatches(self):
             self.setupService(self.nestedUsingDifferentAttributeUsingDN)
             # Data is perfect multiple of size
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130806/d683d31e/attachment-0001.html>


More information about the calendarserver-changes mailing list