[CalendarServer-changes] [11584] CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/ twistedcaldav/directory
source_changes at macosforge.org
source_changes at macosforge.org
Tue Aug 6 12:02:14 PDT 2013
Revision: 11584
http://trac.calendarserver.org//changeset/11584
Author: sagen at apple.com
Date: 2013-08-06 12:02:14 -0700 (Tue, 06 Aug 2013)
Log Message:
-----------
- Put result-count limits on LDAP queries
- Add new record-type specific calendarserver-principal-search contexts
- Normalize guids within delegate assignments from the directory
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py 2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/directory.py 2013-08-06 19:02:14 UTC (rev 11584)
@@ -82,6 +82,9 @@
recordType_resources = "resources"
searchContext_location = "location"
+ searchContext_resource = "resource"
+ searchContext_user = "user"
+ searchContext_group = "group"
searchContext_attendee = "attendee"
aggregateService = None
@@ -261,13 +264,19 @@
"""
Map calendarserver-principal-search REPORT context value to applicable record types
- @param context: The context value to map (either "location" or "attendee")
+ @param context: The context value to map
@type context: C{str}
@returns: The list of record types the context maps to
@rtype: C{list} of C{str}
"""
if context == self.searchContext_location:
recordTypes = [self.recordType_locations]
+ elif context == self.searchContext_resource:
+ recordTypes = [self.recordType_resources]
+ elif context == self.searchContext_user:
+ recordTypes = [self.recordType_users]
+ elif context == self.searchContext_group:
+ recordTypes = [self.recordType_groups]
elif context == self.searchContext_attendee:
recordTypes = [self.recordType_users, self.recordType_groups,
self.recordType_resources]
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py 2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/ldapdirectory.py 2013-08-06 19:02:14 UTC (rev 11584)
@@ -56,7 +56,7 @@
CachingDirectoryRecord)
from twistedcaldav.directory.directory import DirectoryConfigurationError
from twistedcaldav.directory.augment import AugmentRecord
-from twistedcaldav.directory.util import splitIntoBatches
+from twistedcaldav.directory.util import splitIntoBatches, normalizeUUID
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twisted.internet.threads import deferToThread
from twext.web2.http import HTTPError, StatusResponse
@@ -399,12 +399,15 @@
dn = normalizeDNstr(dn)
guid = self._getUniqueLdapAttribute(attrs, guidAttr)
if guid:
+ guid = normalizeUUID(guid)
readDelegate = self._getUniqueLdapAttribute(attrs, readAttr)
if readDelegate:
+ readDelegate = normalizeUUID(readDelegate)
assignments.append(("%s#calendar-proxy-read" % (guid,),
[readDelegate]))
writeDelegate = self._getUniqueLdapAttribute(attrs, writeAttr)
if writeDelegate:
+ writeDelegate = normalizeUUID(writeDelegate)
assignments.append(("%s#calendar-proxy-write" % (guid,),
[writeDelegate]))
@@ -777,6 +780,7 @@
if not guid:
self.log_debug("LDAP data for %s is missing guid attribute %s" % (shortNames, guidAttr))
raise MissingGuidException()
+ guid = normalizeUUID(guid)
# Find or build email
# (The emailAddresses mapping is a list of ldap fields)
@@ -1062,7 +1066,7 @@
% (recordTypes, indexType, indexKey))
- def recordsMatchingTokens(self, tokens, context=None):
+ def recordsMatchingTokens(self, tokens, context=None, limitResults=50, timeoutSeconds=10):
"""
@param tokens: The tokens to search on
@type tokens: C{list} of C{str} (utf-8 bytes)
@@ -1082,29 +1086,35 @@
are considered.
"""
self.log_debug("Peforming calendar user search for %s (%s)" % (tokens, context))
+ startTime = time.time()
records = []
recordTypes = self.recordTypesForSearchContext(context)
recordTypes = [r for r in recordTypes if r in self.recordTypes()]
- guidAttr = self.rdnSchema["guidAttr"]
+ typeCounts = {}
for recordType in recordTypes:
+ if limitResults == 0:
+ self.log_debug("LDAP search aggregate limit reached")
+ break
+ typeCounts[recordType] = 0
base = self.typeDNs[recordType]
scope = ldap.SCOPE_SUBTREE
- filterstr = buildFilterFromTokens(self.rdnSchema[recordType]["mapping"],
+ filterstr = buildFilterFromTokens(recordType, self.rdnSchema[recordType]["mapping"],
tokens)
if filterstr is not None:
# Query the LDAP server
- self.log_debug("LDAP search %s %s %s" %
- (ldap.dn.dn2str(base), scope, filterstr))
+ self.log_debug("LDAP search %s %s (limit=%d)" %
+ (ldap.dn.dn2str(base), filterstr, limitResults))
results = self.timedSearch(ldap.dn.dn2str(base), scope,
filterstr=filterstr, attrlist=self.attrlist,
- timeoutSeconds=self.requestTimeoutSeconds,
- resultLimit=self.requestResultsLimit)
+ timeoutSeconds=timeoutSeconds,
+ resultLimit=limitResults)
self.log_debug("LDAP search returned %d results" % (len(results),))
numMissingGuids = 0
numMissingRecordNames = 0
+ numNotEnabled = 0
for dn, attrs in results:
dn = normalizeDNstr(dn)
# Skip if group restriction is in place and guid is not
@@ -1120,9 +1130,12 @@
# not include in principal property search results
if (recordType != self.recordType_groups):
if not record.enabledForCalendaring:
+ numNotEnabled += 1
continue
records.append(record)
+ typeCounts[recordType] += 1
+ limitResults -= 1
except MissingGuidException:
numMissingGuids += 1
@@ -1130,15 +1143,11 @@
except MissingRecordNameException:
numMissingRecordNames += 1
- if numMissingGuids:
- self.log_warn("%d %s records are missing %s" %
- (numMissingGuids, recordType, guidAttr))
+ self.log_debug("LDAP search returned %d results, %d usable" % (len(results), typeCounts[recordType]))
- if numMissingRecordNames:
- self.log_warn("%d %s records are missing record name" %
- (numMissingRecordNames, recordType))
-
- self.log_debug("Calendar user search matched %d records" % (len(records),))
+ typeCountsStr = ", ".join(["%s:%d" % (rt, ct) for (rt, ct) in typeCounts.iteritems()])
+ totalTime = time.time() - startTime
+ self.log_info("Calendar user search for %s matched %d records (%s) in %.2f seconds" % (tokens, len(records), typeCountsStr, totalTime))
return succeed(records)
@@ -1413,12 +1422,13 @@
return filterstr
-def buildFilterFromTokens(mapping, tokens):
+def buildFilterFromTokens(recordType, mapping, tokens):
"""
Create an LDAP filter string from a list of query tokens. Each token is
searched for in each LDAP attribute corresponding to "fullName" and
"emailAddresses" (could be multiple LDAP fields for either).
+ @param recordType: The recordType to use to customize the filter
@param mapping: A dict mapping internal directory attribute names to ldap names.
@type mapping: C{dict}
@param tokens: The list of tokens to search for
@@ -1432,25 +1442,30 @@
if len(tokens) == 0:
return None
- attributes = ["fullName", "emailAddresses"]
+ attributes = [
+ ("fullName", "(%s=*%s*)"),
+ ("emailAddresses", "(%s=%s*)"),
+ ]
ldapFields = []
- for attribute in attributes:
+ for attribute, template in attributes:
ldapField = mapping.get(attribute, None)
if ldapField:
if isinstance(ldapField, str):
- ldapFields.append(ldapField)
+ ldapFields.append((ldapField, template))
else:
- ldapFields.extend(ldapField)
+ for lf in ldapField:
+ ldapFields.append((lf, template))
if len(ldapFields) == 0:
return None
tokenFragments = []
+
for token in tokens:
fragments = []
- for ldapField in ldapFields:
- fragments.append("(%s=*%s*)" % (ldapField, token))
+ for ldapField, template in ldapFields:
+ fragments.append(template % (ldapField, token))
if len(fragments) == 1:
tokenFragment = fragments[0]
else:
Modified: CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py 2013-08-06 18:37:59 UTC (rev 11583)
+++ CalendarServer/branches/users/cdaboo/CalendarServer-5.0-podfix/twistedcaldav/directory/test/test_ldapdirectory.py 2013-08-06 19:02:14 UTC (rev 11584)
@@ -207,7 +207,7 @@
"fullName" : "cn",
"emailAddresses" : "mail",
},
- "expected" : "(|(cn=*foo*)(mail=*foo*))",
+ "expected" : "(|(cn=*foo*)(mail=foo*))",
},
{
"tokens" : ["foo"],
@@ -215,7 +215,7 @@
"fullName" : "cn",
"emailAddresses" : ["mail", "mailAliases"],
},
- "expected" : "(|(cn=*foo*)(mail=*foo*)(mailAliases=*foo*))",
+ "expected" : "(|(cn=*foo*)(mail=foo*)(mailAliases=foo*))",
},
{
"tokens" : [],
@@ -235,7 +235,7 @@
"mapping" : {
"emailAddresses" : "mail",
},
- "expected" : "(&(mail=*foo*)(mail=*bar*))",
+ "expected" : "(&(mail=foo*)(mail=bar*))",
},
{
"tokens" : ["foo", "bar"],
@@ -243,7 +243,7 @@
"fullName" : "cn",
"emailAddresses" : "mail",
},
- "expected" : "(&(|(cn=*foo*)(mail=*foo*))(|(cn=*bar*)(mail=*bar*)))",
+ "expected" : "(&(|(cn=*foo*)(mail=foo*))(|(cn=*bar*)(mail=bar*)))",
},
{
"tokens" : ["foo", "bar"],
@@ -251,7 +251,7 @@
"fullName" : "cn",
"emailAddresses" : ["mail", "mailAliases"],
},
- "expected" : "(&(|(cn=*foo*)(mail=*foo*)(mailAliases=*foo*))(|(cn=*bar*)(mail=*bar*)(mailAliases=*bar*)))",
+ "expected" : "(&(|(cn=*foo*)(mail=foo*)(mailAliases=foo*))(|(cn=*bar*)(mail=bar*)(mailAliases=bar*)))",
},
{
"tokens" : ["foo", "bar", "baz("],
@@ -259,12 +259,12 @@
"fullName" : "cn",
"emailAddresses" : "mail",
},
- "expected" : "(&(|(cn=*foo*)(mail=*foo*))(|(cn=*bar*)(mail=*bar*))(|(cn=*baz\\28*)(mail=*baz\\28*)))",
+ "expected" : "(&(|(cn=*foo*)(mail=foo*))(|(cn=*bar*)(mail=bar*))(|(cn=*baz\\28*)(mail=baz\\28*)))",
},
]
for entry in entries:
self.assertEquals(
- buildFilterFromTokens(entry["mapping"], entry["tokens"]),
+ buildFilterFromTokens(None, entry["mapping"], entry["tokens"]),
entry["expected"]
)
@@ -330,6 +330,10 @@
key, value = fragment.split("=")
if value in attrs.get(key, []):
results.append(("ignored", (dn, attrs)))
+ break
+ elif value == "*" and key in attrs:
+ results.append(("ignored", (dn, attrs)))
+ break
return results
@@ -401,7 +405,8 @@
"uid=odtestamanda,cn=users,dc=example,dc=com",
{
'uid': ['odtestamanda'],
- 'apple-generateduid': ['9DC04A70-E6DD-11DF-9492-0800200C9A66'],
+ # purposely throw in an un-normalized GUID
+ 'apple-generateduid': ['9dc04a70-e6dd-11df-9492-0800200c9a66'],
'sn': ['Test'],
'mail': ['odtestamanda at example.com', 'alternate at example.com'],
'givenName': ['Amanda'],
@@ -452,6 +457,30 @@
'cn': ['Wilfredo Sanchez']
}
),
+ (
+ "uid=testresource , cn=resources , dc=example,dc=com",
+ {
+ 'uid': ['testresource'],
+ 'apple-generateduid': ['D91B21B9-B856-495A-8E36-0E5AD54EFB3A'],
+ 'sn': ['Resource'],
+ 'givenName': ['Test'],
+ 'cn': ['Test Resource'],
+ # purposely throw in an un-normalized GUID
+ 'read-write-proxy' : ['6423f94a-6b76-4a3a-815b-d52cfd77935d'],
+ 'read-only-proxy' : ['5A985493-EE2C-4665-94CF-4DFEA3A89500'],
+ }
+ ),
+ (
+ "uid=testresource2 , cn=resources , dc=example,dc=com",
+ {
+ 'uid': ['testresource2'],
+ 'apple-generateduid': ['753E5A60-AFFD-45E4-BF2C-31DAB459353F'],
+ 'sn': ['Resource2'],
+ 'givenName': ['Test'],
+ 'cn': ['Test Resource2'],
+ 'read-write-proxy' : ['6423F94A-6B76-4A3A-815B-D52CFD77935D'],
+ }
+ ),
),
{
"augmentService" : None,
@@ -546,8 +575,8 @@
"resourceSchema": {
"resourceInfoAttr": "apple-resource-info", # contains location/resource info
"autoScheduleAttr": None,
- "proxyAttr": None,
- "readOnlyProxyAttr": None,
+ "proxyAttr": "read-write-proxy",
+ "readOnlyProxyAttr": "read-only-proxy",
"autoAcceptGroupAttr": None,
},
"partitionSchema": {
@@ -1468,6 +1497,21 @@
self.assertEquals(record.autoAcceptGroup,
'77A8EB52-AA2A-42ED-8843-B2BEE863AC70')
+ # Record with lowercase guid
+ dn = "uid=odtestamanda,cn=users,dc=example,dc=com"
+ guid = '9dc04a70-e6dd-11df-9492-0800200c9a66'
+ attrs = {
+ 'uid': ['odtestamanda'],
+ 'apple-generateduid': [guid],
+ 'sn': ['Test'],
+ 'mail': ['odtestamanda at example.com', 'alternate at example.com'],
+ 'givenName': ['Amanda'],
+ 'cn': ['Amanda Test']
+ }
+ record = self.service._ldapResultToRecord(dn, attrs,
+ self.service.recordType_users)
+ self.assertEquals(record.guid, guid.upper())
+
def test_listRecords(self):
"""
listRecords makes an LDAP query (with fake results in this test)
@@ -1576,7 +1620,7 @@
@inlineCallbacks
def test_groupMembershipAliases(self):
"""
- Exercise a directory enviornment where group membership does not refer
+ Exercise a directory environment where group membership does not refer
to guids but instead uses LDAP DNs. This example uses the LDAP attribute
"uniqueMember" to specify members of a group. The value of this attribute
is each members' DN. Even though the proxy database deals strictly in
@@ -1608,6 +1652,26 @@
self.assertEquals(groups, (yield record.cachedGroups()))
+ def test_getExternalProxyAssignments(self):
+ """
+ Verify getExternalProxyAssignments can extract assignments from the
+ directory, and that guids are normalized.
+ """
+ self.setupService(self.nestedUsingDifferentAttributeUsingDN)
+ self.assertEquals(
+ self.service.getExternalProxyAssignments(),
+ [
+ ('D91B21B9-B856-495A-8E36-0E5AD54EFB3A#calendar-proxy-read',
+ ['5A985493-EE2C-4665-94CF-4DFEA3A89500']),
+ ('D91B21B9-B856-495A-8E36-0E5AD54EFB3A#calendar-proxy-write',
+ ['6423F94A-6B76-4A3A-815B-D52CFD77935D']),
+ ('753E5A60-AFFD-45E4-BF2C-31DAB459353F#calendar-proxy-write',
+ ['6423F94A-6B76-4A3A-815B-D52CFD77935D'])
+ ]
+ )
+
+
+
def test_splitIntoBatches(self):
self.setupService(self.nestedUsingDifferentAttributeUsingDN)
# Data is perfect multiple of size
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130806/d683d31e/attachment-0001.html>
More information about the calendarserver-changes
mailing list