[CalendarServer-changes] [8998] CalendarServer/branches/users/gaya/ldapdirectorybacker
source_changes at macosforge.org
source_changes at macosforge.org
Mon Apr 9 16:04:40 PDT 2012
Revision: 8998
http://trac.macosforge.org/projects/calendarserver/changeset/8998
Author: gaya at apple.com
Date: 2012-04-09 16:04:40 -0700 (Mon, 09 Apr 2012)
Log Message:
-----------
Filter results in each xxxDirectoryBacker, and retry if results do not match
Modified Paths:
--------------
CalendarServer/branches/users/gaya/ldapdirectorybacker/conf/carddav-odtest.plist
CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/ldapdirectorybacker.py
CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/opendirectorybacker.py
CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/xmldirectorybacker.py
Modified: CalendarServer/branches/users/gaya/ldapdirectorybacker/conf/carddav-odtest.plist
===================================================================
--- CalendarServer/branches/users/gaya/ldapdirectorybacker/conf/carddav-odtest.plist 2012-04-09 18:23:11 UTC (rev 8997)
+++ CalendarServer/branches/users/gaya/ldapdirectorybacker/conf/carddav-odtest.plist 2012-04-09 23:04:40 UTC (rev 8998)
@@ -1114,6 +1114,7 @@
<string>dsAttrTypeStandard:RealName</string>
<string>dsAttrTypeStandard:FirstName</string>
<string>dsAttrTypeStandard:LastName</string>
+ <string>dsAttrTypeStandard:kDSNAttrEMailAddress</string>
<string>dsAttrTypeStandard:PhoneNumber</string>
<string>dsAttrTypeStandard:MobileNumber</string>
<string>dsAttrTypeStandard:Department</string>
Modified: CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/ldapdirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/ldapdirectorybacker.py 2012-04-09 18:23:11 UTC (rev 8997)
+++ CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/ldapdirectorybacker.py 2012-04-09 23:04:40 UTC (rev 8998)
@@ -184,19 +184,88 @@
@inlineCallbacks
+ def _getLdapQueryResults(self, base, queryStr, attributes=None, maxResults=0, ldapAttrToDSAttrMap={} ):
+ """
+ Get a list of filtered ABDirectoryQueryResult for the given query with the given attributes.
+ query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
+ """
+ limited = False
+ resultsDictionary = {}
+
+ # can't resist also using a timeout, 1 sec per request result for now
+ timeout = maxResults
+
+ self.log_debug("_getLdapQueryResults: LDAP query base=%s and filter=%s and attributes=%s timeout=%s resultLimit=%s" % (ldap.dn.dn2str(base), queryStr, attributes, timeout, maxResults))
+
+ ldapSearchResult = (yield self.timedSearch(ldap.dn.dn2str(base), ldap.SCOPE_SUBTREE, filterstr=queryStr, attrlist=attributes, timeoutSeconds=timeout, resultLimit=maxResults))
+ self.log_debug("_getLdapQueryResults: ldapSearchResult=%s" % (ldapSearchResult,))
+
+ if maxResults and len(ldapSearchResult) >= maxResults:
+ limited = True
+ self.log_debug("_getLdapQueryResults: limit (= %d) reached." % (maxResults, ))
+
+ for dn, ldapAttributes in ldapSearchResult:
+ #dn = normalizeDNstr(dn)
+ result = None
+ try:
+ # make a dsRecordAttributes dict from the ldap attributes
+ dsRecordAttributes = {}
+ for ldapAttributeName, ldapAttributeValues in ldapAttributes.iteritems():
+
+ #self.log_debug("inspecting ldapAttributeName %s with values %s" % (ldapAttributeName, ldapAttributeValues,))
+
+ # get rid of '' values
+ ldapAttributeValues = [attr for attr in ldapAttributeValues if len(attr)]
+
+ if len(ldapAttributeValues):
+ dsAttributeNames = ldapAttrToDSAttrMap.get(ldapAttributeName)
+ if dsAttributeNames:
+
+ if not isinstance(dsAttributeNames, list):
+ dsAttributeNames = [dsAttributeNames,]
+
+ for dsAttributeName in dsAttributeNames:
+
+ # base64 encode binary attributes
+ if dsAttributeName in ABDirectoryQueryResult.binaryDSAttrNames:
+ ldapAttributeValues = [attr.encode('base64') for attr in ldapAttributeValues]
+
+ # add to dsRecordAttributes
+ if dsAttributeName not in dsRecordAttributes:
+ dsRecordAttributes[dsAttributeName] = list()
+
+ dsRecordAttributes[dsAttributeName] = list(set(dsRecordAttributes[dsAttributeName] + ldapAttributeValues))
+ self.log_debug("doAddressBookQuery: dsRecordAttributes[%s] = %s" % (dsAttributeName, dsRecordAttributes[dsAttributeName],))
+
+ # get a record for dsRecordAttributes
+ result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, generateSimpleUIDs=self.generateSimpleUIDs, appleInternalServer=self.appleInternalServer)
+ except:
+ traceback.print_exc()
+ self.log_info("Could not get vcard for %s" % (dn,))
+ else:
+ uid = result.vCard().propertyValue("UID")
+
+ if uid in resultsDictionary:
+ self.log_info("Record skipped due to duplicate UID: %s" % (dn,))
+ continue
+
+ self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+ resultsDictionary[uid] = result
+
+ self.log_debug("%s results (limited=%s)." % (len(resultsDictionary), limited))
+ returnValue((resultsDictionary.values(), limited, ))
+
+ @inlineCallbacks
def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
"""
Get vCards for a given addressBookFilter and addressBookQuery
"""
- queryResults = []
+ results = []
limited = False
-
- #calc maxResults from passed in maxResults allowing extra for second stage filtering in caller
- maxResults = int(maxResults * 1.2)
- if self.maxQueryResults and maxResults > self.maxQueryResults:
- maxResults = self.maxQueryResults
-
+ remainingMaxResults = maxResults
+
+ #one ldap query for each rnd in queries
for queryMap in self.rdnSchema["queries"]:
rdn = queryMap["rdn"]
@@ -224,74 +293,59 @@
base = ldap.dn.str2dn(rdn) + self.base
- filterstr = "(cn=*)" # all query
+ queryStr = "(cn=*)" # all results query - should make a param
#add additional filter from config
queryFilter = queryMap.get("filter")
if dsFilter and queryFilter:
- filterstr = "(&%s%s)" % (queryFilter, dsFilter.generate())
+ queryStr = "(&%s%s)" % (queryFilter, dsFilter.generate())
elif queryFilter:
- filterstr = queryFilter
+ queryStr = queryFilter
elif dsFilter:
- filterstr = dsFilter.generate()
+ queryStr = dsFilter.generate()
+
- # can't resist also using a timeout, 1 sec per request result for now
- timeout = maxResults
-
- self.log_debug("doAddressBookQuery:LDAP query base=%s and filter=%s and attributes=%s timeout=%s resultLimit=%s" % (ldap.dn.dn2str(base), filterstr, attributes, timeout, maxResults))
- ldapSearchResult = (yield self.timedSearch(ldap.dn.dn2str(base), ldap.SCOPE_SUBTREE, filterstr=filterstr, attrlist=attributes, timeoutSeconds=timeout, resultLimit=maxResults))
+ # keep trying ldap query till we get results based on filter. Especially when doing "all results" query
+ remainingMaxResults = maxResults - len(results)
+ maxLdapResults = int(remainingMaxResults * 1.2)
- self.log_debug("doAddressBookQuery: ldapSearchResult=%s" % (ldapSearchResult,))
+ while True:
+ ldapQueryResults, ldapQueryLimited = (yield self._getLdapQueryResults(base=base, queryStr=queryStr, attributes=attributes, maxResults=maxLdapResults, ldapAttrToDSAttrMap=ldapAttrToDSAttrMap))
+
+ filteredResults = []
+ for ldapQueryResult in ldapQueryResults:
+ # to do: filter duplicate UIDs
+ if addressBookFilter.match(ldapQueryResult.vCard()):
+ filteredResults.append(ldapQueryResult)
+ else:
+ self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (ldapQueryResult.vCard().propertyValue("FN"), ldapQueryResult.vCard().propertyValue("UID"),))
+
+ #no more results
+ if not ldapQueryLimited:
+ break;
+
+ # more than requested results
+ if maxResults and len(filteredResults) > remainingMaxResults:
+ break
+
+ # more than max report results
+ if len(filteredResults) > config.MaxQueryWithDataResults:
+ break
+
+ # more than self limit
+ if self.maxQueryResults and maxLdapResults >= self.maxQueryResults:
+ break
+
+ # try again with 2x
+ maxLdapResults *= 2
+ if self.maxQueryResults and maxLdapResults > self.maxQueryResults:
+ maxLdapResults = self.maxQueryResults
+
+ results += filteredResults
- for dn, ldapAttributes in ldapSearchResult:
- #dn = normalizeDNstr(dn)
- result = None
- try:
- # make a dsRecordAttributes dict from the ldap attributes
- dsRecordAttributes = {}
- for ldapAttributeName, ldapAttributeValues in ldapAttributes.iteritems():
-
- #self.log_debug("inspecting ldapAttributeName %s with values %s" % (ldapAttributeName, ldapAttributeValues,))
-
- # get rid of '' values
- ldapAttributeValues = [attr for attr in ldapAttributeValues if len(attr)]
-
- if len(ldapAttributeValues):
- dsAttributeNames = ldapAttrToDSAttrMap.get(ldapAttributeName)
- if dsAttributeNames:
-
- if not isinstance(dsAttributeNames, list):
- dsAttributeNames = [dsAttributeNames,]
-
- for dsAttributeName in dsAttributeNames:
-
- # base64 encode binary attributes
- if dsAttributeName in ABDirectoryQueryResult.binaryDSAttrNames:
- ldapAttributeValues = [attr.encode('base64') for attr in ldapAttributeValues]
-
- # add to dsRecordAttributes
- if dsAttributeName not in dsRecordAttributes:
- dsRecordAttributes[dsAttributeName] = list()
-
- dsRecordAttributes[dsAttributeName] = list(set(dsRecordAttributes[dsAttributeName] + ldapAttributeValues))
- self.log_debug("doAddressBookQuery: dsRecordAttributes[%s] = %s" % (dsAttributeName, dsRecordAttributes[dsAttributeName],))
-
- # get a record for dsRecordAttributes
- result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, generateSimpleUIDs=self.generateSimpleUIDs, appleInternalServer=self.appleInternalServer)
- except:
- traceback.print_exc()
- self.log_info("Could not get vcard for %s" % (dn,))
- else:
- self.log_debug("doAddressBookQuery: VCard text =\n%s" % (result.vCardText(),))
- queryResults.append(result)
-
- # only get requested number of record results
- maxResults -= len(ldapSearchResult)
- if maxResults <= 0:
- limited = True
- break
-
+
+ limited = maxResults and len(results) >= maxResults
- self.log_info("limited %s len(queryResults) %s" % (limited,len(queryResults),))
- returnValue((queryResults, limited,))
+ self.log_info("limited %s len(results) %s" % (limited,len(results),))
+ returnValue((results, limited,))
Modified: CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/opendirectorybacker.py 2012-04-09 18:23:11 UTC (rev 8997)
+++ CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/opendirectorybacker.py 2012-04-09 23:04:40 UTC (rev 8998)
@@ -154,8 +154,9 @@
self.fakeETag = fakeETag
self.addDSAttrXProperties = addDSAttrXProperties
- self.generateSimpleUIDs = generateSimpleUIDs
+ self.generateSimpleUIDs = generateSimpleUIDs # for testing
self.appleInternalServer = appleInternalServer
+ self.sortResults = generateSimpleUIDs # for testing: TODO: make separate param
if searchAttributes is None:
@@ -166,6 +167,7 @@
dsattributes.kDS1AttrDistinguishedName,
dsattributes.kDS1AttrFirstName,
dsattributes.kDS1AttrLastName,
+ dsattributes.kDSNAttrEMailAddress,
dsattributes.kDSNAttrPhoneNumber,
dsattributes.kDSNAttrMobileNumber,
dsattributes.kDSNAttrDepartment,
@@ -177,7 +179,7 @@
]
elif not searchAttributes:
# if search Attributes is [], don't restrict searching (but no binary)
- searchAttributes = stringDSAttrNames
+ searchAttributes = ABDirectoryQueryResult.stringDSAttrNames
self.log_debug("self.searchAttributes=%s" % (searchAttributes, ))
# calculate search map
@@ -216,6 +218,8 @@
if ignoreSystemRecords:
returnedAttributes += [dsattributes.kDS1AttrUniqueID,]
+ if not self.queryDSLocal:
+ returnedAttributes += [dsattributes.kDSNAttrMetaNodeLocation,]
self.returnedAttributes = list(set(returnedAttributes))
self.log_debug("self.returnedAttributes=%s" % (self.returnedAttributes, ))
@@ -323,7 +327,7 @@
@inlineCallbacks
def _getDirectoryQueryResults(self, query=None, attributes=None, maxRecords=0 ):
"""
- Get a list of filtered ABDirectoryQueryResult for the given query with the given attributes.
+ Get a list of ABDirectoryQueryResult for the given query with the given attributes.
query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
"""
limited = False
@@ -347,6 +351,12 @@
if self.ignoreSystemRecords:
if self._isSystemRecord(recordShortName, recordAttributes):
continue
+
+ if not self.queryDSLocal:
+ # skip records in local node which happens for non-complex od queries
+ if recordAttributes.get(dsattributes.kDSNAttrMetaNodeLocation, "").startswith("/Local/"):
+ self.log_info("Record from local node %s ignored" % (recordShortName,))
+ continue
result = ABDirectoryQueryResult(self.directoryBackedAddressBook, recordAttributes,
generateSimpleUIDs=self.generateSimpleUIDs,
@@ -367,7 +377,7 @@
self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
resultsDictionary[uid] = result
- self.log_debug("After filtering, %s results (limited=%s)." % (len(resultsDictionary), limited))
+ self.log_debug("_getDirectoryQueryResults: %s results (limited=%s)." % (len(resultsDictionary), limited))
returnValue((resultsDictionary.values(), limited, ))
@@ -492,8 +502,8 @@
"""
allRecords, filterAttributes, dsFilter = dsFilterFromAddressBookFilter( addressBookFilter, vcardPropToDSAttrMap=self.vCardPropToSearchableDSAttrMap );
- #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-
+ self.log_debug("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
+
# testing:
# allRecords = True
@@ -506,7 +516,7 @@
if not clear:
- # add filter to ignore system records rather than post filtering
+ # change query to ignore system records rather than post filtering
# but this appears to be broken in open directory
'''
if self.ignoreSystemRecords:
@@ -516,19 +526,53 @@
filterAttributes = list(set(filterAttributes).union(dsattributes.kDS1AttrGeneratedUID))
dsFilter = dsquery.expression( dsquery.expression.AND, (dsFilter, ignoreExpression,) ) if dsFilter else ignoreExpression
- #dsFilter = ignoreExpression
'''
queryAttributes = self._attributesForAddressBookQuery( addressBookQuery )
attributes = filterAttributes + queryAttributes
- #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
maxRecords = int(maxResults * 1.2)
- if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
- maxRecords = self.maxDSQueryRecords
+
+ # keep trying query till we get results based on filter. Especially when doing "all results" query
+ while True:
+ dsQueryResults, dsQueryLimited = (yield self._getDirectoryQueryResults(dsFilter, attributes, maxRecords))
+
+ filteredResults = []
+ for dsQueryResult in dsQueryResults:
+ if addressBookFilter.match(dsQueryResult.vCard()):
+ filteredResults.append(dsQueryResult)
+ else:
+ self.log_debug("doAddressBookQuery: result did not match filter: %s (%s)" % (dsQueryResult.vCard().propertyValue("FN"), dsQueryResult.vCard().propertyValue("UID"),))
+
+ #no more results
+ if not dsQueryLimited:
+ break;
+
+ # more than requested results
+ if maxResults and len(filteredResults) > maxResults:
+ break
+
+ # more than max report results
+ if len(filteredResults) > config.MaxQueryWithDataResults:
+ break
+
+ # more than self limit
+ if self.maxDSQueryRecords and maxRecords >= self.maxDSQueryRecords:
+ break
+
+ # try again with 2x
+ maxRecords *= 2
+ if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
+ maxRecords = self.maxDSQueryRecords
+
- results, limited = (yield self._getDirectoryQueryResults(dsFilter, attributes, maxRecords))
+ results = filteredResults
+ limited = maxResults and len(results) >= maxResults
+ if self.sortResults:
+ results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+
+ self.log_debug("doAddressBookQuery: %s results (limited=%s)." % (len(results), limited))
returnValue((results, limited,))
@@ -1630,7 +1674,7 @@
return str(self.vCard())
def uriName(self):
- return self.vCard().getProperty("UID").value() + ".vcf"
+ return self.vCard().propertyValue("UID") + ".vcf"
def hRef(self, parentURI="/directory/"):
# FIXME: Get the parent URI from self._directoryBackedAddressBook
Modified: CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/xmldirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/xmldirectorybacker.py 2012-04-09 18:23:11 UTC (rev 8997)
+++ CalendarServer/branches/users/gaya/ldapdirectorybacker/twistedcaldav/directory/xmldirectorybacker.py 2012-04-09 23:04:40 UTC (rev 8998)
@@ -137,14 +137,9 @@
Get vCards for a given addressBookFilter and addressBookQuery
"""
- queryResults = []
+ results = []
limited = False
- #calc maxResults from passed in maxResults allowing extra for second stage filtering in caller
- maxResults = int(maxResults * 1.2)
- if self.maxQueryResults and maxResults > self.maxQueryResults:
- maxResults = self.maxQueryResults
-
for queryType in self.recordTypes():
queryMap = self.rdnSchema[queryType]
@@ -168,7 +163,7 @@
"""
Although this exercises the dsFilter expression tree and recordsMatchingFields() it make little difference to the result of
- a addressbook query because of post filtering.
+ a addressbook query because of filtering.
"""
if not isinstance(dsFilter, dsquery.expression):
@@ -248,17 +243,7 @@
xmlDirectoryRecords = (yield self.listRecords(queryType))
self.log_debug("doAddressBookQuery: all #xmlDirectoryRecords %s" % (len(xmlDirectoryRecords), ))
- #sort so that CalDAVTester can have consistent results when it uses limits
- if self.sortResults:
- xmlDirectoryRecords = sorted(list(xmlDirectoryRecords), key=lambda x:x.guid)
- """ no good reason to use limit here, let caller do it
- # apply limit
- if len(xmlDirectoryRecords) > maxResults:
- xmlDirectoryRecords = xmlDirectoryRecords[:maxResults]
- self.log_debug("doAddressBookQuery: #xmlDirectoryRecords after max %s" % (len(xmlDirectoryRecords), ))
- """
-
for xmlDirectoryRecord in xmlDirectoryRecords:
def dsRecordAttributesFromDirectoryRecord( xmlDirectoryRecord ):
@@ -280,17 +265,21 @@
traceback.print_exc()
self.log_info("Could not get vcard for %s" % (xmlDirectoryRecord,))
else:
- self.log_debug("doAddressBookQuery: VCard text =\n%s" % (result.vCardText(),))
- queryResults.append(result)
+ if addressBookFilter.match(result.vCard()):
+ self.log_debug("doAddressBookQuery: VCard text =\n%s" % (result.vCard(),))
+ results.append(result)
+ else:
+ # should also filter for duplicate UIDs
+ self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (result.vCard().propertyValue("FN"), result.vCard().propertyValue("UID"),))
-
- # only get requested number of record results
- maxResults -= len(xmlDirectoryRecords)
- if maxResults <= 0:
+ if len(results) >= maxResults:
limited = True
break
-
- self.log_info("limited %s len(queryResults) %s" % (limited,len(queryResults),))
- returnValue((queryResults, limited,))
+ #sort results so that CalDAVTester can have consistent results when it uses limits
+ if self.sortResults:
+ results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+ self.log_info("limited %s len(results) %s" % (limited,len(results),))
+ returnValue((results, limited,))
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120409/cda9e365/attachment-0001.html>
More information about the calendarserver-changes
mailing list