[CalendarServer-changes] [4105] CalendarServer/trunk/twistedcaldav/directory
source_changes at macosforge.org
source_changes at macosforge.org
Tue Apr 28 19:02:40 PDT 2009
Revision: 4105
http://trac.macosforge.org/projects/calendarserver/changeset/4105
Author: sagen at apple.com
Date: 2009-04-28 19:02:39 -0700 (Tue, 28 Apr 2009)
Log Message:
-----------
Because opendirectory returns incorrect results when given a query with search terms that don't apply to the requested types, this commit adds a workaround that examines the query and breaks it into multiple queries.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
CalendarServer/trunk/twistedcaldav/directory/cachingappleopendirectory.py
Added Paths:
-----------
CalendarServer/trunk/twistedcaldav/directory/test/test_buildquery.py
Modified: CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2009-04-29 00:14:20 UTC (rev 4104)
+++ CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2009-04-29 02:02:39 UTC (rev 4105)
@@ -362,30 +362,66 @@
_ODFields = {
'fullName' : {
'odField' : dsattributes.kDS1AttrDistinguishedName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
'firstName' : {
'odField' : dsattributes.kDS1AttrFirstName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ ]),
},
'lastName' : {
'odField' : dsattributes.kDS1AttrLastName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ ]),
},
'emailAddresses' : {
'odField' : dsattributes.kDSNAttrEMailAddress,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ ]),
},
'recordName' : {
'odField' : dsattributes.kDSNAttrRecordName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
'guid' : {
'odField' : dsattributes.kDS1AttrGeneratedUID,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
}
+ _toODRecordTypes = {
+ DirectoryService.recordType_users :
+ dsattributes.kDSStdRecordTypeUsers,
+ DirectoryService.recordType_locations :
+ dsattributes.kDSStdRecordTypePlaces,
+ DirectoryService.recordType_groups :
+ dsattributes.kDSStdRecordTypeGroups,
+ DirectoryService.recordType_resources :
+ dsattributes.kDSStdRecordTypeResources,
+ }
+
+ _fromODRecordTypes = dict([(b, a) for a, b in _toODRecordTypes.iteritems()])
+
+
def recordsMatchingFields(self, fields, operand="or", recordType=None):
# Note that OD applies case-sensitivity globally across the entire
@@ -404,75 +440,59 @@
except KeyError:
pass
+ def multiQuery(directory, queries, attrs, operand):
+ results = {}
- operand = (dsquery.expression.OR if operand == "or"
- else dsquery.expression.AND)
+ for query, recordTypes in queries.iteritems():
+ if not query:
+ continue
- excluded = set()
- for field, value, caseless, matchType in fields:
- if field in self._ODFields:
- ODField = self._ODFields[field]['odField']
- excluded = excluded | self._ODFields[field]['excludes']
+ expressions = []
+ for ODField, value, caseless, matchType in query:
+ if matchType == "starts-with":
+ comparison = dsattributes.eDSStartsWith
+ elif matchType == "contains":
+ comparison = dsattributes.eDSContains
+ else:
+ comparison = dsattributes.eDSExact
+ expressions.append(dsquery.match(ODField, value, comparison))
- _ODTypes = {
- self.recordType_users: dsattributes.kDSStdRecordTypeUsers,
- self.recordType_locations: dsattributes.kDSStdRecordTypePlaces,
- self.recordType_groups: dsattributes.kDSStdRecordTypeGroups,
- self.recordType_resources: dsattributes.kDSStdRecordTypeResources,
- }
+ complexExpression = dsquery.expression(operand, expressions).generate()
- if recordType is None:
- # The client is looking for records in any of the four types
- recordTypes = set(_ODTypes.values())
+ self.log_info("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
+ (recordTypes, operand, caseless, complexExpression))
- # Certain query combinations yield invalid results. In particular,
- # any time you query on EMailAddress and are specifying Places
- # and/or Resources in the requested types, you will get all
- # Places/Resources returned. So here we will filter out known
- # invalid combinations:
- excludeFields = False
- recordTypes = list(recordTypes - excluded)
+ results.update(
+ opendirectory.queryRecordsWithAttributes(
+ directory,
+ complexExpression,
+ caseless,
+ recordTypes,
+ attrs,
+ )
+ )
- else:
- # The client is after only one recordType, so let's tailor the
- # query to not include any fields OD has trouble with:
- excludeFields = True
- recordTypes = [_ODTypes[recordType]]
+ return results
- expressions = []
- for field, value, caseless, matchType in fields:
- if field in self._ODFields:
- if (excludeFields and
- _ODTypes[recordType] in self._ODFields[field]['excludes']):
- # This is a field we're excluding because it behaves badly
- # for the record type result we're looking for. Skip it.
- continue
+ operand = (dsquery.expression.OR if operand == "or"
+ else dsquery.expression.AND)
- ODField = self._ODFields[field]['odField']
- if matchType == "starts-with":
- comparison = dsattributes.eDSStartsWith
- elif matchType == "contains":
- comparison = dsattributes.eDSContains
- else:
- comparison = dsattributes.eDSExact
- expressions.append(dsquery.match(ODField, value, comparison))
+ if recordType is None:
+ # The client is looking for records in any of the four types
+ recordTypes = set(self._toODRecordTypes.values())
+ else:
+ # The client is after only one recordType
+ recordTypes = [self._toODRecordTypes[recordType]]
- if not recordTypes or not expressions:
- # If we've excluded all types or all expressions, short circuit.
- self.log_info("Empty query, skipping call to OD")
- return []
+ queries = buildQueries(recordTypes, fields, self._ODFields)
- self.log_info("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
- (recordTypes, operand, caseless, fields))
-
deferred = deferToThread(
- opendirectory.queryRecordsWithAttributes,
+ multiQuery,
self.directory,
- dsquery.expression(operand, expressions).generate(),
- caseless,
- recordTypes,
- [ dsattributes.kDS1AttrGeneratedUID ]
+ queries,
+ [ dsattributes.kDS1AttrGeneratedUID ],
+ operand
)
deferred.addCallback(collectResults)
return deferred
@@ -957,6 +977,29 @@
yield recordGUID, autoSchedule, proxy, readOnlyProxy
+def buildQueries(recordTypes, fields, mapping):
+ """
+ Determine how many queries need to be performed in order to work around opendirectory
+ quirks, where searching on fields that don't apply to a given recordType returns incorrect
+ results (either none, or all records).
+ """
+
+ fieldLists = {}
+ for recordType in recordTypes:
+ fieldLists[recordType] = []
+ for field, value, caseless, matchType in fields:
+ if field in mapping:
+ if recordType in mapping[field]['appliesTo']:
+ ODField = mapping[field]['odField']
+ fieldLists[recordType].append((ODField, value, caseless, matchType))
+
+ queries = {}
+ for recordType, fieldList in fieldLists.iteritems():
+ key = tuple(fieldList)
+ queries.setdefault(key, []).append(recordType)
+ return queries
+
+
class OpenDirectoryRecord(DirectoryRecord):
"""
Open Directory implementation of L{IDirectoryRecord}.
Modified: CalendarServer/trunk/twistedcaldav/directory/cachingappleopendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/cachingappleopendirectory.py 2009-04-29 00:14:20 UTC (rev 4104)
+++ CalendarServer/trunk/twistedcaldav/directory/cachingappleopendirectory.py 2009-04-29 02:02:39 UTC (rev 4105)
@@ -294,27 +294,49 @@
_ODFields = {
'fullName' : {
'odField' : dsattributes.kDS1AttrDistinguishedName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
'firstName' : {
'odField' : dsattributes.kDS1AttrFirstName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ ]),
},
'lastName' : {
'odField' : dsattributes.kDS1AttrLastName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ ]),
},
'emailAddresses' : {
'odField' : dsattributes.kDSNAttrEMailAddress,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ ]),
},
'recordName' : {
'odField' : dsattributes.kDSNAttrRecordName,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
'guid' : {
'odField' : dsattributes.kDS1AttrGeneratedUID,
- 'excludes' : set(),
+ 'appliesTo' : set([
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces,
+ dsattributes.kDSStdRecordTypeResources,
+ ]),
},
}
@@ -331,6 +353,7 @@
_fromODRecordTypes = dict([(b, a) for a, b in _toODRecordTypes.iteritems()])
+
def recordsMatchingFields(self, fields, operand="or", recordType=None):
# Note that OD applies case-sensitivity globally across the entire
@@ -349,68 +372,59 @@
except KeyError:
pass
+ def multiQuery(directory, queries, attrs, operand):
+ results = {}
+ for query, recordTypes in queries.iteritems():
+ if not query:
+ continue
+
+ expressions = []
+ for ODField, value, caseless, matchType in query:
+ if matchType == "starts-with":
+ comparison = dsattributes.eDSStartsWith
+ elif matchType == "contains":
+ comparison = dsattributes.eDSContains
+ else:
+ comparison = dsattributes.eDSExact
+ expressions.append(dsquery.match(ODField, value, comparison))
+
+ complexExpression = dsquery.expression(operand, expressions).generate()
+
+ self.log_info("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
+ (recordTypes, operand, caseless, complexExpression))
+
+ results.update(
+ opendirectory.queryRecordsWithAttributes(
+ directory,
+ complexExpression,
+ caseless,
+ recordTypes,
+ attrs,
+ )
+ )
+
+ return results
+
+
operand = (dsquery.expression.OR if operand == "or"
else dsquery.expression.AND)
- excluded = set()
- for field, value, caseless, matchType in fields:
- if field in self._ODFields:
- ODField = self._ODFields[field]['odField']
- excluded = excluded | self._ODFields[field]['excludes']
-
if recordType is None:
# The client is looking for records in any of the four types
recordTypes = set(self._toODRecordTypes.values())
-
- # Certain query combinations yield invalid results. In particular,
- # any time you query on EMailAddress and are specifying Places
- # and/or Resources in the requested types, you will get all
- # Places/Resources returned. So here we will filter out known
- # invalid combinations:
- excludeFields = False
- recordTypes = list(recordTypes - excluded)
-
else:
- # The client is after only one recordType, so let's tailor the
- # query to not include any fields OD has trouble with:
- excludeFields = True
+ # The client is after only one recordType
recordTypes = [self._toODRecordTypes[recordType]]
- expressions = []
- for field, value, caseless, matchType in fields:
- if field in self._ODFields:
+ queries = buildQueries(recordTypes, fields, self._ODFields)
- if (excludeFields and
- self._toODRecordTypes[recordType] in self._ODFields[field]['excludes']):
- # This is a field we're excluding because it behaves badly
- # for the record type result we're looking for. Skip it.
- continue
-
- ODField = self._ODFields[field]['odField']
- if matchType == "starts-with":
- comparison = dsattributes.eDSStartsWith
- elif matchType == "contains":
- comparison = dsattributes.eDSContains
- else:
- comparison = dsattributes.eDSExact
- expressions.append(dsquery.match(ODField, value, comparison))
-
- if not recordTypes or not expressions:
- # If we've excluded all types or all expressions, short circuit.
- self.log_info("Empty query, skipping call to OD")
- return []
-
- self.log_info("Calling OD: Types %s, Operand %s, Caseless %s, %s" %
- (recordTypes, operand, caseless, fields))
-
deferred = deferToThread(
- opendirectory.queryRecordsWithAttributes,
+ multiQuery,
self.directory,
- dsquery.expression(operand, expressions).generate(),
- caseless,
- recordTypes,
- [ dsattributes.kDS1AttrGeneratedUID ]
+ queries,
+ [ dsattributes.kDS1AttrGeneratedUID ],
+ operand
)
deferred.addCallback(collectResults)
return deferred
@@ -713,6 +727,29 @@
yield recordGUID, autoSchedule, proxy, readOnlyProxy
+def buildQueries(recordTypes, fields, mapping):
+ """
+ Determine how many queries need to be performed in order to work around opendirectory
+ quirks, where searching on fields that don't apply to a given recordType returns incorrect
+ results (either none, or all records).
+ """
+
+ fieldLists = {}
+ for recordType in recordTypes:
+ fieldLists[recordType] = []
+ for field, value, caseless, matchType in fields:
+ if field in mapping:
+ if recordType in mapping[field]['appliesTo']:
+ ODField = mapping[field]['odField']
+ fieldLists[recordType].append((ODField, value, caseless, matchType))
+
+ queries = {}
+ for recordType, fieldList in fieldLists.iteritems():
+ key = tuple(fieldList)
+ queries.setdefault(key, []).append(recordType)
+ return queries
+
+
class OpenDirectoryRecord(CachingDirectoryRecord):
"""
Open Directory implementation of L{IDirectoryRecord}.
Added: CalendarServer/trunk/twistedcaldav/directory/test/test_buildquery.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_buildquery.py (rev 0)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_buildquery.py 2009-04-29 02:02:39 UTC (rev 4105)
@@ -0,0 +1,115 @@
+##
+# Copyright (c) 2009 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twistedcaldav.test.util import TestCase
+from twistedcaldav.directory.cachingappleopendirectory import buildQueries, OpenDirectoryService
+import dsattributes
+
+class BuildQueryTests(TestCase):
+
+ def test_buildQuery(self):
+ self.assertEquals(
+ buildQueries(
+ [dsattributes.kDSStdRecordTypeUsers],
+ (
+ ("firstName", "morgen", True, "starts-with"),
+ ("lastName", "sagen", True, "starts-with"),
+ ),
+ OpenDirectoryService._ODFields
+ ),
+ {
+ (
+ ('dsAttrTypeStandard:FirstName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:LastName', 'sagen', True, 'starts-with')
+ ): ['dsRecTypeStandard:Users']
+ }
+ )
+ self.assertEquals(
+ buildQueries(
+ [
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypePlaces
+ ],
+ (
+ ("firstName", "morgen", True, "starts-with"),
+ ("emailAddresses", "morgen", True, "contains"),
+ ),
+ OpenDirectoryService._ODFields
+ ),
+ {
+ (
+ ('dsAttrTypeStandard:FirstName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:EMailAddress', 'morgen', True, 'contains'),
+ ): ['dsRecTypeStandard:Users'],
+ (): ['dsRecTypeStandard:Places']
+ }
+ )
+ self.assertEquals(
+ buildQueries(
+ [
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypePlaces
+ ],
+ (
+ ("firstName", "morgen", True, "starts-with"),
+ ("lastName", "morgen", True, "starts-with"),
+ ("fullName", "morgen", True, "starts-with"),
+ ("emailAddresses", "morgen", True, "contains"),
+ ),
+ OpenDirectoryService._ODFields
+ ),
+ {
+ (
+ ('dsAttrTypeStandard:RealName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:EMailAddress', 'morgen', True, 'contains'),
+ ): ['dsRecTypeStandard:Groups'],
+ (
+ ('dsAttrTypeStandard:RealName', 'morgen', True, 'starts-with'),
+ ): ['dsRecTypeStandard:Places']
+ }
+ )
+ self.assertEquals(
+ buildQueries(
+ [
+ dsattributes.kDSStdRecordTypeUsers,
+ dsattributes.kDSStdRecordTypeGroups,
+ dsattributes.kDSStdRecordTypeResources,
+ dsattributes.kDSStdRecordTypePlaces
+ ],
+ (
+ ("firstName", "morgen", True, "starts-with"),
+ ("lastName", "morgen", True, "starts-with"),
+ ("fullName", "morgen", True, "starts-with"),
+ ("emailAddresses", "morgen", True, "contains"),
+ ),
+ OpenDirectoryService._ODFields
+ ),
+ {
+ (
+ ('dsAttrTypeStandard:RealName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:EMailAddress', 'morgen', True, 'contains')
+ ): ['dsRecTypeStandard:Groups'],
+ (
+ ('dsAttrTypeStandard:RealName', 'morgen', True, 'starts-with'),
+ ): ['dsRecTypeStandard:Resources', 'dsRecTypeStandard:Places'],
+ (
+ ('dsAttrTypeStandard:FirstName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:LastName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:RealName', 'morgen', True, 'starts-with'),
+ ('dsAttrTypeStandard:EMailAddress', 'morgen', True, 'contains')
+ ): ['dsRecTypeStandard:Users']
+ }
+ )
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090428/0a9d4c20/attachment-0001.html>
More information about the calendarserver-changes
mailing list