[CalendarServer-changes] [9765] CalendarServer/branches/users/gaya/directorybacker

source_changes at macosforge.org source_changes at macosforge.org
Fri Aug 31 17:56:45 PDT 2012


Revision: 9765
          http://trac.macosforge.org/projects/calendarserver/changeset/9765
Author:   gaya at apple.com
Date:     2012-08-31 17:56:43 -0700 (Fri, 31 Aug 2012)
Log Message:
-----------
merge in most of ldapdirectorybacker branch

Modified Paths:
--------------
    CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py
    CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml
    CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py

Added Paths:
-----------
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py
    CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py

Modified: CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -34,6 +34,7 @@
 from twext.python.log import Logger
 from twext.web2.auth.basic import BasicCredentialFactory
 from twext.web2.dav import auth
+from twext.web2.dav.util import joinURL
 from twext.web2.http_headers import Headers
 from twext.web2.static import File as FileResource
 
@@ -483,12 +484,12 @@
             newStore,
         )
 
-        directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
         if config.DirectoryAddressBook.Enabled and config.EnableSearchAddressBook:
             log.info("Setting up directory address book: %r" % (directoryBackedAddressBookResourceClass,))
 
             directoryBackedAddressBookCollection = directoryBackedAddressBookResourceClass(
-                principalCollections=(principalCollection,)
+                principalCollections=(principalCollection,),
+                uri=joinURL("/", config.DirectoryAddressBook.name, "/")
             )
             if _reactor._started:
                 directoryBackedAddressBookCollection.provisionDirectory()
@@ -496,6 +497,7 @@
                 addSystemEventTrigger("after", "startup", directoryBackedAddressBookCollection.provisionDirectory)
         else:
             # remove /directory from previous runs that may have created it
+            directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
             try:
                 FilePath(directoryPath).remove()
                 log.info("Deleted: %s" %    directoryPath)

Modified: CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml	2012-09-01 00:56:43 UTC (rev 9765)
@@ -106,6 +106,7 @@
     <name>Public %02d</name>
     <first-name>Public</first-name>
     <last-name>%02d</last-name>
+     <email-address>public%02d at example.com</email-address>
   </user>
   <group>
     <uid>group01</uid>

Modified: CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist	2012-09-01 00:56:43 UTC (rev 9765)
@@ -1015,6 +1015,30 @@
       <string>English</string>
     </dict>
 
-
+    <!--
+        Directory Address Book
+      -->
+    
+    <!--  Disable Directory Address Book -->
+    <!--
+      <key>DirectoryAddressBook</key>
+      <false/>
+    -->
+    
+    <!-- XML Directory-backed Directory Address Book -->
+    <key>EnableSearchAddressBook</key>
+    <true/>
+    <key>DirectoryAddressBook</key>
+    <dict>
+        <key>Enabled</key>
+        <true/>
+      <key>type</key>
+      <string>twistedcaldav.directory.xmldirectorybacker.XMLDirectoryBackingService</string>
+      <key>params</key>
+      <dict>
+        <key>xmlFile</key>
+        <string>./conf/auth/accounts-test.xml</string>
+      </dict>
+   </dict>
   </dict>
 </plist>

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -228,7 +228,7 @@
         # config.
         attrSet = set()
 
-        if self.rdnSchema["guidAttr"]:
+        if self.rdnSchema.get("guidAttr"):
             attrSet.add(self.rdnSchema["guidAttr"])
         for recordType in self.recordTypes():
             if self.rdnSchema[recordType]["attr"]:
@@ -255,7 +255,7 @@
             attrSet.add(self.groupSchema["nestedGroupsAttr"])
         if self.groupSchema["memberIdAttr"]:
             attrSet.add(self.groupSchema["memberIdAttr"])
-        if self.rdnSchema["users"]["loginEnabledAttr"]:
+        if self.rdnSchema.get("users") and self.rdnSchema["users"]["loginEnabledAttr"]:
             attrSet.add(self.rdnSchema["users"]["loginEnabledAttr"])
         if self.resourceSchema["resourceInfoAttr"]:
             attrSet.add(self.resourceSchema["resourceInfoAttr"])

Added: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py	                        (rev 0)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -0,0 +1,376 @@
+##
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+"""
+Apple Open Directory directory service implementation for backing up directory-backed address books
+"""
+
+__all__ = [
+    "LdapDirectoryBackingService",
+]
+
+import traceback
+import ldap
+
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+
+from twistedcaldav.config import config
+from twistedcaldav.directory.ldapdirectory import LdapDirectoryService, normalizeDNstr
+from twistedcaldav.directory.opendirectorybacker import ABDirectoryQueryResult, dsFilterFromAddressBookFilter, propertiesInAddressBookQuery
+
+
+class LdapDirectoryBackingService(LdapDirectoryService):
+    """
+    Directory backer for L{LdapDirectoryService}.
+    """
+
+    def __init__(self, params):
+        self._actuallyConfigure(**params)
+
+    def _actuallyConfigure(self, **params):
+        
+        self.log_debug("_actuallyConfigure: params=%s" % (params,))
+        defaults = {
+            "recordTypes": (), # for super
+            "rdnSchema": {
+                "base": "dc=example,dc=com",
+                "queries": (
+                    { #people
+                        "rdn":"ou=people",
+                        "vcardPropToLdapAttrMap" : { # maps vCard properties to searchable ldap attributes
+                            "FN" : "cn",
+                         },
+                        "ldapAttrToDSAttrMap" : { # maps ldap attributes to ds attribute types
+                            "cn" : "dsAttrTypeStandard:RealName",
+                         },
+                        "additionalVCardProps":None,
+                    },
+                ),
+
+            },
+            "removeDuplicateUIDs":True,      # remove vCards with duplicate UIDs
+            "appleInternalServer":False,    # does magic in ABDirectoryQueryResult
+            "maxQueryResults":0,            # max records returned
+            "fakeETag":True,                # eTag is fake, otherwise it is md5(all attributes)
+       }
+
+        #params = self.getParams(params, defaults, ignored)
+        def addDefaults(params, defaults, remove=None):
+            for key in defaults:
+                if not key in params:
+                    params[key] = defaults[key]
+            return params
+            
+        params = addDefaults(params, defaults)
+        self.log_debug("_actuallyConfigure after addDefaults: params=%s" % (params,))
+        
+        # super does not like these extra params
+        directoryBackedAddressBook=params["directoryBackedAddressBook"]
+        del params["directoryBackedAddressBook"]
+        appleInternalServer=params["appleInternalServer"]
+        del params["appleInternalServer"] 
+        maxQueryResults=params["maxQueryResults"]
+        del params["maxQueryResults"]
+        fakeETag=params["fakeETag"]
+        del params["fakeETag"]
+        removeDuplicateUIDs=params["removeDuplicateUIDs"]
+        del params["removeDuplicateUIDs"]
+
+        
+        #standardize ds attributes type names
+        # or we could just require dsAttrTypeStandard: prefix in the plist
+        rdnSchema = params["rdnSchema"];
+        for query in rdnSchema["queries"]:
+            ldapAttrToDSAttrMap = query["ldapAttrToDSAttrMap"]
+            for ldapAttrName, dsAttrNames in ldapAttrToDSAttrMap.iteritems():
+                if not isinstance(dsAttrNames, list):
+                    dsAttrNames = [dsAttrNames,]
+                
+                normalizedDSAttrNames = []
+                for dsAttrName in dsAttrNames:
+                    if not dsAttrName.startswith("dsAttrTypeStandard:") and not dsAttrName.startswith("dsAttrTypeNative:"):
+                        normalizedDSAttrNames.append("dsAttrTypeStandard:" + dsAttrName)
+                    else:
+                        normalizedDSAttrNames.append(dsAttrName)
+                
+                # not needed, but tests code paths
+                if len(normalizedDSAttrNames) > 1:
+                    ldapAttrToDSAttrMap[ldapAttrName] = normalizedDSAttrNames
+                else:
+                    ldapAttrToDSAttrMap[ldapAttrName] = normalizedDSAttrNames[0]
+               
+                
+        self.log_debug("_actuallyConfigure after clean: params=%s" % (params,))
+ 
+        assert directoryBackedAddressBook is not None
+        self.directoryBackedAddressBook = directoryBackedAddressBook
+       
+        self.maxQueryResults = maxQueryResults
+        
+        ### params for ABDirectoryQueryResult()
+        self.fakeETag = fakeETag
+        self.appleInternalServer = appleInternalServer
+        self.removeDuplicateUIDs = removeDuplicateUIDs
+ 
+        super(LdapDirectoryBackingService, self).__init__(params)
+        
+ 
+    def createCache(self):
+         succeed(None)
+                        
+
+    @inlineCallbacks
+    def _getLdapQueryResults(self, base, queryStr, attributes=None, maxResults=0, ldapAttrToDSAttrMap=None, ldapAttrTransforms=None, additionalVCardProps=None, kind=None ):
+        """
+        Get a list of ABDirectoryQueryResult for the given query with the given attributes.
+        query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
+        """
+        limited = False
+        resultsDictionary = {}
+        
+        # can't resist also using a timeout, 1 sec per request result for now
+        timeout = maxResults
+
+        self.log_debug("_getLdapQueryResults: LDAP query base=%s and filter=%s and attributes=%s timeout=%s resultLimit=%s" % (ldap.dn.dn2str(base), queryStr, attributes, timeout, maxResults))
+        
+        ldapSearchResult = (yield self.timedSearch(ldap.dn.dn2str(base), ldap.SCOPE_SUBTREE, filterstr=queryStr, attrlist=attributes, timeoutSeconds=timeout, resultLimit=maxResults))
+        self.log_debug("_getLdapQueryResults: ldapSearchResult=%s" % (ldapSearchResult,))
+        
+        if maxResults and len(ldapSearchResult) >= maxResults:
+            limited = True
+            self.log_debug("_getLdapQueryResults: limit (= %d) reached." % (maxResults, ))
+
+        for dn, ldapAttributes in ldapSearchResult:
+            #dn = normalizeDNstr(dn)
+            result = None
+            try:
+                if "dn" not in ldapAttributes:
+                    ldapAttributes["dn"] = [normalizeDNstr(dn),]
+                
+                # make a dsRecordAttributes dict from the ldap attributes
+                dsRecordAttributes = {}
+                for ldapAttributeName, ldapAttributeValues in ldapAttributes.iteritems():
+
+                    #self.log_debug("inspecting ldapAttributeName %s with values %s" % (ldapAttributeName, ldapAttributeValues,))
+
+                    # get rid of '' values
+                    ldapAttributeValues = [attr for attr in ldapAttributeValues if len(attr)]
+                    
+                    if len(ldapAttributeValues):
+                                                
+                        
+                        dsAttributeNames = ldapAttrToDSAttrMap.get(ldapAttributeName)
+                        if dsAttributeNames:
+                            
+                            if ldapAttrTransforms:
+                            
+                                # do value transforms
+                                # need to expand this to cover all cases
+                                # All this does now is to pull part of an ldap string out
+                                # e.g: uid=renuka,ou=People,o=apple.com,o=email -> renuka
+                                transforms = ldapAttrTransforms.get(ldapAttributeName)
+                                if transforms:
+                                    if not isinstance(transforms, list):
+                                        transforms = [transforms,]
+                                    
+                                    transformedValues = []
+                                    for ldapAttributeValue in ldapAttributeValues:
+                                        transformedValue = ldapAttributeValue
+                                        for valuePart in normalizeDNstr(ldapAttributeValue).split(","):
+                                            kvPair = valuePart.split("=")
+                                            if len(kvPair) == 2:
+                                                for transform in transforms:
+                                                    if transform.lower() == kvPair[0]:
+                                                        transformedValue = kvPair[1]
+                                                        break
+                                                    
+                                        transformedValues += [transformedValue,]
+                                    
+                                    if (ldapAttributeValues != transformedValues):
+                                        self.log_debug("_getLdapQueryResults: %s %s transformed to %s" % (ldapAttributeName, ldapAttributeValues, transformedValues))
+                                        ldapAttributeValues = transformedValues
+                                                
+                                        
+                            
+                            if not isinstance(dsAttributeNames, list):
+                                dsAttributeNames = [dsAttributeNames,]
+                                
+                            for dsAttributeName in dsAttributeNames:
+                                
+                                # base64 encode binary attributes
+                                if dsAttributeName in ABDirectoryQueryResult.binaryDSAttrNames:
+                                    ldapAttributeValues = [attr.encode('base64') for attr in ldapAttributeValues]
+                                
+                                # add to dsRecordAttributes
+                                if dsAttributeName not in dsRecordAttributes:
+                                    dsRecordAttributes[dsAttributeName] = list()
+                                    
+                                dsRecordAttributes[dsAttributeName] = list(set(dsRecordAttributes[dsAttributeName] + ldapAttributeValues))
+                                self.log_debug("doAddressBookQuery: dsRecordAttributes[%s] = %s" % (dsAttributeName, dsRecordAttributes[dsAttributeName],))
+
+                # get a record for dsRecordAttributes 
+                result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, kind=kind, additionalVCardProps=additionalVCardProps, appleInternalServer=self.appleInternalServer)
+            except:
+                traceback.print_exc()
+                self.log_info("Could not get vcard for %s" % (dn,))
+            else:
+                uid = result.vCard().propertyValue("UID")
+
+                if uid in resultsDictionary:
+                    self.log_info("Record skipped due to duplicate UID: %s" % (dn,))
+                    continue
+                    
+                self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+                resultsDictionary[uid] = result                   
+
+        self.log_debug("%s results (limited=%s)." % (len(resultsDictionary), limited))
+        returnValue((resultsDictionary, limited, ))
+
+    @inlineCallbacks
+    def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
+        """
+        Get vCards for a given addressBookFilter and addressBookQuery
+        """
+    
+        results = {} if self.removeDuplicateUIDs else []
+        limited = False
+        
+        #one ldap query for each rnd in queries
+        for queryMap in self.rdnSchema["queries"]:
+
+            rdn = queryMap["rdn"]
+            vcardPropToLdapAttrMap = queryMap["vcardPropToLdapAttrMap"]
+            ldapAttrToDSAttrMap = queryMap["ldapAttrToDSAttrMap"]
+            additionalVCardProps = queryMap.get("additionalVCardProps")
+            ldapAttrTransforms = queryMap.get("ldapAttrTransforms")
+            kind = queryMap.get("kind", "individual")
+            
+            # add constants and KIND
+            constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+            if additionalVCardProps:
+                for key, value in additionalVCardProps.iteritems():
+                    if key not in constantProperties:
+                        constantProperties[key] = value
+                        
+            # add KIND as constant so that query can be skipped if addressBookFilter needs a different kind
+            constantProperties["KIND"] = kind
+            
+
+            filterPropertyNames, dsFilter  = dsFilterFromAddressBookFilter( addressBookFilter, vcardPropToLdapAttrMap, constantProperties=constantProperties );
+            self.log_debug("doAddressBookQuery: rdn=%s, query=%s, propertyNames=%s" % (rdn, dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames))
+
+            if dsFilter:
+                if dsFilter is True:
+                    dsFilter = None
+                
+                # calculate minimum attributes needed for this query
+                etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+            
+                if (etagRequested and not self.fakeETag) or not queryPropNames:
+                    queryAttributes = ldapAttrToDSAttrMap.keys()
+                elif queryPropNames:
+                    '''
+                    # To DO:  Need mapping from properties to returned attributes 
+                    queryPropNames += filterPropertyNames
+                    queryAttributes = []
+                    for prop in queryPropNames:
+                        attributes = ABDirectoryQueryResult.vcardPropToDSAttrMap.get(prop)
+                        if attributes:
+                            queryAttributes += attributes
+                    '''
+                            
+                    queryAttributes =  ldapAttrToDSAttrMap.keys()
+                    
+                self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s, queryAttributes=%s" % (etagRequested, queryPropNames, queryAttributes,))
+                
+                #get all ldap attributes -- for debug
+                if queryMap.get("getAllAttributes"):
+                    queryAttributes = None
+                   
+                base =  ldap.dn.str2dn(rdn) + self.base
+                
+                queryStr = "(cn=*)"    # all results query  - should make a param
+                #add additional filter from config
+                queryFilter = queryMap.get("filter")
+                if dsFilter and queryFilter:
+                    queryStr = "(&%s%s)" % (queryFilter, dsFilter.generate())
+                elif queryFilter:
+                    queryStr = queryFilter
+                elif dsFilter:
+                    queryStr = dsFilter.generate()
+
+                
+                # keep trying ldap query till we get results based on filter.  Especially when doing "all results" query
+                remainingMaxResults = maxResults - len(results) if maxResults else 0
+                maxLdapResults = int(remainingMaxResults * 1.2)
+    
+                while True:
+                    ldapQueryResultsDictionary, ldapQueryLimited = (yield self._getLdapQueryResults(base=base, 
+                                                                                                    queryStr=queryStr, 
+                                                                                                    attributes=queryAttributes, 
+                                                                                                    maxResults=maxLdapResults, 
+                                                                                                    kind=kind, 
+                                                                                                    ldapAttrToDSAttrMap=ldapAttrToDSAttrMap, 
+                                                                                                    ldapAttrTransforms=ldapAttrTransforms, 
+                                                                                                    additionalVCardProps=additionalVCardProps))
+                    
+                    for uid, ldapQueryResult in ldapQueryResultsDictionary.iteritems():
+
+                        if self.removeDuplicateUIDs and uid in results:
+                            self.log_info("Record skipped due to duplicate UID: %s" % (uid,))
+                            continue
+                    
+                        if not addressBookFilter.match(ldapQueryResult.vCard()):
+                            self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (ldapQueryResult.vCard().propertyValue("FN"), uid,))
+                            continue
+                        
+                        if self.removeDuplicateUIDs:
+                            results[uid] = ldapQueryResult
+                        else:
+                            results += [ldapQueryResult,]              
+
+                    
+                    #no more results    
+                    if not ldapQueryLimited:
+                        break;
+                    
+                    # more than requested results
+                    if maxResults and len(results) >= maxResults:
+                        break
+                    
+                    # more than max report results
+                    if len(results) >= config.MaxQueryWithDataResults:
+                        break
+                    
+                    # more than self limit
+                    if self.maxQueryResults and maxLdapResults >= self.maxQueryResults:
+                        break
+                    
+                    # try again with 2x
+                    maxLdapResults *= 2
+                    if self.maxQueryResults and maxLdapResults > self.maxQueryResults:
+                        maxLdapResults = self.maxQueryResults
+                    
+                if maxResults and len(results) >= maxResults:
+                    break
+                
+        
+        limited = maxResults and len(results) >= maxResults
+                         
+        self.log_info("limited %s len(results) %s" % (limited,len(results),))
+        returnValue((results.values() if self.removeDuplicateUIDs else results, limited,))        
+

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,53 +20,45 @@
 """
 
 __all__ = [
-    "OpenDirectoryBackingService", "VCardRecord",
+    "OpenDirectoryBackingService", "ABDirectoryQueryResult",
 ]
 
 import traceback
 import hashlib
-
-import os
-import sys
 import time
-
-from os import listdir
-from os.path import join, abspath
-from tempfile import mkstemp, gettempdir
 from random import random
 
+from calendarserver.platform.darwin.od import dsattributes, dsquery
 from pycalendar.n import N
 from pycalendar.adr import Adr
 from pycalendar.datetime import PyCalendarDateTime
 
-from socket import getfqdn
-
-from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, returnValue, deferredGenerator, succeed
-from twext.python.filepath import CachingFilePath as FilePath
+from twisted.python.reflect import namedModule
+
 from txdav.xml import element as davxml
 from txdav.xml.base import twisted_dav_namespace, dav_namespace, parse_date, twisted_private_namespace
+
+from twext.python.log import LoggingMixIn, Logger
 from twext.web2.dav.resource import DAVPropertyMixIn
 from twext.web2.dav.util import joinURL
 from twext.web2.http_headers import MimeType, generateContentType, ETag
 
-
-from twistedcaldav import customxml, carddavxml
-from twistedcaldav.customxml import calendarserver_namespace
+from twistedcaldav import carddavxml
 from twistedcaldav.config import config
-from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
-from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
+from twistedcaldav.directory.directory import DirectoryService
 from twistedcaldav.query import addressbookqueryfilter
 from twistedcaldav.vcard import Component, Property, vCardProductID
 
 from xmlrpclib import datetime
 
-from calendarserver.platform.darwin.od import dsattributes, dsquery
-from twisted.python.reflect import namedModule
+log = Logger()
 
+addSourceProperty = False
+
 class OpenDirectoryBackingService(DirectoryService):
     """
-    Open Directory implementation of L{IDirectoryService}.
+    Directory backer for L{IDirectoryService}.
     """
 
     baseGUID = "BF07A1A2-5BB5-4A4D-A59A-67260EA7E143"
@@ -82,24 +74,22 @@
         peopleNode = "/Search/Contacts",
         queryUserRecords=True, 
         userNode = "/Search",
+        queryGroupRecords=True, 
+        groupNode = "/Search",
         maxDSQueryRecords = 0,            # maximum number of records requested for any ds query
         
-        queryDSLocal = False,              #query in DSLocal -- debug
+        queryDSLocal = False,             #query in DSLocal -- debug
         dsLocalCacheTimeout = 30,
         ignoreSystemRecords = True,
         
-        liveQuery = True,                    # query directory service as needed
-        fakeETag = True,                # eTag is not reliable if True 
-        
-        cacheQuery = False,
-        cacheTimeout=30,                # cache timeout            
-        
-        addDSAttrXProperties=False,        # add dsattributes to vcards as "X-" attributes
-        standardizeSyntheticUIDs = False,  # use simple synthetic UIDs --- good for testing
+        fakeETag = True,                  # eTag is not reliable if True 
+                
+        addDSAttrXProperties=False,       # add dsattributes to vcards as "X-" attributes
         appleInternalServer=False,
         
-        additionalAttributes=[],
-        allowedAttributes=[],
+        additionalAttributes=None,
+        allowedAttributes=None,
+        searchAttributes=None,
         directoryBackedAddressBook=None
     ):
         """
@@ -108,12 +98,7 @@
         @maxDSQueryRecords: maximum number of (unfiltered) ds records retrieved before raising 
             NumberOfMatchesWithinLimits exception or returning results
         @dsLocalCacheTimeout: how log to keep cache of DSLocal records
-        @liveQuery: C{True} to query the directory as needed
         @fakeETag: C{True} to use a fake eTag; allows ds queries with partial attributes
-        @cacheQuery: C{True} to query the directory and cache results
-        @cacheTimeout: if caching, the average cache timeout
-        @standardizeSyntheticUIDs: C{True} when creating synthetic UID (==f(Node, Type, Record Name)), 
-            use a standard Node name. This allows testing with the same UID on different hosts
         @allowedAttributes: list of DSAttributes that are used to create VCards
 
         """
@@ -125,425 +110,292 @@
         self.userDirectory = None
         self.userNode = None
         
-        self.realmName = None # needed for super
 
-        self.odModule = namedModule(config.OpenDirectoryModule)
+        # get node to record type map
+        def addNodesToNodeRecordTypeMap(nodeList, recordType):
+            for node in nodeList if isinstance(nodeList, list) else (nodeList,):
+                if not node in nodeRecordTypeMap:
+                     nodeRecordTypeMap[node] = []
+                nodeRecordTypeMap[node] += [recordType,]
+            self.recordTypes += [recordType,]
+
+        nodeRecordTypeMap = {}
+        self.recordTypes = []
+        if queryPeopleRecords:
+            addNodesToNodeRecordTypeMap(peopleNode, dsattributes.kDSStdRecordTypePeople,)
+        if queryUserRecords:
+             addNodesToNodeRecordTypeMap(userNode, dsattributes.kDSStdRecordTypeUsers,)
+        if queryGroupRecords:
+            addNodesToNodeRecordTypeMap(groupNode, dsattributes.kDSStdRecordTypeGroups,)
         
-        if queryPeopleRecords or not queryUserRecords:
-            self.peopleNode = peopleNode
+        # get query info
+        nodeDirectoryRecordTypeMap = {}
+        self.odModule = namedModule(config.OpenDirectoryModule)
+        for node in nodeRecordTypeMap:
+            queryInfo = {"recordTypes":nodeRecordTypeMap[node],}
             try:
-                self.peopleDirectory = self.odModule.odInit(peopleNode)
+                queryInfo["directory"] = self.odModule.odInit(node)
             except self.odModule.ODError, e:
-                self.log_error("Open Directory (node=%s) Initialization error: %s" % (peopleNode, e))
+                self.log_error("Open Directory (node=%s) Initialization error: %s" % (node, e))
                 raise
-            self.realmName = peopleNode
-
-        if queryUserRecords:
-            if self.peopleNode == userNode:          # use sane directory and node if they are equal
-                self.userNode = self.peopleNode
-                self.userDirectory = self.peopleDirectory
-            else:
-                self.userNode = userNode
-                try:
-                    self.userDirectory = self.odModule.odInit(userNode)
-                except self.odModule.ODError, e:
-                    self.log_error("Open Directory (node=%s) Initialization error: %s" % (userNode, e))
-                    raise
-                if self.realmName:
-                    self.realmName += "+" + userNode
-                else:
-                    self.realmName = userNode
+            
+            nodeDirectoryRecordTypeMap[node] = queryInfo
         
+        self.nodeDirectoryRecordTypeMap = nodeDirectoryRecordTypeMap
         
+        
+        # calc realm name
+        self.realmName = "+".join(nodeDirectoryRecordTypeMap.keys())
+        
+        self.queryPeopleRecords = queryPeopleRecords
+        self.queryUserRecords = queryUserRecords
+        self.queryGroupRecords = queryGroupRecords
         self.maxDSQueryRecords = maxDSQueryRecords
 
         self.ignoreSystemRecords = ignoreSystemRecords
         self.queryDSLocal = queryDSLocal
         self.dsLocalCacheTimeout = dsLocalCacheTimeout
 
-        self.liveQuery = liveQuery or not cacheQuery
         self.fakeETag = fakeETag
-
-        self.cacheQuery = cacheQuery
-        
-        self.cacheTimeout = cacheTimeout if cacheTimeout > 0 else 30
-        
+                
         self.addDSAttrXProperties = addDSAttrXProperties
-        self.standardizeSyntheticUIDs = standardizeSyntheticUIDs
         self.appleInternalServer = appleInternalServer
         
-        self.additionalAttributes = additionalAttributes
-        # filter allows attributes, but make sure there are a minimum of attributes for functionality
+        
+        if searchAttributes is None:
+            # this is the intersection of ds default indexed attributes and ABDirectoryQueryResult.vcardPropToDSAttrMap.values()
+            # so, not all indexed attributes are below
+            searchAttributes = [
+                dsattributes.kDS1AttrGeneratedUID,
+                dsattributes.kDS1AttrDistinguishedName,
+                dsattributes.kDS1AttrFirstName,
+                dsattributes.kDS1AttrLastName,
+                dsattributes.kDSNAttrEMailAddress,
+                dsattributes.kDSNAttrPhoneNumber,
+                dsattributes.kDSNAttrMobileNumber,
+                dsattributes.kDSNAttrDepartment,
+                dsattributes.kDSNAttrCompany,
+                dsattributes.kDSNAttrStreet,
+                dsattributes.kDSNAttrState,
+                dsattributes.kDSNAttrCity,
+                dsattributes.kDSNAttrCountry,
+                ]
+        elif not searchAttributes:
+            # if search Attributes is [], don't restrict searching (but no binary)
+            searchAttributes = ABDirectoryQueryResult.stringDSAttrNames
+        self.log_debug("self.searchAttributes=%s" % (searchAttributes, ))
+        
+        # calculate search map
+        vcardPropToSearchableDSAttrMap = {}
+        for prop, dsAttributeList in ABDirectoryQueryResult.vcardPropToDSAttrMap.iteritems():
+            dsIndexedAttributeList = [attr for attr in dsAttributeList if attr in searchAttributes]
+            if len(dsIndexedAttributeList):
+                vcardPropToSearchableDSAttrMap[prop] = dsIndexedAttributeList
+        
+        self.vcardPropToSearchableDSAttrMap = vcardPropToSearchableDSAttrMap
+        self.log_debug("self.vcardPropToSearchableDSAttrMap=%s" % (self.vcardPropToSearchableDSAttrMap, ))
+ 
+        #get attributes required for needed for valid vCard
+        requiredAttributes = [attr for prop in ("UID", "FN", "N") for attr in ABDirectoryQueryResult.vcardPropToDSAttrMap[prop]]
+        requiredAttributes += [dsattributes.kDS1AttrModificationTimestamp, dsattributes.kDS1AttrCreationTimestamp,] # for VCardResult DAVPropertyMixIn
+        self.requiredAttributes = list(set(requiredAttributes))
+        self.log_debug("self.requiredAttributes=%s" % (self.requiredAttributes, ))
+           
+        # get returned attributes
+        #allowedAttributes = [dsattributes.kDS1AttrUniqueID,]
         if allowedAttributes:
-            self.allowedDSQueryAttributes = sorted(list(set(
-                                                [attr for attr in VCardRecord.allDSQueryAttributes
+            
+            returnedAttributes = [attr for attr in ABDirectoryQueryResult.allDSQueryAttributes
                                                     if (isinstance(attr, str) and attr in allowedAttributes) or
-                                                       (isinstance(attr, tuple) and attr[0] in allowedAttributes)] +
-                                                VCardRecord.dsqueryAttributesForProperty.get("X-INTERNAL-REQUIRED")
-                                                )))
-            if (self.allowedDSQueryAttributes != VCardRecord.allDSQueryAttributes):
-                self.log_info("Allowed DS query attributes = %r" % (self.allowedDSQueryAttributes, ))
+                                                       (isinstance(attr, tuple) and attr[0] in allowedAttributes)]
+            self.log_debug("allowedAttributes%s" % (allowedAttributes, ))
         else:
-            self.allowedDSQueryAttributes = VCardRecord.allDSQueryAttributes
-        
-        #self.returnedAttributes = VCardRecord.allDSQueryAttributes
-        self.returnedAttributes = self.allowedDSQueryAttributes
-        
+            returnedAttributes = ABDirectoryQueryResult.allDSQueryAttributes
             
+        # add required
+        returnedAttributes += self.requiredAttributes
         
+        if additionalAttributes:
+            returnedAttributes += additionalAttributes
         
-        self._dsLocalRecords = []
-        self._nextDSLocalQueryTime = 0
+        if ignoreSystemRecords:
+            returnedAttributes += [dsattributes.kDS1AttrUniqueID,]
+        if not queryDSLocal:
+            returnedAttributes += [dsattributes.kDSNAttrMetaNodeLocation,]
+        if queryGroupRecords:
+            returnedAttributes += [dsattributes.kDSNAttrGroupMembers,]
         
-        # get this now once
-        hostname = getfqdn()
-        if hostname:
-            self.defaultNodeName = "/LDAPv3/" + hostname
-        else:
-            self.defaultNodeName = None
-        
-        #cleanup
-        self._cleanupTime = time.time()
-        
-        # file system locks
-        self._initLockPath = join(config.DocumentRoot, ".directory_address_book_create_lock")
-        self._createdLockPath = join(config.DocumentRoot, ".directory_address_book_created_lock")
-        self._updateLockPath = join(config.DocumentRoot, ".directory_address_book_update_lock")
-        self._tmpDirAddressBookLockPath = join(config.DocumentRoot, ".directory_address_book_tmpFolder_lock")
-        
-        self._updateLock = MemcacheLock("OpenDirectoryBacker", self._updateLockPath)
-        self._tmpDirAddressBookLock = MemcacheLock("OpenDirectoryBacker", self._tmpDirAddressBookLockPath)        
-                
-        # optimization so we don't have to always get create lock
-        self._triedCreateLock = False
-        self._created = False
+        #for debugging
+        returnedAttributes += [dsattributes.kDSNAttrRecordType,]
 
-
-    def __cmp__(self, other):
-        if not isinstance(other, DirectoryRecord):
-            return super(DirectoryRecord, self).__eq__(other)
-
-        for attr in ("directory", "node"):
-            diff = cmp(getattr(self, attr), getattr(other, attr))
-            if diff != 0:
-                return diff
-        return 0
-
-    def __hash__(self):
-        h = hash(self.__class__.__name__)
-        for attr in ("node",):
-            h = (h + hash(getattr(self, attr))) & sys.maxint
-        return h
-    
-    @inlineCallbacks
-    def available(self):
-        if not self._triedCreateLock:
-            returnValue( False )
-        elif not self._created:
-            createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
-            self.log_debug("blocking on lock of: \"%s\")" % self._createdLockPath)
-            self._created = (yield createdLock.locked())
+        self.returnedAttributes = list(set(returnedAttributes))
+        self.log_debug("self.returnedAttributes=%s" % (self.returnedAttributes, ))
+              
         
-        returnValue(self._created)
+        self._dsLocalResults = {}
+        self._nextDSLocalQueryTime = 0
         
-    
-    def updateLock(self):
-        return self._updateLock
 
-    
-    @inlineCallbacks
     def createCache(self):
-        """
-        If caching, create the cache for the first time.
-        """
-        
-        if not self.liveQuery:
-            self.log_info("loading directory address book")
-    
-            # get init lock
-            initLock = MemcacheLock("OpenDirectoryBacker", self._initLockPath, timeout=0)
-            self.log_debug("Attempt lock of: \"%s\")" % self._initLockPath)
-            gotCreateLock = False
-            try:
-                yield initLock.acquire()
-                gotCreateLock = True
-            except MemcacheLockTimeoutError:
-                pass
-                
-            self._triedCreateLock = True
-            
-            if gotCreateLock:
-                self.log_debug("Got lock!")
-                yield self._refreshCache( flushCache=False, creating=True )
-            else:
-                self.log_debug("Could not get lock - directory address book will be filled by peer")
-                        
-        
+        succeed(None)
 
-    @inlineCallbacks
-    def _refreshCache(self, flushCache=False, creating=False, reschedule=True, query=None, attributes=None, keepLock=False, clear=False, maxRecords=0 ):
-        """
-        refresh the cache.
-        """
-
-        #print("_refreshCache:, flushCache=%s, creating=%s, reschedule=%s, query = %s" % (flushCache, creating, reschedule, "None" if query is None else query.generate(),))
-
-        def refreshLater():
-            #
-            # Add jitter/fuzz factor to avoid stampede for large OD query
-            #
-            cacheTimeout = min(self.cacheTimeout, 60) * 60
-            cacheTimeout = (cacheTimeout * random()) - (cacheTimeout / 2)
-            cacheTimeout += self.cacheTimeout * 60
-            reactor.callLater(cacheTimeout, self._refreshCache) #@UndefinedVariable
-            self.log_info("Refresh directory address book in %d minutes %d seconds" % divmod(cacheTimeout, 60))          
-
-        def cleanupLater():
-            
-            # try to cancel previous call if last clean up was less than 15 minutes ago
-            if (time.time() - self._cleanupTime) < 15*60:
-                try:
-                    self._lastCleanupCall.cancel()
-                except:
-                    pass
-            
-            #
-            # Add jitter/fuzz factor 
-            #
-            nom = 120
-            later = nom* (random() + .5)
-            self._lastCleanupCall = reactor.callLater(later, removeTmpAddressBooks) #@UndefinedVariable
-            self.log_info("Remove temporary directory address books in %d minutes %d seconds" % divmod(later, 60))          
-
-
-        def getTmpDirAndTmpFilePrefixSuffix():
-            # need to have temp file on same volumes as documents so that move works
-            absDocPath = abspath(config.DocumentRoot)
-            if absDocPath.startswith("/Volumes/"):
-                tmpDir = absDocPath
-                prefix = ".directoryAddressBook-"
-            else:
-                tmpDir = gettempdir()
-                prefix = "directoryAddressBook-"
-            
-            return (tmpDir, prefix, ".tmp")
-            
-        def makeTmpFilename():
-            tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
-            fd, fname = mkstemp(suffix=suffix, prefix=prefix, dir=tmpDir)
-            os.close(fd)
-            os.remove(fname)
-            return fname
+    def _isSystemRecord(self, recordShortName, recordAttributes):
         
-        @inlineCallbacks
-        def removeTmpAddressBooks():
-            self.log_info("Checking for temporary directory address books")
-            tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
-
-            tmpDirLock = self._tmpDirAddressBookLock
-            self.log_debug("blocking on lock of: \"%s\")" % self._tmpDirAddressBookLockPath)
-            yield tmpDirLock.acquire()
-            
-            try:
-                for name in listdir(tmpDir):
-                    if name.startswith(prefix) and name.endswith(suffix):
-                        try:
-                            path = join(tmpDir, name)
-                            self.log_info("Deleting temporary directory address book at: %s" %    path)
-                            FilePath(path).remove()
-                            self.log_debug("Done deleting")
-                        except:
-                            self.log_info("Deletion failed")
-            finally:
-                self.log_debug("unlocking: \"%s\")" % self._tmpDirAddressBookLockPath)
-                yield tmpDirLock.release()
-            
-            self._cleanupTime = time.time()
-
-        
-        updateLock = None
-        limited = False
-        try:
-            
-            try:
-                # get the records
-                if clear:
-                    records = {}
-                else:
-                    records, limited = (yield self._getDirectoryRecords(query, attributes, maxRecords))
-                
-                # calculate the hash
-                # simple for now, could use MD5 digest if too many collisions     
-                newAddressBookCTag = customxml.GETCTag(str(hash(self.baseGUID + ":" + self.realmName + ":" + "".join(str(hash(records[key])) for key in records.keys()))))
-                
-                # get the old hash
-                oldAddressBookCTag = ""
-                updateLock = self.updateLock()
-                self.log_debug("blocking on lock of: \"%s\")" % self._updateLockPath)
-                yield updateLock.acquire()
-                
-                if not flushCache:
-                    # get update lock
-                    try:
-                        oldAddressBookCTag = self.directoryBackedAddressBook.readDeadProperty((calendarserver_namespace, "getctag"))
-                    except:
-                        oldAddressBookCTag = ""
+        recordType = recordAttributes.get(dsattributes.kDSNAttrRecordType)
+        guid = recordAttributes.get(dsattributes.kDS1AttrGeneratedUID)
+        if guid and guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
+            self.log_info("Ignoring record %s (type %s) with %s %s"  % (recordShortName, recordType, dsattributes.kDS1AttrGeneratedUID, guid,))
+            return True
     
-                self.log_debug("Comparing {http://calendarserver.org/ns/}getctag: new = %s, old = %s" % (newAddressBookCTag, oldAddressBookCTag))
-                if str(newAddressBookCTag) != str(oldAddressBookCTag):
-                    
-                    self.log_debug("unlocking: \"%s\")" % self._updateLockPath)
-                    yield updateLock.release()
-                    updateLock = None
-                    
+        uniqueID = recordAttributes.get(dsattributes.kDS1AttrUniqueID)
+        if uniqueID and (int(uniqueID) < 500 or (recordType == dsattributes.kDSStdRecordTypeUsers and int(uniqueID) == 1000)):
+            self.log_info("Ignoring record %s (type %s) with %s %s"  % (recordShortName, recordType, dsattributes.kDS1AttrUniqueID, uniqueID,))
+            return True
 
-                if not keepLock:
-                    self.log_debug("unlocking: \"%s\")" % self._updateLockPath)
-                    yield updateLock.release()
-                    updateLock = None
-                    
-            except:
-                cleanupLater()
-                if reschedule:
-                    refreshLater() 
-                raise
-            
-            if creating:
-                createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
-                self.log_debug("blocking on lock of: \"%s\")" % self._createdLockPath)
-                yield createdLock.acquire()
-            
-            cleanupLater()
-            if reschedule:
-                refreshLater() 
+        if recordShortName.startswith("_"):
+            self.log_info("Ignoring record %s (type %s) with %s %s"  % (recordShortName, recordType, dsattributes.kDSNAttrRecordName, recordShortName,))
+            return True
         
-        except:
-            if updateLock:
-                yield updateLock.release()
-            raise
+        return False
 
-        returnValue( (updateLock, limited) )
-
-
-
-    def _getDSLocalRecords(self):
-        
-        def generateDSLocalRecords():
+  
+    def _getAllDSLocalResults(self):
+        """
+        Get a dictionary of ABDirectoryQueryResult by enumerating the local directory
+        """
+       
+        def generateDSLocalResults():
                         
-            records = {}
+            resultsDictionary = {}
             
-            recordTypes = [dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers, ]
             try:
                 localNodeDirectory = self.odModule.odInit("/Local/Default")
                 self.log_debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r)" % (
                         "/DSLocal",
-                        recordTypes,
+                        self.recordTypes,
                         self.returnedAttributes,
                     ))
-                results = list(self.odModule.listAllRecordsWithAttributes_list(
+                records = list(self.odModule.listAllRecordsWithAttributes_list(
                         localNodeDirectory,
-                        recordTypes,
+                        self.recordTypes,
                         self.returnedAttributes,
                     ))
             except self.odModule.ODError, ex:
                 self.log_error("Open Directory (node=%s) error: %s" % ("/Local/Default", str(ex)))
                 raise
             
-            self._dsLocalRecords = []        
-            for (recordShortName, value) in results: #@UnusedVariable
+            for (recordShortName, recordAttributes) in records: #@UnusedVariable
                 
-                record = VCardRecord(self, value, "/Local/Default")
+                try:
+                    self.log_info("Inspecting record %s"  % (recordAttributes,))
+                    if self.ignoreSystemRecords:
+                        if self._isSystemRecord(recordShortName, recordAttributes):
+                            continue
 
-                if self.ignoreSystemRecords:
-                    # remove system users and people
-                    if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
-                        self.log_info("Ignoring vcard for system record %s"  % (record,))
+                    result = ABDirectoryQueryResult(self.directoryBackedAddressBook, recordAttributes)
+                    
+                except:
+                    traceback.print_exc()
+                    self.log_info("Could not get vcard for record %s" % (recordShortName,))
+                    
+                else:
+                    uid = result.vCard().propertyValue("UID")
+
+                    if uid in resultsDictionary:
+                        self.log_info("Record %s skipped due to duplicate UID: %s" % (recordShortName, uid,))
                         continue
+                        
+                    self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+                    resultsDictionary[uid] = result                   
 
-                if record.guid in records:
-                    self.log_info("Record skipped due to conflict (duplicate uuid): %s" % (record,))
-                else:
-                    try:
-                        vCardText = record.vCardText()
-                    except:
-                        traceback.print_exc()
-                        self.log_info("Could not get vcard for record %s" % (record,))
-                    else:
-                        self.log_debug("VCard text =\n%s" % (vCardText, ))
-                        records[record.guid] = record                   
     
-            return records
+            return resultsDictionary
         
 
-        if not self.liveQuery or not self.queryDSLocal:
+        if not self.queryDSLocal:
             return {}
         
         if time.time() > self._nextDSLocalQueryTime:
-            self._dsLocalRecords = generateDSLocalRecords()
+            self._dsLocalResults = generateDSLocalResults()
             # Add jitter/fuzz factor 
             self._nextDSLocalQueryTime = time.time() + self.dsLocalCacheTimeout * (random() + 0.5)  * 60
 
-        return self._dsLocalRecords
+        return self._dsLocalResults
     
 
     @inlineCallbacks
-    def _getDirectoryRecords(self, query=None, attributes=None, maxRecords=0 ):
+    def _getDirectoryQueryResults(self, query=None, attributes=None, maxRecords=0, allowedRecordTypes=None ):
         """
-        Get a list of filtered VCardRecord for the given query with the given attributes.
-        query == None gets all records. attribute == None gets VCardRecord.allDSQueryAttributes
+        Get a list of ABDirectoryQueryResult for the given query with the given attributes.
+        query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
         """
         limited = False
-        queryResults = (yield self._queryDirectory(query, attributes, maxRecords ))
-        if maxRecords and len(queryResults) >= maxRecords:
+        records = (yield self._queryDirectory(query, attributes, maxRecords, allowedRecordTypes=allowedRecordTypes ))
+        if maxRecords and len(records) >= maxRecords:
             limited = True
             self.log_debug("Directory address book record limit (= %d) reached." % (maxRecords, ))
 
-        self.log_debug("Query done. Inspecting %s results" % len(queryResults))
+        self.log_debug("Query done. Inspecting %s records" % (len(records),))
 
-        records = self._getDSLocalRecords().copy()
-        self.log_debug("Adding %s DSLocal results" % len(records.keys()))
+        resultsDictionary = self._getAllDSLocalResults().copy()
+        self.log_debug("Adding %s DSLocal results" % len(resultsDictionary.keys()))
         
-        for (recordShortName, value) in queryResults: #@UnusedVariable
+        for (recordShortName, recordAttributes) in records: #@UnusedVariable
             
-            record = VCardRecord(self, value, self.defaultNodeName)
+            try:
+                # fix ds strangeness
+                if recordAttributes.get(dsattributes.kDS1AttrLastName, "") == "99":
+                    del recordAttributes[dsattributes.kDS1AttrLastName]
+        
+                if self.ignoreSystemRecords:
+                    if self._isSystemRecord(recordShortName, recordAttributes):
+                        continue
+                    
+                if not self.queryDSLocal:
+                    # skip records in local node which happens for non-complex od queries
+                    node = recordAttributes.get(dsattributes.kDSNAttrMetaNodeLocation)
+                    if node and node.startswith("/Local/"):
+                        recordType = recordAttributes.get(dsattributes.kDSNAttrRecordType)
+                        self.log_info("Ignoring record %s (type %s) with %s %s"  % (recordShortName, recordType, dsattributes.kDSNAttrMetaNodeLocation, recordAttributes.get(dsattributes.kDSNAttrMetaNodeLocation),))
+                        continue
 
-            if self.ignoreSystemRecords:
-                # remove system users and people
-                if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
-                    self.log_info("Ignoring vcard for system record %s"  % (record,))
+                result = ABDirectoryQueryResult(self.directoryBackedAddressBook, recordAttributes, 
+                                     addDSAttrXProperties=self.addDSAttrXProperties,
+                                     appleInternalServer=self.appleInternalServer,
+                                     )
+            except:
+                traceback.print_exc()
+                self.log_info("Could not get vcard for record %s" % (recordShortName,))
+                
+            else:
+                uid = result.vCard().propertyValue("UID")
+
+                if uid in resultsDictionary:
+                    self.log_info("Record skipped due to duplicate UID: %s" % (recordShortName,))
                     continue
+                    
+                self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+                resultsDictionary[uid] = result                   
         
-            if record.guid in records:
-                self.log_info("Ignoring vcard for record due to conflict (duplicate uuid): %s" % (record,))
-            else:
-                records[record.guid] = record                   
-        
-        self.log_debug("After filtering, %s records (limited=%s)." % (len(records), limited))
-        returnValue((records, limited, ))
+        self.log_debug("_getDirectoryQueryResults: %s results (limited=%s)." % (len(resultsDictionary), limited))
+        returnValue((resultsDictionary.values(), limited, ))
 
 
-    def _queryDirectory(self, query=None, attributes=None, maxRecords=0 ):
+    def _queryDirectory(self, query=None, attributes=None, maxRecords=0, allowedRecordTypes=None ):
         
         startTime = time.time()
-
-        
         if not attributes:
             attributes = self.returnedAttributes
-            
-        attributes = list(set(attributes + self.additionalAttributes)) # remove duplicates
-        
-        directoryAndRecordTypes = []
-        if self.peopleDirectory == self.userDirectory:
-            # use single ds query if possible for best performance
-            directoryAndRecordTypes.append( (self.peopleDirectory, self.peopleNode, (dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers) ) )
-        else:
-            if self.peopleDirectory:
-                directoryAndRecordTypes.append( (self.peopleDirectory, self.peopleNode, dsattributes.kDSStdRecordTypePeople) )
-            if self.userDirectory:
-                directoryAndRecordTypes.append( (self.userDirectory, self.userNode, dsattributes.kDSStdRecordTypeUsers) )
-        
+                    
         allResults = []
-        for directory, node, recordType in directoryAndRecordTypes:
+        for node, queryInfo in self.nodeDirectoryRecordTypeMap.iteritems():
+            directory = queryInfo["directory"]
+            recordTypes = queryInfo["recordTypes"]
+            if not allowedRecordTypes is None:
+                recordTypes = list(set(recordTypes).intersection(set(allowedRecordTypes)))
+                if not recordTypes:
+                    continue
+            
             try:
                 if query:
                     if isinstance(query, dsquery.match) and query.value is not "":
@@ -553,7 +405,7 @@
                             query.value,
                             query.matchType,
                             False,
-                            recordType,
+                            recordTypes,
                             attributes,
                             maxRecords,
                         ))
@@ -564,7 +416,7 @@
                                 query.value,
                                 query.matchType,
                                 False,
-                                recordType,
+                                recordTypes,
                                 attributes,
                                 maxRecords,
                             ))
@@ -573,7 +425,7 @@
                             node,
                             query.generate(),
                             False,
-                            recordType,
+                            recordTypes,
                             attributes,
                             maxRecords,
                         ))
@@ -582,21 +434,21 @@
                                 directory,
                                 query.generate(),
                                 False,
-                                recordType,
+                                recordTypes,
                                 attributes,
                                 maxRecords,
                             ))
                 else:
                     self.log_debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r,%r)" % (
                         node,
-                        recordType,
+                        recordTypes,
                         attributes,
                         maxRecords,
                     ))
                     results = list(
                         self.odModule.listAllRecordsWithAttributes_list(
                             directory,
-                            recordType,
+                            recordTypes,
                             attributes,
                             maxRecords,
                         ))
@@ -616,443 +468,443 @@
         self.log_info("Timing: Directory query: %.1f ms (%d records, %.2f records/sec)" % (elaspedTime*1000, len(allResults), len(allResults)/elaspedTime))
         return succeed(allResults)
     
-    def _getDSFilter(self, addressBookFilter):
+
+    @inlineCallbacks
+    def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
         """
-        Convert the supplied addressbook-query into an expression tree.
-    
-        @param filter: the L{Filter} for the addressbook-query to convert.
-        @return: (needsAllRecords, espressionAttributes, expression) tuple
+        Get vCards for a given addressBookFilter and addressBookQuery
         """
-        def propFilterListQuery(filterAllOf, propFilters):
+        
+        def allowedRecordTypes():
+            constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+    
+            # optimization: use KIND as constant to filter record type list
+            dsRecordTypeToKindMap = {
+                           dsattributes.kDSStdRecordTypeGroups:"group",
+                           dsattributes.kDSStdRecordTypeLocations:"location",
+                           dsattributes.kDSStdRecordTypeResources:"device",
+                           }
+            
+            allowedRecordTypes = []
+            for recordType in set(self.recordTypes):
+                kind = dsRecordTypeToKindMap.get(recordType, "individual")
+                constantProperties["KIND"] = kind
+           
+                filterPropertyNames, dsFilter  = dsFilterFromAddressBookFilter( addressBookFilter, 
+                                                                                         self.vcardPropToSearchableDSAttrMap,
+                                                                                         constantProperties=constantProperties );
+                if not dsFilter is False:
+                    allowedRecordTypes += [recordType,]
+            return set(allowedRecordTypes)
+        
 
-            def propFilterExpression(filterAllOf, propFilter):
-                #print("propFilterExpression")
-                """
-                Create an expression for a single prop-filter element.
+        filterPropertyNames, dsFilter  = dsFilterFromAddressBookFilter( addressBookFilter, 
+                                                                                 self.vcardPropToSearchableDSAttrMap,
+                                                                                 constantProperties=ABDirectoryQueryResult.constantProperties );
+        self.log_debug("doAddressBookQuery: query=%s, propertyNames=%s" % (dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames,))
+
+        results = []
+        limited = False
+        if dsFilter:
+            
+            if dsFilter is True:
+                dsFilter = None  # None means get all records hereafter
+            
+            # calculate minimum attributes needed for this query
+            etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+        
+            if (etagRequested and not self.fakeETag) or not queryPropNames:
+                queryAttributes = self.returnedAttributes
+            elif queryPropNames:
+                queryPropNames += filterPropertyNames
+                queryAttributes = []
+                for prop in queryPropNames:
+                    attributes = ABDirectoryQueryResult.vcardPropToDSAttrMap.get(prop)
+                    if attributes:
+                        queryAttributes += attributes
+                        
+                queryAttributes =  list(set(queryAttributes + self.requiredAttributes).intersection(self.returnedAttributes))
+            
+            self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s, queryAttributes=%s" % (etagRequested, queryPropNames, queryAttributes,))
+
+            '''
+            # change query to ignore system records rather than post filtering
+            # but this is broken in open directory client
+            if self.ignoreSystemRecords:
+                ignoreExpression = dsquery.expression( dsquery.expression.NOT, 
+                                                       dsquery.match(dsattributes.kDS1AttrGeneratedUID, "FFFFEEEE-DDDD-CCCC-BBBB-AAAA", dsattributes.eDSStartsWith)
+                                                       )
+                filterAttributes = list(set(filterAttributes).union(dsattributes.kDS1AttrGeneratedUID))
                 
-                @param propFilter: the L{PropertyFilter} element.
-                @return: (needsAllRecords, espressionAttributes, expressions) tuple
-                """
+                dsFilter = dsquery.expression( dsquery.expression.AND, (dsFilter, ignoreExpression,) ) if dsFilter else ignoreExpression
+            '''
+            maxRecords = int(maxResults * 1.2)
+
+            # keep trying query till we get results based on filter.  Especially when doing "all results" query
+            while True:
+                dsQueryResults, dsQueryLimited = (yield self._getDirectoryQueryResults(dsFilter, queryAttributes, maxRecords, allowedRecordTypes=allowedRecordTypes()))
                 
-                def definedExpression( defined, allOf, filterName, constant, queryAttributes, allAttrStrings):
-                    if constant or filterName in ("N" , "FN", "UID", ):
-                        return (defined, [], [])     # all records have this property so no records do not have it
+                filteredResults = []
+                for dsQueryResult in dsQueryResults:
+                    if addressBookFilter.match(dsQueryResult.vCard()):
+                        filteredResults.append(dsQueryResult)
                     else:
-                        matchList = list(set([dsquery.match(attrName, "", dsattributes.eDSStartsWith) for attrName in allAttrStrings]))
-                        if defined:
-                            return andOrExpression(allOf, queryAttributes, matchList)
-                        else:
-                            if len(matchList) > 1:
-                                expr = dsquery.expression( dsquery.expression.OR, matchList )
-                            else:
-                                expr = matchList
-                            return (False, queryAttributes, [dsquery.expression( dsquery.expression.NOT, expr),])
-                    #end isNotDefinedExpression()
+                        self.log_debug("doAddressBookQuery: result did not match filter: %s (%s)" % (dsQueryResult.vCard().propertyValue("FN"), dsQueryResult.vCard().propertyValue("UID"),))
+                
+                #no more results    
+                if not dsQueryLimited:
+                    break;
+                
+                # more than requested results
+                if maxResults and len(filteredResults) >= maxResults:
+                    break
+                
+                # more than max report results
+                if len(filteredResults) >= config.MaxQueryWithDataResults:
+                    break
+                
+                # more than self limit
+                if self.maxDSQueryRecords and maxRecords >= self.maxDSQueryRecords:
+                    break
+                
+                # try again with 2x
+                maxRecords *= 2
+                if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
+                    maxRecords = self.maxDSQueryRecords
+                
+            
+            results = filteredResults
+            limited = maxResults and len(results) >= maxResults
+                        
+        #if self.sortResults:
+        #    results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+
+        self.log_debug("doAddressBookQuery: %s results (limited=%s)." % (len(results), limited))
+        returnValue((results, limited,))        
+
+
+#utility
+def propertiesInAddressBookQuery( addressBookQuery ):
+    """
+    Get the vCard properties requested by a given query
+    """
     
+    etagRequested = False
+    propertyNames = [] 
+    if addressBookQuery.qname() == ("DAV:", "prop"):
     
-                def andOrExpression(propFilterAllOf, queryAttributes, matchList):
-                    #print("andOrExpression(propFilterAllOf=%r, queryAttributes%r, matchList%r)" % (propFilterAllOf, queryAttributes, matchList))
-                    if propFilterAllOf and len(matchList):
-                        # add OR expression because parent will AND
-                        return (False, queryAttributes, [dsquery.expression( dsquery.expression.OR, matchList),])
-                    else:
-                        return (False, queryAttributes, matchList)
-                    #end andOrExpression()
+        for property in addressBookQuery.children:                
+            if isinstance(property, carddavxml.AddressData):
+                for addressProperty in property.children:
+                    if isinstance(addressProperty, carddavxml.Property):
+                        propertyNames += [addressProperty.attributes["name"],]
                     
+            elif property.qname() == ("DAV:", "getetag"):
+                # for a real etag == md5(vCard), we need all attributes
+                etagRequested = True
     
-                # short circuit parameter filters
-                def supportedParamter( filterName, paramFilters, propFilterAllOf ):
-                    
-                    def supported( paramFilterName, paramFilterDefined, params ):
-                        paramFilterName = paramFilterName.upper()
-                        if len(params.keys()) and ((paramFilterName in params.keys()) != paramFilterDefined):
-                            return False
-                        if len(params[paramFilterName]) and str(paramFilter.qualifier).upper() not in params[paramFilterName]:
-                            return False
-                        return True 
-                        #end supported()
+    return (etagRequested, propertyNames if len(propertyNames) else None)
+
+
+def dsFilterFromAddressBookFilter(addressBookFilter, vcardPropToSearchableAttrMap, constantProperties={}):
+    """
+    Convert the supplied addressbook-query into a ds expression tree.
+
+    @param addressBookFilter: the L{Filter} for the addressbook-query to convert.
+    @param vcardPropToSearchableAttrMap: a mapping from vcard properties to searchable query attributes.
+    @param constantProperties: a mapping of constant properties.  A query on a constant property will return all or None
+    @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+    """
+    def propFilterListQuery(filterAllOf, propFilters):
+
+        def combineExpressionLists(expressionList, allOf, addedExpressions):
+            """
+            deal with the 4-state logic
+                addedExpressions=None means ignore
+                addedExpressions=True means all records
+                addedExpressions=False means no records
+                addedExpressions=[expressionlist] add to expression list
+            """
+            #def explen(exp): return len(exp) if isinstance(exp, list) else 0
+            #log.debug("propFilterListQuery(): allOf=%s, expressionList=%s (%s), addedExpressions=%s (%s)" % (allOf, expressionList, explen(expressionList), addedExpressions, explen(addedExpressions)))
+            if expressionList is None:
+                expressionList = addedExpressions
+            elif addedExpressions is not None:
+                if addedExpressions is True:
+                    if not allOf:
+                        expressionList = True # expressionList or True is True
+                    #else  expressionList and True is expressionList
+                elif addedExpressions is False:
+                    if allOf:
+                        expressionList = False # expressionList and False is False
+                    #else expressionList or False is expressionList
+                else:
+                    if expressionList is False:
+                        if not allOf:
+                            expressionList = addedExpressions # False or addedExpressions is addedExpressions
+                        #else False and addedExpressions is False
+                    elif expressionList is True:
+                        if allOf:
+                            expressionList = addedExpressions # False or addedExpressions is addedExpressions
+                        #else False and addedExpressions is False
+                    else:
+                        expressionList += addedExpressions
+            #log.debug("propFilterListQuery(): out expressionList=%s (%s)" % (expressionList, explen(expressionList)))
+            return expressionList
+            
+
+        def propFilterExpression(filterAllOf, propFilter):
+            """
+            Create an expression for a single prop-filter element.
+            
+            @param propFilter: the L{PropertyFilter} element.
+            @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+            """
+            
+            def definedExpression( defined, allOf ):
+                if constant or propFilter.filter_name in ("N" , "FN", "UID", "SOURCE",):
+                    return defined     # all records have this property so no records do not have it
+                else:
+                    matchList = [dsquery.match(attrName, "", dsattributes.eDSStartsWith) for attrName in searchablePropFilterAttrNames]
+                    if defined:
+                        return andOrExpression(allOf, matchList)
+                    else:
+                        if len(matchList) > 1:
+                            expr = dsquery.expression( dsquery.expression.OR, matchList )
+                        else:
+                            expr = matchList[0]
+                        return [dsquery.expression( dsquery.expression.NOT, expr),]
+                #end definedExpression()
+
+
+            def andOrExpression(propFilterAllOf, matchList):
+                if propFilterAllOf and len(matchList) > 1:
+                    # add OR expression because parent will AND
+                    return [dsquery.expression( dsquery.expression.OR, matchList),]
+                else:
+                    return matchList
+                #end andOrExpression()
                 
+
+            def paramFilterElementExpression(propFilterAllOf, paramFilterElement):
+
+                params = ABDirectoryQueryResult.vcardPropToParamMap.get(propFilter.filter_name.upper())
+                defined = params and paramFilterElement.filter_name.upper() in params
+                
+                #defined test
+                if defined != paramFilterElement.defined:
+                    return False
+                
+                #parameter value text match
+                if defined and paramFilterElement.filters:
+                    paramValues = params[paramFilterElement.filter_name.upper()]
+                    if paramValues and paramFilterElement.filters[0].text.upper() not in paramValues:
+                        return False
+                
+                return True
+
+            
+            def textMatchElementExpression( propFilterAllOf, textMatchElement ):
+
+                # pre process text match strings for ds query 
+                def getMatchStrings( propFilter, matchString ):
+                                        
+                    if propFilter.filter_name in ("REV" , "BDAY", ):
+                        rawString = matchString
+                        matchString = ""
+                        for c in rawString:
+                            if not c in "TZ-:":
+                                matchString += c
+                    elif propFilter.filter_name == "GEO":
+                        matchString = ",".join(matchString.split(";"))
                     
-                    oneSupported = False
-                    for paramFilter in paramFilters:
-                        if filterName == "PHOTO":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "ENCODING": ["B",], "TYPE": ["JPEG",], }):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "ADR":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": ["WORK", "PREF", "POSTAL", "PARCEL",], }):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "LABEL":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": ["POSTAL", "PARCEL",]}):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "TEL":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": [], }): # has params derived from ds attributes
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "EMAIL":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": [], }): # has params derived from ds attributes
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "URL":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, {}):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "KEY":
-                            if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "ENCODING": ["B",], "TYPE": ["PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE",] }):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif not filterName.startswith("X-"): #X- IMHandles X-ABRELATEDNAMES excepted, no other params are used
-                            if propFilterAllOf == paramFilter.defined:
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
+                    if propFilter.filter_name in ("N" , "ADR", "ORG", ):
+                        # for structured properties, change into multiple strings for ds query
+                        if propFilter.filter_name == "ADR":
+                            #split by newline and comma
+                            rawStrings = ",".join( matchString.split("\n") ).split(",")
+                        else:
+                            #split by space
+                            rawStrings = matchString.split(" ")
+                        
+                        # remove empty strings
+                        matchStrings = []
+                        for oneString in rawStrings:
+                            if len(oneString):
+                                matchStrings += [oneString,]
+                        return matchStrings
                     
-                    if propFilterAllOf:
-                        return True
+                    elif len(matchString):
+                        return [matchString,]
                     else:
-                        return oneSupported
-                    #end supportedParamter()
-                    
-                    
-                def textMatchElementExpression( propFilterAllOf, textMatchElement ):
-    
-                    # pre process text match strings for ds query 
-                    def getMatchStrings( propFilter, matchString ):
-                                            
-                        if propFilter.filter_name in ("REV" , "BDAY", ):
-                            rawString = matchString
-                            matchString = ""
-                            for c in rawString:
-                                if not c in "TZ-:":
-                                    matchString += c
-                        elif propFilter.filter_name == "GEO":
-                            matchString = ",".join(matchString.split(";"))
-                        
-                        if propFilter.filter_name in ("N" , "ADR", "ORG", ):
-                            # for structured properties, change into multiple strings for ds query
-                            if propFilter.filter_name == "ADR":
-                                #split by newline and comma
-                                rawStrings = ",".join( matchString.split("\n") ).split(",")
-                            else:
-                                #split by space
-                                rawStrings = matchString.split(" ")
-                            
-                            # remove empty strings
-                            matchStrings = []
-                            for oneString in rawStrings:
-                                if len(oneString):
-                                    matchStrings += [oneString,]
-                            return matchStrings
-                        
-                        elif len(matchString):
-                            return [matchString,]
-                        else:
-                            return []
-                        # end getMatchStrings
-    
-                    if constant:
-                        # do the match right now!  Return either all or none.
-                        return( textMatchElement.test([constant,]), [], [] )
-                    else:
+                        return []
+                    # end getMatchStrings
 
-                        matchStrings = getMatchStrings(propFilter, textMatchElement.text)
+                if constant:
+                    #FIXME: match is not implemented in twisteddaldav.query.addressbookqueryfilter.TextMatch so use _match for now
+                    return textMatchElement._match([constant,])
+                else:
 
-                        if not len(matchStrings) or binaryAttrStrs:
-                            # no searching text in binary ds attributes, so change to defined/not defined case
-                            if textMatchElement.negate:
-                                return definedExpression(False, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                            # else fall through to attribute exists case below
-                        else:
-                            
-                            # special case UID's formed from node and record name
-                            if propFilter.filter_name == "UID":
-                                matchString = matchStrings[0]
-                                seperatorIndex = matchString.find(VCardRecord.peopleUIDSeparator)
-                                if seperatorIndex > 1:
-                                    recordNameStart = seperatorIndex + len(VCardRecord.peopleUIDSeparator)
-                                else:
-                                    seperatorIndex = matchString.find(VCardRecord.userUIDSeparator)                        
-                                    if seperatorIndex > 1:
-                                        recordNameStart = seperatorIndex + len(VCardRecord.userUIDSeparator)
-                                    else:
-                                        recordNameStart = sys.maxint
-        
+                    matchStrings = getMatchStrings(propFilter, textMatchElement.text)
+
+                    if not len(matchStrings):
+                        # no searching text in binary ds attributes, so change to defined/not defined case
+                        if textMatchElement.negate:
+                            return definedExpression(False, propFilterAllOf)
+                        # else fall through to attribute exists case below
+                    else:
+                        
+                        # special case UID's formed from node and record name
+                        if propFilter.filter_name == "UID":
+                            matchString = matchStrings[0]
+                            seperatorIndex = matchString.find(ABDirectoryQueryResult.uidSeparator)
+                            if seperatorIndex > 1:
+                                recordNameStart = seperatorIndex + len(ABDirectoryQueryResult.uidSeparator)
+                                
                                 if recordNameStart < len(matchString)-1:
                                     try:
                                         recordNameQualifier = matchString[recordNameStart:].decode("base64").decode("utf8")
                                     except Exception, e:
-                                        self.log_debug("Could not decode UID string %r in %r: %r" % (matchString[recordNameStart:], matchString, e,))
+                                        log.debug("Could not decode UID string %r in %r: %r" % (matchString[recordNameStart:], matchString, e,))
                                     else:
                                         if textMatchElement.negate:
-                                            return (False, queryAttributes, 
-                                                    [dsquery.expression(dsquery.expression.NOT, dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact)),]
-                                                    )
+                                            return [dsquery.expression(dsquery.expression.NOT, dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact)),]
                                         else:
-                                            return (False, queryAttributes, 
-                                                    [dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact),]
-                                                    )
-                            
-                            # use match_type where possible depending on property/attribute mapping
-                            # Note that case sensitive negate will not work
-                            #        Should return all records in that case
-                            matchType = dsattributes.eDSContains
-                            if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV",  "LABEL", ):
-                                if textMatchElement.match_type == "equals":
-                                        matchType = dsattributes.eDSExact
-                                elif textMatchElement.match_type == "starts-with":
-                                        matchType = dsattributes.eDSStartsWith
-                                elif textMatchElement.match_type == "ends-with":
-                                        matchType = dsattributes.eDSEndsWith
-                            
-                            matchList = []
-                            for matchString in matchStrings:
-                                matchList += [dsquery.match(attrName, matchString, matchType) for attrName in stringAttrStrs]
-                            
-                            matchList = list(set(matchList))
-    
-                            if textMatchElement.negate:
-                                if len(matchList) > 1:
-                                    expr = dsquery.expression( dsquery.expression.OR, matchList )
-                                else:
-                                    expr = matchList
-                                return (False, queryAttributes, [dsquery.expression( dsquery.expression.NOT, expr),])
+                                            return [dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact),]
+                        
+                        # use match_type where possible depending on property/attribute mapping
+                        # FIXME: case-sensitive negate will not work.  This should return all all records in that case
+                        matchType = dsattributes.eDSContains
+                        if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV",  "LABEL", ):
+                            if textMatchElement.match_type == "equals":
+                                    matchType = dsattributes.eDSExact
+                            elif textMatchElement.match_type == "starts-with":
+                                    matchType = dsattributes.eDSStartsWith
+                            elif textMatchElement.match_type == "ends-with":
+                                    matchType = dsattributes.eDSEndsWith
+                        
+                        matchList = []
+                        for matchString in matchStrings:
+                            matchList += [dsquery.match(attrName, matchString, matchType) for attrName in searchableAttributes]
+                        
+                        matchList = list(set(matchList))
+
+                        if textMatchElement.negate:
+                            if len(matchList) > 1:
+                                expr = dsquery.expression( dsquery.expression.OR, matchList )
                             else:
-                                return andOrExpression(propFilterAllOf, queryAttributes, matchList)
-    
-                    # attribute exists search
-                    return definedExpression(True, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                    #end textMatchElementExpression()
-                    
-    
-                # get attribute strings from dsqueryAttributesForProperty list 
-                queryAttributes = list(set(VCardRecord.dsqueryAttributesForProperty.get(propFilter.filter_name, [])).intersection(set(self.allowedDSQueryAttributes)))
-                
-                binaryAttrStrs = []
-                stringAttrStrs = []
-                for attr in queryAttributes:
-                    if isinstance(attr, tuple):
-                        binaryAttrStrs.append(attr[0])
-                    else:
-                        stringAttrStrs.append(attr)
-                allAttrStrings = stringAttrStrs + binaryAttrStrs
-                                        
-                constant = VCardRecord.constantProperties.get(propFilter.filter_name)
-                if not constant and not allAttrStrings: 
-                    return (False, [], [])
-                
-                if propFilter.qualifier and isinstance(propFilter.qualifier, addressbookqueryfilter.IsNotDefined):
-                    return definedExpression(False, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                
-                paramFilterElements = [paramFilterElement for paramFilterElement in propFilter.filters if isinstance(paramFilterElement, addressbookqueryfilter.ParameterFilter)]
-                textMatchElements = [textMatchElement for textMatchElement in propFilter.filters if isinstance(textMatchElement, addressbookqueryfilter.TextMatch)]
-                propFilterAllOf = propFilter.propfilter_test == "allof"
-                
-                # handle parameter filter elements
-                if len(paramFilterElements) > 0:
-                    if supportedParamter(propFilter.filter_name, paramFilterElements, propFilterAllOf ):
-                        if len(textMatchElements) == 0:
-                            return definedExpression(True, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                    else:
-                        if propFilterAllOf:
-                            return (False, [], [])
-                
-                # handle text match elements
-                propFilterNeedsAllRecords = propFilterAllOf
-                propFilterAttributes = []
-                propFilterExpressionList = []
-                for textMatchElement in textMatchElements:
-                    
-                    textMatchNeedsAllRecords, textMatchExpressionAttributes, textMatchExpression = textMatchElementExpression(propFilterAllOf, textMatchElement)
-                    if propFilterAllOf:
-                        propFilterNeedsAllRecords &= textMatchNeedsAllRecords
-                    else:
-                        propFilterNeedsAllRecords |= textMatchNeedsAllRecords
-                    propFilterAttributes += textMatchExpressionAttributes
-                    propFilterExpressionList += textMatchExpression
-    
+                                expr = matchList[0]
+                            return [dsquery.expression( dsquery.expression.NOT, expr),]
+                        else:
+                            return andOrExpression(propFilterAllOf, matchList)
 
-                if (len(propFilterExpressionList) > 1) and (filterAllOf != propFilterAllOf):
-                    propFilterExpressions = [dsquery.expression(dsquery.expression.AND if propFilterAllOf else dsquery.expression.OR , list(set(propFilterExpressionList)))] # remove duplicates
-                else:
-                    propFilterExpressions = list(set(propFilterExpressionList))
+                # attribute exists search
+                return definedExpression(True, propFilterAllOf)
+                #end textMatchElementExpression()
                 
-                return (propFilterNeedsAllRecords, propFilterAttributes, propFilterExpressions)
-                #end propFilterExpression
 
-            #print("propFilterListQuery: filterAllOf=%r, propFilters=%r" % (filterAllOf, propFilters,))
-            """
-            Create an expression for a list of prop-filter elements.
+            # searchablePropFilterAttrNames are attributes to be used by this propfilter's expression
+            searchableAttributes = vcardPropToSearchableAttrMap.get(propFilter.filter_name, [])
+            if isinstance(searchableAttributes, str):
+                searchableAttributes = [searchableAttributes,]
+            searchablePropFilterAttrNames = list(searchableAttributes)
             
-            @param filterAllOf: the C{True} if parent filter test is "allof"
-            @param propFilters: the C{list} of L{ComponentFilter} elements.
-            @return: (needsAllRecords, espressionAttributes, expression) tuple
-            """
-            needsAllRecords = filterAllOf
-            attributes = []
-            expressions = []
-            for propFilter in propFilters:
+            constant = constantProperties.get(propFilter.filter_name)
+            if not searchablePropFilterAttrNames and not constant:
+                # not allAttrNames means propFilter.filter_name is not mapped
+                # return None to try to match all items if this is the only property filter
+                return None
+            
+            #create a textMatchElement for the IsNotDefined qualifier
+            if isinstance(propFilter.qualifier, addressbookqueryfilter.IsNotDefined):
+                textMatchElement = addressbookqueryfilter.TextMatch(carddavxml.TextMatch.fromString(""))
+                textMatchElement.negate = True
+                propFilter.filters.append(textMatchElement)
+
+            # if only one propFilter, then use filterAllOf as propFilterAllOf to reduce subexpressions and simplify generated query string
+            if len(propFilter.filters) == 1:
+                propFilterAllOf = filterAllOf
+            else:
+                propFilterAllOf = propFilter.propfilter_test == "allof"
+
+            propFilterExpressions = None
+            for propFilterElement in propFilter.filters:
+                propFilterExpression = None
+                if isinstance(propFilterElement, addressbookqueryfilter.ParameterFilter):
+                    propFilterExpression = paramFilterElementExpression(propFilterAllOf, propFilterElement)
+                elif isinstance(propFilterElement, addressbookqueryfilter.TextMatch):
+                    propFilterExpression = textMatchElementExpression(propFilterAllOf, propFilterElement)
+                propFilterExpressions = combineExpressionLists(propFilterExpressions, propFilterAllOf, propFilterExpression)
+                if isinstance(propFilterExpressions, bool) and propFilterAllOf != propFilterExpression:
+                    break
                 
-                propNeedsAllRecords, propExpressionAttributes, propExpression = propFilterExpression(filterAllOf, propFilter)
-                if filterAllOf:
-                    needsAllRecords &= propNeedsAllRecords
-                else:
-                    needsAllRecords |= propNeedsAllRecords
-                attributes += propExpressionAttributes
-                expressions += propExpression
+            if isinstance(propFilterExpressions, list):
+                propFilterExpressions = list(set(propFilterExpressions))
+                if propFilterExpressions and (filterAllOf != propFilterAllOf):
+                    propFilterExpressions = [dsquery.expression(dsquery.expression.AND if propFilterAllOf else dsquery.expression.OR , propFilterExpressions)]
+            
+            return propFilterExpressions
+            #end propFilterExpression
 
+        """
+        Create an expression for a list of prop-filter elements.
+        
+        @param filterAllOf: the C{True} if parent filter test is "allof"
+        @param propFilters: the C{list} of L{ComponentFilter} elements.
+        @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+        """
+        expressions = None
+        for propFilter in propFilters:
+            
+            propExpressions = propFilterExpression(filterAllOf, propFilter)
+            expressions = combineExpressionLists(expressions, filterAllOf, propExpressions)
+        
+            # early loop exit
+            if isinstance(expressions, bool) and filterAllOf != expressions:
+                break
+            
+        # convert to needsAllRecords to return
+        if isinstance(expressions, list):
+            expressions = list(set(expressions))
             if len(expressions) > 1:
-                expr = dsquery.expression(dsquery.expression.AND if filterAllOf else dsquery.expression.OR , list(set(expressions))) # remove duplicates
+                expr = dsquery.expression(dsquery.expression.AND if filterAllOf else dsquery.expression.OR , expressions)
             elif len(expressions):
                 expr = expressions[0]
             else:
-                expr = None
-            
-            return (needsAllRecords, attributes, expr)
-        
-                
-        #print("_getDSFilter")
-        # Lets assume we have a valid filter from the outset
-        
-        # Top-level filter contains zero or more prop-filters
-        if addressBookFilter:
-            filterAllOf = addressBookFilter.filter_test == "allof"
-            if len(addressBookFilter.children) > 0:
-                return propFilterListQuery(filterAllOf, addressBookFilter.children)
-            else:
-                return (filterAllOf, [], [])
+                expr = not filterAllOf # empty expression list. should not happen
+        elif expressions is None:
+            expr = expr = not filterAllOf
         else:
-            return (False, [], [])    
-    
-                        
+            # True or False
+            expr = expressions
+            
+        properties = [propFilter.filter_name for propFilter in propFilters]
 
-    def _attributesForAddressBookQuery(self, addressBookQuery ):
-                        
-        propertyNames = [] 
-        #print( "addressBookQuery.qname=%r" % addressBookQuery.qname)
-        if addressBookQuery.qname() == ("DAV:", "prop"):
-        
-            for property in addressBookQuery.children:                
-                #print("property = %r" % property )
-                if isinstance(property, carddavxml.AddressData):
-                    for addressProperty in property.children:
-                        #print("addressProperty = %r" % addressProperty )
-                        if isinstance(addressProperty, carddavxml.Property):
-                            #print("Adding property %r", addressProperty.attributes["name"])
-                            propertyNames.append(addressProperty.attributes["name"])
-                        
-                elif not self.fakeETag and property.qname() == ("DAV:", "getetag"):
-                    # for a real etag == md5(vCard), we need all attributes
-                    propertyNames = None
-                    break;
-                            
-        
-        if not len(propertyNames):
-            #print("using all attributes")
-            return self.returnedAttributes
-        
-        else:
-            propertyNames.append("X-INTERNAL-MINIMUM-VCARD-PROPERTIES") # these properties are required to make a vCard
-            queryAttributes = []
-            for prop in propertyNames:
-                if prop in VCardRecord.dsqueryAttributesForProperty:
-                    #print("adding attributes %r" % VCardRecord.dsqueryAttributesForProperty.get(prop))
-                    queryAttributes += VCardRecord.dsqueryAttributesForProperty.get(prop)
-
-            return list(set(queryAttributes).intersection(set(self.returnedAttributes)))
-
+        return (list(set(properties)), expr)
     
-    @inlineCallbacks
-    def cacheVCardsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
-        """
-        Cache the vCards for a given addressBookFilder and addressBookQuery
-        """
-        startTime = time.time()
-        #print("Timing: cacheVCardsForAddressBookQuery.starttime=%f" % startTime)
-        
+    # Lets assume we have a valid filter from the outset
     
-        allRecords, filterAttributes, dsFilter  = self._getDSFilter( addressBookFilter );
-        #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-    
-        if allRecords:
-            dsFilter = None #  None expression == all Records
-        clear = not allRecords and not dsFilter
-        
-        #get unique list of requested attributes
-        if clear:
-            attributes = None
+    # Top-level filter contains zero or more prop-filters
+    if addressBookFilter:
+        filterAllOf = addressBookFilter.filter_test == "allof"
+        if len(addressBookFilter.children):
+            return propFilterListQuery(filterAllOf, addressBookFilter.children)
         else:
-            queryAttributes = self._attributesForAddressBookQuery( addressBookQuery )
-            attributes = filterAttributes + queryAttributes
-        
-        #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
-        maxRecords = int(maxResults * 1.2)
-        if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
-            maxRecords = self.maxDSQueryRecords
-            
-        updateLock, limited = (yield self._refreshCache(reschedule=False, query=dsFilter, attributes=attributes, keepLock=True, clear=clear, maxRecords=maxRecords ))
-
-        elaspedTime = time.time()-startTime
-        self.log_info("Timing: Cache fill: %.1f ms" % (elaspedTime*1000,))
-        
-
-        returnValue((updateLock, limited))
-
-
-    @inlineCallbacks
-    def vCardRecordsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
-        """
-        Get vCards for a given addressBookFilder and addressBookQuery
-        """
+            return ([], not filterAllOf)
+    else:
+        return ([], False)    
     
-        allRecords, filterAttributes, dsFilter  = self._getDSFilter( addressBookFilter );
-        #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-        
-        # testing:
-        # allRecords = True
-        
-        if allRecords:
-            dsFilter = None #  None expression == all Records
-        clear = not allRecords and not dsFilter
-        
-        queryRecords = []
-        limited = False
-
-        if not clear:
-            queryAttributes = self._attributesForAddressBookQuery( addressBookQuery )
-            attributes = filterAttributes + queryAttributes
-            
-            #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
-            maxRecords = int(maxResults * 1.2)
-            if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
-                maxRecords = self.maxDSQueryRecords
-
-            records, limited = (yield self._getDirectoryRecords(dsFilter, attributes, maxRecords))
-            
-            #filter out bad records --- should only happen during development
-            for record in records.values():
-                try:
-                    vCardText = record.vCardText()
-                except:
-                    traceback.print_exc()
-                    self.log_info("Could not get vcard for record %s" % (record,))
-                else:
-                    if not record.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation).startswith("/Local"):
-                        self.log_debug("VCard text =\n%s" % (vCardText, ))
-                    queryRecords.append(record)
                         
-        returnValue((queryRecords, limited,))        
 
-
-class VCardRecord(DirectoryRecord, DAVPropertyMixIn):
+class ABDirectoryQueryResult(DAVPropertyMixIn, LoggingMixIn):
     """
-    Open Directory implementation of L{IDirectoryRecord}.
+    Result from ab query report or multiget on directory
     """
 
     # od attributes that may contribute to vcard properties
     # will be used to translate vCard queries to od queries
 
-    dsqueryAttributesForProperty = {
+    vcardPropToDSAttrMap = {
                              
         "FN" : [
                dsattributes.kDS1AttrFirstName, 
@@ -1130,10 +982,7 @@
                 ],
          "UID" : [
                 dsattributes.kDS1AttrGeneratedUID,
-                # special cased
-                #dsattributes.kDSNAttrMetaNodeLocation,
-                #dsattributes.kDSNAttrRecordName,
-                #dsattributes.kDS1AttrDistinguishedName,
+                dsattributes.kDSNAttrRecordName,
                 ],
          "URL" : [
                 dsattributes.kDS1AttrWeblogURI,
@@ -1146,66 +995,44 @@
                 (dsattributes.kDS1AttrUserPKCS12Data, "base64"),
                 (dsattributes.kDS1AttrUserSMIMECertificate, "base64"),
                 ],
-         # too bad this is not one X-Attribute with params.     Would make searching easier
-         "X-AIM" : [
+         "IMPP" : [
                 dsattributes.kDSNAttrIMHandle,
                 ],
-         "X-JABBER" :    [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-MSN" :    [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-YAHOO" :  [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-ICQ" :    [
-                dsattributes.kDSNAttrIMHandle,
-                ],
          "X-ABRELATEDNAMES" :  [
                 dsattributes.kDSNAttrRelationships,
-                ],
-          "X-INTERNAL-MINIMUM-VCARD-PROPERTIES" : [
+                ],  
+         "SOURCE" : [
                 dsattributes.kDS1AttrGeneratedUID,
-                dsattributes.kDSNAttrMetaNodeLocation,
-                dsattributes.kDS1AttrFirstName, 
-                 dsattributes.kDS1AttrLastName, 
-                dsattributes.kDS1AttrMiddleName,
-                   dsattributes.kDSNAttrNamePrefix,
-                  dsattributes.kDSNAttrNameSuffix,
-                 dsattributes.kDS1AttrDistinguishedName,
                 dsattributes.kDSNAttrRecordName,
-                dsattributes.kDSNAttrRecordType,
-                dsattributes.kDS1AttrModificationTimestamp,
-                dsattributes.kDS1AttrCreationTimestamp,
                 ],
-          "X-INTERNAL-REQUIRED" : [
-                dsattributes.kDS1AttrGeneratedUID,
-                dsattributes.kDSNAttrMetaNodeLocation,
-                 dsattributes.kDS1AttrDistinguishedName,
-                dsattributes.kDSNAttrRecordName,
-                dsattributes.kDS1AttrFirstName, 
-                 dsattributes.kDS1AttrLastName, 
-                dsattributes.kDSNAttrRecordType,
-                ],
-  
     }
     
-
-    allDSQueryAttributes = sorted(list(set([attr for lookupAttributes in dsqueryAttributesForProperty.values()
-                                      for attr in lookupAttributes])))
-
-    binaryDSAttributeStrs = [attr[0] for attr in allDSQueryAttributes
+    allDSQueryAttributes = list(set([attr for lookupAttributes in vcardPropToDSAttrMap.values()
+                                      for attr in lookupAttributes]))
+    binaryDSAttrNames = [attr[0] for attr in allDSQueryAttributes
                                 if isinstance(attr, tuple) ]
-
-    stringDSAttributeStrs = [attr for attr in allDSQueryAttributes
+    stringDSAttrNames = [attr for attr in allDSQueryAttributes
                                 if isinstance(attr, str) ]
+    allDSAttrNames = stringDSAttrNames + binaryDSAttrNames
+   
+    # all possible generated parameters.
+    vcardPropToParamMap = {
+        "PHOTO": { "ENCODING": ("B",), "TYPE": ("JPEG",), },
+        "ADR": { "TYPE": ("WORK", "PREF", "POSTAL", "PARCEL",), },
+        "LABEL": { "TYPE": ("POSTAL", "PARCEL",)},
+        "TEL": { "TYPE": None, }, # None means param can contain can be anything
+        "EMAIL": { "TYPE": None, },
+        "KEY": { "ENCODING": ("B",), "TYPE": ("PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE",) },
+        "URL": { "TYPE": ("WEBLOG", "HOMEPAGE",) },
+        "IMPP": { "TYPE": ("PREF",), "X-SERVICE-TYPE": None, },
+        "X-ABRELATEDNAMES" : { "TYPE":None, },
+        "X-AIM": { "TYPE": ("PREF",), },
+        "X-JABBER": { "TYPE": ("PREF",), },
+        "X-MSN": { "TYPE": ("PREF",), },
+        "X-ICQ": { "TYPE": ("PREF",), },
+    }
 
-    allDSAttributeStrs = stringDSAttributeStrs + binaryDSAttributeStrs
-    
-    #peopleUIDSeparator = "-" + OpenDirectoryBackingService.baseGUID + "-"
-    userUIDSeparator = "-bf07a1a2-"
-    peopleUIDSeparator = "-cf07a1a2-"
+    uidSeparator = "-cf07a1a2-"
 
     
     constantProperties = {
@@ -1216,93 +1043,75 @@
         }
 
     
-    def __init__(self, service, recordAttributes, defaultNodeName=None):
+    def __init__(self, directoryBackedAddressBook, recordAttributes, 
+                 kind=None, 
+                 additionalVCardProps=None, 
+                 addDSAttrXProperties=False, 
+                 appleInternalServer=False, 
+                 ):
+
+        self.log_debug("directoryBackedAddressBook=%s, attributes=%s, additionalVCardProps=%s" % (directoryBackedAddressBook, recordAttributes, additionalVCardProps,))
         
+        constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+        if additionalVCardProps:
+            for key, value in additionalVCardProps.iteritems():
+                if key not in constantProperties:
+                    constantProperties[key] = value
+        self.constantProperties = constantProperties
+        self.log_debug("directoryBackedAddressBook=%s, attributes=%s, self.constantProperties=%s" % (directoryBackedAddressBook, recordAttributes, self.constantProperties,))
 
-        self.log_debug("service=%s, attributes=%s"    % (service, recordAttributes))
-
         #save off for debugging
-        if service.addDSAttrXProperties:
+        self.addDSAttrXProperties = addDSAttrXProperties;
+        if addDSAttrXProperties:
             self.originalAttributes = recordAttributes.copy()
+        self.appleInternalServer = appleInternalServer
 
-        self.directoryBackedAddressBook = service.directoryBackedAddressBook
+        self._directoryBackedAddressBook = directoryBackedAddressBook
         self._vCard = None
-        self._vCardText = None
-        self._uriName = None
-        self._hRef = None
         
+        #clean attributes
         self.attributes = {}
         for key, values in recordAttributes.items():
-            if key in VCardRecord.stringDSAttributeStrs:
+            if key in ABDirectoryQueryResult.stringDSAttrNames:
                 if isinstance(values, list):
                     self.attributes[key] = [removeControlChars(val).decode("utf8") for val in values]
                 else:
                     self.attributes[key] = removeControlChars(values).decode("utf8")
             else:
                 self.attributes[key] = values
-                                                        
-        # fill in  missing essential attributes used for filtering
-        fullName = self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)
-        if not fullName:
-            fullName = self.firstValueForAttribute(dsattributes.kDSNAttrRecordName)
-            self.attributes[dsattributes.kDS1AttrDistinguishedName] = fullName
-            
-        node = self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation)
-        
-        # use a better node name -- makes better synthetic GUIDS
-        if not node or node == "/LDAPv3/127.0.0.1":
-            node = defaultNodeName if defaultNodeName else service.realmName
-            self.attributes[dsattributes.kDSNAttrMetaNodeLocation] = node
-        
+                
+        # find or create guid 
         guid = self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)
         if not guid:
-            if service.standardizeSyntheticUIDs:
-                nodeUUIDStr = "00000000"
-            else:
-                nodeUUIDStr = "%x" % abs(hash(node))
-            nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("utf8").encode("base64").split("\n"))
-            if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
-                guid =  VCardRecord.userUIDSeparator.join([nodeUUIDStr, nameUUIDStr,])
-            else:
-                guid =  VCardRecord.peopleUIDSeparator.join([nodeUUIDStr, nameUUIDStr,])
-
+            nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("base64").split("\n"))
+            guid =  ABDirectoryQueryResult.uidSeparator.join(["00000000", nameUUIDStr,])
+            #guid =  ABDirectoryQueryResult.uidSeparator.join(["d9a8e41b", nameUUIDStr,])
             
-        # since guid is used as file name, normalize so uid uniqueness == fine name uniqueness
-        #guid = "/".join(guid.split(":")).upper()
-        self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
+            self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
         
-        if self.firstValueForAttribute(dsattributes.kDS1AttrLastName) == "99":
-            del self.attributes[dsattributes.kDS1AttrLastName]
+        if not kind:
+            dsRecordTypeToKindMap = {
+                           #dsattributes.kDSStdRecordTypePeople:"individual",
+                           #dsattributes.kDSStdRecordTypeUsers:"individual",
+                           dsattributes.kDSStdRecordTypeGroups:"group",
+                           dsattributes.kDSStdRecordTypeLocations:"location",
+                           dsattributes.kDSStdRecordTypeResources:"device",
+                           }
+            recordType = self.firstValueForAttribute(dsattributes.kDSNAttrRecordType)
+            kind = dsRecordTypeToKindMap.get(recordType, "individual")
+        self.kind = kind.lower()
+
+
+       #generate a vCard here.  May throw an exception
+        self.vCard()
         
-        if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
-            recordType = DirectoryService.recordType_users
-        else:
-            recordType = DirectoryService.recordType_people
-                        
-        super(VCardRecord, self).__init__(
-            service                  = service,
-            recordType              = recordType,
-            guid                  = guid,
-            shortNames              = tuple(self.valuesForAttribute(dsattributes.kDSNAttrRecordName)),
-            fullName              = fullName,
-            firstName              = self.firstValueForAttribute(dsattributes.kDS1AttrFirstName, None),
-            lastName              = self.firstValueForAttribute(dsattributes.kDS1AttrLastName, None),
-            emailAddresses        = (),
-            calendarUserAddresses = (),
-            autoSchedule          = False,
-            enabledForCalendaring = False,
-        )
-        
 
 
     def __repr__(self):
-        return "<%s[%s(%s)] %s(%s) %r>" % (
+        return "<%s[%s(%s)]>" % (
             self.__class__.__name__,
-            self.firstValueForAttribute(dsattributes.kDSNAttrRecordType),
-            self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation),
-            self.guid,
-            self.shortNames,
-            self.fullName
+            self.vCard().propertyValue("FN"),
+            self.vCard().propertyValue("UID")
         )
     
     def __hash__(self):
@@ -1312,12 +1121,6 @@
               ])
         return hash(s)
 
-    """
-    def nextFileName(self):
-        self.renameCounter += 1
-        self.fileName = self.baseFileName + "-" + str(self.renameCounter)
-        self.fileNameLower = self.fileName.lower()
-    """
     
     def hasAttribute(self, attributeName ):
         return self.valuesForAttribute(attributeName, None) is not None
@@ -1399,15 +1202,17 @@
                 vcard.addProperty(Property("X-ABLabel", label, group=groupPrefix))
 
             # for attributes of the form  param:value
-            def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, defaultLabel, nolabelParamTypes, labelMap, attrType):
+            def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, attrType, defaultLabel, nolabelParamTypes=(), labelMap={}, specialParamType=None):
                 preferred = True
                 for attrValue in self.valuesForAttribute(attrType):
                     try:
                         # special case for Apple
-                        if self.service.appleInternalServer and attrType == dsattributes.kDSNAttrIMHandle:
+                        if self.appleInternalServer and attrType == dsattributes.kDSNAttrIMHandle:
                             splitValue = attrValue.split("|")
                             if len (splitValue) > 1:
                                 attrValue = splitValue[0]
+                                if splitValue[1].upper() in nolabelParamTypes:
+                                    defaultLabel = splitValue[1]
 
                         colonIndex = attrValue.find(":")
                         if (colonIndex > len(attrValue)-2):
@@ -1417,11 +1222,16 @@
                         labelString = attrValue[:colonIndex] if colonIndex > 0 else defaultLabel
                         paramTypeString = labelString.upper()
                         
-                        # add PREF to first prop's parameters
-                        paramTypeStrings = [paramTypeString,]
-                        if preferred and "PREF" != paramTypeString:
-                            paramTypeStrings += ["PREF",]
-                        parameters = { "TYPE": paramTypeStrings, }
+                        if specialParamType:
+                            parameters = { specialParamType: (paramTypeString,) }
+                            if preferred:
+                                parameters["TYPE"] = ("PREF",)
+                        else:
+                            # add PREF to first prop's parameters
+                            paramTypeStrings = [paramTypeString,]
+                            if preferred and "PREF" != paramTypeString:
+                                paramTypeStrings += ["PREF",]
+                            parameters = { "TYPE": paramTypeStrings, }
 
                         #special case for IMHandles which the param is the last part of the property like X-AIM or X-JABBER 
                         if propertyPrefix:
@@ -1432,8 +1242,8 @@
                             addUniqueProperty(vcard, Property(propertyName, attrValue[colonIndex+1:], params=parameters), None, attrValue, attrType)
                         else:
                             # use special localizable addressbook labels where possible
-                            abLabelString = labelMap.get(labelString, labelString)
-                            addPropertyAndLabel(groupCount, abLabelString, propertyName, propertyValue, parameters)
+                            localizedABLabelString = labelMap.get(labelString, labelString)
+                            addPropertyAndLabel(groupCount, localizedABLabelString, propertyName, propertyValue, parameters)
                         preferred = False
 
                     except Exception, e:
@@ -1442,15 +1252,14 @@
                         self.log_error("addPropertiesAndLabelsForPrefixedAttribute(): Trouble parsing attribute %s, with value \"%s\".  Error = %s" % (attrType, attrValue, e,))
 
             
-            #print("VCardRecord.vCard")
             # create vCard
             vcard = Component("VCARD")
             groupCount = [0]
             
             # add constant properties - properties that are the same regardless of the record attributes
-            for key, value in VCardRecord.constantProperties.items():
+            for key, value in self.constantProperties.items():
                 vcard.addProperty(Property(key, value))
-    
+                
             # 3.1 IDENTIFICATION TYPES http://tools.ietf.org/html/rfc2426#section-3.1
             # 3.1.1 FN Type Definition
             # dsattributes.kDS1AttrDistinguishedName,      # Users distinguished or real name
@@ -1459,14 +1268,14 @@
             #vcard.addProperty(Property("FN", self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)))
             
             # 3.1.2 N Type Definition
-            # dsattributes.kDS1AttrFirstName,            # Used for first name of user or person record.
+            # dsattributes.kDS1AttrFirstName,           # Used for first name of user or person record.
             # dsattributes.kDS1AttrLastName,            # Used for the last name of user or person record.
-            # dsattributes.kDS1AttrMiddleName,            # Used for the middle name of user or person record.
-            # dsattributes.kDSNAttrNameSuffix,            # Represents the name suffix of a user or person.
+            # dsattributes.kDS1AttrMiddleName,          #Used for the middle name of user or person record.
+            # dsattributes.kDSNAttrNameSuffix,          # Represents the name suffix of a user or person.
                                                         #      ie. Jr., Sr., etc.
                                                         #      Usually found in user or people records (kDSStdRecordTypeUsers or 
                                                         #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrNamePrefix,            # Represents the title prefix of a user or person.
+            # dsattributes.kDSNAttrNamePrefix,          # Represents the title prefix of a user or person.
                                                         #      ie. Mr., Ms., Mrs., Dr., etc.
                                                         #      Usually found in user or people records (kDSStdRecordTypeUsers or 
                                                         #      dsattributes.kDSStdRecordTypePeople).
@@ -1500,12 +1309,13 @@
                 addUniqueProperty(vcard, Property("NICKNAME", nickname), None, dsattributes.kDSNAttrNickName, nickname)
             
             # 3.1.4 PHOTO Type Definition
-            # dsattributes.kDSNAttrJPEGPhoto,            # Used to store binary picture data in JPEG format. 
+            # dsattributes.kDSNAttrJPEGPhoto,           # Used to store binary picture data in JPEG format. 
                                                         #      Usually found in user, people or group records (kDSStdRecordTypeUsers, 
                                                         #      dsattributes.kDSStdRecordTypePeople,dsattributes.kDSStdRecordTypeGroups).
             # pyOpenDirectory always returns binary-encoded string                                       
                                                         
             for photo in self.valuesForAttribute(dsattributes.kDSNAttrJPEGPhoto):
+                photo = "".join("".join(photo.split("\r")).split("\n")) #get rid of line folding: for PHOTO
                 addUniqueProperty(vcard, Property("PHOTO", photo, params={"ENCODING": ["b",], "TYPE": ["JPEG",],}), None, dsattributes.kDSNAttrJPEGPhoto, photo)
     
     
@@ -1549,8 +1359,8 @@
     
             # 3.2.2 LABEL Type Definition
             
-            # dsattributes.kDSNAttrPostalAddress,            # The postal address usually excluding postal code.
-            # dsattributes.kDSNAttrPostalAddressContacts,    # multi-valued attribute that defines a record's alternate postal addresses .
+            # dsattributes.kDSNAttrPostalAddress,           # The postal address usually excluding postal code.
+            # dsattributes.kDSNAttrPostalAddressContacts,   # multi-valued attribute that defines a record's alternate postal addresses .
                                                             #      found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
             # dsattributes.kDSNAttrAddressLine1,            # Line one of multiple lines of address data for a user.
             # dsattributes.kDSNAttrAddressLine2,            # Line two of multiple lines of address data for a user.
@@ -1577,18 +1387,18 @@
             # 3.3.1 TEL Type Definition
             #          TEL;TYPE=work,voice,pref,msg:+1-213-555-1234
     
-            # dsattributes.kDSNAttrPhoneNumber,            # Telephone number of a user.
+            # dsattributes.kDSNAttrPhoneNumber,         # Telephone number of a user.
             # dsattributes.kDSNAttrMobileNumber,        # Represents the mobile numbers of a user or person.
                                                         #      Usually found in user or people records (kDSStdRecordTypeUsers or 
                                                         #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrFaxNumber,            # Represents the FAX numbers of a user or person.
+            # dsattributes.kDSNAttrFaxNumber,           # Represents the FAX numbers of a user or person.
                                                         # Usually found in user or people records (kDSStdRecordTypeUsers or 
                                                         # kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrPagerNumber,            # Represents the pager numbers of a user or person.
+            # dsattributes.kDSNAttrPagerNumber,         # Represents the pager numbers of a user or person.
                                                         #      Usually found in user or people records (kDSStdRecordTypeUsers or 
                                                         #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrHomePhoneNumber,        # Home telephone number of a user or person.
-            # dsattributes.kDSNAttrPhoneContacts,        # multi-valued attribute that defines a record's custom phone numbers .
+            # dsattributes.kDSNAttrHomePhoneNumber,     # Home telephone number of a user or person.
+            # dsattributes.kDSNAttrPhoneContacts,       # multi-valued attribute that defines a record's custom phone numbers .
                                                         #      found in user records (kDSStdRecordTypeUsers). 
                                                         #      Example: home fax:408-555-4444
             
@@ -1617,9 +1427,9 @@
                 addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrHomePhoneNumber)
                 params = { "TYPE": ["HOME", "VOICE",], }
                     
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "TEL", "work",
-                                                        ["VOICE", "CELL", "FAX", "PAGER",], {},
-                                                        dsattributes.kDSNAttrPhoneContacts, )
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="TEL", defaultLabel="work",
+                                                        nolabelParamTypes=("VOICE", "CELL", "FAX", "PAGER",),
+                                                        attrType=dsattributes.kDSNAttrPhoneContacts, )
 
             """
             # EXTEND:  Use this attribute
@@ -1637,14 +1447,14 @@
                 addUniqueProperty(vcard, Property("EMAIL", emailAddress, params=params), (("TYPE", "PREF"),), emailAddress, dsattributes.kDSNAttrEMailAddress)
                 params = workParams
                 
-            # dsattributes.kDSNAttrEMailContacts,        # multi-valued attribute that defines a record's custom email addresses .
+            # dsattributes.kDSNAttrEMailContacts,       # multi-valued attribute that defines a record's custom email addresses .
                                                         #    found in user records (kDSStdRecordTypeUsers). 
                                                         #      Example: home:johndoe at mymail.com
     
             # check to see if parameters type are open ended. Could be any string
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "EMAIL", "work",
-                                                        ["WORK", "HOME",], {}, 
-                                                        dsattributes.kDSNAttrEMailContacts, )
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="EMAIL", defaultLabel="work",
+                                                        nolabelParamTypes=("WORK", "HOME",), 
+                                                        attrType=dsattributes.kDSNAttrEMailContacts, )
     
             """
             # UNIMPLEMENTED:
@@ -1656,7 +1466,7 @@
             # 3.4.1 TZ Type Definition
             """
             # 3.4.2 GEO Type Definition
-            #dsattributes.kDSNAttrMapCoordinates,        # attribute that defines coordinates for a user's location .
+            #dsattributes.kDSNAttrMapCoordinates,       # attribute that defines coordinates for a user's location .
                                                         #      Found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
                                                         #      Example: 7.7,10.6
             for coordinate in self.valuesForAttribute(dsattributes.kDSNAttrMapCoordinates):
@@ -1693,14 +1503,12 @@
             # 3.6.1 CATEGORIES Type Definition
             """
             # 3.6.2 NOTE Type Definition
-            # dsattributes.kDS1AttrComment,                  # Attribute used for unformatted comment.
+            # dsattributes.kDS1AttrComment,               # Attribute used for unformatted comment.
             # dsattributes.kDS1AttrNote,                  # Note attribute. Commonly used in printer records.
-            for comment in self.valuesForAttribute(dsattributes.kDS1AttrComment):
-                addUniqueProperty(vcard, Property("NOTE", comment), None, dsattributes.kDS1AttrComment, comment)
+            notes = self.valuesForAttribute(dsattributes.kDS1AttrComment, []) + self.valuesForAttribute(dsattributes.kDS1AttrNote, []);
+            if len(notes):
+                vcard.addProperty(Property("NOTE", "\n".join(notes),))
     
-            for note in self.valuesForAttribute(dsattributes.kDS1AttrNote):
-                addUniqueProperty(vcard, Property("NOTE", note), None, dsattributes.kDS1AttrNote, note)
-            
             # 3.6.3 PRODID Type Definition
             #vcard.addProperty(Property("PRODID", vCardProductID + "//BUILD %s" % twistedcaldav.__version__))
             #vcard.addProperty(Property("PRODID", vCardProductID))
@@ -1720,19 +1528,20 @@
             # dsattributes.kDS1AttrGeneratedUID,        # Used for 36 character (128 bit) unique ID. Usually found in user, 
                                                         #      group, and computer records. An example value is "A579E95E-CDFE-4EBC-B7E7-F2158562170F".
                                                         #      The standard format contains 32 hex characters and four hyphen characters.
-            # !! don't use self.guid which is URL encoded
+                                                        
             vcard.addProperty(Property("UID", self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)))
     
+    
             # 3.6.8 URL Type Definition 
-            # dsattributes.kDSNAttrURL,                    # List of URLs.
-            # dsattributes.kDS1AttrWeblogURI,            # Single-valued attribute that defines the URI of a user's weblog.
+            # dsattributes.kDSNAttrURL,                 # List of URLs.
+            # dsattributes.kDS1AttrWeblogURI,           # Single-valued attribute that defines the URI of a user's weblog.
                                                         #     Usually found in user records (kDSStdRecordTypeUsers). 
                                                         #      Example: http://example.com/blog/jsmith
             for url in self.valuesForAttribute(dsattributes.kDS1AttrWeblogURI):
-                addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters = {"TYPE": ["Weblog",]})
+                addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters = {"TYPE": ["WEBLOG",]})
     
             for url in self.valuesForAttribute(dsattributes.kDSNAttrURL):
-                addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters = {"TYPE": ["Homepage",]})
+                addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters = {"TYPE": ["HOMEPAGE",]})
     
     
             # 3.6.9 VERSION Type Definition
@@ -1745,11 +1554,11 @@
             # 3.7.2 KEY Type Definition
             
             # dsattributes.kDSNAttrPGPPublicKey,        # Pretty Good Privacy public encryption key.
-            # dsattributes.kDS1AttrUserCertificate,        # Attribute containing the binary of the user's certificate.
+            # dsattributes.kDS1AttrUserCertificate,     # Attribute containing the binary of the user's certificate.
                                                         #       Usually found in user records. The certificate is data which identifies a user.
                                                         #       This data is attested to by a known party, and can be independently verified 
                                                         #       by a third party.
-            # dsattributes.kDS1AttrUserPKCS12Data,        # Attribute containing binary data in PKCS #12 format. 
+            # dsattributes.kDS1AttrUserPKCS12Data,      # Attribute containing binary data in PKCS #12 format. 
                                                         #       Usually found in user records. The value can contain keys, certificates,
                                                         #      and other related information and is encrypted with a passphrase.
             # dsattributes.kDS1AttrUserSMIMECertificate,# Attribute containing the binary of the user's SMIME certificate.
@@ -1779,19 +1588,27 @@
                                                         #      Values should be prefixed with the appropriate IM type
                                                         #       ie. AIM:, Jabber:, MSN:, Yahoo:, or ICQ:
                                                         #       Usually found in user records (kDSStdRecordTypeUsers).
-    
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, "X-", None, "aim",
-                                                        ["AIM", "JABBER", "MSN", "YAHOO", "ICQ"], 
-                                                        {}, 
-                                                        dsattributes.kDSNAttrIMHandle,)
+            imNolabelParamTypes=("AIM", "FACEBOOK", "GAGU-GAGU", "GOOGLE TALK", "ICQ", "JABBER", "MSN", "QQ", "SKYPE", "YAHOO",)
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix="X-", propertyName=None, defaultLabel="aim",
+                                                        nolabelParamTypes=imNolabelParamTypes, 
+                                                        attrType=dsattributes.kDSNAttrIMHandle,)
+            
+            
+
+            # IMPP
+            # Address Book's implementation of http://tools.ietf.org/html/rfc6350#section-6.4.3
+            # adding IMPP property allows ab query report search on one property
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="IMPP", defaultLabel="aim",
+                                                        specialParamType = "X-SERVICE-TYPE",
+                                                        nolabelParamTypes=imNolabelParamTypes, 
+                                                        attrType=dsattributes.kDSNAttrIMHandle,)
                     
             # X-ABRELATEDNAMES
-            # dsattributes.kDSNAttrRelationships,        #      multi-valued attribute that defines the relationship to the record type .
+            # dsattributes.kDSNAttrRelationships,       #      multi-valued attribute that defines the relationship to the record type .
                                                         #      found in user records (kDSStdRecordTypeUsers). 
                                                         #      Example: brother:John
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "X-ABRELATEDNAMES", "friend",
-                                                        [],  
-                                                        {   "FATHER":"_$!<Father>!$_",
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="X-ABRELATEDNAMES", defaultLabel="friend",
+                                                        labelMap={   "FATHER":"_$!<Father>!$_",
                                                             "MOTHER":"_$!<Mother>!$_",
                                                             "PARENT":"_$!<Parent>!$_",
                                                             "BROTHER":"_$!<Brother>!$_",
@@ -1802,11 +1619,11 @@
                                                             "PARTNER":"_$!<Partner>!$_",
                                                             "ASSISTANT":"_$!<Assistant>!$_",
                                                             "MANAGER":"_$!<Manager>!$_", },
-                                                        dsattributes.kDSNAttrRelationships, )
+                                                        attrType=dsattributes.kDSNAttrRelationships, )
             
             
             # special case for Apple
-            if self.service.appleInternalServer:
+            if self.appleInternalServer:
                 for manager in self.valuesForAttribute("dsAttrTypeNative:appleManager"):
                     splitManager = manager.split("|")
                     if len(splitManager) >= 4:
@@ -1815,8 +1632,17 @@
                         managerValue = "%s %s" % (splitManager[0], splitManager[1])
                     else:
                         managerValue = manager
-                    addPropertyAndLabel( groupCount, "_$!<Manager>!$_", "X-ABRELATEDNAMES", managerValue, parameters={ "TYPE": ["Manager",]} )
+                    addPropertyAndLabel( groupCount, "_$!<Manager>!$_", "X-ABRELATEDNAMES", managerValue, parameters={ "TYPE": ["MANAGER",]} )
             
+ 
+            # add apple-defined group vcard properties if record type is group
+            if self.kind == "group":
+                vcard.addProperty(Property("X-ADDRESSBOOKSERVER-KIND", "group"))
+            
+            # add members
+            for memberguid in self.valuesForAttribute(dsattributes.kDSNAttrGroupMembers):
+                vcard.addProperty(Property("X-ADDRESSBOOKSERVER-MEMBER", "urn:uuid:" + memberguid))
+    
             """
             # UNIMPLEMENTED: X- attributes
             
@@ -1828,18 +1654,51 @@
             sattributes.kDS1AttrPicture,                # Represents the path of the picture for each user displayed in the login window.
                                                         #      Found in user records (kDSStdRecordTypeUsers).
            
-            dsattributes.kDS1AttrMapGUID,                # Represents the GUID for a record's map.
+            dsattributes.kDS1AttrMapGUID,               # Represents the GUID for a record's map.
             dsattributes.kDSNAttrMapURI,                # attribute that defines the URI of a user's location.
     
-            dsattributes.kDSNAttrOrganizationInfo,        # Usually the organization info of a user.
-            dsattributes.kDSNAttrAreaCode,                # Area code of a user's phone number.
+            dsattributes.kDSNAttrOrganizationInfo,      # Usually the organization info of a user.
+            dsattributes.kDSNAttrAreaCode,              # Area code of a user's phone number.
     
-            dsattributes.kDSNAttrMIME,                    # Data contained in this attribute type is a fully qualified MIME Type. 
+            dsattributes.kDSNAttrMIME,                  # Data contained in this attribute type is a fully qualified MIME Type. 
             
             """
             
-            # debug, create x attributes for all ds attributes
-            if self.service.addDSAttrXProperties:
+            # 2.1.4 SOURCE Type http://tools.ietf.org/html/rfc2426#section-2.1.4
+            #    If the SOURCE type is present, then its value provides information
+            #    how to find the source for the vCard.
+            
+            # add the source, so that if the SOURCE is copied out and preserved, the client can refresh information
+            # However, client should really do a ab-query report matching UID on /directory/ not a multiget.
+            uri = joinURL(self._directoryBackedAddressBook.uri, vcard.propertyValue("UID") + ".vcf")
+            
+            # seems like this should be in some standard place.
+            if config.EnableSSL and config.SSLPort:
+                if config.SSLPort == 443:
+                    source = "https://%s%s" % (config.ServerHostName, uri)
+                else:
+                    source = "https://%s:%s%s" % (config.ServerHostName, config.SSLPort, uri)
+            elif config.HTTPPort:
+                if config.HTTPPort == 80:
+                    source = "http://%s%s" % (config.ServerHostName, uri) 
+                else:
+                    source = "http://%s:%s%s" % (config.ServerHostName, config.HTTPPort, uri)
+            vcard.addProperty(Property("SOURCE", source))
+                       
+            #  in 4.0 spec: 
+            # 6.1.4.  KIND http://tools.ietf.org/html/rfc6350#section-6.1.4
+            # 
+            # see also: http://www.iana.org/assignments/vcard-elements/vcard-elements.xml
+            #
+            vcard.addProperty(Property("KIND", self.kind))
+            
+            # one more X- related to kind
+            if self.kind == "org":
+                vcard.addProperty(Property("X-ABShowAs", "COMPANY"))
+
+
+            # debug, create X-attributes for all ds attributes
+            if self.addDSAttrXProperties:
                 for attribute in self.originalAttributes:
                     for value in self.valuesForAttribute(attribute):
                         vcard.addProperty(Property("X-"+"-".join(attribute.split(":")), removeControlChars(value)))
@@ -1853,53 +1712,36 @@
         return self._vCard
     
     def vCardText(self):
-        if not self._vCardText:
-            self._vCardText = str(self.vCard())
-        
-        return self._vCardText
-
-    def uriName(self):
-        if not self._uriName:
-            self._uriName = self.vCard().getProperty("UID").value() + ".vcf"
-        #print("uriName():self._uriName=%s" % self._uriName)
-        return self._uriName
-        
+        return str(self.vCard())
     
-    def hRef(self, parentURI="/directory/"):
-        if not self._hRef:
-            self._hRef = davxml.HRef.fromString(joinURL(parentURI, self.uriName()))
-            
-        return self._hRef
-
-
+    def uri(self):
+        return self.vCard().propertyValue("UID") + ".vcf"
+        
+    def hRef(self, parentURI=None):
+        return davxml.HRef.fromString(joinURL(parentURI if parentURI else  self._directoryBackedAddressBook.uri, self.uri()))
+ 
+                       
     def readProperty(self, property, request):
         
         if type(property) is tuple:
             qname = property
         else:
             qname = property.qname()
-
         namespace, name = qname
-        
-        #print("VCardResource.readProperty: qname = %s" % (qname, ))
-        
+                
         if namespace == dav_namespace:
             if name == "resourcetype":
                 result = davxml.ResourceType.empty #@UndefinedVariable
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "getetag":
                 result = davxml.GETETag( ETag(hashlib.md5(self.vCardText()).hexdigest()).generate() )
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "getcontenttype":
                 mimeType = MimeType('text', 'vcard', {})
                 result = davxml.GETContentType(generateContentType(mimeType))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "getcontentlength":
                 result = davxml.GETContentLength.fromString(str(len(self.vCardText())))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "getlastmodified":
                 if self.vCard().hasProperty("REV"):
@@ -1917,7 +1759,6 @@
                 t = modDatetime.time()
                 modDatetimeNoTZ = datetime.datetime(d.year, d.month, d.day, t.hour, t.minute, t.second, t.microsecond, None)
                 result = davxml.GETLastModified.fromDate(modDatetimeNoTZ)
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "creationdate":
                 creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
@@ -1928,22 +1769,18 @@
                 else:
                     creationDatetime = datetime.datetime.utcnow()
                 result = davxml.CreationDate.fromDate(creationDatetime)
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
             elif name == "displayname":
                 # AddressBook.app uses N. Use FN or UID instead?
                 result = davxml.DisplayName.fromString(self.vCard().propertyValue("N"))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
                 return result
 
         elif namespace == twisted_dav_namespace:
-            return super(VCardRecord, self).readProperty(property, request)
-            #return DAVPropertyMixIn.readProperty(self, property, request)
+            return super(ABDirectoryQueryResult, self).readProperty(property, request)
 
-        return self.directoryBackedAddressBook.readProperty(property, request)
+        return self._directoryBackedAddressBook.readProperty(property, request)
 
     def listProperties(self, request):
-        #print("VCardResource.listProperties()")
         qnames = set(self.liveProperties())
 
         # Add dynamic live properties that exist
@@ -1952,33 +1789,20 @@
             (dav_namespace, "quota-used-bytes"          ),
         )
         for dqname in dynamicLiveProperties:
-            #print("VCardResource.listProperties: removing dqname=%s" % (dqname,))
             qnames.remove(dqname)
 
         for qname in self.deadProperties().list():
             if (qname not in qnames) and (qname[0] != twisted_private_namespace):
-                #print("listProperties: adding qname=%s" % (qname,))
                 qnames.add(qname)
 
-        #for qn in qnames: print("VCardResource.listProperties: qn=%s" % (qn,))
-
         yield qnames
         
     listProperties = deferredGenerator(listProperties)
     
 # utility
-#remove control characters because vCard does not support them
+#remove illegal XML
 def removeControlChars( utf8String ):
-    result = utf8String
-    for a in utf8String:
-        if '\x00' <= a <= '\x1F':
-            result = ""
-            for c in utf8String:
-                if '\x00' <= c <= '\x1F':
-                    pass 
-                else:
-                    result += c
-    #if utf8String != result: print ("changed %r to %r" % (utf8String, result))
+    result = ''.join([c for c in utf8String if c not in "\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"])
     return result
 
 

Added: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py	                        (rev 0)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -0,0 +1,308 @@
+##
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+"""
+Apple Open Directory directory service implementation for backing up directory-backed address books
+"""
+
+__all__ = [
+    "XMLDirectoryBackingService",
+]
+
+import traceback
+
+from calendarserver.platform.darwin.od import dsattributes, dsquery
+
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.directory.opendirectorybacker import ABDirectoryQueryResult, dsFilterFromAddressBookFilter, propertiesInAddressBookQuery
+
+
+class XMLDirectoryBackingService(XMLDirectoryService):
+    """
+    Directory backer for L{XMLDirectoryService}.
+    """
+    
+    def __init__(self, params):
+        self._actuallyConfigure(**params)
+
+    def _actuallyConfigure(self, **params):
+        
+        self.log_debug("_actuallyConfigure: params=%s" % (params,))
+        defaults = {
+            "recordTypes": (self.recordType_users, self.recordType_groups, ), 
+            "rdnSchema": {
+                self.recordType_users : { 
+                    "vcardPropToDirRecordAttrMap" : { 
+                        "FN" : (
+                                "fullName",
+                                "shortNames",
+                                "firstName",
+                                "lastName",
+                                ),
+                        "N" : (
+                                "fullName",
+                                "shortNames",
+                                "firstName",
+                                "lastName",
+                                ),
+                        "EMAIL" : "emailAddresses",
+                        "UID" : "guid",
+                     },
+                     "dirRecordAttrToDSAttrMap" : { 
+                        "guid" :            dsattributes.kDS1AttrGeneratedUID,
+                        "fullName" :        dsattributes.kDS1AttrDistinguishedName,
+                        "firstName" :       dsattributes.kDS1AttrFirstName,
+                        "lastName" :        dsattributes.kDS1AttrLastName,
+                        "emailAddresses" :  dsattributes.kDSNAttrEMailAddress,
+                     },
+                },
+                self.recordType_groups : { 
+                    "vcardPropToDirRecordAttrMap" : { 
+                        "FN" : (
+                                "fullName",
+                                "shortNames",
+                                "firstName",
+                                "lastName",
+                                ),
+                        "N" : (
+                                "fullName",
+                                "shortNames",
+                                "firstName",
+                                "lastName",
+                                ),
+                        "EMAIL" : "emailAddresses",
+                        "UID" : "guid",
+                        "X-ADDRESSBOOKSERVER-MEMBER" : "members",
+                     },
+                     "dirRecordAttrToDSAttrMap" : { 
+                        "guid" :            dsattributes.kDS1AttrGeneratedUID,
+                        "fullName" :        dsattributes.kDS1AttrDistinguishedName,
+                        "firstName" :       dsattributes.kDS1AttrFirstName,
+                        "lastName" :        dsattributes.kDS1AttrLastName,
+                        "emailAddresses" :  dsattributes.kDSNAttrEMailAddress,
+                        "members" :         dsattributes.kDSNAttrGroupMembers,
+                     },
+                },
+            },
+            "maxQueryResults":0,            # max records returned
+            "sortResults":True,             # sort results by UID
+            "implementNot":True,            # implement Not query by listing all records and subtracting
+       }
+
+        #params = self.getParams(params, defaults, ignored)
+        def addDefaults(params, defaults, remove=None):
+            for key in defaults:
+                if not key in params:
+                    params[key] = defaults[key]
+            return params
+                    
+        params = addDefaults(params, defaults)
+        self.log_debug("_actuallyConfigure after addDefaults: params=%s" % (params,))
+                
+        # super does not like these extra params
+        directoryBackedAddressBook=params["directoryBackedAddressBook"]
+        #del params["directoryBackedAddressBook"]
+        rdnSchema=params["rdnSchema"]
+        del params["rdnSchema"]
+        maxQueryResults=params["maxQueryResults"]
+        del params["maxQueryResults"]
+        sortResults=params["sortResults"]
+        del params["sortResults"]
+        implementNot=params["implementNot"]
+        del params["implementNot"]
+        
+        
+        assert directoryBackedAddressBook is not None
+        self.directoryBackedAddressBook = directoryBackedAddressBook
+        
+        self.maxQueryResults = maxQueryResults
+        self.sortResults = sortResults
+        self.implementNot = implementNot
+        self.rdnSchema = rdnSchema
+
+        
+        super(XMLDirectoryBackingService, self).__init__(params)
+        
+ 
+    def createCache(self):
+        succeed(None)
+                        
+
+    @inlineCallbacks
+    def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
+        """
+        Get vCards for a given addressBookFilter and addressBookQuery
+        """
+    
+        results = []
+        limited = False
+
+        for recordType in self.recordTypes():
+
+            queryMap = self.rdnSchema[recordType]
+            vcardPropToDirRecordAttrMap = queryMap["vcardPropToDirRecordAttrMap"]
+            dirRecordAttrToDSAttrMap = queryMap["dirRecordAttrToDSAttrMap"]
+            
+            kind = {self.recordType_groups:"group",
+                    self.recordType_locations:"location",
+                    self.recordType_resources:"calendarresource",
+                    }.get(recordType, "individual")
+        
+            constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+            constantProperties["KIND"] = kind
+            # add KIND as constant so that query can be skipped if addressBookFilter needs a different kind
+
+            filterPropertyNames, dsFilter  = dsFilterFromAddressBookFilter( addressBookFilter, vcardPropToDirRecordAttrMap, constantProperties=constantProperties );
+            self.log_debug("doAddressBookQuery: rdn=%s, query=%s, propertyNames=%s" % (recordType, dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames))
+
+            if dsFilter:
+                                
+                @inlineCallbacks
+                def recordsForDSFilter(dsFilter, recordType):
+                    
+                    """
+                        Athough recordsForDSFilter() exercises the dsFilter expression tree and recordsMatchingFields(),
+                        it make little difference to the result of a addressbook query because of filtering.
+                    """
+
+                    if not isinstance(dsFilter, dsquery.expression):
+                        #change  match list  into an expression and recurse
+                        returnValue((yield recordsForDSFilter(dsquery.expression( dsquery.expression.OR, (dsFilter,)), recordType)))
+
+                    else:
+                        #self.log_debug("recordsForDSFilter:  dsFilter=%s" % (dsFilter.generate(), ))
+                        dsFilterSubexpressions = dsFilter.subexpressions if isinstance(dsFilter.subexpressions, list) else (dsFilter.subexpressions,)
+                        #self.log_debug("recordsForDSFilter: #subs %s" % (len(dsFilterSubexpressions), ))
+                        
+                        # evaluate matches
+                        matches = [match for match in dsFilterSubexpressions if isinstance(match, dsquery.match)]
+                        fields = []
+                        for match in matches:
+                            #self.log_debug("recordsForDSFilter: match=%s" % (match.generate(), ))
+                            xmlMatchType = {
+                                dsattributes.eDSExact :        "exact",
+                                dsattributes.eDSStartsWith :   "starts-with",
+                                dsattributes.eDSContains :     "contains",
+                            }.get(match.matchType)
+                            if not xmlMatchType:
+                                self.log_debug("recordsForDSFilter: match type=%s match not supported" % (match.generate(), ))
+                                returnValue(None) # match type not supported by recordsMatchingFields()
+                            
+                            fields += ((match.attribute, match.value, True, xmlMatchType,),)
+                            #self.log_debug("recordsForDSFilter: fields=%s" % (fields,))
+                        
+                        # if there were matches, call get records that match
+                        result = None
+                        if len(fields):
+                            operand = "and" if dsFilter.operator == dsquery.expression.AND else "or"
+                            #self.log_debug("recordsForDSFilter: recordsMatchingFields(fields=%s, operand=%s, recordType=%s)" % (fields, operand, recordType,))
+                            result = set((yield self.recordsMatchingFields(fields, operand=operand, recordType=recordType)))
+                            #self.log_debug("recordsForDSFilter: result=%s" % (result,))
+                            if dsFilter.operator == dsquery.expression.NOT:
+                                if self.implementNot:
+                                    result = (yield self.listRecords(recordType)).difference(result)
+                                else:
+                                    self.log_debug("recordsForDSFilter: NOT expression not supported" % (match.generate(), ))
+                                    returnValue(None)
+                                    
+
+                        # evaluate subexpressions
+                        subexpressions = [subexpression for subexpression in dsFilterSubexpressions if isinstance(subexpression, dsquery.expression)]
+                        for subexpression in subexpressions:
+                            #self.log_debug("recordsForDSFilter: subexpression=%s" % (subexpression.generate(), ))
+                            subresult = (yield recordsForDSFilter(subexpression, recordType))
+                            #self.log_debug("recordsForDSFilter: subresult=%s" % (subresult,))
+                            if subresult is None:
+                                returnValue(None)
+                            
+                            if dsFilter.operator == dsquery.expression.NOT:
+                                if self.implementNot:
+                                    result = (yield self.listRecords(recordType)).difference(subresult)
+                                else:
+                                    self.log_debug("recordsForDSFilter: NOT expression not supported" % (match.generate(), ))
+                                    returnValue(None)
+                            elif result is None:
+                                result = subresult
+                            elif dsFilter.operator == dsquery.expression.OR:
+                                result = result.union(subresult)
+                            else:
+                                result = result.intersection(subresult)
+
+                    #self.log_debug("recordsForDSFilter:  dsFilter=%s returning %s" % (dsFilter.generate(), result, ))
+                    returnValue(result)
+                                
+                # calculate minimum attributes needed for this query: results unused
+                etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+                self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s" % (etagRequested, queryPropNames,))
+
+                # walk the expression tree
+                if dsFilter is True:
+                    xmlDirectoryRecords = None
+                else:
+                    xmlDirectoryRecords = (yield recordsForDSFilter(dsFilter, recordType))
+                self.log_debug("doAddressBookQuery: #xmlDirectoryRecords %s" % (len(xmlDirectoryRecords) if xmlDirectoryRecords is not None else xmlDirectoryRecords, ))
+                
+                if xmlDirectoryRecords is None:
+                    xmlDirectoryRecords = (yield self.listRecords(recordType))
+                    self.log_debug("doAddressBookQuery: all #xmlDirectoryRecords %s" % (len(xmlDirectoryRecords), ))
+                
+                
+                for xmlDirectoryRecord in xmlDirectoryRecords:
+                    
+                    def dsRecordAttributesFromDirectoryRecord( xmlDirectoryRecord ):
+                        #FIXME should filter based on request
+                        dsRecordAttributes = {}
+                        for attr in dirRecordAttrToDSAttrMap:
+                            try:
+                                if attr == "members":
+                                    value = [member.guid for member in xmlDirectoryRecord.members()]
+                                else:
+                                    value = getattr(xmlDirectoryRecord, attr)
+                                if value:
+                                    dsRecordAttributes[dirRecordAttrToDSAttrMap[attr]] = value
+                            except AttributeError:
+                                self.log_info("Could not get attribute %s from record %s" % (attr, xmlDirectoryRecord,))
+                                pass
+                        return dsRecordAttributes
+
+                    result = None
+                    dsRecordAttributes = dsRecordAttributesFromDirectoryRecord( xmlDirectoryRecord )
+                    try:
+                        result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, kind=kind)
+                    except:
+                        traceback.print_exc()
+                        self.log_info("Could not get vcard for %s" % (xmlDirectoryRecord,))
+                    else:
+                        self.log_debug("doAddressBookQuery: VCard text =\n%s" % (result.vCard(),))
+                        if addressBookFilter.match(result.vCard()):
+                            results.append(result)
+                        else:
+                            # should also filter for duplicate UIDs
+                            self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (result.vCard().propertyValue("FN"), result.vCard().propertyValue("UID"),))
+                
+                if len(results) >= maxResults:
+                    limited = True
+                    break
+                         
+        #sort results so that CalDAVTester can have consistent results when it uses limits
+        if self.sortResults:
+            results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+
+        self.log_info("limited  %s len(results) %s" % (limited,len(results),))
+        returnValue((results, limited,))        
+

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2008-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2008-2012 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -45,10 +45,11 @@
     Directory-backed address book
     """
 
-    def __init__(self, principalCollections):
+    def __init__(self, principalCollections, uri):
 
         CalDAVResource.__init__(self, principalCollections=principalCollections)
-
+        
+        self.uri = uri
         self.directory = None       # creates directory attribute
 
         # create with permissions, similar to CardDAVOptions in tap.py
@@ -93,14 +94,11 @@
                 return succeed(None)
 
             return self.directory.createCache()
-            
-            #print ("DirectoryBackedAddressBookResource.provisionDirectory: provisioned")
-        
+                    
         return succeed(None)
 
 
     def defaultAccessControlList(self):
-        #print( "DirectoryBackedAddressBookResource.defaultAccessControlList" )
         if config.AnonymousDirectoryAddressBookAccess:
             # DAV:Read for all principals (includes anonymous)
             accessPrincipal = davxml.All()
@@ -141,7 +139,6 @@
         return True
 
     def isAddressBookCollection(self):
-        #print( "DirectoryBackedAddressBookResource.isAddressBookCollection: return True" )
         return True
 
     def isCollection(self):
@@ -155,22 +152,6 @@
     def renderHTTP(self, request):
         if not self.directory:
             raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,"Service is starting up" ))
-        elif self.directory.liveQuery:
-            response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
-            returnValue(response)
-        else:
-            available = (yield maybeDeferred(self.directory.available, ))
-        
-            if not available:
-                raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,"Service is starting up" ))
-            else:
-                updateLock = self.directory.updateLock()
-                yield updateLock.acquire()
-                try:
-                    response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
-    
-                finally:
-                    yield updateLock.release()
-                
-                returnValue(response)
 
+        response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
+        returnValue(response)

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -118,7 +118,7 @@
         @param addrresource: the L{CalDAVResource} for an address book collection.
         @param uri: the uri for the address book collecton resource.
         """
-        
+                
         def checkMaxResults():
             matchcount[0] += 1
             if max_number_of_results[0] is not None and matchcount[0] > max_number_of_results[0]:
@@ -159,142 +159,109 @@
         def queryDirectoryBackedAddressBook(directoryBackedAddressBook, addressBookFilter):
             """
             """
-            records, limited[0] = (yield directoryBackedAddressBook.directory.vCardRecordsForAddressBookQuery( addressBookFilter, query, max_number_of_results[0] ))
-            for vCardRecord in records:
+            results, limited[0] = (yield directoryBackedAddressBook.directory.doAddressBookQuery( addressBookFilter, query, max_number_of_results[0] ))
+            for vCardResult in results:
                 
-                # match against original filter
-                if filter.match((yield vCardRecord.vCard())):
+                # match against original filter if different from addressBookFilter
+                if addressBookFilter is filter or filter.match((yield vCardResult.vCard())):
  
                     # Check size of results is within limit
                     checkMaxResults()
                    
                     try:
-                        yield report_common.responseForHref(request, responses, vCardRecord.hRef(), vCardRecord, propertiesForResource, query, vcard=(yield vCardRecord.vCard()))
+                        yield report_common.responseForHref(request, responses, vCardResult.hRef(), vCardResult, propertiesForResource, query, vcard=(yield vCardResult.vCard()))
                     except ConcurrentModification:
                         # This can happen because of a race-condition between the
                         # time we determine which resources exist and the deletion
                         # of one of these resources in another request.  In this
                         # case, we ignore the now missing resource rather
                         # than raise an error for the entire report.
-                        log.err("Missing resource during sync: %s" % (vCardRecord.hRef(),))
+                        log.err("Missing resource during sync: %s" % (vCardResult.hRef(),))
  
             
-        directoryAddressBookLock = None
-        try:                
 
-            if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.cacheQuery:
-                
-                directory = addrresource.directory
-                if directory.liveQuery:
-                    # if liveQuery and cacheQuery, get vCards into the directory address book on disk
-                    directoryAddressBookLock, limited[0] = (yield  directory.cacheVCardsForAddressBookQuery( filter, query, max_number_of_results[0] ) )
- 
-                elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
-                    max_number_of_results[0] = directory.maxDSQueryRecords
-                   
-                
-            elif not addrresource.isAddressBookCollection():
- 
-                #do UID lookup on last part of uri
-                resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
-                if resource_name.endswith(".vcf") and len(resource_name) > 4:
-    
-                    # see if parent is directory backed address book
-                    parent = (yield  addrresource.locateParent( request, uri ) )
-    
-                    if parent.isDirectoryBackedAddressBookCollection() and parent.directory.cacheQuery:
-                        
-                        directory = parent.directory
-                        if directory.liveQuery:
-                            vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
-                                                        carddavxml.TextMatch.fromString(resource_name[:-4]), 
-                                                        name="UID", # attributes
-                                                        ),] )
-                            vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
-                            
-                            directoryAddressBookLock, limited[0] = (yield  directory.cacheVCardsForAddressBookQuery( vCardFilter, query, max_number_of_results[0] ) )
+        if not addrresource.isAddressBookCollection():
 
-                        elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
-                            max_number_of_results[0] = directory.maxDSQueryRecords
-   
+            #do UID lookup on last part of uri
+            resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+            if resource_name.endswith(".vcf") and len(resource_name) > 4:
+
+                # see if parent is directory backed address book
+                parent = (yield  addrresource.locateParent( request, uri ) )
     
-        
-            # Check whether supplied resource is an address book or an address book object resource
-            if addrresource.isAddressBookCollection():
     
-                if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.liveQuery and not addrresource.directory.cacheQuery:
-                    yield  maybeDeferred( queryDirectoryBackedAddressBook, addrresource, filter )
-                
-                else:
+        # Check whether supplied resource is an address book or an address book object resource
+        if addrresource.isAddressBookCollection():
 
-                    # Do some optimisation of access control calculation by determining any inherited ACLs outside of
-                    # the child resource loop and supply those to the checkPrivileges on each child.
-                    filteredaces = (yield addrresource.inheritedACEsforChildren(request))
+            if addrresource.isDirectoryBackedAddressBookCollection():
+                yield  maybeDeferred( queryDirectoryBackedAddressBook, addrresource, filter )
+            
+            else:
+
+                # Do some optimisation of access control calculation by determining any inherited ACLs outside of
+                # the child resource loop and supply those to the checkPrivileges on each child.
+                filteredaces = (yield addrresource.inheritedACEsforChildren(request))
+            
+                # Check for disabled access
+                if filteredaces is not None:
+                    # See whether the filter is valid for an index only query
+                    index_query_ok = addrresource.index().searchValid(filter)
                 
-                    # Check for disabled access
-                    if filteredaces is not None:
-                        # See whether the filter is valid for an index only query
-                        index_query_ok = addrresource.index().searchValid(filter)
-                    
-                        # Get list of children that match the search and have read access
-                        names = [name for name, ignore_uid in (yield addrresource.index().search(filter))] #@UnusedVariable
-                        if not names:
-                            return
-                          
-                        # Now determine which valid resources are readable and which are not
-                        ok_resources = []
-                        yield addrresource.findChildrenFaster(
-                            "1",
-                            request,
-                            lambda x, y: ok_resources.append((x, y)),
-                            None,
-                            None,
-                            names,
-                            (davxml.Read(),),
-                            inherited_aces=filteredaces
-                        )
-                        for child, child_uri in ok_resources:
-                            child_uri_name = child_uri[child_uri.rfind("/") + 1:]
-                            
-                            if generate_address_data or not index_query_ok:
-                                vcard = yield child.vCard()
-                                assert vcard is not None, "vCard %s is missing from address book collection %r" % (child_uri_name, self)
-                            else:
-                                vcard = None
-                            
-                            yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok = index_query_ok)
+                    # Get list of children that match the search and have read access
+                    names = [name for name, ignore_uid in (yield addrresource.index().search(filter))] #@UnusedVariable
+                    if not names:
+                        return
+                      
+                    # Now determine which valid resources are readable and which are not
+                    ok_resources = []
+                    yield addrresource.findChildrenFaster(
+                        "1",
+                        request,
+                        lambda x, y: ok_resources.append((x, y)),
+                        None,
+                        None,
+                        names,
+                        (davxml.Read(),),
+                        inherited_aces=filteredaces
+                    )
+                    for child, child_uri in ok_resources:
+                        child_uri_name = child_uri[child_uri.rfind("/") + 1:]
                         
-            else:
+                        if generate_address_data or not index_query_ok:
+                            vcard = yield child.vCard()
+                            assert vcard is not None, "vCard %s is missing from address book collection %r" % (child_uri_name, self)
+                        else:
+                            vcard = None
+                        
+                        yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok = index_query_ok)
+                    
+        else:
+            
+            handled = False;
+            resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+            if resource_name.endswith(".vcf") and len(resource_name) > 4:
                 
-                handled = False;
-                resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
-                if resource_name.endswith(".vcf") and len(resource_name) > 4:
+                # see if parent is directory backed address book
+                parent = (yield  addrresource.locateParent( request, uri ) )
+
+                if parent.isDirectoryBackedAddressBookCollection():
+
+                    vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
+                                                carddavxml.TextMatch.fromString(resource_name[:-4]), 
+                                                name="UID", # attributes
+                                                ),] )
+                    vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
                     
-                    # see if parent is directory backed address book
-                    parent = (yield  addrresource.locateParent( request, uri ) )
+                    yield  maybeDeferred( queryDirectoryBackedAddressBook, parent, vCardFilter )
+                    handled = True
+
+            if not handled:
+                vcard = yield addrresource.vCard()
+                yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
     
-                    if parent.isDirectoryBackedAddressBookCollection() and parent.directory.liveQuery and not parent.directory.cacheQuery:
- 
-                        vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
-                                                    carddavxml.TextMatch.fromString(resource_name[:-4]), 
-                                                    name="UID", # attributes
-                                                    ),] )
-                        vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
-                        
-                        yield  maybeDeferred( queryDirectoryBackedAddressBook, parent, vCardFilter )
-                        handled = True
+        if limited[0]:
+            raise NumberOfMatchesWithinLimits(matchcount[0])
 
-                if not handled:
-                    vcard = yield addrresource.vCard()
-                    yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
-        
-            if limited[0]:
-                raise NumberOfMatchesWithinLimits(matchcount[0])
-
-        finally:
-            if directoryAddressBookLock:
-                yield directoryAddressBookLock.release()
-                
     
     # Run report taking depth into account
     try:

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -170,7 +170,7 @@
             
             # Special for addressbooks
             if collection_type == COLLECTION_TYPE_ADDRESSBOOK:
-                if self.isDirectoryBackedAddressBookCollection() and self.directory.liveQuery:
+                if self.isDirectoryBackedAddressBookCollection():
                     result = (yield doDirectoryAddressBookResponse())
                     returnValue(result)
 
@@ -249,11 +249,13 @@
                     resource_name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
                     if self._isChildURI(request, resource_uri) and resource_name.endswith(".vcf") and len(resource_name) > 4:
                         valid_hrefs.append(href)
+                        textMatchElement = carddavxml.TextMatch.fromString(resource_name[:-4])
+                        textMatchElement.attributes["match-type"] = "equals" # do equals compare. Default is "contains"
                         vCardFilters.append(carddavxml.PropertyFilter(
-                                                carddavxml.TextMatch.fromString(resource_name[:-4]), 
+                                                textMatchElement, 
                                                 name="UID", # attributes
                                             ))
-                    elif not self.directory.cacheQuery:
+                    else:
                         responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
                        
                 # exit if not valid           
@@ -262,40 +264,29 @@
                      
                 addressBookFilter = carddavxml.Filter( *vCardFilters )
                 addressBookFilter = addressbookqueryfilter.Filter(addressBookFilter)
-                if self.directory.cacheQuery:
-                    # add vcards to directory address book and run "normal case" below
-                    limit = config.DirectoryAddressBook.MaxQueryResults
-                    directoryAddressBookLock, limited = (yield  self.directory.cacheVCardsForAddressBookQuery(addressBookFilter, propertyreq, limit) )
-                    if limited:
-                        log.err("Too many results in multiget report: %d" % len(resources))
-                        raise HTTPError(ErrorResponse(
-                            responsecode.FORBIDDEN,
-                            (dav_namespace, "number-of-matches-within-limits"),
-                            "Too many results",
-                        ))
-                else:
-                    #get vCards and filter
-                    limit = config.DirectoryAddressBook.MaxQueryResults
-                    vCardRecords, limited = (yield self.directory.vCardRecordsForAddressBookQuery( addressBookFilter, propertyreq, limit ))
-                    if limited:
-                        log.err("Too many results in multiget report: %d" % len(resources))
-                        raise HTTPError(ErrorResponse(
-                            responsecode.FORBIDDEN,
-                            (dav_namespace, "number-of-matches-within-limits"),
-                            "Too many results",
-                        ))
-                   
-                    for href in valid_hrefs:
-                        matchingRecord = None
-                        for vCardRecord in vCardRecords:
-                            if href == vCardRecord.hRef(): # might need to compare urls instead - also case sens ok?
-                                matchingRecord = vCardRecord
-                                break;
+                
+                #get vCards and filter
+                limit = config.DirectoryAddressBook.MaxQueryResults
+                results, limited = (yield self.directory.doAddressBookQuery( addressBookFilter, propertyreq, limit ))
+                if limited:
+                    log.err("Too many results in multiget report: %d" % len(resources))
+                    raise HTTPError(ErrorResponse(
+                        responsecode.FORBIDDEN,
+                        (dav_namespace, "number-of-matches-within-limits"),
+                        "Too many results",
+                    ))
+               
+                for href in valid_hrefs:
+                    matchingResource = None
+                    for vCardResource in results:
+                        if href == vCardResource.hRef(): # might need to compare urls instead - also case sens ok?
+                            matchingResource = vCardResource
+                            break;
 
-                        if matchingRecord:
-                            yield report_common.responseForHref(request, responses, href, matchingRecord, propertiesForResource, propertyreq, vcard=matchingRecord.vCard())
-                        else:
-                            responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+                    if matchingResource:
+                        yield report_common.responseForHref(request, responses, href, matchingResource, propertiesForResource, propertyreq, vcard=matchingResource.vCard())
+                    else:
+                        responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
             finally:
                 if directoryAddressBookLock:
                     yield directoryAddressBookLock.release()

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2011 Apple Inc. All rights reserved.
+# Copyright (c) 2011-2012 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,7 +73,7 @@
                     return not allof
             return allof
         else:
-            return True
+            return not allof
 
     def valid(self):
         """
@@ -120,9 +120,6 @@
             else:
                 raise ValueError("Unknown child element: %s" % (qname,))
 
-        if qualifier and isinstance(qualifier, IsNotDefined) and (len(filters) != 0):
-            raise ValueError("No other tests allowed when CardDAV:is-not-defined is present")
-            
         if xml_element.qname() == (carddav_namespace, "prop-filter"):
             propfilter_test = xml_element.attributes.get("test", "anyof")
             if propfilter_test not in ("anyof", "allof"):
@@ -130,13 +127,16 @@
         else:
             propfilter_test = "anyof"
 
+        if qualifier and isinstance(qualifier, IsNotDefined) and (len(filters) != 0) and propfilter_test == "allof":
+            raise ValueError("When test is allof, no other tests allowed when CardDAV:is-not-defined is present")
+            
         self.propfilter_test = propfilter_test
         self.qualifier = qualifier
         self.filters = filters
         self.filter_name = xml_element.attributes["name"]
         if isinstance(self.filter_name, unicode):
             self.filter_name = self.filter_name.encode("utf-8")
-        self.defined = not self.qualifier or not isinstance(qualifier, IsNotDefined)
+        self.defined = not self.qualifier or not isinstance(qualifier, IsNotDefined) or len(filters)
 
     def match(self, item):
         """
@@ -144,20 +144,17 @@
         matches this filter, False otherwise.
         """
         
-        # Always return True for the is-not-defined case as the result of this will
-        # be negated by the caller
-        if not self.defined: return True
+        allof = self.propfilter_test == "allof"
+        if self.qualifier and allof != self.qualifier.match(item):
+            return not allof
 
-        if self.qualifier and not self.qualifier.match(item): return False
-
         if len(self.filters) > 0:
-            allof = self.propfilter_test == "allof"
             for filter in self.filters:
                 if allof != filter._match(item):
                     return not allof
             return allof
         else:
-            return True
+            return not allof
 
 class PropertyFilter (FilterChildBase):
     """

Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py	2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py	2012-09-01 00:56:43 UTC (rev 9765)
@@ -207,28 +207,64 @@
 
 
 directoryAddressBookBackingServiceDefaultParams = {
-    "twistedcaldav.directory.xmlfile.XMLDirectoryService": {
-        "xmlFile": "/etc/carddavd/accounts.xml",
+    "twistedcaldav.directory.xmldirectorybacker.XMLDirectoryBackingService": {
+        "xmlFile": "accounts.xml",
+        "recordTypes": ("users", "groups"),
+        "statSeconds" : 15,
     },
     "twistedcaldav.directory.opendirectorybacker.OpenDirectoryBackingService": {
         "queryPeopleRecords": True,
         "peopleNode": "/Search/Contacts",
         "queryUserRecords": True,
-        "userNode": "/Search/Contacts",
+        "userNode": "/Search",
+        "queryGroupRecords": True,
+        "groupNode": "/Search",
         "maxDSQueryRecords":0,
         "queryDSLocal": False,
         "ignoreSystemRecords": True,
         "dsLocalCacheTimeout":30,
-        "liveQuery": True,
         "fakeETag": True,
-        "cacheQuery": False,
-        "cacheTimeout": 30,
-        "standardizeSyntheticUIDs": False,
         "addDSAttrXProperties": False,
         "appleInternalServer": False,
-        "additionalAttributes" : [],
-        "allowedAttributes" : [],
+        "additionalAttributes" : None,
+        "allowedAttributes" : None,
+        "searchAttributes" : None,
     },
+    "twistedcaldav.directory.ldapdirectorybacker.LdapDirectoryBackingService": {
+        "appleInternalServer": False,
+        "warningThresholdSeconds": 3,
+        "uri": "ldap://localhost/",
+        "tls": False,
+        "tlsCACertFile": None,
+        "tlsCACertDir": None,
+        "tlsRequireCert": None, # never, allow, try, demand, hard
+        "credentials": {
+            "dn": None,
+            "password": None,
+        },
+        "authMethod": "LDAP",
+        "rdnSchema": {
+            "base": "dc=example,dc=com",
+            "queryTypes": ("people", ),
+            "people": {
+                "rdn": "ou=People",
+                "attr": "uid", # used only to synthesize email address
+                "filter": None, # additional filter for this type
+                "vcardPropToLdapAttrMap" : { # maps address book query vCard properties to ldap attributes
+                    "givenName" : "FirstName",
+                    "sn" : "LastName",
+                    "cn" : "RealName",
+                    "uid" : "GeneratedUID",
+                    "mail" : "EMailAddress",
+                },
+                "ldapAttrToDSAttrMap" : { # maps ldap attributes to ds record attributes
+                    "FN" : "cn",
+                    "EMAIL" : "mail",
+                    "UID" : "uid",
+                },
+            },
+        },
+    },
 }
 
 DEFAULT_CONFIG = {
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120831/87743a7e/attachment-0001.html>


More information about the calendarserver-changes mailing list