[CalendarServer-changes] [13037] CalendarServer/branches/users/sagen/move2who-4

source_changes at macosforge.org source_changes at macosforge.org
Fri Mar 28 14:51:51 PDT 2014


Revision: 13037
          http://trac.calendarserver.org//changeset/13037
Author:   gaya at apple.com
Date:     2014-03-28 14:51:51 -0700 (Fri, 28 Mar 2014)
Log Message:
-----------
First checkin of directory gateway based on twext.who

Modified Paths:
--------------
    CalendarServer/branches/users/sagen/move2who-4/calendarserver/tap/util.py
    CalendarServer/branches/users/sagen/move2who-4/conf/caldavd-test.plist
    CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directorybackedaddressbook.py
    CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_addressbook_query.py
    CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_multiget_common.py

Removed Paths:
-------------
    CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/opendirectorybacker.py
    CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/test/test_opendirectorybacker.py

Modified: CalendarServer/branches/users/sagen/move2who-4/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/calendarserver/tap/util.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/calendarserver/tap/util.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -35,6 +35,7 @@
 from twext.python.log import Logger
 from txweb2.auth.basic import BasicCredentialFactory
 from txweb2.dav import auth
+from txweb2.dav.util import joinURL
 from txweb2.http_headers import Headers
 from txweb2.resource import Resource
 from txweb2.static import File as FileResource
@@ -46,25 +47,25 @@
 from twisted.internet.reactor import addSystemEventTrigger
 from twisted.internet.tcp import Connection
 
+from calendarserver.push.applepush import APNSubscriptionResource
+from calendarserver.push.notifier import NotifierFactory
+from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
+from twext.enterprise.ienterprise import ORACLE_DIALECT
+from twext.enterprise.ienterprise import POSTGRES_DIALECT
 from twistedcaldav.bind import doBind
 from twistedcaldav.cache import CacheStoreNotifierFactory
 from twistedcaldav.directory.addressbook import DirectoryAddressBookHomeProvisioningResource
 from twistedcaldav.directory.calendar import DirectoryCalendarHomeProvisioningResource
 from twistedcaldav.directory.digest import QopDigestCredentialFactory
 from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
-from calendarserver.push.notifier import NotifierFactory
-from calendarserver.push.applepush import APNSubscriptionResource
 from twistedcaldav.directorybackedaddressbook import DirectoryBackedAddressBookResource
 from twistedcaldav.resource import AuthenticationWrapper
-from txdav.caldav.datastore.scheduling.ischedule.dkim import DKIMUtils, DomainKeyResource
-from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
 from twistedcaldav.simpleresource import SimpleResource, SimpleRedirectResource
 from twistedcaldav.timezones import TimezoneCache
 from twistedcaldav.timezoneservice import TimezoneServiceResource
 from twistedcaldav.timezonestdservice import TimezoneStdServiceResource
-from twext.enterprise.ienterprise import POSTGRES_DIALECT
-from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
+from txdav.caldav.datastore.scheduling.ischedule.dkim import DKIMUtils, DomainKeyResource
+from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
 
 
 try:
@@ -237,9 +238,9 @@
         quota = None
     if txnFactory is not None:
         if config.EnableSSL:
-            uri = "https://%s:%s" % (config.ServerHostName, config.SSLPort,)
+            uri = "https://{config.ServerHostName}:{config.SSLPort}".format(config=config)
         else:
-            uri = "http://%s:%s" % (config.ServerHostName, config.HTTPPort,)
+            uri = "https://{config.ServerHostName}:{config.HTTPPort}".format(config=config)
         attachments_uri = uri + "/calendars/__uids__/%(home)s/dropbox/%(dropbox_id)s/%(name)s"
         store = CommonSQLDataStore(
             txnFactory, notifierFactories,
@@ -283,7 +284,11 @@
         credentials = IPrincipalCredentials(credentials)
 
         if credentials.authnPrincipal is None:
-            raise UnauthorizedLogin("No such user: %s" % (credentials.credentials.username,))
+            raise UnauthorizedLogin(
+                "No such user: {user}".format(
+                    user=credentials.credentials.username
+                )
+        )
 
         # See if record is enabledForLogin
         if not credentials.authnPrincipal.record.isLoginEnabled():
@@ -321,9 +326,14 @@
                     )
                 )
             else:
-                raise UnauthorizedLogin("Incorrect credentials for %s" % (credentials.credentials.username,))
+                raise UnauthorizedLogin(
+                    "Incorrect credentials for user: {user}".format(
+                        user=credentials.credentials.username
+                    )
+                )
 
 
+
 def getRootResource(config, newStore, resources=None):
     """
     Set up directory service and resource hierarchy based on config.
@@ -468,13 +478,14 @@
             newStore,
         )
 
-        directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
         if config.DirectoryAddressBook.Enabled and config.EnableSearchAddressBook:
             log.info("Setting up directory address book: {cls}",
                 cls=directoryBackedAddressBookResourceClass)
 
             directoryBackedAddressBookCollection = directoryBackedAddressBookResourceClass(
-                principalCollections=(principalCollection,)
+                principalCollections=(principalCollection,),
+                principalDirectory=directory,
+                uri=joinURL("/", config.DirectoryAddressBook.name, "/")
             )
             if _reactor._started:
                 directoryBackedAddressBookCollection.provisionDirectory()
@@ -482,6 +493,7 @@
                 addSystemEventTrigger("after", "startup", directoryBackedAddressBookCollection.provisionDirectory)
         else:
             # remove /directory from previous runs that may have created it
+            directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
             try:
                 FilePath(directoryPath).remove()
                 log.info("Deleted: {path}", path=directoryPath)

Modified: CalendarServer/branches/users/sagen/move2who-4/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/conf/caldavd-test.plist	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/conf/caldavd-test.plist	2014-03-28 21:51:51 UTC (rev 13037)
@@ -577,7 +577,7 @@
 
     <!-- Log levels -->
     <key>DefaultLogLevel</key>
-    <string>info</string> <!-- debug, info, warn, error -->
+    <string>debug</string> <!-- debug, info, warn, error -->
 
     <!-- Log level overrides for specific functionality -->
     <key>LogLevels</key>
@@ -1017,6 +1017,8 @@
       <string>en</string>
     </dict>
 
-
+    <!-- Directory Address Book -->
+    <key>EnableSearchAddressBook</key>
+    <true/>
   </dict>
 </plist>

Deleted: CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/opendirectorybacker.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/opendirectorybacker.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -1,1960 +0,0 @@
-##
-# Copyright (c) 2006-2014 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-
-"""
-Apple Open Directory directory service implementation for backing up directory-backed address books
-"""
-
-__all__ = [
-    "OpenDirectoryBackingService", "VCardRecord",
-]
-
-from calendarserver.platform.darwin.od import opendirectory, dsattributes, dsquery
-
-from pycalendar.datetime import DateTime
-from pycalendar.vcard.adr import Adr
-from pycalendar.vcard.n import N
-
-
-from twext.python.filepath import CachingFilePath as FilePath
-
-from txweb2.dav.resource import DAVPropertyMixIn
-from txweb2.dav.util import joinURL
-from txweb2.http_headers import MimeType, generateContentType, ETag
-
-from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks, returnValue, deferredGenerator, succeed
-
-from twistedcaldav import customxml, carddavxml
-from twistedcaldav.config import config
-from twistedcaldav.customxml import calendarserver_namespace
-from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
-from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
-from twistedcaldav.vcard import Component, Property, vCardProductID
-
-from txdav.carddav.datastore.query.filter import IsNotDefined, ParameterFilter, \
-    TextMatch
-from txdav.xml import element as davxml
-from txdav.xml.base import twisted_dav_namespace, dav_namespace, parse_date, twisted_private_namespace
-
-from os import listdir
-from os.path import join, abspath
-from random import random
-from socket import getfqdn
-from tempfile import mkstemp, gettempdir
-from xmlrpclib import datetime
-import hashlib
-import os
-import sys
-import time
-import traceback
-
-class OpenDirectoryBackingService(DirectoryService):
-    """
-    Open Directory implementation of L{IDirectoryService}.
-    """
-
-    baseGUID = "BF07A1A2-5BB5-4A4D-A59A-67260EA7E143"
-
-    def __repr__(self):
-        return "<%s %r>" % (self.__class__.__name__, self.realmName,)
-
-
-    def __init__(self, params):
-        self._actuallyConfigure(**params)
-
-
-    def _actuallyConfigure(
-        self, queryPeopleRecords=True,
-        peopleNode="/Search/Contacts",
-        queryUserRecords=True,
-        userNode="/Search",
-        maxDSQueryRecords=0,            # maximum number of records requested for any ds query
-
-        queryDSLocal=False,             # query in DSLocal -- debug
-        dsLocalCacheTimeout=30,
-        ignoreSystemRecords=True,
-
-        liveQuery=True,                 # query directory service as needed
-        fakeETag=True,                  # eTag is not reliable if True
-
-        cacheQuery=False,
-        cacheTimeout=30,                # cache timeout
-
-        addDSAttrXProperties=False,        # add dsattributes to vcards as "X-" attributes
-        standardizeSyntheticUIDs=False,  # use simple synthetic UIDs --- good for testing
-        appleInternalServer=False,
-
-        additionalAttributes=[],
-        allowedAttributes=[],
-        directoryBackedAddressBook=None
-    ):
-        """
-        @queryPeopleRecords: C{True} to query for People records
-        @queryUserRecords: C{True} to query for User records
-        @maxDSQueryRecords: maximum number of (unfiltered) ds records retrieved before raising
-            NumberOfMatchesWithinLimits exception or returning results
-        @dsLocalCacheTimeout: how log to keep cache of DSLocal records
-        @liveQuery: C{True} to query the directory as needed
-        @fakeETag: C{True} to use a fake eTag; allows ds queries with partial attributes
-        @cacheQuery: C{True} to query the directory and cache results
-        @cacheTimeout: if caching, the average cache timeout
-        @standardizeSyntheticUIDs: C{True} when creating synthetic UID (==f(Node, Type, Record Name)),
-            use a standard Node name. This allows testing with the same UID on different hosts
-        @allowedAttributes: list of DSAttributes that are used to create VCards
-
-        """
-        assert directoryBackedAddressBook is not None
-        self.directoryBackedAddressBook = directoryBackedAddressBook
-
-        self.peopleDirectory = None
-        self.peopleNode = None
-        self.userDirectory = None
-        self.userNode = None
-
-        self.realmName = None # needed for super
-
-        if queryPeopleRecords or not queryUserRecords:
-            self.peopleNode = peopleNode
-            try:
-                self.peopleDirectory = opendirectory.odInit(peopleNode)
-            except opendirectory.ODError, e:
-                self.log.error("Open Directory (node=%s) Initialization error: %s" % (peopleNode, e))
-                raise
-            self.realmName = peopleNode
-
-        if queryUserRecords:
-            if self.peopleNode == userNode:          # use sane directory and node if they are equal
-                self.userNode = self.peopleNode
-                self.userDirectory = self.peopleDirectory
-            else:
-                self.userNode = userNode
-                try:
-                    self.userDirectory = opendirectory.odInit(userNode)
-                except opendirectory.ODError, e:
-                    self.log.error("Open Directory (node=%s) Initialization error: %s" % (userNode, e))
-                    raise
-                if self.realmName:
-                    self.realmName += "+" + userNode
-                else:
-                    self.realmName = userNode
-
-        self.maxDSQueryRecords = maxDSQueryRecords
-
-        self.ignoreSystemRecords = ignoreSystemRecords
-        self.queryDSLocal = queryDSLocal
-        self.dsLocalCacheTimeout = dsLocalCacheTimeout
-
-        self.liveQuery = liveQuery or not cacheQuery
-        self.fakeETag = fakeETag
-
-        self.cacheQuery = cacheQuery
-
-        self.cacheTimeout = cacheTimeout if cacheTimeout > 0 else 30
-
-        self.addDSAttrXProperties = addDSAttrXProperties
-        self.standardizeSyntheticUIDs = standardizeSyntheticUIDs
-        self.appleInternalServer = appleInternalServer
-
-        self.additionalAttributes = additionalAttributes
-        # filter allows attributes, but make sure there are a minimum of attributes for functionality
-        if allowedAttributes:
-            self.allowedDSQueryAttributes = sorted(list(set(
-                                                [attr for attr in VCardRecord.allDSQueryAttributes
-                                                    if (isinstance(attr, str) and attr in allowedAttributes) or
-                                                       (isinstance(attr, tuple) and attr[0] in allowedAttributes)] +
-                                                VCardRecord.dsqueryAttributesForProperty.get("X-INTERNAL-REQUIRED")
-                                                )))
-            if (self.allowedDSQueryAttributes != VCardRecord.allDSQueryAttributes):
-                self.log.info("Allowed DS query attributes = %r" % (self.allowedDSQueryAttributes,))
-        else:
-            self.allowedDSQueryAttributes = VCardRecord.allDSQueryAttributes
-
-        #self.returnedAttributes = VCardRecord.allDSQueryAttributes
-        self.returnedAttributes = self.allowedDSQueryAttributes
-
-        self._dsLocalRecords = []
-        self._nextDSLocalQueryTime = 0
-
-        # get this now once
-        hostname = getfqdn()
-        if hostname:
-            self.defaultNodeName = "/LDAPv3/" + hostname
-        else:
-            self.defaultNodeName = None
-
-        #cleanup
-        self._cleanupTime = time.time()
-
-        # file system locks
-        self._initLockPath = join(config.DocumentRoot, ".directory_address_book_create_lock")
-        self._createdLockPath = join(config.DocumentRoot, ".directory_address_book_created_lock")
-        self._updateLockPath = join(config.DocumentRoot, ".directory_address_book_update_lock")
-        self._tmpDirAddressBookLockPath = join(config.DocumentRoot, ".directory_address_book_tmpFolder_lock")
-
-        self._updateLock = MemcacheLock("OpenDirectoryBacker", self._updateLockPath)
-        self._tmpDirAddressBookLock = MemcacheLock("OpenDirectoryBacker", self._tmpDirAddressBookLockPath)
-
-        # optimization so we don't have to always get create lock
-        self._triedCreateLock = False
-        self._created = False
-
-
-    def __cmp__(self, other):
-        if not isinstance(other, DirectoryRecord):
-            return super(DirectoryRecord, self).__eq__(other)
-
-        for attr in ("directory", "node"):
-            diff = cmp(getattr(self, attr), getattr(other, attr))
-            if diff != 0:
-                return diff
-        return 0
-
-
-    def __hash__(self):
-        h = hash(self.__class__.__name__)
-        for attr in ("node",):
-            h = (h + hash(getattr(self, attr))) & sys.maxint
-        return h
-
-
-    @inlineCallbacks
-    def available(self):
-        if not self._triedCreateLock:
-            returnValue(False)
-        elif not self._created:
-            createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
-            self.log.debug("blocking on lock of: \"%s\")" % self._createdLockPath)
-            self._created = (yield createdLock.locked())
-
-        returnValue(self._created)
-
-
-    def updateLock(self):
-        return self._updateLock
-
-
-    @inlineCallbacks
-    def createCache(self):
-        """
-        If caching, create the cache for the first time.
-        """
-
-        if not self.liveQuery:
-            self.log.info("loading directory address book")
-
-            # get init lock
-            initLock = MemcacheLock("OpenDirectoryBacker", self._initLockPath, timeout=0)
-            self.log.debug("Attempt lock of: \"%s\")" % self._initLockPath)
-            gotCreateLock = False
-            try:
-                yield initLock.acquire()
-                gotCreateLock = True
-            except MemcacheLockTimeoutError:
-                pass
-
-            self._triedCreateLock = True
-
-            if gotCreateLock:
-                self.log.debug("Got lock!")
-                yield self._refreshCache(flushCache=False, creating=True)
-            else:
-                self.log.debug("Could not get lock - directory address book will be filled by peer")
-
-
-    @inlineCallbacks
-    def _refreshCache(self, flushCache=False, creating=False, reschedule=True, query=None, attributes=None, keepLock=False, clear=False, maxRecords=0):
-        """
-        refresh the cache.
-        """
-
-        #print("_refreshCache:, flushCache=%s, creating=%s, reschedule=%s, query = %s" % (flushCache, creating, reschedule, "None" if query is None else query.generate(),))
-
-        def refreshLater():
-            #
-            # Add jitter/fuzz factor to avoid stampede for large OD query
-            #
-            cacheTimeout = min(self.cacheTimeout, 60) * 60
-            cacheTimeout = (cacheTimeout * random()) - (cacheTimeout / 2)
-            cacheTimeout += self.cacheTimeout * 60
-            reactor.callLater(cacheTimeout, self._refreshCache) #@UndefinedVariable
-            self.log.info("Refresh directory address book in %d minutes %d seconds" % divmod(cacheTimeout, 60))
-
-        def cleanupLater():
-
-            # try to cancel previous call if last clean up was less than 15 minutes ago
-            if (time.time() - self._cleanupTime) < 15 * 60:
-                try:
-                    self._lastCleanupCall.cancel()
-                except:
-                    pass
-
-            #
-            # Add jitter/fuzz factor
-            #
-            nom = 120
-            later = nom * (random() + .5)
-            self._lastCleanupCall = reactor.callLater(later, removeTmpAddressBooks) #@UndefinedVariable
-            self.log.info("Remove temporary directory address books in %d minutes %d seconds" % divmod(later, 60))
-
-
-        def getTmpDirAndTmpFilePrefixSuffix():
-            # need to have temp file on same volumes as documents so that move works
-            absDocPath = abspath(config.DocumentRoot)
-            if absDocPath.startswith("/Volumes/"):
-                tmpDir = absDocPath
-                prefix = ".directoryAddressBook-"
-            else:
-                tmpDir = gettempdir()
-                prefix = "directoryAddressBook-"
-
-            return (tmpDir, prefix, ".tmp")
-
-        def makeTmpFilename():
-            tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
-            fd, fname = mkstemp(suffix=suffix, prefix=prefix, dir=tmpDir)
-            os.close(fd)
-            os.remove(fname)
-            return fname
-
-        @inlineCallbacks
-        def removeTmpAddressBooks():
-            self.log.info("Checking for temporary directory address books")
-            tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
-
-            tmpDirLock = self._tmpDirAddressBookLock
-            self.log.debug("blocking on lock of: \"%s\")" % self._tmpDirAddressBookLockPath)
-            yield tmpDirLock.acquire()
-
-            try:
-                for name in listdir(tmpDir):
-                    if name.startswith(prefix) and name.endswith(suffix):
-                        try:
-                            path = join(tmpDir, name)
-                            self.log.info("Deleting temporary directory address book at: %s" % path)
-                            FilePath(path).remove()
-                            self.log.debug("Done deleting")
-                        except:
-                            self.log.info("Deletion failed")
-            finally:
-                self.log.debug("unlocking: \"%s\")" % self._tmpDirAddressBookLockPath)
-                yield tmpDirLock.release()
-
-            self._cleanupTime = time.time()
-
-        updateLock = None
-        limited = False
-        try:
-
-            try:
-                # get the records
-                if clear:
-                    records = {}
-                else:
-                    records, limited = (yield self._getDirectoryRecords(query, attributes, maxRecords))
-
-                # calculate the hash
-                # simple for now, could use MD5 digest if too many collisions
-                newAddressBookCTag = customxml.GETCTag(str(hash(self.baseGUID + ":" + self.realmName + ":" + "".join(str(hash(records[key])) for key in records.keys()))))
-
-                # get the old hash
-                oldAddressBookCTag = ""
-                updateLock = self.updateLock()
-                self.log.debug("blocking on lock of: \"%s\")" % self._updateLockPath)
-                yield updateLock.acquire()
-
-                if not flushCache:
-                    # get update lock
-                    try:
-                        oldAddressBookCTag = self.directoryBackedAddressBook.readDeadProperty((calendarserver_namespace, "getctag"))
-                    except:
-                        oldAddressBookCTag = ""
-
-                self.log.debug("Comparing {http://calendarserver.org/ns/}getctag: new = %s, old = %s" % (newAddressBookCTag, oldAddressBookCTag))
-                if str(newAddressBookCTag) != str(oldAddressBookCTag):
-
-                    self.log.debug("unlocking: \"%s\")" % self._updateLockPath)
-                    yield updateLock.release()
-                    updateLock = None
-
-                if not keepLock:
-                    self.log.debug("unlocking: \"%s\")" % self._updateLockPath)
-                    yield updateLock.release()
-                    updateLock = None
-
-            except:
-                cleanupLater()
-                if reschedule:
-                    refreshLater()
-                raise
-
-            if creating:
-                createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
-                self.log.debug("blocking on lock of: \"%s\")" % self._createdLockPath)
-                yield createdLock.acquire()
-
-            cleanupLater()
-            if reschedule:
-                refreshLater()
-
-        except:
-            if updateLock:
-                yield updateLock.release()
-            raise
-
-        returnValue((updateLock, limited))
-
-
-    def _getDSLocalRecords(self):
-
-        def generateDSLocalRecords():
-
-            records = {}
-
-            recordTypes = [dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers, ]
-            try:
-                localNodeDirectory = opendirectory.odInit("/Local/Default")
-                self.log.debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r)" % (
-                        "/DSLocal",
-                        recordTypes,
-                        self.returnedAttributes,
-                    ))
-                results = list(opendirectory.listAllRecordsWithAttributes_list(
-                        localNodeDirectory,
-                        recordTypes,
-                        self.returnedAttributes,
-                    ))
-            except opendirectory.ODError, ex:
-                self.log.error("Open Directory (node=%s) error: %s" % ("/Local/Default", str(ex)))
-                raise
-
-            self._dsLocalRecords = []
-            for (recordShortName, value) in results: #@UnusedVariable
-
-                record = VCardRecord(self, value, "/Local/Default")
-
-                if self.ignoreSystemRecords:
-                    # remove system users and people
-                    if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
-                        self.log.info("Ignoring vcard for system record %s" % (record,))
-                        continue
-
-                if record.guid in records:
-                    self.log.info("Record skipped due to conflict (duplicate uuid): %s" % (record,))
-                else:
-                    try:
-                        vCardText = record.vCardText()
-                    except:
-                        traceback.print_exc()
-                        self.log.info("Could not get vcard for record %s" % (record,))
-                    else:
-                        self.log.debug("VCard text =\n%s" % (vCardText,))
-                        records[record.guid] = record
-
-            return records
-
-        if not self.liveQuery or not self.queryDSLocal:
-            return {}
-
-        if time.time() > self._nextDSLocalQueryTime:
-            self._dsLocalRecords = generateDSLocalRecords()
-            # Add jitter/fuzz factor
-            self._nextDSLocalQueryTime = time.time() + self.dsLocalCacheTimeout * (random() + 0.5) * 60
-
-        return self._dsLocalRecords
-
-
-    @inlineCallbacks
-    def _getDirectoryRecords(self, query=None, attributes=None, maxRecords=0):
-        """
-        Get a list of filtered VCardRecord for the given query with the given attributes.
-        query == None gets all records. attribute == None gets VCardRecord.allDSQueryAttributes
-        """
-        limited = False
-        queryResults = (yield self._queryDirectory(query, attributes, maxRecords))
-        if maxRecords and len(queryResults) >= maxRecords:
-            limited = True
-            self.log.debug("Directory address book record limit (= %d) reached." % (maxRecords,))
-
-        self.log.debug("Query done. Inspecting %s results" % len(queryResults))
-
-        records = self._getDSLocalRecords().copy()
-        self.log.debug("Adding %s DSLocal results" % len(records.keys()))
-
-        for (recordShortName, value) in queryResults: #@UnusedVariable
-
-            record = VCardRecord(self, value, self.defaultNodeName)
-
-            if self.ignoreSystemRecords:
-                # remove system users and people
-                if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
-                    self.log.info("Ignoring vcard for system record %s" % (record,))
-                    continue
-
-            if record.guid in records:
-                self.log.info("Ignoring vcard for record due to conflict (duplicate uuid): %s" % (record,))
-            else:
-                records[record.guid] = record
-
-        self.log.debug("After filtering, %s records (limited=%s)." % (len(records), limited))
-        returnValue((records, limited,))
-
-
-    def _queryDirectory(self, query=None, attributes=None, maxRecords=0):
-
-        startTime = time.time()
-
-        if not attributes:
-            attributes = self.returnedAttributes
-
-        attributes = list(set(attributes + self.additionalAttributes)) # remove duplicates
-
-        directoryAndRecordTypes = []
-        if self.peopleDirectory == self.userDirectory:
-            # use single ds query if possible for best performance
-            directoryAndRecordTypes.append((self.peopleDirectory, self.peopleNode, (dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers)))
-        else:
-            if self.peopleDirectory:
-                directoryAndRecordTypes.append((self.peopleDirectory, self.peopleNode, dsattributes.kDSStdRecordTypePeople))
-            if self.userDirectory:
-                directoryAndRecordTypes.append((self.userDirectory, self.userNode, dsattributes.kDSStdRecordTypeUsers))
-
-        allResults = []
-        for directory, node, recordType in directoryAndRecordTypes:
-            try:
-                if query:
-                    if isinstance(query, dsquery.match) and query.value is not "":
-                        self.log.debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r,%r)" % (
-                            node,
-                            query.attribute,
-                            query.value,
-                            query.matchType,
-                            False,
-                            recordType,
-                            attributes,
-                            maxRecords,
-                        ))
-                        results = list(
-                            opendirectory.queryRecordsWithAttribute_list(
-                                directory,
-                                query.attribute,
-                                query.value,
-                                query.matchType,
-                                False,
-                                recordType,
-                                attributes,
-                                maxRecords,
-                            ))
-                    else:
-                        self.log.debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r)" % (
-                            node,
-                            query.generate(),
-                            False,
-                            recordType,
-                            attributes,
-                            maxRecords,
-                        ))
-                        results = list(
-                            opendirectory.queryRecordsWithAttributes_list(
-                                directory,
-                                query.generate(),
-                                False,
-                                recordType,
-                                attributes,
-                                maxRecords,
-                            ))
-                else:
-                    self.log.debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r,%r)" % (
-                        node,
-                        recordType,
-                        attributes,
-                        maxRecords,
-                    ))
-                    results = list(
-                        opendirectory.listAllRecordsWithAttributes_list(
-                            directory,
-                            recordType,
-                            attributes,
-                            maxRecords,
-                        ))
-            except opendirectory.ODError, ex:
-                self.log.error("Open Directory (node=%s) error: %s" % (self.realmName, str(ex)))
-                raise
-
-            allResults.extend(results)
-
-            if maxRecords:
-                maxRecords -= len(results)
-                if maxRecords <= 0:
-                    break
-
-        elaspedTime = time.time() - startTime
-        self.log.info("Timing: Directory query: %.1f ms (%d records, %.2f records/sec)" % (elaspedTime * 1000, len(allResults), len(allResults) / elaspedTime))
-        return succeed(allResults)
-
-
-    def _getDSFilter(self, addressBookFilter):
-        """
-        Convert the supplied addressbook-query into an expression tree.
-
-        @param filter: the L{Filter} for the addressbook-query to convert.
-        @return: (needsAllRecords, espressionAttributes, expression) tuple
-        """
-        def propFilterListQuery(filterAllOf, propFilters):
-
-            def propFilterExpression(filterAllOf, propFilter):
-                #print("propFilterExpression")
-                """
-                Create an expression for a single prop-filter element.
-
-                @param propFilter: the L{PropertyFilter} element.
-                @return: (needsAllRecords, espressionAttributes, expressions) tuple
-                """
-
-                def definedExpression(defined, allOf, filterName, constant, queryAttributes, allAttrStrings):
-                    if constant or filterName in ("N" , "FN", "UID",):
-                        return (defined, [], [])     # all records have this property so no records do not have it
-                    else:
-                        matchList = list(set([dsquery.match(attrName, "", dsattributes.eDSStartsWith) for attrName in allAttrStrings]))
-                        if defined:
-                            return andOrExpression(allOf, queryAttributes, matchList)
-                        else:
-                            if len(matchList) > 1:
-                                expr = dsquery.expression(dsquery.expression.OR, matchList)
-                            else:
-                                expr = matchList
-                            return (False, queryAttributes, [dsquery.expression(dsquery.expression.NOT, expr), ])
-                    #end isNotDefinedExpression()
-
-
-                def andOrExpression(propFilterAllOf, queryAttributes, matchList):
-                    #print("andOrExpression(propFilterAllOf=%r, queryAttributes%r, matchList%r)" % (propFilterAllOf, queryAttributes, matchList))
-                    if propFilterAllOf and len(matchList):
-                        # add OR expression because parent will AND
-                        return (False, queryAttributes, [dsquery.expression(dsquery.expression.OR, matchList), ])
-                    else:
-                        return (False, queryAttributes, matchList)
-                    #end andOrExpression()
-
-
-                # short circuit parameter filters
-                def supportedParamter(filterName, paramFilters, propFilterAllOf):
-
-                    def supported(paramFilterName, paramFilterDefined, params):
-                        paramFilterName = paramFilterName.upper()
-                        if len(params.keys()) and ((paramFilterName in params.keys()) != paramFilterDefined):
-                            return False
-                        if len(params[paramFilterName]) and str(paramFilter.qualifier).upper() not in params[paramFilterName]:
-                            return False
-                        return True
-                        #end supported()
-
-                    oneSupported = False
-                    for paramFilter in paramFilters:
-                        if filterName == "PHOTO":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"ENCODING": ["B", ], "TYPE": ["JPEG", ], }):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "ADR":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"TYPE": ["WORK", "PREF", "POSTAL", "PARCEL", ], }):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "LABEL":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"TYPE": ["POSTAL", "PARCEL", ]}):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "TEL":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"TYPE": [], }): # has params derived from ds attributes
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "EMAIL":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"TYPE": [], }): # has params derived from ds attributes
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "URL":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {}):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif filterName == "KEY":
-                            if propFilterAllOf != supported(paramFilter.filter_name, paramFilter.defined, {"ENCODING": ["B", ], "TYPE": ["PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE", ]}):
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-                        elif not filterName.startswith("X-"): #X- IMHandles X-ABRELATEDNAMES excepted, no other params are used
-                            if propFilterAllOf == paramFilter.defined:
-                                return not propFilterAllOf
-                            oneSupported |= propFilterAllOf
-
-                    if propFilterAllOf:
-                        return True
-                    else:
-                        return oneSupported
-                    #end supportedParamter()
-
-
-                def textMatchElementExpression(propFilterAllOf, textMatchElement):
-
-                    # pre process text match strings for ds query
-                    def getMatchStrings(propFilter, matchString):
-
-                        if propFilter.filter_name in ("REV" , "BDAY",):
-                            rawString = matchString
-                            matchString = ""
-                            for c in rawString:
-                                if not c in "TZ-:":
-                                    matchString += c
-                        elif propFilter.filter_name == "GEO":
-                            matchString = ",".join(matchString.split(";"))
-
-                        if propFilter.filter_name in ("N" , "ADR", "ORG",):
-                            # for structured properties, change into multiple strings for ds query
-                            if propFilter.filter_name == "ADR":
-                                #split by newline and comma
-                                rawStrings = ",".join(matchString.split("\n")).split(",")
-                            else:
-                                #split by space
-                                rawStrings = matchString.split(" ")
-
-                            # remove empty strings
-                            matchStrings = []
-                            for oneString in rawStrings:
-                                if len(oneString):
-                                    matchStrings += [oneString, ]
-                            return matchStrings
-
-                        elif len(matchString):
-                            return [matchString, ]
-                        else:
-                            return []
-                        # end getMatchStrings
-
-                    if constant:
-                        # do the match right now!  Return either all or none.
-                        return(textMatchElement.test([constant, ]), [], [])
-                    else:
-
-                        matchStrings = getMatchStrings(propFilter, textMatchElement.text)
-
-                        if not len(matchStrings) or binaryAttrStrs:
-                            # no searching text in binary ds attributes, so change to defined/not defined case
-                            if textMatchElement.negate:
-                                return definedExpression(False, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                            # else fall through to attribute exists case below
-                        else:
-
-                            # special case UID's formed from node and record name
-                            if propFilter.filter_name == "UID":
-                                matchString = matchStrings[0]
-                                seperatorIndex = matchString.find(VCardRecord.peopleUIDSeparator)
-                                if seperatorIndex > 1:
-                                    recordNameStart = seperatorIndex + len(VCardRecord.peopleUIDSeparator)
-                                else:
-                                    seperatorIndex = matchString.find(VCardRecord.userUIDSeparator)
-                                    if seperatorIndex > 1:
-                                        recordNameStart = seperatorIndex + len(VCardRecord.userUIDSeparator)
-                                    else:
-                                        recordNameStart = sys.maxint
-
-                                if recordNameStart < len(matchString) - 1:
-                                    try:
-                                        recordNameQualifier = matchString[recordNameStart:].decode("base64").decode("utf8")
-                                    except Exception, e:
-                                        self.log.debug("Could not decode UID string %r in %r: %r" % (matchString[recordNameStart:], matchString, e,))
-                                    else:
-                                        if textMatchElement.negate:
-                                            return (False, queryAttributes,
-                                                    [dsquery.expression(dsquery.expression.NOT, dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact)), ]
-                                                    )
-                                        else:
-                                            return (False, queryAttributes,
-                                                    [dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact), ]
-                                                    )
-
-                            # use match_type where possible depending on property/attribute mapping
-                            # Note that case sensitive negate will not work
-                            #        Should return all records in that case
-                            matchType = dsattributes.eDSContains
-                            if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV", "LABEL",):
-                                if textMatchElement.match_type == "equals":
-                                        matchType = dsattributes.eDSExact
-                                elif textMatchElement.match_type == "starts-with":
-                                        matchType = dsattributes.eDSStartsWith
-                                elif textMatchElement.match_type == "ends-with":
-                                        matchType = dsattributes.eDSEndsWith
-
-                            matchList = []
-                            for matchString in matchStrings:
-                                matchList += [dsquery.match(attrName, matchString, matchType) for attrName in stringAttrStrs]
-
-                            matchList = list(set(matchList))
-
-                            if textMatchElement.negate:
-                                if len(matchList) > 1:
-                                    expr = dsquery.expression(dsquery.expression.OR, matchList)
-                                else:
-                                    expr = matchList
-                                return (False, queryAttributes, [dsquery.expression(dsquery.expression.NOT, expr), ])
-                            else:
-                                return andOrExpression(propFilterAllOf, queryAttributes, matchList)
-
-                    # attribute exists search
-                    return definedExpression(True, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                    #end textMatchElementExpression()
-
-                # get attribute strings from dsqueryAttributesForProperty list
-                queryAttributes = list(set(VCardRecord.dsqueryAttributesForProperty.get(propFilter.filter_name, [])).intersection(set(self.allowedDSQueryAttributes)))
-
-                binaryAttrStrs = []
-                stringAttrStrs = []
-                for attr in queryAttributes:
-                    if isinstance(attr, tuple):
-                        binaryAttrStrs.append(attr[0])
-                    else:
-                        stringAttrStrs.append(attr)
-                allAttrStrings = stringAttrStrs + binaryAttrStrs
-
-                constant = VCardRecord.constantProperties.get(propFilter.filter_name)
-                if not constant and not allAttrStrings:
-                    return (False, [], [])
-
-                if propFilter.qualifier and isinstance(propFilter.qualifier, IsNotDefined):
-                    return definedExpression(False, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-
-                paramFilterElements = [paramFilterElement for paramFilterElement in propFilter.filters if isinstance(paramFilterElement, ParameterFilter)]
-                textMatchElements = [textMatchElement for textMatchElement in propFilter.filters if isinstance(textMatchElement, TextMatch)]
-                propFilterAllOf = propFilter.propfilter_test == "allof"
-
-                # handle parameter filter elements
-                if len(paramFilterElements) > 0:
-                    if supportedParamter(propFilter.filter_name, paramFilterElements, propFilterAllOf):
-                        if len(textMatchElements) == 0:
-                            return definedExpression(True, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-                    else:
-                        if propFilterAllOf:
-                            return (False, [], [])
-
-                # handle text match elements
-                propFilterNeedsAllRecords = propFilterAllOf
-                propFilterAttributes = []
-                propFilterExpressionList = []
-                for textMatchElement in textMatchElements:
-
-                    textMatchNeedsAllRecords, textMatchExpressionAttributes, textMatchExpression = textMatchElementExpression(propFilterAllOf, textMatchElement)
-                    if propFilterAllOf:
-                        propFilterNeedsAllRecords &= textMatchNeedsAllRecords
-                    else:
-                        propFilterNeedsAllRecords |= textMatchNeedsAllRecords
-                    propFilterAttributes += textMatchExpressionAttributes
-                    propFilterExpressionList += textMatchExpression
-
-                if (len(propFilterExpressionList) > 1) and (filterAllOf != propFilterAllOf):
-                    propFilterExpressions = [dsquery.expression(dsquery.expression.AND if propFilterAllOf else dsquery.expression.OR , list(set(propFilterExpressionList)))] # remove duplicates
-                else:
-                    propFilterExpressions = list(set(propFilterExpressionList))
-
-                return (propFilterNeedsAllRecords, propFilterAttributes, propFilterExpressions)
-                #end propFilterExpression
-
-            #print("propFilterListQuery: filterAllOf=%r, propFilters=%r" % (filterAllOf, propFilters,))
-            """
-            Create an expression for a list of prop-filter elements.
-
-            @param filterAllOf: the C{True} if parent filter test is "allof"
-            @param propFilters: the C{list} of L{ComponentFilter} elements.
-            @return: (needsAllRecords, espressionAttributes, expression) tuple
-            """
-            needsAllRecords = filterAllOf
-            attributes = []
-            expressions = []
-            for propFilter in propFilters:
-
-                propNeedsAllRecords, propExpressionAttributes, propExpression = propFilterExpression(filterAllOf, propFilter)
-                if filterAllOf:
-                    needsAllRecords &= propNeedsAllRecords
-                else:
-                    needsAllRecords |= propNeedsAllRecords
-                attributes += propExpressionAttributes
-                expressions += propExpression
-
-            if len(expressions) > 1:
-                expr = dsquery.expression(dsquery.expression.AND if filterAllOf else dsquery.expression.OR , list(set(expressions))) # remove duplicates
-            elif len(expressions):
-                expr = expressions[0]
-            else:
-                expr = None
-
-            return (needsAllRecords, attributes, expr)
-
-        #print("_getDSFilter")
-        # Lets assume we have a valid filter from the outset
-
-        # Top-level filter contains zero or more prop-filters
-        if addressBookFilter:
-            filterAllOf = addressBookFilter.filter_test == "allof"
-            if len(addressBookFilter.children) > 0:
-                return propFilterListQuery(filterAllOf, addressBookFilter.children)
-            else:
-                return (filterAllOf, [], [])
-        else:
-            return (False, [], [])
-
-
-    def _attributesForAddressBookQuery(self, addressBookQuery):
-
-        propertyNames = []
-        #print( "addressBookQuery.qname=%r" % addressBookQuery.qname)
-        if addressBookQuery.qname() == ("DAV:", "prop"):
-
-            for property in addressBookQuery.children:
-                #print("property = %r" % property )
-                if isinstance(property, carddavxml.AddressData):
-                    for addressProperty in property.children:
-                        #print("addressProperty = %r" % addressProperty )
-                        if isinstance(addressProperty, carddavxml.Property):
-                            #print("Adding property %r", addressProperty.attributes["name"])
-                            propertyNames.append(addressProperty.attributes["name"])
-
-                elif not self.fakeETag and property.qname() == ("DAV:", "getetag"):
-                    # for a real etag == md5(vCard), we need all attributes
-                    propertyNames = None
-                    break
-
-        if not len(propertyNames):
-            #print("using all attributes")
-            return self.returnedAttributes
-
-        else:
-            propertyNames.append("X-INTERNAL-MINIMUM-VCARD-PROPERTIES") # these properties are required to make a vCard
-            queryAttributes = []
-            for prop in propertyNames:
-                if prop in VCardRecord.dsqueryAttributesForProperty:
-                    #print("adding attributes %r" % VCardRecord.dsqueryAttributesForProperty.get(prop))
-                    queryAttributes += VCardRecord.dsqueryAttributesForProperty.get(prop)
-
-            return list(set(queryAttributes).intersection(set(self.returnedAttributes)))
-
-
-    @inlineCallbacks
-    def cacheVCardsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults):
-        """
-        Cache the vCards for a given addressBookFilder and addressBookQuery
-        """
-        startTime = time.time()
-        #print("Timing: cacheVCardsForAddressBookQuery.starttime=%f" % startTime)
-
-        allRecords, filterAttributes, dsFilter = self._getDSFilter(addressBookFilter)
-        #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-
-        if allRecords:
-            dsFilter = None #  None expression == all Records
-        clear = not allRecords and not dsFilter
-
-        #get unique list of requested attributes
-        if clear:
-            attributes = None
-        else:
-            queryAttributes = self._attributesForAddressBookQuery(addressBookQuery)
-            attributes = filterAttributes + queryAttributes
-
-        #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
-        maxRecords = int(maxResults * 1.2)
-        if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
-            maxRecords = self.maxDSQueryRecords
-
-        updateLock, limited = (yield self._refreshCache(reschedule=False, query=dsFilter, attributes=attributes, keepLock=True, clear=clear, maxRecords=maxRecords))
-
-        elaspedTime = time.time() - startTime
-        self.log.info("Timing: Cache fill: %.1f ms" % (elaspedTime * 1000,))
-
-        returnValue((updateLock, limited))
-
-
-    @inlineCallbacks
-    def vCardRecordsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults):
-        """
-        Get vCards for a given addressBookFilder and addressBookQuery
-        """
-
-        allRecords, filterAttributes, dsFilter = self._getDSFilter(addressBookFilter)
-        #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-
-        # testing:
-        # allRecords = True
-
-        if allRecords:
-            dsFilter = None #  None expression == all Records
-        clear = not allRecords and not dsFilter
-
-        queryRecords = []
-        limited = False
-
-        if not clear:
-            queryAttributes = self._attributesForAddressBookQuery(addressBookQuery)
-            attributes = filterAttributes + queryAttributes
-
-            #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
-            maxRecords = int(maxResults * 1.2)
-            if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
-                maxRecords = self.maxDSQueryRecords
-
-            records, limited = (yield self._getDirectoryRecords(dsFilter, attributes, maxRecords))
-
-            #filter out bad records --- should only happen during development
-            for record in records.values():
-                try:
-                    vCardText = record.vCardText()
-                except:
-                    traceback.print_exc()
-                    self.log.info("Could not get vcard for record %s" % (record,))
-                else:
-                    if not record.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation).startswith("/Local"):
-                        self.log.debug("VCard text =\n%s" % (vCardText,))
-                    queryRecords.append(record)
-
-        returnValue((queryRecords, limited,))
-
-
-
-class VCardRecord(DirectoryRecord, DAVPropertyMixIn):
-    """
-    Open Directory implementation of L{IDirectoryRecord}.
-    """
-
-    # od attributes that may contribute to vcard properties
-    # will be used to translate vCard queries to od queries
-
-    dsqueryAttributesForProperty = {
-
-        "FN" : [
-               dsattributes.kDS1AttrFirstName,
-               dsattributes.kDS1AttrLastName,
-               dsattributes.kDS1AttrMiddleName,
-               dsattributes.kDSNAttrNamePrefix,
-               dsattributes.kDSNAttrNameSuffix,
-               dsattributes.kDS1AttrDistinguishedName,
-               dsattributes.kDSNAttrRecordName,
-               ],
-        "N" : [
-               dsattributes.kDS1AttrFirstName,
-               dsattributes.kDS1AttrLastName,
-               dsattributes.kDS1AttrMiddleName,
-               dsattributes.kDSNAttrNamePrefix,
-               dsattributes.kDSNAttrNameSuffix,
-               dsattributes.kDS1AttrDistinguishedName,
-               dsattributes.kDSNAttrRecordName,
-               ],
-        "NICKNAME" : [
-                dsattributes.kDSNAttrNickName,
-                ],
-        # no binary searching
-        "PHOTO" : [
-                (dsattributes.kDSNAttrJPEGPhoto, "base64"),
-                ],
-        "BDAY" : [
-                dsattributes.kDS1AttrBirthday,
-                ],
-        "ADR" : [
-                dsattributes.kDSNAttrBuilding,
-                dsattributes.kDSNAttrStreet,
-                dsattributes.kDSNAttrCity,
-                dsattributes.kDSNAttrState,
-                dsattributes.kDSNAttrPostalCode,
-                dsattributes.kDSNAttrCountry,
-                ],
-        "LABEL" : [
-                dsattributes.kDSNAttrPostalAddress,
-                dsattributes.kDSNAttrPostalAddressContacts,
-                dsattributes.kDSNAttrAddressLine1,
-                dsattributes.kDSNAttrAddressLine2,
-                dsattributes.kDSNAttrAddressLine3,
-                ],
-         "TEL" : [
-                dsattributes.kDSNAttrPhoneNumber,
-                dsattributes.kDSNAttrMobileNumber,
-                dsattributes.kDSNAttrPagerNumber,
-                dsattributes.kDSNAttrHomePhoneNumber,
-                dsattributes.kDSNAttrPhoneContacts,
-                dsattributes.kDSNAttrFaxNumber,
-                #dsattributes.kDSNAttrAreaCode,
-                ],
-         "EMAIL" : [
-                dsattributes.kDSNAttrEMailAddress,
-                dsattributes.kDSNAttrEMailContacts,
-                ],
-         "GEO" : [
-                dsattributes.kDSNAttrMapCoordinates,
-                ],
-         "TITLE" : [
-                dsattributes.kDSNAttrJobTitle,
-                ],
-         "ORG" : [
-                dsattributes.kDSNAttrCompany,
-                dsattributes.kDSNAttrOrganizationName,
-                dsattributes.kDSNAttrDepartment,
-                ],
-         "NOTE" : [
-                dsattributes.kDS1AttrComment,
-                dsattributes.kDS1AttrNote,
-                ],
-         "REV" : [
-                dsattributes.kDS1AttrModificationTimestamp,
-                ],
-         "UID" : [
-                dsattributes.kDS1AttrGeneratedUID,
-                # special cased
-                #dsattributes.kDSNAttrMetaNodeLocation,
-                #dsattributes.kDSNAttrRecordName,
-                #dsattributes.kDS1AttrDistinguishedName,
-                ],
-         "URL" : [
-                dsattributes.kDS1AttrWeblogURI,
-                dsattributes.kDSNAttrURL,
-                ],
-         "KEY" : [
-                # check on format, are these all binary?
-                (dsattributes.kDSNAttrPGPPublicKey, "base64"),
-                (dsattributes.kDS1AttrUserCertificate, "base64"),
-                (dsattributes.kDS1AttrUserPKCS12Data, "base64"),
-                (dsattributes.kDS1AttrUserSMIMECertificate, "base64"),
-                ],
-         # too bad this is not one X-Attribute with params.     Would make searching easier
-         "X-AIM" : [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-JABBER" : [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-MSN" : [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-YAHOO" : [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-ICQ" : [
-                dsattributes.kDSNAttrIMHandle,
-                ],
-         "X-ABRELATEDNAMES" : [
-                dsattributes.kDSNAttrRelationships,
-                ],
-          "X-INTERNAL-MINIMUM-VCARD-PROPERTIES" : [
-                dsattributes.kDS1AttrGeneratedUID,
-                dsattributes.kDSNAttrMetaNodeLocation,
-                dsattributes.kDS1AttrFirstName,
-                 dsattributes.kDS1AttrLastName,
-                dsattributes.kDS1AttrMiddleName,
-                   dsattributes.kDSNAttrNamePrefix,
-                  dsattributes.kDSNAttrNameSuffix,
-                 dsattributes.kDS1AttrDistinguishedName,
-                dsattributes.kDSNAttrRecordName,
-                dsattributes.kDSNAttrRecordType,
-                dsattributes.kDS1AttrModificationTimestamp,
-                dsattributes.kDS1AttrCreationTimestamp,
-                ],
-          "X-INTERNAL-REQUIRED" : [
-                dsattributes.kDS1AttrGeneratedUID,
-                dsattributes.kDSNAttrMetaNodeLocation,
-                 dsattributes.kDS1AttrDistinguishedName,
-                dsattributes.kDSNAttrRecordName,
-                dsattributes.kDS1AttrFirstName,
-                 dsattributes.kDS1AttrLastName,
-                dsattributes.kDSNAttrRecordType,
-                ],
-
-    }
-
-    allDSQueryAttributes = sorted(list(set([attr for lookupAttributes in dsqueryAttributesForProperty.values()
-                                      for attr in lookupAttributes])))
-
-    binaryDSAttributeStrs = [attr[0] for attr in allDSQueryAttributes
-                                if isinstance(attr, tuple)]
-
-    stringDSAttributeStrs = [attr for attr in allDSQueryAttributes
-                                if isinstance(attr, str)]
-
-    allDSAttributeStrs = stringDSAttributeStrs + binaryDSAttributeStrs
-
-    #peopleUIDSeparator = "-" + OpenDirectoryBackingService.baseGUID + "-"
-    userUIDSeparator = "-bf07a1a2-"
-    peopleUIDSeparator = "-cf07a1a2-"
-
-    constantProperties = {
-        # 3.6.3 PRODID Type Definition
-        "PRODID": vCardProductID,
-        # 3.6.9 VERSION Type Definition
-        "VERSION": "3.0",
-        }
-
-
-    def __init__(self, service, recordAttributes, defaultNodeName=None):
-
-        self.log.debug("service=%s, attributes=%s" % (service, recordAttributes))
-
-        #save off for debugging
-        if service.addDSAttrXProperties:
-            self.originalAttributes = recordAttributes.copy()
-
-        self.directoryBackedAddressBook = service.directoryBackedAddressBook
-        self._vCard = None
-        self._vCardText = None
-        self._uriName = None
-        self._hRef = None
-
-        self.attributes = {}
-        for key, values in recordAttributes.items():
-            if key in VCardRecord.stringDSAttributeStrs:
-                if isinstance(values, list):
-                    self.attributes[key] = [removeControlChars(val).decode("utf8") for val in values]
-                else:
-                    self.attributes[key] = removeControlChars(values).decode("utf8")
-            else:
-                self.attributes[key] = values
-
-        # fill in  missing essential attributes used for filtering
-        fullName = self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)
-        if not fullName:
-            fullName = self.firstValueForAttribute(dsattributes.kDSNAttrRecordName)
-            self.attributes[dsattributes.kDS1AttrDistinguishedName] = fullName
-
-        node = self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation)
-
-        # use a better node name -- makes better synthetic GUIDS
-        if not node or node == "/LDAPv3/127.0.0.1":
-            node = defaultNodeName if defaultNodeName else service.realmName
-            self.attributes[dsattributes.kDSNAttrMetaNodeLocation] = node
-
-        guid = self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)
-        if not guid:
-            if service.standardizeSyntheticUIDs:
-                nodeUUIDStr = "00000000"
-            else:
-                nodeUUIDStr = "%x" % abs(hash(node))
-            nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("utf8").encode("base64").split("\n"))
-            if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
-                guid = VCardRecord.userUIDSeparator.join([nodeUUIDStr, nameUUIDStr, ])
-            else:
-                guid = VCardRecord.peopleUIDSeparator.join([nodeUUIDStr, nameUUIDStr, ])
-
-        # since guid is used as file name, normalize so uid uniqueness == fine name uniqueness
-        #guid = "/".join(guid.split(":")).upper()
-        self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
-
-        if self.firstValueForAttribute(dsattributes.kDS1AttrLastName) == "99":
-            del self.attributes[dsattributes.kDS1AttrLastName]
-
-        if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
-            recordType = DirectoryService.recordType_users
-        else:
-            recordType = DirectoryService.recordType_people
-
-        super(VCardRecord, self).__init__(
-            service=service,
-            recordType=recordType,
-            guid=guid,
-            shortNames=tuple(self.valuesForAttribute(dsattributes.kDSNAttrRecordName)),
-            fullName=fullName,
-            firstName=self.firstValueForAttribute(dsattributes.kDS1AttrFirstName, None),
-            lastName=self.firstValueForAttribute(dsattributes.kDS1AttrLastName, None),
-            emailAddresses=(),
-            calendarUserAddresses=(),
-            autoSchedule=False,
-            enabledForCalendaring=False,
-        )
-
-
-    def __repr__(self):
-        return "<%s[%s(%s)] %s(%s) %r>" % (
-            self.__class__.__name__,
-            self.firstValueForAttribute(dsattributes.kDSNAttrRecordType),
-            self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation),
-            self.guid,
-            self.shortNames,
-            self.fullName
-        )
-
-
-    def __hash__(self):
-        s = "".join([
-              "%s:%s" % (attribute, self.valuesForAttribute(attribute),)
-              for attribute in self.attributes
-              ])
-        return hash(s)
-
-    """
-    def nextFileName(self):
-        self.renameCounter += 1
-        self.fileName = self.baseFileName + "-" + str(self.renameCounter)
-        self.fileNameLower = self.fileName.lower()
-    """
-
-    def hasAttribute(self, attributeName):
-        return self.valuesForAttribute(attributeName, None) is not None
-
-
-    def valuesForAttribute(self, attributeName, default_values=[]):
-        values = self.attributes.get(attributeName)
-        if (values is None):
-            return default_values
-        elif not isinstance(values, list):
-            values = [values, ]
-
-        # ds templates often return empty attribute values
-        #     get rid of them here
-        nonEmptyValues = [(value.encode("utf-8") if isinstance(value, unicode) else value) for value in values if len(value) > 0]
-
-        if len(nonEmptyValues) > 0:
-            return nonEmptyValues
-        else:
-            return default_values
-
-
-    def firstValueForAttribute(self, attributeName, default_value=""):
-        values = self.attributes.get(attributeName)
-        if values is None:
-            return default_value
-        elif isinstance(values, list):
-            return values[0].encode("utf_8") if isinstance(values[0], unicode) else values[0]
-        else:
-            return values.encode("utf_8") if isinstance(values, unicode) else values
-
-
-    def joinedValuesForAttribute(self, attributeName, separator=",", default_string=""):
-        values = self.valuesForAttribute(attributeName, None)
-        if not values:
-            return default_string
-        else:
-            return separator.join(values)
-
-
-    def isoDateStringForDateAttribute(self, attributeName, default_string=""):
-        modDate = self.firstValueForAttribute(attributeName, default_string)
-        revDate = None
-        if modDate:
-            if len(modDate) >= len("YYYYMMDD") and modDate[:8].isdigit():
-                revDate = "%s-%s-%s" % (modDate[:4], modDate[4:6], modDate[6:8],)
-            if len(modDate) >= len("YYYYMMDDHHMMSS") and modDate[8:14].isdigit():
-                revDate += "T%s:%s:%sZ" % (modDate[8:10], modDate[10:12], modDate[12:14],)
-        return revDate
-
-
-    def vCard(self):
-
-
-        def generateVCard():
-
-            def isUniqueProperty(vcard, newProperty, ignoreParams=None):
-                existingProperties = vcard.properties(newProperty.name())
-                for existingProperty in existingProperties:
-                    if ignoreParams:
-                        existingProperty = existingProperty.duplicate()
-                        for paramname, paramvalue in ignoreParams:
-                            existingProperty.removeParameterValue(paramname, paramvalue)
-                    if existingProperty == newProperty:
-                        return False
-                return True
-
-            def addUniqueProperty(vcard, newProperty, ignoreParams=None, attrType=None, attrValue=None):
-                if isUniqueProperty(vcard, newProperty, ignoreParams):
-                    vcard.addProperty(newProperty)
-                else:
-                    if attrType and attrValue:
-                        self.log.info("Ignoring attribute %r with value %r in creating property %r. A duplicate property already exists." % (attrType, attrValue, newProperty,))
-
-            def addPropertyAndLabel(groupCount, label, propertyName, propertyValue, parameters=None):
-                groupCount[0] += 1
-                groupPrefix = "item%d" % groupCount[0]
-                vcard.addProperty(Property(propertyName, propertyValue, params=parameters, group=groupPrefix))
-                vcard.addProperty(Property("X-ABLabel", label, group=groupPrefix))
-
-            # for attributes of the form  param:value
-            def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, defaultLabel, nolabelParamTypes, labelMap, attrType):
-                preferred = True
-                for attrValue in self.valuesForAttribute(attrType):
-                    try:
-                        # special case for Apple
-                        if self.service.appleInternalServer and attrType == dsattributes.kDSNAttrIMHandle:
-                            splitValue = attrValue.split("|")
-                            if len(splitValue) > 1:
-                                attrValue = splitValue[0]
-
-                        colonIndex = attrValue.find(":")
-                        if (colonIndex > len(attrValue) - 2):
-                            raise ValueError("Nothing after colon.")
-
-                        propertyValue = attrValue[colonIndex + 1:]
-                        labelString = attrValue[:colonIndex] if colonIndex > 0 else defaultLabel
-                        paramTypeString = labelString.upper()
-
-                        # add PREF to first prop's parameters
-                        paramTypeStrings = [paramTypeString, ]
-                        if preferred and "PREF" != paramTypeString:
-                            paramTypeStrings += ["PREF", ]
-                        parameters = {"TYPE": paramTypeStrings, }
-
-                        #special case for IMHandles which the param is the last part of the property like X-AIM or X-JABBER
-                        if propertyPrefix:
-                            propertyName = propertyPrefix + paramTypeString
-
-                        # only add label prop if needed
-                        if paramTypeString in nolabelParamTypes:
-                            addUniqueProperty(vcard, Property(propertyName, attrValue[colonIndex + 1:], params=parameters), None, attrValue, attrType)
-                        else:
-                            # use special localizable addressbook labels where possible
-                            abLabelString = labelMap.get(labelString, labelString)
-                            addPropertyAndLabel(groupCount, abLabelString, propertyName, propertyValue, parameters)
-                        preferred = False
-
-                    except Exception, e:
-                        traceback.print_exc()
-                        self.log.debug("addPropertiesAndLabelsForPrefixedAttribute(): groupCount=%r, propertyPrefix=%r, propertyName=%r, nolabelParamTypes=%r, labelMap=%r, attrType=%r" % (groupCount[0], propertyPrefix, propertyName, nolabelParamTypes, labelMap, attrType,))
-                        self.log.error("addPropertiesAndLabelsForPrefixedAttribute(): Trouble parsing attribute %s, with value \"%s\".  Error = %s" % (attrType, attrValue, e,))
-
-            #print("VCardRecord.vCard")
-            # create vCard
-            vcard = Component("VCARD")
-            groupCount = [0]
-
-            # add constant properties - properties that are the same regardless of the record attributes
-            for key, value in VCardRecord.constantProperties.items():
-                vcard.addProperty(Property(key, value))
-
-            # 3.1 IDENTIFICATION TYPES http://tools.ietf.org/html/rfc2426#section-3.1
-            # 3.1.1 FN Type Definition
-            # dsattributes.kDS1AttrDistinguishedName,      # Users distinguished or real name
-            #
-            # full name is required but this is set in OpenDiretoryBackingRecord.__init__
-            #vcard.addProperty(Property("FN", self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)))
-
-            # 3.1.2 N Type Definition
-            # dsattributes.kDS1AttrFirstName,            # Used for first name of user or person record.
-            # dsattributes.kDS1AttrLastName,            # Used for the last name of user or person record.
-            # dsattributes.kDS1AttrMiddleName,            # Used for the middle name of user or person record.
-            # dsattributes.kDSNAttrNameSuffix,            # Represents the name suffix of a user or person.
-                                                        #      ie. Jr., Sr., etc.
-                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrNamePrefix,            # Represents the title prefix of a user or person.
-                                                        #      ie. Mr., Ms., Mrs., Dr., etc.
-                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        #      dsattributes.kDSStdRecordTypePeople).
-
-            # name is required, so make sure we have one
-            # vcard says: Each name attribute can be a string or a list of strings.
-            if not self.hasAttribute(dsattributes.kDS1AttrFirstName) and not self.hasAttribute(dsattributes.kDS1AttrLastName):
-                familyName = self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)
-            else:
-                familyName = self.valuesForAttribute(dsattributes.kDS1AttrLastName, "")
-
-            nameObject = N(
-                first=self.valuesForAttribute(dsattributes.kDS1AttrFirstName, ""),
-                last=familyName,
-                middle=self.valuesForAttribute(dsattributes.kDS1AttrMiddleName, ""),
-                prefix=self.valuesForAttribute(dsattributes.kDSNAttrNamePrefix, ""),
-                suffix=self.valuesForAttribute(dsattributes.kDSNAttrNameSuffix, ""),
-            )
-            vcard.addProperty(Property("N", nameObject))
-
-            # set full name to Name with contiguous spaces stripped
-            # it turns out that Address Book.app ignores FN and creates it fresh from N in ABRecord
-            # so no reason to have FN distinct from N
-            vcard.addProperty(Property("FN", nameObject.getFullName()))
-
-            # 3.1.3 NICKNAME Type Definition
-            # dsattributes.kDSNAttrNickName,            # Represents the nickname of a user or person.
-                                                        #    Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        #    dsattributes.kDSStdRecordTypePeople).
-            for nickname in self.valuesForAttribute(dsattributes.kDSNAttrNickName):
-                addUniqueProperty(vcard, Property("NICKNAME", nickname), None, dsattributes.kDSNAttrNickName, nickname)
-
-            # 3.1.4 PHOTO Type Definition
-            # dsattributes.kDSNAttrJPEGPhoto,            # Used to store binary picture data in JPEG format.
-                                                        #      Usually found in user, people or group records (kDSStdRecordTypeUsers,
-                                                        #      dsattributes.kDSStdRecordTypePeople,dsattributes.kDSStdRecordTypeGroups).
-            # pyOpenDirectory always returns binary-encoded string
-
-            for photo in self.valuesForAttribute(dsattributes.kDSNAttrJPEGPhoto):
-                addUniqueProperty(vcard, Property("PHOTO", photo, params={"ENCODING": ["b", ], "TYPE": ["JPEG", ], }), None, dsattributes.kDSNAttrJPEGPhoto, photo)
-
-            # 3.1.5 BDAY Type Definition
-            # dsattributes.kDS1AttrBirthday,            # Single-valued attribute that defines the user's birthday.
-                                                        #      Format is x.208 standard YYYYMMDDHHMMSSZ which we will require as GMT time.
-                                                        #                               012345678901234
-
-            birthdate = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrBirthday)
-            if birthdate:
-                vcard.addProperty(Property("BDAY", DateTime.parseText(birthdate, fullISO=True)))
-
-            # 3.2 Delivery Addressing Types http://tools.ietf.org/html/rfc2426#section-3.2
-            #
-            # 3.2.1 ADR Type Definition
-
-            #address
-            # vcard says: Each address attribute can be a string or a list of strings.
-            extended = self.valuesForAttribute(dsattributes.kDSNAttrBuilding, "")
-            street = self.valuesForAttribute(dsattributes.kDSNAttrStreet, "")
-            city = self.valuesForAttribute(dsattributes.kDSNAttrCity, "")
-            region = self.valuesForAttribute(dsattributes.kDSNAttrState, "")
-            code = self.valuesForAttribute(dsattributes.kDSNAttrPostalCode, "")
-            country = self.valuesForAttribute(dsattributes.kDSNAttrCountry, "")
-
-            if len(extended) > 0 or len(street) > 0 or len(city) > 0 or len(region) > 0 or len(code) > 0 or len(country) > 0:
-                vcard.addProperty(Property("ADR",
-                    Adr(
-                        #pobox = box,
-                        extended=extended,
-                        street=street,
-                        locality=city,
-                        region=region,
-                        postalcode=code,
-                        country=country,
-                    ),
-                    params={"TYPE": ["WORK", "PREF", "POSTAL", "PARCEL", ], }
-                ))
-
-            # 3.2.2 LABEL Type Definition
-
-            # dsattributes.kDSNAttrPostalAddress,            # The postal address usually excluding postal code.
-            # dsattributes.kDSNAttrPostalAddressContacts,    # multi-valued attribute that defines a record's alternate postal addresses .
-                                                            #      found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
-            # dsattributes.kDSNAttrAddressLine1,            # Line one of multiple lines of address data for a user.
-            # dsattributes.kDSNAttrAddressLine2,            # Line two of multiple lines of address data for a user.
-            # dsattributes.kDSNAttrAddressLine3,            # Line three of multiple lines of address data for a user.
-
-            for label in self.valuesForAttribute(dsattributes.kDSNAttrPostalAddress):
-                addUniqueProperty(vcard, Property("LABEL", label, params={"TYPE": ["POSTAL", "PARCEL", ]}), None, dsattributes.kDSNAttrPostalAddress, label)
-
-            for label in self.valuesForAttribute(dsattributes.kDSNAttrPostalAddressContacts):
-                addUniqueProperty(vcard, Property("LABEL", label, params={"TYPE": ["POSTAL", "PARCEL", ]}), None, dsattributes.kDSNAttrPostalAddressContacts, label)
-
-            address = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine1)
-            addressLine2 = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine2)
-            if len(addressLine2) > 0:
-                address += "\n" + addressLine2
-            addressLine3 = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine3)
-            if len(addressLine3) > 0:
-                address += "\n" + addressLine3
-
-            if len(address) > 0:
-                vcard.addProperty(Property("LABEL", address, params={"TYPE": ["POSTAL", "PARCEL", ]}))
-
-            # 3.3 TELECOMMUNICATIONS ADDRESSING TYPES http://tools.ietf.org/html/rfc2426#section-3.3
-            # 3.3.1 TEL Type Definition
-            #          TEL;TYPE=work,voice,pref,msg:+1-213-555-1234
-
-            # dsattributes.kDSNAttrPhoneNumber,            # Telephone number of a user.
-            # dsattributes.kDSNAttrMobileNumber,        # Represents the mobile numbers of a user or person.
-                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrFaxNumber,            # Represents the FAX numbers of a user or person.
-                                                        # Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        # kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrPagerNumber,            # Represents the pager numbers of a user or person.
-                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
-                                                        #      dsattributes.kDSStdRecordTypePeople).
-            # dsattributes.kDSNAttrHomePhoneNumber,        # Home telephone number of a user or person.
-            # dsattributes.kDSNAttrPhoneContacts,        # multi-valued attribute that defines a record's custom phone numbers .
-                                                        #      found in user records (kDSStdRecordTypeUsers).
-                                                        #      Example: home fax:408-555-4444
-
-            params = {"TYPE": ["WORK", "PREF", "VOICE", ], }
-            for phone in self.valuesForAttribute(dsattributes.kDSNAttrPhoneNumber):
-                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrPhoneNumber)
-                params = {"TYPE": ["WORK", "VOICE", ], }
-
-            params = {"TYPE": ["WORK", "PREF", "CELL", ], }
-            for phone in self.valuesForAttribute(dsattributes.kDSNAttrMobileNumber):
-                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrMobileNumber)
-                params = {"TYPE": ["WORK", "CELL", ], }
-
-            params = {"TYPE": ["WORK", "PREF", "FAX", ], }
-            for phone in self.valuesForAttribute(dsattributes.kDSNAttrFaxNumber):
-                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrFaxNumber)
-                params = {"TYPE": ["WORK", "FAX", ], }
-
-            params = {"TYPE": ["WORK", "PREF", "PAGER", ], }
-            for phone in self.valuesForAttribute(dsattributes.kDSNAttrPagerNumber):
-                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrPagerNumber)
-                params = {"TYPE": ["WORK", "PAGER", ], }
-
-            params = {"TYPE": ["HOME", "PREF", "VOICE", ], }
-            for phone in self.valuesForAttribute(dsattributes.kDSNAttrHomePhoneNumber):
-                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrHomePhoneNumber)
-                params = {"TYPE": ["HOME", "VOICE", ], }
-
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "TEL", "work",
-                                                        ["VOICE", "CELL", "FAX", "PAGER", ], {},
-                                                        dsattributes.kDSNAttrPhoneContacts,)
-
-            """
-            # EXTEND:  Use this attribute
-            # dsattributes.kDSNAttrAreaCode,            # Area code of a user's phone number.
-            """
-
-            # 3.3.2 EMAIL Type Definition
-            # dsattributes.kDSNAttrEMailAddress,        # Email address of usually a user record.
-
-            # setup some params
-            preferredWorkParams = {"TYPE": ["WORK", "PREF", "INTERNET", ], }
-            workParams = {"TYPE": ["WORK", "INTERNET", ], }
-            params = preferredWorkParams
-            for emailAddress in self.valuesForAttribute(dsattributes.kDSNAttrEMailAddress):
-                addUniqueProperty(vcard, Property("EMAIL", emailAddress, params=params), (("TYPE", "PREF"),), emailAddress, dsattributes.kDSNAttrEMailAddress)
-                params = workParams
-
-            # dsattributes.kDSNAttrEMailContacts,        # multi-valued attribute that defines a record's custom email addresses .
-                                                        #    found in user records (kDSStdRecordTypeUsers).
-                                                        #      Example: home:johndoe at mymail.com
-
-            # check to see if parameters type are open ended. Could be any string
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "EMAIL", "work",
-                                                        ["WORK", "HOME", ], {},
-                                                        dsattributes.kDSNAttrEMailContacts,)
-
-            """
-            # UNIMPLEMENTED:
-            # 3.3.3 MAILER Type Definition
-            """
-            # 3.4 GEOGRAPHICAL TYPES http://tools.ietf.org/html/rfc2426#section-3.4
-            """
-            # UNIMPLEMENTED:
-            # 3.4.1 TZ Type Definition
-            """
-            # 3.4.2 GEO Type Definition
-            #dsattributes.kDSNAttrMapCoordinates,        # attribute that defines coordinates for a user's location .
-                                                        #      Found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
-                                                        #      Example: 7.7,10.6
-            for coordinate in self.valuesForAttribute(dsattributes.kDSNAttrMapCoordinates):
-                parts = coordinate.split(",")
-                if (len(parts) == 2):
-                    vcard.addProperty(Property("GEO", parts))
-                else:
-                    self.log.info("Ignoring malformed attribute %r with value %r. Well-formed example: 7.7,10.6." % (dsattributes.kDSNAttrMapCoordinates, coordinate))
-            #
-            # 3.5 ORGANIZATIONAL TYPES http://tools.ietf.org/html/rfc2426#section-3.5
-            #
-            # 3.5.1 TITLE Type Definition
-            for jobTitle in self.valuesForAttribute(dsattributes.kDSNAttrJobTitle):
-                addUniqueProperty(vcard, Property("TITLE", jobTitle), None, dsattributes.kDSNAttrJobTitle, jobTitle)
-
-            """
-            # UNIMPLEMENTED:
-            # 3.5.2 ROLE Type Definition
-            # 3.5.3 LOGO Type Definition
-            # 3.5.4 AGENT Type Definition
-            """
-            # 3.5.5 ORG Type Definition
-            company = self.joinedValuesForAttribute(dsattributes.kDSNAttrCompany)
-            if len(company) == 0:
-                company = self.joinedValuesForAttribute(dsattributes.kDSNAttrOrganizationName)
-            department = self.joinedValuesForAttribute(dsattributes.kDSNAttrDepartment)
-            extra = self.joinedValuesForAttribute(dsattributes.kDSNAttrOrganizationInfo)
-            if len(company) > 0 or len(department) > 0:
-                vcard.addProperty(Property("ORG", (company, department, extra,),))
-
-            # 3.6 EXPLANATORY TYPES http://tools.ietf.org/html/rfc2426#section-3.6
-            """
-            # UNIMPLEMENTED:
-            # 3.6.1 CATEGORIES Type Definition
-            """
-            # 3.6.2 NOTE Type Definition
-            # dsattributes.kDS1AttrComment,                  # Attribute used for unformatted comment.
-            # dsattributes.kDS1AttrNote,                  # Note attribute. Commonly used in printer records.
-            for comment in self.valuesForAttribute(dsattributes.kDS1AttrComment):
-                addUniqueProperty(vcard, Property("NOTE", comment), None, dsattributes.kDS1AttrComment, comment)
-
-            for note in self.valuesForAttribute(dsattributes.kDS1AttrNote):
-                addUniqueProperty(vcard, Property("NOTE", note), None, dsattributes.kDS1AttrNote, note)
-
-            # 3.6.3 PRODID Type Definition
-            #vcard.addProperty(Property("PRODID", vCardProductID + "//BUILD %s" % twistedcaldav.__version__))
-            #vcard.addProperty(Property("PRODID", vCardProductID))
-            # ADDED WITH CONTSTANT PROPERTIES
-
-            # 3.6.4 REV Type Definition
-            revDate = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrModificationTimestamp)
-            if revDate:
-                vcard.addProperty(Property("REV", DateTime.parseText(revDate, fullISO=True)))
-
-            """
-            # UNIMPLEMENTED:
-            # 3.6.5 SORT-STRING Type Definition
-            # 3.6.6 SOUND Type Definition
-            """
-            # 3.6.7 UID Type Definition
-            # dsattributes.kDS1AttrGeneratedUID,        # Used for 36 character (128 bit) unique ID. Usually found in user,
-                                                        #      group, and computer records. An example value is "A579E95E-CDFE-4EBC-B7E7-F2158562170F".
-                                                        #      The standard format contains 32 hex characters and four hyphen characters.
-            # !! don't use self.guid which is URL encoded
-            vcard.addProperty(Property("UID", self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)))
-
-            # 3.6.8 URL Type Definition
-            # dsattributes.kDSNAttrURL,                    # List of URLs.
-            # dsattributes.kDS1AttrWeblogURI,            # Single-valued attribute that defines the URI of a user's weblog.
-                                                        #     Usually found in user records (kDSStdRecordTypeUsers).
-                                                        #      Example: http://example.com/blog/jsmith
-            for url in self.valuesForAttribute(dsattributes.kDS1AttrWeblogURI):
-                addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters={"TYPE": ["Weblog", ]})
-
-            for url in self.valuesForAttribute(dsattributes.kDSNAttrURL):
-                addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters={"TYPE": ["Homepage", ]})
-
-            # 3.6.9 VERSION Type Definition
-            # ALREADY ADDED
-
-            # 3.7 SECURITY TYPES http://tools.ietf.org/html/rfc2426#section-3.7
-            # 3.7.1 CLASS Type Definition
-            # ALREADY ADDED
-
-            # 3.7.2 KEY Type Definition
-
-            # dsattributes.kDSNAttrPGPPublicKey,        # Pretty Good Privacy public encryption key.
-            # dsattributes.kDS1AttrUserCertificate,        # Attribute containing the binary of the user's certificate.
-                                                        #       Usually found in user records. The certificate is data which identifies a user.
-                                                        #       This data is attested to by a known party, and can be independently verified
-                                                        #       by a third party.
-            # dsattributes.kDS1AttrUserPKCS12Data,        # Attribute containing binary data in PKCS #12 format.
-                                                        #       Usually found in user records. The value can contain keys, certificates,
-                                                        #      and other related information and is encrypted with a passphrase.
-            # dsattributes.kDS1AttrUserSMIMECertificate,# Attribute containing the binary of the user's SMIME certificate.
-                                                        #       Usually found in user records. The certificate is data which identifies a user.
-                                                        #       This data is attested to by a known party, and can be independently verified
-                                                        #       by a third party. SMIME certificates are often used for signed or encrypted
-                                                        #       emails.
-
-            for key in self.valuesForAttribute(dsattributes.kDSNAttrPGPPublicKey):
-                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["PGPPublicKey", ]}), None, dsattributes.kDSNAttrPGPPublicKey, key)
-
-            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserCertificate):
-                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserCertificate", ]}), None, dsattributes.kDS1AttrUserCertificate, key)
-
-            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserPKCS12Data):
-                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserPKCS12Data", ]}), None, dsattributes.kDS1AttrUserPKCS12Data, key)
-
-            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserSMIMECertificate):
-                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserSMIMECertificate", ]}), None, dsattributes.kDS1AttrUserSMIMECertificate, key)
-
-            """
-            X- attributes, Address Book support
-            """
-            # X-AIM, X-JABBER, X-MSN, X-YAHOO, X-ICQ
-            # instant messaging
-            # dsattributes.kDSNAttrIMHandle,            # Represents the Instant Messaging handles of a user.
-                                                        #      Values should be prefixed with the appropriate IM type
-                                                        #       ie. AIM:, Jabber:, MSN:, Yahoo:, or ICQ:
-                                                        #       Usually found in user records (kDSStdRecordTypeUsers).
-
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, "X-", None, "aim",
-                                                        ["AIM", "JABBER", "MSN", "YAHOO", "ICQ"],
-                                                        {},
-                                                        dsattributes.kDSNAttrIMHandle,)
-
-            # X-ABRELATEDNAMES
-            # dsattributes.kDSNAttrRelationships,        #      multi-valued attribute that defines the relationship to the record type .
-                                                        #      found in user records (kDSStdRecordTypeUsers).
-                                                        #      Example: brother:John
-            addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "X-ABRELATEDNAMES", "friend",
-                                                        [],
-                                                        {"FATHER": "_$!<Father>!$_",
-                                                         "MOTHER": "_$!<Mother>!$_",
-                                                         "PARENT": "_$!<Parent>!$_",
-                                                         "BROTHER": "_$!<Brother>!$_",
-                                                         "SISTER": "_$!<Sister>!$_",
-                                                         "CHILD": "_$!<Child>!$_",
-                                                         "FRIEND": "_$!<Friend>!$_",
-                                                         "SPOUSE": "_$!<Spouse>!$_",
-                                                         "PARTNER": "_$!<Partner>!$_",
-                                                         "ASSISTANT": "_$!<Assistant>!$_",
-                                                         "MANAGER": "_$!<Manager>!$_", },
-                                                        dsattributes.kDSNAttrRelationships,)
-
-            # special case for Apple
-            if self.service.appleInternalServer:
-                for manager in self.valuesForAttribute("dsAttrTypeNative:appleManager"):
-                    splitManager = manager.split("|")
-                    if len(splitManager) >= 4:
-                        managerValue = "%s %s, %s" % (splitManager[0], splitManager[1], splitManager[3],)
-                    elif len(splitManager) >= 2:
-                        managerValue = "%s %s" % (splitManager[0], splitManager[1])
-                    else:
-                        managerValue = manager
-                    addPropertyAndLabel(groupCount, "_$!<Manager>!$_", "X-ABRELATEDNAMES", managerValue, parameters={"TYPE": ["Manager", ]})
-
-            """
-            # UNIMPLEMENTED: X- attributes
-
-            X-MAIDENNAME
-            X-PHONETIC-FIRST-NAME
-            X-PHONETIC-MIDDLE-NAME
-            X-PHONETIC-LAST-NAME
-
-            sattributes.kDS1AttrPicture,                # Represents the path of the picture for each user displayed in the login window.
-                                                        #      Found in user records (kDSStdRecordTypeUsers).
-
-            dsattributes.kDS1AttrMapGUID,                # Represents the GUID for a record's map.
-            dsattributes.kDSNAttrMapURI,                # attribute that defines the URI of a user's location.
-
-            dsattributes.kDSNAttrOrganizationInfo,        # Usually the organization info of a user.
-            dsattributes.kDSNAttrAreaCode,                # Area code of a user's phone number.
-
-            dsattributes.kDSNAttrMIME,                    # Data contained in this attribute type is a fully qualified MIME Type.
-
-            """
-
-            # debug, create x attributes for all ds attributes
-            if self.service.addDSAttrXProperties:
-                for attribute in self.originalAttributes:
-                    for value in self.valuesForAttribute(attribute):
-                        vcard.addProperty(Property("X-" + "-".join(attribute.split(":")), removeControlChars(value)))
-
-            return vcard
-
-        if not self._vCard:
-            self._vCard = generateVCard()
-
-        return self._vCard
-
-
-    def vCardText(self):
-        if not self._vCardText:
-            self._vCardText = str(self.vCard())
-
-        return self._vCardText
-
-
-    def uriName(self):
-        if not self._uriName:
-            self._uriName = self.vCard().getProperty("UID").value() + ".vcf"
-        #print("uriName():self._uriName=%s" % self._uriName)
-        return self._uriName
-
-
-    def hRef(self, parentURI="/directory/"):
-        if not self._hRef:
-            self._hRef = davxml.HRef.fromString(joinURL(parentURI, self.uriName()))
-
-        return self._hRef
-
-
-    def readProperty(self, property, request):
-
-        if type(property) is tuple:
-            qname = property
-        else:
-            qname = property.qname()
-
-        namespace, name = qname
-
-        #print("VCardResource.readProperty: qname = %s" % (qname, ))
-
-        if namespace == dav_namespace:
-            if name == "resourcetype":
-                result = davxml.ResourceType.empty #@UndefinedVariable
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "getetag":
-                result = davxml.GETETag(ETag(hashlib.md5(self.vCardText()).hexdigest()).generate())
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "getcontenttype":
-                mimeType = MimeType('text', 'vcard', {})
-                result = davxml.GETContentType(generateContentType(mimeType))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "getcontentlength":
-                result = davxml.GETContentLength.fromString(str(len(self.vCardText())))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "getlastmodified":
-                if self.vCard().hasProperty("REV"):
-                    modDatetime = parse_date(self.vCard().propertyValue("REV"))
-                else:
-                    # use creation date attribute if it exists
-                    creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
-                    if creationDateString:
-                        modDatetime = parse_date(creationDateString)
-                    else:
-                        modDatetime = datetime.datetime.utcnow()
-
-                #strip time zone because time zones are unimplemented in davxml.GETLastModified.fromDate
-                d = modDatetime.date()
-                t = modDatetime.time()
-                modDatetimeNoTZ = datetime.datetime(d.year, d.month, d.day, t.hour, t.minute, t.second, t.microsecond, None)
-                result = davxml.GETLastModified.fromDate(modDatetimeNoTZ)
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "creationdate":
-                creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
-                if creationDateString:
-                    creationDatetime = parse_date(creationDateString)
-                elif self.vCard().hasProperty("REV"):    # use modification date property if it exists
-                    creationDatetime = parse_date(self.vCard().propertyValue("REV"))
-                else:
-                    creationDatetime = datetime.datetime.utcnow()
-                result = davxml.CreationDate.fromDate(creationDatetime)
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-            elif name == "displayname":
-                # AddressBook.app uses N. Use FN or UID instead?
-                result = davxml.DisplayName.fromString(self.vCard().propertyValue("N"))
-                #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
-                return result
-
-        elif namespace == twisted_dav_namespace:
-            return super(VCardRecord, self).readProperty(property, request)
-            #return DAVPropertyMixIn.readProperty(self, property, request)
-
-        return self.directoryBackedAddressBook.readProperty(property, request)
-
-
-    def listProperties(self, request):
-        #print("VCardResource.listProperties()")
-        qnames = set(self.liveProperties())
-
-        # Add dynamic live properties that exist
-        dynamicLiveProperties = (
-            (dav_namespace, "quota-available-bytes"),
-            (dav_namespace, "quota-used-bytes"),
-        )
-        for dqname in dynamicLiveProperties:
-            #print("VCardResource.listProperties: removing dqname=%s" % (dqname,))
-            qnames.remove(dqname)
-
-        for qname in self.deadProperties().list():
-            if (qname not in qnames) and (qname[0] != twisted_private_namespace):
-                #print("listProperties: adding qname=%s" % (qname,))
-                qnames.add(qname)
-
-        #for qn in qnames: print("VCardResource.listProperties: qn=%s" % (qn,))
-
-        yield qnames
-
-    listProperties = deferredGenerator(listProperties)
-
-
-
-# utility
-#remove control characters because vCard does not support them
-def removeControlChars(utf8String):
-    result = utf8String
-    for a in utf8String:
-        if '\x00' <= a <= '\x1F':
-            result = ""
-            for c in utf8String:
-                if '\x00' <= c <= '\x1F':
-                    pass
-                else:
-                    result += c
-    #if utf8String != result: print ("changed %r to %r" % (utf8String, result))
-    return result

Deleted: CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/test/test_opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/test/test_opendirectorybacker.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directory/test/test_opendirectorybacker.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -1,55 +0,0 @@
-##
-# Copyright (c) 2011-2014 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-try:
-    from twistedcaldav.directory.opendirectorybacker import VCardRecord
-except ImportError:
-    pass
-else:
-    from twistedcaldav.test.util import TestCase
-
-    class VCardRecordTestCase(TestCase):
-
-
-        def test_multiplePhoneNumbersAndEmailAddresses(self):
-            attributes = {
-                u'dsAttrTypeStandard:AppleMetaRecordName': ['uid=odtestamanda,cn=users,dc=dalek,dc=example,dc=com'],
-                u'dsAttrTypeStandard:ModificationTimestamp': '20111017170937Z',
-                u'dsAttrTypeStandard:PhoneNumber': ['408 555-1212', '415 555-1212'],
-                u'dsAttrTypeStandard:RecordType': ['dsRecTypeStandard:Users'],
-                u'dsAttrTypeStandard:AppleMetaNodeLocation': ['/LDAPv3/127.0.0.1'],
-                u'dsAttrTypeStandard:RecordName': ['odtestamanda'],
-                u'dsAttrTypeStandard:FirstName': 'Amanda',
-                u'dsAttrTypeStandard:GeneratedUID': '9DC04A70-E6DD-11DF-9492-0800200C9A66',
-                u'dsAttrTypeStandard:LastName': 'Test',
-                u'dsAttrTypeStandard:CreationTimestamp': '20110927182945Z',
-                u'dsAttrTypeStandard:EMailAddress': ['amanda at example.com', 'second at example.com'],
-                u'dsAttrTypeStandard:RealName': 'Amanda Test',
-            }
-            vcardRecord = VCardRecord(StubService(), attributes)
-            vcard = vcardRecord.vCard()
-            properties = set([prop.value() for prop in vcard.properties("TEL")])
-            self.assertEquals(properties, set(["408 555-1212", "415 555-1212"]))
-            properties = set([prop.value() for prop in vcard.properties("EMAIL")])
-            self.assertEquals(properties, set(["amanda at example.com", "second at example.com"]))
-
-
-
-    class StubService(object):
-        addDSAttrXProperties = False
-        directoryBackedAddressBook = None
-        appleInternalServer = False
-        realmName = "testing"

Modified: CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directorybackedaddressbook.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directorybackedaddressbook.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/directorybackedaddressbook.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -23,17 +23,23 @@
 ]
 
 from twext.python.log import Logger
+from twext.who.expression import Operand, MatchType, MatchFlags, \
+    MatchExpression, CompoundExpression
+from twext.who.idirectory import FieldName, RecordType
+from twisted.internet.defer import succeed, inlineCallbacks, maybeDeferred, \
+    returnValue
+from twisted.python.constants import NamedConstant
+from twistedcaldav import carddavxml
+from twistedcaldav.config import config
+from twistedcaldav.resource import CalDAVResource
+from txdav.carddav.datastore.query.filter import IsNotDefined, TextMatch, \
+    ParameterFilter
+from txdav.who.idirectory import FieldName as CalFieldName, \
+    RecordType as CalRecordType
+from txdav.xml import element as davxml
 from txweb2 import responsecode
-from txdav.xml import element as davxml
 from txweb2.dav.resource import TwistedACLInheritable
 from txweb2.http import HTTPError, StatusResponse
-
-from twisted.internet.defer import succeed, inlineCallbacks, maybeDeferred, returnValue
-from twisted.python.reflect import namedClass
-
-from twistedcaldav.config import config
-from twistedcaldav.resource import CalDAVResource
-
 import uuid
 
 log = Logger()
@@ -45,66 +51,31 @@
     Directory-backed address book
     """
 
-    def __init__(self, principalCollections):
+    def __init__(self, principalCollections, principalDirectory, uri):
 
         CalDAVResource.__init__(self, principalCollections=principalCollections)
 
-        self.directory = None       # creates directory attribute
+        self.principalDirectory = principalDirectory
+        self.uri = uri
+        self.directory = None
 
-        # create with permissions, similar to CardDAVOptions in tap.py
-        # FIXME:  /Directory does not need to be in file system unless debug-only caching options are used
-#        try:
-#            os.mkdir(path)
-#            os.chmod(path, 0750)
-#            if config.UserName and config.GroupName:
-#                import pwd
-#                import grp
-#                uid = pwd.getpwnam(config.UserName)[2]
-#                gid = grp.getgrnam(config.GroupName)[2]
-#                os.chown(path, uid, gid)
-#
-#            log.info("Created %s" % (path,))
-#
-#        except (OSError,), e:
-#            # this is caused by multiprocessor race and is harmless
-#            if e.errno != errno.EEXIST:
-#                raise
 
-
     def makeChild(self, name):
-        from twistedcaldav.simpleresource import SimpleCalDAVResource
-        return SimpleCalDAVResource(principalCollections=self.principalCollections())
+        return self.directory
 
 
     def provisionDirectory(self):
         if self.directory is None:
-            directoryClass = namedClass(config.DirectoryAddressBook.type)
-
             log.info(
-                "Configuring: {t}:{p}",
-                t=config.DirectoryAddressBook.type,
-                p=config.DirectoryAddressBook.params,
-            )
+                "Setting search directory to {principalDirectory}",
+                principalDirectory=self.principalDirectory)
+            self.directory = self.principalDirectory
+            # future: instantiate another directory based on /Search/Contacts (?)
 
-            #add self as "directoryBackedAddressBook" parameter
-            params = config.DirectoryAddressBook.params.copy()
-            params["directoryBackedAddressBook"] = self
-
-            try:
-                self.directory = directoryClass(params)
-            except ImportError, e:
-                log.error("Unable to set up directory address book: %s" % (e,))
-                return succeed(None)
-
-            return self.directory.createCache()
-
-            #print ("DirectoryBackedAddressBookResource.provisionDirectory: provisioned")
-
         return succeed(None)
 
 
     def defaultAccessControlList(self):
-        #print( "DirectoryBackedAddressBookResource.defaultAccessControlList" )
         if config.AnonymousDirectoryAddressBookAccess:
             # DAV:Read for all principals (includes anonymous)
             accessPrincipal = davxml.All()
@@ -152,7 +123,6 @@
 
 
     def isAddressBookCollection(self):
-        #print( "DirectoryBackedAddressBookResource.isAddressBookCollection: return True" )
         return True
 
 
@@ -169,21 +139,1320 @@
     def renderHTTP(self, request):
         if not self.directory:
             raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE, "Service is starting up"))
-        elif self.directory.liveQuery:
-            response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
-            returnValue(response)
+
+        response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
+        returnValue(response)
+
+
+    @inlineCallbacks
+    def doAddressBookDirectoryQuery(self, addressBookFilter, addressBookQuery, maxResults):
+        """
+        Get vCards for a given addressBookFilter and addressBookQuery
+        """
+
+        log.debug("doAddressBookDirectoryQuery: directory={directory} addressBookFilter={addressBookFilter}, addressBookQuery={addressBookQuery}, maxResults={maxResults}",
+                  directory=self.directory, addressBookFilter=addressBookFilter, addressBookQuery=addressBookQuery, maxResults=maxResults)
+        results = []
+        limited = False
+        maxQueryRecords = 0
+
+        schema = {
+            RecordType.user: {
+                "FN": (
+                        FieldName.fullNames,
+                        FieldName.shortNames,
+                        ),
+                "N": (
+                        FieldName.fullNames,
+                        FieldName.shortNames,
+                        ),
+                "EMAIL": FieldName.emailAddresses,
+                "UID": FieldName.uid,
+                "ADR": CalFieldName.streetAddress,
+             },
+            RecordType.group: {
+                "FN": (
+                        FieldName.fullNames,
+                        FieldName.shortNames,
+                        ),
+                "N": (
+                        FieldName.fullNames,
+                        FieldName.shortNames,
+                        ),
+                "EMAIL": FieldName.emailAddresses,
+                "UID": FieldName.uid,
+                "ADR": CalFieldName.streetAddress,
+                # LATER "X-ADDRESSBOOKSERVER-MEMBER": FieldName.members,
+             },
+        }
+
+        recordTypeToKindMap = {
+                       RecordType.user: "individual",
+                       RecordType.group: "group",
+                       CalRecordType.location: "location",
+                       CalRecordType.resource: "device",
+                       }
+
+        allowedRecordTypes = set(self.directory.recordTypes()) & set(recordTypeToKindMap.keys()) & set(schema.keys())
+        log.debug("doAddressBookDirectoryQuery: allowedRecordTypes={allowedRecordTypes}", allowedRecordTypes=allowedRecordTypes,)
+
+        expressions = []
+        for recordType in allowedRecordTypes:
+
+            #log.debug("doAddressBookDirectoryQuery: recordType={recordType}", recordType=recordType,)
+
+            vcardPropToRecordFieldMap = schema[recordType]
+            kind = recordTypeToKindMap[recordType]
+            constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+            constantProperties["KIND"] = kind
+            # add KIND as constant so that query can be skipped if addressBookFilter needs a different kind
+
+            propNames, expression = expressionFromABFilter(addressBookFilter, vcardPropToRecordFieldMap, recordType=recordType, constantProperties=constantProperties)
+            log.debug("doAddressBookDirectoryQuery: recordType={recordType}, expression={expression!r}, propNames={propNames}", recordType=recordType, expression=expression, propNames=propNames)
+            if expression:
+                expressions.append(expression)
+
+        if expressions:
+            if expressions > 1:
+                expression = CompoundExpression(expressions, Operand.OR)
+            else:
+                expression = expressions[0]
+
+            maxRecords = int(maxResults * 1.2)
+
+            # keep trying query till we get results based on filter.  Especially when doing "all results" query
+            while True:
+
+                log.debug("doAddressBookDirectoryQuery: expression={expression!r}, ", expression=expression)
+
+                records = yield self.directory.recordsFromExpression(expression)
+                log.debug("doAddressBookDirectoryQuery: #records={n}, records={records!r}", n=len(records), records=records)
+                queryLimited = False
+
+                vCardsResults = []#[ABDirectoryQueryResult(self, record) for record in records]
+
+                filteredResults = []
+                for vCardResult in vCardsResults:
+                    if addressBookFilter.match(vCardResult.vCard()):
+                        filteredResults.append(vCardResult)
+                    else:
+                        log.debug("doAddressBookQuery: vCard did not match filter: {vCard}", vcard=vCardResult.vCard())
+
+                #no more results
+                if not queryLimited:
+                    break
+
+                # more than requested results
+                if maxResults and len(filteredResults) >= maxResults:
+                    break
+
+                # more than max report results
+                if len(filteredResults) >= config.MaxQueryWithDataResults:
+                    break
+
+                # more than self limit
+                if maxQueryRecords and maxRecords >= maxQueryRecords:
+                    break
+
+                # try again with 2x
+                maxRecords *= 2
+                if maxQueryRecords and maxRecords > maxQueryRecords:
+                    maxRecords = maxQueryRecords
+
+            results = sorted(list(filteredResults), key=lambda result: result.vCard().propertyValue("UID"))
+            limited = maxResults and len(results) >= maxResults
+
+        log.info("limited={l} result count={n}", l=limited, n=len(results))
+        returnValue((results, limited,))
+
+
+
+def propertiesInAddressBookQuery(addressBookQuery):
+    """
+    Get the vCard properties requested by a given query
+    """
+
+    etagRequested = False
+    propertyNames = []
+    if addressBookQuery.qname() == ("DAV:", "prop"):
+
+        for property in addressBookQuery.children:
+            if isinstance(property, carddavxml.AddressData):
+                for addressProperty in property.children:
+                    if isinstance(addressProperty, carddavxml.Property):
+                        propertyNames += [addressProperty.attributes["name"], ]
+
+            elif property.qname() == ("DAV:", "getetag"):
+                # for a real etag == md5(vCard), we need all properties
+                etagRequested = True
+
+    return (etagRequested, propertyNames if len(propertyNames) else None)
+
+
+
+def expressionFromABFilter(addressBookFilter, vcardPropToSearchableFieldMap, recordType, constantProperties={}):
+    """
+    Convert the supplied addressbook-query into a ds expression tree.
+
+    @param addressBookFilter: the L{Filter} for the addressbook-query to convert.
+    @param vcardPropToSearchableFieldMap: a mapping from vcard properties to searchable query attributes.
+    @param constantProperties: a mapping of constant properties.  A query on a constant property will return all or None
+    @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+    """
+
+    def propFilterListQuery(filterAllOf, propFilters):
+
+        def combineExpressionLists(expressionList, allOf, addedExpressions):
+            """
+            deal with the 4-state logic
+                addedExpressions=None means ignore
+                addedExpressions=True means all records
+                addedExpressions=False means no records
+                addedExpressions=[expressionlist] add to expression list
+            """
+            if expressionList is None:
+                expressionList = addedExpressions
+            elif addedExpressions is not None:
+                if addedExpressions is True:
+                    if not allOf:
+                        expressionList = True  # expressionList or True is True
+                    #else  expressionList and True is expressionList
+                elif addedExpressions is False:
+                    if allOf:
+                        expressionList = False  # expressionList and False is False
+                    #else expressionList or False is expressionList
+                else:
+                    if expressionList is False:
+                        if not allOf:
+                            expressionList = addedExpressions  # False or addedExpressions is addedExpressions
+                        #else False and addedExpressions is False
+                    elif expressionList is True:
+                        if allOf:
+                            expressionList = addedExpressions  # False or addedExpressions is addedExpressions
+                        #else False and addedExpressions is False
+                    else:
+                        expressionList += addedExpressions
+            return expressionList
+
+
+        def propFilterExpression(filterAllOf, propFilter):
+            """
+            Create an expression for a single prop-filter element.
+
+            @param propFilter: the L{PropertyFilter} element.
+            @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+            """
+
+            def definedExpression(defined, allOf):
+                if constant or propFilter.filter_name in ("N" , "FN", "UID", "SOURCE",):
+                    return defined  # all records have this property so no records do not have it
+                else:
+                    if defined:
+                        matchList = [MatchExpression(fieldName, u"", MatchType.startsWith) for fieldName in searchableFields]
+                    else:
+                        # this may generate inefficient LDAP query stirng
+                        matchList = [MatchExpression(fieldName, u"", MatchType.startsWith, MatchFlags.NOT) for fieldName in searchableFields]
+                    return andOrExpression(allOf, matchList)
+
+            def andOrExpression(propFilterAllOf, matchList):
+                matchList = list(set(matchList))
+                if propFilterAllOf and len(matchList) > 1:
+                    # add OR expression because parent will AND
+                    return [CompoundExpression(matchList, Operand.OR), ]
+                else:
+                    return matchList
+
+
+            def paramFilterElementExpression(propFilterAllOf, paramFilterElement): #@UnusedVariable
+
+                params = ABDirectoryQueryResult.vcardPropToParamMap.get(propFilter.filter_name.upper())
+                defined = params and paramFilterElement.filter_name.upper() in params
+
+                #defined test
+                if defined != paramFilterElement.defined:
+                    return False
+
+                #parameter value text match
+                if defined and paramFilterElement.filters:
+                    paramValues = params[paramFilterElement.filter_name.upper()]
+                    if paramValues and paramFilterElement.filters[0].text.upper() not in paramValues:
+                        return False
+
+                return True
+
+
+            def textMatchElementExpression(propFilterAllOf, textMatchElement):
+
+                # pre process text match strings for ds query
+                def getMatchStrings(propFilter, matchString):
+
+                    if propFilter.filter_name in ("REV" , "BDAY",):
+                        rawString = matchString
+                        matchString = ""
+                        for c in rawString:
+                            if not c in "TZ-:":
+                                matchString += c
+                    elif propFilter.filter_name == "GEO":
+                        matchString = ",".join(matchString.split(";"))
+
+                    if propFilter.filter_name in ("N" , "ADR", "ORG",):
+                        # for structured properties, change into multiple strings for ds query
+                        if propFilter.filter_name == "ADR":
+                            #split by newline and comma
+                            rawStrings = ",".join(matchString.split("\n")).split(",")
+                        else:
+                            #split by space
+                            rawStrings = matchString.split(" ")
+
+                        # remove empty strings
+                        matchStrings = []
+                        for oneString in rawStrings:
+                            if len(oneString):
+                                matchStrings += [oneString, ]
+                        return matchStrings
+
+                    elif len(matchString):
+                        return [matchString, ]
+                    else:
+                        return []
+                    # end getMatchStrings
+
+                if constant:
+                    #FIXME: match is not implemented in twisteddaldav.query.Filter.TextMatch so use _match for now
+                    return textMatchElement._match([constant, ])
+                else:
+
+                    matchStrings = getMatchStrings(propFilter, textMatchElement.text)
+
+                    if not len(matchStrings):
+                        # no searching text in binary ds attributes, so change to defined/not defined case
+                        if textMatchElement.negate:
+                            return definedExpression(False, propFilterAllOf)
+                        # else fall through to attribute exists case below
+                    else:
+
+                        # use match_type where possible depending on property/attribute mapping
+                        # FIXME: case-sensitive negate will not work.  This should return all all records in that case
+                        matchType = MatchType.contains
+                        if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV", "LABEL",):
+                            if textMatchElement.match_type == "equals":
+                                matchType = MatchType.equals
+                            elif textMatchElement.match_type == "starts-with":
+                                matchType = MatchType.startsWith
+                            elif textMatchElement.match_type == "ends-with":
+                                matchType = MatchType.endsWith
+
+                        matchList = []
+                        for matchString in matchStrings:
+                            if textMatchElement.negate:
+                                matchList = [MatchExpression(fieldName, matchString.decode("utf-8"), matchType, MatchFlags.NOT) for fieldName in searchableFields]
+                            else:
+                                matchList = [MatchExpression(fieldName, matchString.decode("utf-8"), matchType) for fieldName in searchableFields]
+                            matchList.extend(matchList)
+                        return andOrExpression(propFilterAllOf, matchList)
+
+                # attribute exists search
+                return definedExpression(True, propFilterAllOf)
+                #end textMatchElementExpression()
+
+            # searchablePropFilterAttrNames are attributes to be used by this propfilter's expression
+            searchableFields = vcardPropToSearchableFieldMap.get(propFilter.filter_name, [])
+            if isinstance(searchableFields, NamedConstant):
+                searchableFields = (searchableFields,)
+
+            constant = constantProperties.get(propFilter.filter_name)
+            if not searchableFields and not constant:
+                # not allAttrNames means propFilter.filter_name is not mapped
+                # return None to try to match all items if this is the only property filter
+                return None
+
+            #create a textMatchElement for the IsNotDefined qualifier
+            if isinstance(propFilter.qualifier, IsNotDefined):
+                textMatchElement = TextMatch(carddavxml.TextMatch.fromString(""))
+                textMatchElement.negate = True
+                propFilter.filters.append(textMatchElement)
+
+            # if only one propFilter, then use filterAllOf as propFilterAllOf to reduce subexpressions and simplify generated query string
+            if len(propFilter.filters) == 1:
+                propFilterAllOf = filterAllOf
+            else:
+                propFilterAllOf = propFilter.propfilter_test == "allof"
+
+            propFilterExpressions = None
+            for propFilterElement in propFilter.filters:
+                propFilterExpression = None
+                if isinstance(propFilterElement, ParameterFilter):
+                    propFilterExpression = paramFilterElementExpression(propFilterAllOf, propFilterElement)
+                elif isinstance(propFilterElement, TextMatch):
+                    propFilterExpression = textMatchElementExpression(propFilterAllOf, propFilterElement)
+                propFilterExpressions = combineExpressionLists(propFilterExpressions, propFilterAllOf, propFilterExpression)
+                if isinstance(propFilterExpressions, bool) and propFilterAllOf != propFilterExpression:
+                    break
+
+            if isinstance(propFilterExpressions, list):
+                propFilterExpressions = list(set(propFilterExpressions))
+                if propFilterExpressions and (filterAllOf != propFilterAllOf):
+                    propFilterExpressions = [CompoundExpression(propFilterExpressions, Operand.AND if propFilterAllOf else Operand.OR)]
+
+            return propFilterExpressions
+            #end propFilterExpression
+
+        """
+        Create an expression for a list of prop-filter elements.
+
+        @param filterAllOf: the C{True} if parent filter test is "allof"
+        @param propFilters: the C{list} of L{ComponentFilter} elements.
+        @return: (filterProperyNames, expressions) tuple.  expression==True means list all results, expression==False means no results
+        """
+        expressions = None
+        for propFilter in propFilters:
+
+            propExpressions = propFilterExpression(filterAllOf, propFilter)
+            expressions = combineExpressionLists(expressions, filterAllOf, propExpressions)
+
+            # early loop exit
+            if isinstance(expressions, bool) and filterAllOf != expressions:
+                break
+
+        # convert to needsAllRecords to return
+        log.debug("expressionFromABFilter: expressions={q!r}", q=expressions,)
+        if isinstance(expressions, list):
+            expressions = list(set(expressions))
+            if len(expressions) > 1:
+                expr = CompoundExpression(expressions, Operand.AND if filterAllOf else Operand.OR)
+            elif len(expressions):
+                expr = expressions[0]
+            else:
+                expr = not filterAllOf  # empty expression list. should not happen
+        elif expressions is None:
+            expr = not filterAllOf
         else:
-            available = (yield maybeDeferred(self.directory.available,))
+            # True or False
+            expr = expressions
 
-            if not available:
-                raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE, "Service is starting up"))
+        properties = [propFilter.filter_name for propFilter in propFilters]
+
+        return (tuple(set(properties)), expr)
+
+    # Assume the filter is valid
+
+    # Top-level filter contains zero or more prop-filters
+    properties = tuple()
+    expression = None
+    if addressBookFilter:
+        filterAllOf = addressBookFilter.filter_test == "allof"
+        if len(addressBookFilter.children):
+            properties, expression = propFilterListQuery(filterAllOf, addressBookFilter.children)
+        else:
+            expression = not filterAllOf
+
+    #log.debug("expressionFromABFilter: recordType={rdn!r}, expression={q!r}, properties={pn}", rdn=recordType, q=expression, pn=properties)
+    if expression:
+        recordTypeExpression = MatchExpression(FieldName.recordType, recordType, MatchType.equals)
+        if expression is True:
+            expression = recordTypeExpression
+        else:
+            expression = CompoundExpression((expression, recordTypeExpression,), Operand.AND)
+
+    #log.debug("expressionFromABFilter: expression={q!r}, properties={pn}", q=expression, pn=properties)
+    return((properties, expression))
+
+
+
+#===============================================================================
+# Taken from obsolete twistedcaldav.directory.opendirctorybacker
+# Work in Progress
+#===============================================================================
+
+from calendarserver.platform.darwin.od import dsattributes
+from pycalendar.datetime import DateTime
+from pycalendar.vcard.adr import Adr
+from pycalendar.vcard.n import N
+from twisted.internet.defer import deferredGenerator
+from twistedcaldav.vcard import Component, Property, vCardProductID
+from txdav.xml.base import twisted_dav_namespace, dav_namespace, parse_date, twisted_private_namespace
+from txweb2.dav.resource import DAVPropertyMixIn
+from txweb2.dav.util import joinURL
+from txweb2.http_headers import MimeType, generateContentType, ETag
+from xmlrpclib import datetime
+import hashlib
+
+addSourceProperty = False
+
+
+class ABDirectoryQueryResult(DAVPropertyMixIn):
+    """
+    Result from ab query report or multiget on directory
+    """
+
+    log = Logger()
+
+    # od attributes that may contribute to vcard properties
+    # will be used to translate vCard queries to od queries
+
+    vcardPropToDSAttrMap = {
+
+        "FN": [
+               dsattributes.kDS1AttrFirstName,
+               dsattributes.kDS1AttrLastName,
+               dsattributes.kDS1AttrMiddleName,
+               dsattributes.kDSNAttrNamePrefix,
+               dsattributes.kDSNAttrNameSuffix,
+               dsattributes.kDS1AttrDistinguishedName,
+               dsattributes.kDSNAttrRecordName,
+               ],
+        "N": [
+               dsattributes.kDS1AttrFirstName,
+               dsattributes.kDS1AttrLastName,
+               dsattributes.kDS1AttrMiddleName,
+               dsattributes.kDSNAttrNamePrefix,
+               dsattributes.kDSNAttrNameSuffix,
+               dsattributes.kDS1AttrDistinguishedName,
+               dsattributes.kDSNAttrRecordName,
+               ],
+        "NICKNAME": [
+                dsattributes.kDSNAttrNickName,
+                ],
+        # no binary searching
+        "PHOTO": [
+                (dsattributes.kDSNAttrJPEGPhoto, "base64"),
+                ],
+        "BDAY": [
+                dsattributes.kDS1AttrBirthday,
+                ],
+        "ADR": [
+                dsattributes.kDSNAttrBuilding,
+                dsattributes.kDSNAttrStreet,
+                dsattributes.kDSNAttrCity,
+                dsattributes.kDSNAttrState,
+                dsattributes.kDSNAttrPostalCode,
+                dsattributes.kDSNAttrCountry,
+                ],
+        "LABEL": [
+                dsattributes.kDSNAttrPostalAddress,
+                dsattributes.kDSNAttrPostalAddressContacts,
+                dsattributes.kDSNAttrAddressLine1,
+                dsattributes.kDSNAttrAddressLine2,
+                dsattributes.kDSNAttrAddressLine3,
+                ],
+         "TEL": [
+                dsattributes.kDSNAttrPhoneNumber,
+                dsattributes.kDSNAttrMobileNumber,
+                dsattributes.kDSNAttrPagerNumber,
+                dsattributes.kDSNAttrHomePhoneNumber,
+                dsattributes.kDSNAttrPhoneContacts,
+                dsattributes.kDSNAttrFaxNumber,
+                #dsattributes.kDSNAttrAreaCode,
+                ],
+         "EMAIL": [
+                dsattributes.kDSNAttrEMailAddress,
+                dsattributes.kDSNAttrEMailContacts,
+                ],
+         "GEO": [
+                dsattributes.kDSNAttrMapCoordinates,
+                ],
+         "TITLE": [
+                dsattributes.kDSNAttrJobTitle,
+                ],
+         "ORG": [
+                dsattributes.kDSNAttrCompany,
+                dsattributes.kDSNAttrOrganizationName,
+                dsattributes.kDSNAttrDepartment,
+                ],
+         "NOTE": [
+                dsattributes.kDS1AttrComment,
+                dsattributes.kDS1AttrNote,
+                ],
+         "REV": [
+                dsattributes.kDS1AttrModificationTimestamp,
+                ],
+         "UID": [
+                dsattributes.kDS1AttrGeneratedUID,
+                dsattributes.kDSNAttrRecordName,
+                ],
+         "URL": [
+                dsattributes.kDS1AttrWeblogURI,
+                dsattributes.kDSNAttrURL,
+                ],
+         "KEY": [
+                (dsattributes.kDSNAttrPGPPublicKey, "base64"),
+                (dsattributes.kDS1AttrUserCertificate, "base64"),
+                (dsattributes.kDS1AttrUserPKCS12Data, "base64"),
+                (dsattributes.kDS1AttrUserSMIMECertificate, "base64"),
+                ],
+         "IMPP": [
+                dsattributes.kDSNAttrIMHandle,
+                ],
+         "X-ABRELATEDNAMES": [
+                dsattributes.kDSNAttrRelationships,
+                ],
+         "SOURCE": [
+                dsattributes.kDS1AttrGeneratedUID,
+                dsattributes.kDSNAttrRecordName,
+                ],
+    }
+
+    allDSQueryAttributes = list(set([attr for lookupAttributes in vcardPropToDSAttrMap.values()
+                                      for attr in lookupAttributes]))
+    binaryDSAttrNames = [attr[0] for attr in allDSQueryAttributes
+                                if isinstance(attr, tuple)]
+    stringDSAttrNames = [attr for attr in allDSQueryAttributes
+                                if isinstance(attr, str)]
+    allDSAttrNames = stringDSAttrNames + binaryDSAttrNames
+
+    # all possible generated parameters.
+    vcardPropToParamMap = {
+        "PHOTO": {"ENCODING": ("B",), "TYPE": ("JPEG",), },
+        "ADR": {"TYPE": ("WORK", "PREF", "POSTAL", "PARCEL",), },
+        "LABEL": {"TYPE": ("POSTAL", "PARCEL",)},
+        "TEL": {"TYPE": None, },  # None means param can contain can be anything
+        "EMAIL": {"TYPE": None, },
+        "KEY": {"ENCODING": ("B",), "TYPE": ("PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE",)},
+        "URL": {"TYPE": ("WEBLOG", "HOMEPAGE",)},
+        "IMPP": {"TYPE": ("PREF",), "X-SERVICE-TYPE": None, },
+        "X-ABRELATEDNAMES": {"TYPE": None, },
+        "X-AIM": {"TYPE": ("PREF",), },
+        "X-JABBER": {"TYPE": ("PREF",), },
+        "X-MSN": {"TYPE": ("PREF",), },
+        "X-ICQ": {"TYPE": ("PREF",), },
+    }
+
+    uidSeparator = "-cf07a1a2-"
+
+    constantProperties = {
+        # 3.6.3 PRODID Type Definition
+        "PRODID": vCardProductID,
+        # 3.6.9 VERSION Type Definition
+        "VERSION": "3.0",
+        }
+
+
+    def __init__(self, directoryBackedAddressBook, recordAttributes,
+                 kind=None,
+                 additionalVCardProps=None,
+                 ):
+
+        self.log.debug("directoryBackedAddressBook={directoryBackedAddressBook}, attributes={attributes}, additionalVCardProps={additionalVCardProps}",
+                       directoryBackedAddressBook=directoryBackedAddressBook, attributes=recordAttributes, additionalVCardProps=additionalVCardProps,)
+
+        constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+        if additionalVCardProps:
+            for key, value in additionalVCardProps.iteritems():
+                if key not in constantProperties:
+                    constantProperties[key] = value
+        self.constantProperties = constantProperties
+        self.log.debug("directoryBackedAddressBook={directoryBackedAddressBook}, attributes={attributes}, constantProperties={constantProperties}",
+                       directoryBackedAddressBook=directoryBackedAddressBook, attributes=recordAttributes, constantProperties=self.constantProperties,)
+
+        self._directoryBackedAddressBook = directoryBackedAddressBook
+        self._vCard = None
+
+        #clean attributes
+        self.attributes = {}
+        for key, values in recordAttributes.items():
+            if key in ABDirectoryQueryResult.stringDSAttrNames:
+                if isinstance(values, list):
+                    self.attributes[key] = [removeControlChars(val).decode("utf8") for val in values]
+                else:
+                    self.attributes[key] = removeControlChars(values).decode("utf8")
             else:
-                updateLock = self.directory.updateLock()
-                yield updateLock.acquire()
-                try:
-                    response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
+                self.attributes[key] = values
 
-                finally:
-                    yield updateLock.release()
+        # find or create guid
+        guid = self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)
+        if not guid:
+            nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("base64").split("\n"))
+            guid = ABDirectoryQueryResult.uidSeparator.join(["00000000", nameUUIDStr, ])
+            #guid =  ABDirectoryQueryResult.uidSeparator.join(["d9a8e41b", nameUUIDStr,])
 
-                returnValue(response)
+            self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
+
+        if not kind:
+            dsRecordTypeToKindMap = {
+                           #dsattributes.kDSStdRecordTypePeople:"individual",
+                           #dsattributes.kDSStdRecordTypeUsers:"individual",
+                           dsattributes.kDSStdRecordTypeGroups: "group",
+                           dsattributes.kDSStdRecordTypeLocations: "location",
+                           dsattributes.kDSStdRecordTypeResources: "device",
+                           }
+            recordType = self.firstValueForAttribute(dsattributes.kDSNAttrRecordType)
+            kind = dsRecordTypeToKindMap.get(recordType, "individual")
+        self.kind = kind.lower()
+
+        #generate a vCard here.  May throw an exception
+        self.vCard()
+
+
+    def __repr__(self):
+        return "<{self.__class__.__name__}[{rn}({uid})]>".format(
+            self=self,
+            fn=self.vCard().propertyValue("FN"),
+            uid=self.vCard().propertyValue("UID")
+        )
+
+
+    def __hash__(self):
+        s = "".join([
+              "{attr}:{values}".format(attr=attribute, values=self.valuesForAttribute(attribute),)
+              for attribute in self.attributes
+              ])
+        return hash(s)
+
+
+    def hasAttribute(self, attributeName):
+        return self.valuesForAttribute(attributeName, None) is not None
+
+
+    def valuesForAttribute(self, attributeName, default_values=[]):
+        values = self.attributes.get(attributeName)
+        if (values is None):
+            return default_values
+        elif not isinstance(values, list):
+            values = [values, ]
+
+        # ds templates often return empty attribute values
+        #     get rid of them here
+        nonEmptyValues = [(value.encode("utf-8") if isinstance(value, unicode) else value) for value in values if len(value) > 0]
+
+        if len(nonEmptyValues) > 0:
+            return nonEmptyValues
+        else:
+            return default_values
+
+
+    def firstValueForAttribute(self, attributeName, default_value=""):
+        values = self.attributes.get(attributeName)
+        if values is None:
+            return default_value
+        elif isinstance(values, list):
+            return values[0].encode("utf_8") if isinstance(values[0], unicode) else values[0]
+        else:
+            return values.encode("utf_8") if isinstance(values, unicode) else values
+
+
+    def joinedValuesForAttribute(self, attributeName, separator=",", default_string=""):
+        values = self.valuesForAttribute(attributeName, None)
+        if not values:
+            return default_string
+        else:
+            return separator.join(values)
+
+
+    def isoDateStringForDateAttribute(self, attributeName, default_string=""):
+        modDate = self.firstValueForAttribute(attributeName, default_string)
+        revDate = None
+        if modDate:
+            if len(modDate) >= len("YYYYMMDD") and modDate[:8].isdigit():
+                revDate = "{YYYY}-{MM}-{DD}".format(YYYY=modDate[:4], MM=modDate[4:6], DD=modDate[6:8],)
+            if len(modDate) >= len("YYYYMMDDHHMMSS") and modDate[8:14].isdigit():
+                revDate += "T{HH}:{MM}:{SS}Z".format(HH=modDate[8:10], MM=modDate[10:12], SS=modDate[12:14],)
+        return revDate
+
+
+    def vCard(self):
+
+        def generateVCard():
+
+            def isUniqueProperty(vcard, newProperty, ignoreParams=None):
+                existingProperties = vcard.properties(newProperty.name())
+                for existingProperty in existingProperties:
+                    if ignoreParams:
+                        existingProperty = existingProperty.duplicate()
+                        for paramname, paramvalue in ignoreParams:
+                            existingProperty.removeParameterValue(paramname, paramvalue)
+                    if existingProperty == newProperty:
+                        return False
+                return True
+
+
+            def addUniqueProperty(vcard, newProperty, ignoreParams=None, attrType=None, attrValue=None):
+                if isUniqueProperty(vcard, newProperty, ignoreParams):
+                    vcard.addProperty(newProperty)
+                else:
+                    if attrType and attrValue:
+                        self.log.info("Ignoring attribute %r with value %r in creating property %r. A duplicate property already exists." % (attrType, attrValue, newProperty,))
+
+
+            def addPropertyAndLabel(groupCount, label, propertyName, propertyValue, parameters=None):
+                groupCount[0] += 1
+                groupPrefix = "item%d" % groupCount[0]
+                vcard.addProperty(Property(propertyName, propertyValue, params=parameters, group=groupPrefix))
+                vcard.addProperty(Property("X-ABLabel", label, group=groupPrefix))
+
+
+            # for attributes of the form  param:value
+            def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, attrType, defaultLabel, nolabelParamTypes=(), labelMap={}, specialParamType=None):
+                preferred = True
+                for attrValue in self.valuesForAttribute(attrType):
+                    try:
+                        colonIndex = attrValue.find(":")
+                        if (colonIndex > len(attrValue) - 2):
+                            raise ValueError("Nothing after colon.")
+
+                        propertyValue = attrValue[colonIndex + 1:]
+                        labelString = attrValue[:colonIndex] if colonIndex > 0 else defaultLabel
+                        paramTypeString = labelString.upper()
+
+                        if specialParamType:
+                            parameters = {specialParamType: (paramTypeString,)}
+                            if preferred:
+                                parameters["TYPE"] = ("PREF",)
+                        else:
+                            # add PREF to first prop's parameters
+                            paramTypeStrings = [paramTypeString, ]
+                            if preferred and "PREF" != paramTypeString:
+                                paramTypeStrings += ["PREF", ]
+                            parameters = {"TYPE": paramTypeStrings, }
+
+                        #special case for IMHandles which the param is the last part of the property like X-AIM or X-JABBER
+                        if propertyPrefix:
+                            propertyName = propertyPrefix + paramTypeString
+
+                        # only add label prop if needed
+                        if paramTypeString in nolabelParamTypes:
+                            addUniqueProperty(vcard, Property(propertyName, attrValue[colonIndex + 1:], params=parameters), None, attrValue, attrType)
+                        else:
+                            # use special localizable addressbook labels where possible
+                            localizedABLabelString = labelMap.get(labelString, labelString)
+                            addPropertyAndLabel(groupCount, localizedABLabelString, propertyName, propertyValue, parameters)
+                        preferred = False
+
+                    except Exception, e:
+                        self.log.debug(
+                            "addPropertiesAndLabelsForPrefixedAttribute(): groupCount={groupCount}, propertyPrefix={propertyPrefix}, propertyName={propertyName}, nolabelParamTypes={nolabelParamTypes}, labelMap={labelMap}, attrType={attrType}",
+                            groupCount=groupCount[0], propertyPrefix=propertyPrefix, propertyName=propertyName, nolabelParamTypes=nolabelParamTypes, labelMap=labelMap, attrType=attrType,
+                        )
+                        self.log.error(
+                            "addPropertiesAndLabelsForPrefixedAttribute(): Trouble parsing attribute {attrType}, with value \"{attrValue}\".  Error = {e}",
+                            attrType=attrType, attrValue=attrValue, e=e
+                        )
+
+            # create vCard
+            vcard = Component("VCARD")
+            groupCount = [0]
+
+            # add constant properties - properties that are the same regardless of the record attributes
+            for key, value in self.constantProperties.items():
+                vcard.addProperty(Property(key, value))
+
+            # 3.1 IDENTIFICATION TYPES http://tools.ietf.org/html/rfc2426#section-3.1
+            # 3.1.1 FN Type Definition
+            # dsattributes.kDS1AttrDistinguishedName,      # Users distinguished or real name
+            #
+            # full name is required but this is set in OpenDiretoryBackingRecord.__init__
+            #vcard.addProperty(Property("FN", self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)))
+
+            # 3.1.2 N Type Definition
+            # dsattributes.kDS1AttrFirstName,           # Used for first name of user or person record.
+            # dsattributes.kDS1AttrLastName,            # Used for the last name of user or person record.
+            # dsattributes.kDS1AttrMiddleName,          #Used for the middle name of user or person record.
+            # dsattributes.kDSNAttrNameSuffix,          # Represents the name suffix of a user or person.
+                                                        #      ie. Jr., Sr., etc.
+                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        #      dsattributes.kDSStdRecordTypePeople).
+            # dsattributes.kDSNAttrNamePrefix,          # Represents the title prefix of a user or person.
+                                                        #      ie. Mr., Ms., Mrs., Dr., etc.
+                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        #      dsattributes.kDSStdRecordTypePeople).
+
+            # name is required, so make sure we have one
+            # vcard says: Each name attribute can be a string or a list of strings.
+            if not self.hasAttribute(dsattributes.kDS1AttrFirstName) and not self.hasAttribute(dsattributes.kDS1AttrLastName):
+                familyName = self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)
+            else:
+                familyName = self.valuesForAttribute(dsattributes.kDS1AttrLastName, "")
+
+            nameObject = N(
+                first=self.valuesForAttribute(dsattributes.kDS1AttrFirstName, ""),
+                last=familyName,
+                middle=self.valuesForAttribute(dsattributes.kDS1AttrMiddleName, ""),
+                prefix=self.valuesForAttribute(dsattributes.kDSNAttrNamePrefix, ""),
+                suffix=self.valuesForAttribute(dsattributes.kDSNAttrNameSuffix, ""),
+            )
+            vcard.addProperty(Property("N", nameObject))
+
+            # set full name to Name with contiguous spaces stripped
+            # it turns out that Address Book.app ignores FN and creates it fresh from N in ABRecord
+            # so no reason to have FN distinct from N
+            vcard.addProperty(Property("FN", nameObject.getFullName()))
+
+            # 3.1.3 NICKNAME Type Definition
+            # dsattributes.kDSNAttrNickName,            # Represents the nickname of a user or person.
+                                                        #    Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        #    dsattributes.kDSStdRecordTypePeople).
+            for nickname in self.valuesForAttribute(dsattributes.kDSNAttrNickName):
+                addUniqueProperty(vcard, Property("NICKNAME", nickname), None, dsattributes.kDSNAttrNickName, nickname)
+
+            # 3.1.4 PHOTO Type Definition
+            # dsattributes.kDSNAttrJPEGPhoto,           # Used to store binary picture data in JPEG format.
+                                                        #      Usually found in user, people or group records (kDSStdRecordTypeUsers,
+                                                        #      dsattributes.kDSStdRecordTypePeople,dsattributes.kDSStdRecordTypeGroups).
+            # pyOpenDirectory always returns binary-encoded string
+
+            for photo in self.valuesForAttribute(dsattributes.kDSNAttrJPEGPhoto):
+                photo = "".join("".join(photo.split("\r")).split("\n")) # get rid of line folding: for PHOTO
+                addUniqueProperty(vcard, Property("PHOTO", photo, params={"ENCODING": ["b", ], "TYPE": ["JPEG", ], }), None, dsattributes.kDSNAttrJPEGPhoto, photo)
+
+            # 3.1.5 BDAY Type Definition
+            # dsattributes.kDS1AttrBirthday,            # Single-valued attribute that defines the user's birthday.
+                                                        #      Format is x.208 standard YYYYMMDDHHMMSSZ which we will require as GMT time.
+                                                        #                               012345678901234
+
+            birthdate = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrBirthday)
+            if birthdate:
+                vcard.addProperty(Property("BDAY", DateTime.parseText(birthdate, fullISO=True)))
+
+            # 3.2 Delivery Addressing Types http://tools.ietf.org/html/rfc2426#section-3.2
+            #
+            # 3.2.1 ADR Type Definition
+
+            #address
+            # vcard says: Each address attribute can be a string or a list of strings.
+            extended = self.valuesForAttribute(dsattributes.kDSNAttrBuilding, "")
+            street = self.valuesForAttribute(dsattributes.kDSNAttrStreet, "")
+            city = self.valuesForAttribute(dsattributes.kDSNAttrCity, "")
+            region = self.valuesForAttribute(dsattributes.kDSNAttrState, "")
+            code = self.valuesForAttribute(dsattributes.kDSNAttrPostalCode, "")
+            country = self.valuesForAttribute(dsattributes.kDSNAttrCountry, "")
+
+            if len(extended) > 0 or len(street) > 0 or len(city) > 0 or len(region) > 0 or len(code) > 0 or len(country) > 0:
+                vcard.addProperty(Property("ADR",
+                    Adr(
+                        #pobox = box,
+                        extended=extended,
+                        street=street,
+                        locality=city,
+                        region=region,
+                        postalcode=code,
+                        country=country,
+                    ),
+                    params={"TYPE": ["WORK", "PREF", "POSTAL", "PARCEL", ], }
+                ))
+
+            # 3.2.2 LABEL Type Definition
+            #
+            # dsattributes.kDSNAttrPostalAddress,           # The postal address usually excluding postal code.
+            # dsattributes.kDSNAttrPostalAddressContacts,   # multi-valued attribute that defines a record's alternate postal addresses .
+                                                            #      found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
+            # dsattributes.kDSNAttrAddressLine1,            # Line one of multiple lines of address data for a user.
+            # dsattributes.kDSNAttrAddressLine2,            # Line two of multiple lines of address data for a user.
+            # dsattributes.kDSNAttrAddressLine3,            # Line three of multiple lines of address data for a user.
+
+            for label in self.valuesForAttribute(dsattributes.kDSNAttrPostalAddress):
+                addUniqueProperty(vcard, Property("LABEL", label, params={"TYPE": ["POSTAL", "PARCEL", ]}), None, dsattributes.kDSNAttrPostalAddress, label)
+
+            for label in self.valuesForAttribute(dsattributes.kDSNAttrPostalAddressContacts):
+                addUniqueProperty(vcard, Property("LABEL", label, params={"TYPE": ["POSTAL", "PARCEL", ]}), None, dsattributes.kDSNAttrPostalAddressContacts, label)
+
+            address = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine1)
+            addressLine2 = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine2)
+            if len(addressLine2) > 0:
+                address += "\n" + addressLine2
+            addressLine3 = self.joinedValuesForAttribute(dsattributes.kDSNAttrAddressLine3)
+            if len(addressLine3) > 0:
+                address += "\n" + addressLine3
+
+            if len(address) > 0:
+                vcard.addProperty(Property("LABEL", address, params={"TYPE": ["POSTAL", "PARCEL", ]}))
+
+            # 3.3 TELECOMMUNICATIONS ADDRESSING TYPES http://tools.ietf.org/html/rfc2426#section-3.3
+            # 3.3.1 TEL Type Definition
+            #          TEL;TYPE=work,voice,pref,msg:+1-213-555-1234
+
+            # dsattributes.kDSNAttrPhoneNumber,         # Telephone number of a user.
+            # dsattributes.kDSNAttrMobileNumber,        # Represents the mobile numbers of a user or person.
+                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        #      dsattributes.kDSStdRecordTypePeople).
+            # dsattributes.kDSNAttrFaxNumber,           # Represents the FAX numbers of a user or person.
+                                                        # Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        # kDSStdRecordTypePeople).
+            # dsattributes.kDSNAttrPagerNumber,         # Represents the pager numbers of a user or person.
+                                                        #      Usually found in user or people records (kDSStdRecordTypeUsers or
+                                                        #      dsattributes.kDSStdRecordTypePeople).
+            # dsattributes.kDSNAttrHomePhoneNumber,     # Home telephone number of a user or person.
+            # dsattributes.kDSNAttrPhoneContacts,       # multi-valued attribute that defines a record's custom phone numbers .
+                                                        #      found in user records (kDSStdRecordTypeUsers).
+                                                        #      Example: home fax:408-555-4444
+
+            params = {"TYPE": ["WORK", "PREF", "VOICE", ], }
+            for phone in self.valuesForAttribute(dsattributes.kDSNAttrPhoneNumber):
+                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrPhoneNumber)
+                params = {"TYPE": ["WORK", "VOICE", ], }
+
+            params = {"TYPE": ["WORK", "PREF", "CELL", ], }
+            for phone in self.valuesForAttribute(dsattributes.kDSNAttrMobileNumber):
+                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrMobileNumber)
+                params = {"TYPE": ["WORK", "CELL", ], }
+
+            params = {"TYPE": ["WORK", "PREF", "FAX", ], }
+            for phone in self.valuesForAttribute(dsattributes.kDSNAttrFaxNumber):
+                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrFaxNumber)
+                params = {"TYPE": ["WORK", "FAX", ], }
+
+            params = {"TYPE": ["WORK", "PREF", "PAGER", ], }
+            for phone in self.valuesForAttribute(dsattributes.kDSNAttrPagerNumber):
+                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrPagerNumber)
+                params = {"TYPE": ["WORK", "PAGER", ], }
+
+            params = {"TYPE": ["HOME", "PREF", "VOICE", ], }
+            for phone in self.valuesForAttribute(dsattributes.kDSNAttrHomePhoneNumber):
+                addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrHomePhoneNumber)
+                params = {"TYPE": ["HOME", "VOICE", ], }
+
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="TEL", defaultLabel="work",
+                                                        nolabelParamTypes=("VOICE", "CELL", "FAX", "PAGER",),
+                                                        attrType=dsattributes.kDSNAttrPhoneContacts,)
+
+            """
+            # EXTEND:  Use this attribute
+            # dsattributes.kDSNAttrAreaCode,            # Area code of a user's phone number.
+            """
+
+            # 3.3.2 EMAIL Type Definition
+            # dsattributes.kDSNAttrEMailAddress,        # Email address of usually a user record.
+
+            # setup some params
+            preferredWorkParams = {"TYPE": ["WORK", "PREF", "INTERNET", ], }
+            workParams = {"TYPE": ["WORK", "INTERNET", ], }
+            params = preferredWorkParams
+            for emailAddress in self.valuesForAttribute(dsattributes.kDSNAttrEMailAddress):
+                addUniqueProperty(vcard, Property("EMAIL", emailAddress, params=params), (("TYPE", "PREF"),), emailAddress, dsattributes.kDSNAttrEMailAddress)
+                params = workParams
+
+            # dsattributes.kDSNAttrEMailContacts,       # multi-valued attribute that defines a record's custom email addresses .
+                                                        #    found in user records (kDSStdRecordTypeUsers).
+                                                        #      Example: home:johndoe at mymail.com
+
+            # check to see if parameters type are open ended. Could be any string
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="EMAIL", defaultLabel="work",
+                                                        nolabelParamTypes=("WORK", "HOME",),
+                                                        attrType=dsattributes.kDSNAttrEMailContacts,)
+
+            """
+            # UNIMPLEMENTED:
+            # 3.3.3 MAILER Type Definition
+            """
+            # 3.4 GEOGRAPHICAL TYPES http://tools.ietf.org/html/rfc2426#section-3.4
+            """
+            # UNIMPLEMENTED:
+            # 3.4.1 TZ Type Definition
+            """
+            # 3.4.2 GEO Type Definition
+            #dsattributes.kDSNAttrMapCoordinates,       # attribute that defines coordinates for a user's location .
+                                                        #      Found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
+                                                        #      Example: 7.7,10.6
+            for coordinate in self.valuesForAttribute(dsattributes.kDSNAttrMapCoordinates):
+                parts = coordinate.split(",")
+                if (len(parts) == 2):
+                    vcard.addProperty(Property("GEO", parts))
+                else:
+                    log.info("Ignoring malformed attribute %r with value %r." % (dsattributes.kDSNAttrMapCoordinates, coordinate))
+            #
+            # 3.5 ORGANIZATIONAL TYPES http://tools.ietf.org/html/rfc2426#section-3.5
+            #
+            # 3.5.1 TITLE Type Definition
+            for jobTitle in self.valuesForAttribute(dsattributes.kDSNAttrJobTitle):
+                addUniqueProperty(vcard, Property("TITLE", jobTitle), None, dsattributes.kDSNAttrJobTitle, jobTitle)
+
+            """
+            # UNIMPLEMENTED:
+            # 3.5.2 ROLE Type Definition
+            # 3.5.3 LOGO Type Definition
+            # 3.5.4 AGENT Type Definition
+            """
+            # 3.5.5 ORG Type Definition
+            company = self.joinedValuesForAttribute(dsattributes.kDSNAttrCompany)
+            if len(company) == 0:
+                company = self.joinedValuesForAttribute(dsattributes.kDSNAttrOrganizationName)
+            department = self.joinedValuesForAttribute(dsattributes.kDSNAttrDepartment)
+            extra = self.joinedValuesForAttribute(dsattributes.kDSNAttrOrganizationInfo)
+            if len(company) > 0 or len(department) > 0:
+                vcard.addProperty(Property("ORG", (company, department, extra,),))
+
+            # 3.6 EXPLANATORY TYPES http://tools.ietf.org/html/rfc2426#section-3.6
+            """
+            # UNIMPLEMENTED:
+            # 3.6.1 CATEGORIES Type Definition
+            """
+            # 3.6.2 NOTE Type Definition
+            # dsattributes.kDS1AttrComment,               # Attribute used for unformatted comment.
+            # dsattributes.kDS1AttrNote,                  # Note attribute. Commonly used in printer records.
+            notes = self.valuesForAttribute(dsattributes.kDS1AttrComment, []) + self.valuesForAttribute(dsattributes.kDS1AttrNote, [])
+            if len(notes):
+                vcard.addProperty(Property("NOTE", "\n".join(notes),))
+
+            # 3.6.3 PRODID Type Definition
+            #vcard.addProperty(Property("PRODID", vCardProductID + "//BUILD {build}".format(build=twistedcaldav.__version__))
+            #vcard.addProperty(Property("PRODID", vCardProductID))
+            # ADDED WITH CONTSTANT PROPERTIES
+
+            # 3.6.4 REV Type Definition
+            revDate = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrModificationTimestamp)
+            if revDate:
+                vcard.addProperty(Property("REV", DateTime.parseText(revDate, fullISO=True)))
+
+            """
+            # UNIMPLEMENTED:
+            # 3.6.5 SORT-STRING Type Definition
+            # 3.6.6 SOUND Type Definition
+            """
+            # 3.6.7 UID Type Definition
+            # dsattributes.kDS1AttrGeneratedUID,        # Used for 36 character (128 bit) unique ID. Usually found in user,
+                                                        #      group, and computer records. An example value is "A579E95E-CDFE-4EBC-B7E7-F2158562170F".
+                                                        #      The standard format contains 32 hex characters and four hyphen characters.
+
+            vcard.addProperty(Property("UID", self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)))
+
+            # 3.6.8 URL Type Definition
+            # dsattributes.kDSNAttrURL,                 # List of URLs.
+            # dsattributes.kDS1AttrWeblogURI,           # Single-valued attribute that defines the URI of a user's weblog.
+                                                        #     Usually found in user records (kDSStdRecordTypeUsers).
+                                                        #      Example: http://example.com/blog/jsmith
+            for url in self.valuesForAttribute(dsattributes.kDS1AttrWeblogURI):
+                addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters={"TYPE": ["WEBLOG", ]})
+
+            for url in self.valuesForAttribute(dsattributes.kDSNAttrURL):
+                addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters={"TYPE": ["HOMEPAGE", ]})
+
+
+            # 3.6.9 VERSION Type Definition
+            # ALREADY ADDED
+            #
+            # 3.7 SECURITY TYPES http://tools.ietf.org/html/rfc2426#section-3.7
+            # 3.7.1 CLASS Type Definition
+            # ALREADY ADDED
+            #
+            # 3.7.2 KEY Type Definition
+            #
+            # dsattributes.kDSNAttrPGPPublicKey,        # Pretty Good Privacy public encryption key.
+            # dsattributes.kDS1AttrUserCertificate,     # Attribute containing the binary of the user's certificate.
+                                                        #       Usually found in user records. The certificate is data which identifies a user.
+                                                        #       This data is attested to by a known party, and can be independently verified
+                                                        #       by a third party.
+            # dsattributes.kDS1AttrUserPKCS12Data,      # Attribute containing binary data in PKCS #12 format.
+                                                        #       Usually found in user records. The value can contain keys, certificates,
+                                                        #      and other related information and is encrypted with a passphrase.
+            # dsattributes.kDS1AttrUserSMIMECertificate,# Attribute containing the binary of the user's SMIME certificate.
+                                                        #       Usually found in user records. The certificate is data which identifies a user.
+                                                        #       This data is attested to by a known party, and can be independently verified
+                                                        #       by a third party. SMIME certificates are often used for signed or encrypted
+                                                        #       emails.
+
+            for key in self.valuesForAttribute(dsattributes.kDSNAttrPGPPublicKey):
+                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["PGPPublicKey", ]}), None, dsattributes.kDSNAttrPGPPublicKey, key)
+
+            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserCertificate):
+                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserCertificate", ]}), None, dsattributes.kDS1AttrUserCertificate, key)
+
+            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserPKCS12Data):
+                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserPKCS12Data", ]}), None, dsattributes.kDS1AttrUserPKCS12Data, key)
+
+            for key in self.valuesForAttribute(dsattributes.kDS1AttrUserSMIMECertificate):
+                addUniqueProperty(vcard, Property("KEY", key, params={"ENCODING": ["b", ], "TYPE": ["UserSMIMECertificate", ]}), None, dsattributes.kDS1AttrUserSMIMECertificate, key)
+
+            """
+            X- attributes, Address Book support
+            """
+            # X-AIM, X-JABBER, X-MSN, X-YAHOO, X-ICQ
+            # instant messaging
+            # dsattributes.kDSNAttrIMHandle,            # Represents the Instant Messaging handles of a user.
+                                                        #      Values should be prefixed with the appropriate IM type
+                                                        #       ie. AIM:, Jabber:, MSN:, Yahoo:, or ICQ:
+                                                        #       Usually found in user records (kDSStdRecordTypeUsers).
+            imNolabelParamTypes = ("AIM", "FACEBOOK", "GAGU-GAGU", "GOOGLE TALK", "ICQ", "JABBER", "MSN", "QQ", "SKYPE", "YAHOO",)
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix="X-", propertyName=None, defaultLabel="aim",
+                                                        nolabelParamTypes=imNolabelParamTypes,
+                                                        attrType=dsattributes.kDSNAttrIMHandle,)
+
+            # IMPP
+            # Address Book's implementation of http://tools.ietf.org/html/rfc6350#section-6.4.3
+            # adding IMPP property allows ab query report search on one property
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="IMPP", defaultLabel="aim",
+                                                        specialParamType="X-SERVICE-TYPE",
+                                                        nolabelParamTypes=imNolabelParamTypes,
+                                                        attrType=dsattributes.kDSNAttrIMHandle,)
+
+            # X-ABRELATEDNAMES
+            # dsattributes.kDSNAttrRelationships,       #      multi-valued attribute that defines the relationship to the record type .
+                                                        #      found in user records (kDSStdRecordTypeUsers).
+                                                        #      Example: brother:John
+            addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="X-ABRELATEDNAMES", defaultLabel="friend",
+                                                        labelMap={"FATHER": "_$!<Father>!$_",
+                                                            "MOTHER": "_$!<Mother>!$_",
+                                                            "PARENT": "_$!<Parent>!$_",
+                                                            "BROTHER": "_$!<Brother>!$_",
+                                                            "SISTER": "_$!<Sister>!$_",
+                                                            "CHILD": "_$!<Child>!$_",
+                                                            "FRIEND": "_$!<Friend>!$_",
+                                                            "SPOUSE": "_$!<Spouse>!$_",
+                                                            "PARTNER": "_$!<Partner>!$_",
+                                                            "ASSISTANT": "_$!<Assistant>!$_",
+                                                            "MANAGER": "_$!<Manager>!$_", },
+                                                        attrType=dsattributes.kDSNAttrRelationships,)
+
+            # add apple-defined group vcard properties if record type is group
+            if self.kind == "group":
+                vcard.addProperty(Property("X-ADDRESSBOOKSERVER-KIND", "group"))
+
+            # add members
+            for memberguid in self.valuesForAttribute(dsattributes.kDSNAttrGroupMembers):
+                vcard.addProperty(Property("X-ADDRESSBOOKSERVER-MEMBER", "urn:uuid:" + memberguid))
+
+            """
+            # UNIMPLEMENTED: X- attributes
+
+            X-MAIDENNAME
+            X-PHONETIC-FIRST-NAME
+            X-PHONETIC-MIDDLE-NAME
+            X-PHONETIC-LAST-NAME
+
+            sattributes.kDS1AttrPicture,                # Represents the path of the picture for each user displayed in the login window.
+                                                        #      Found in user records (kDSStdRecordTypeUsers).
+
+            dsattributes.kDS1AttrMapGUID,               # Represents the GUID for a record's map.
+            dsattributes.kDSNAttrMapURI,                # attribute that defines the URI of a user's location.
+
+            dsattributes.kDSNAttrOrganizationInfo,      # Usually the organization info of a user.
+            dsattributes.kDSNAttrAreaCode,              # Area code of a user's phone number.
+
+            dsattributes.kDSNAttrMIME,                  # Data contained in this attribute type is a fully qualified MIME Type.
+
+            """
+
+            # 2.1.4 SOURCE Type http://tools.ietf.org/html/rfc2426#section-2.1.4
+            #    If the SOURCE type is present, then its value provides information
+            #    how to find the source for the vCard.
+
+            # add the source, so that if the SOURCE is copied out and preserved, the client can refresh information
+            # However, client should really do a ab-query report matching UID on /directory/ not a multiget.
+            uri = joinURL(self._directoryBackedAddressBook.uri, vcard.propertyValue("UID") + ".vcf")
+
+            # seems like this should be in some standard place.
+            if config.EnableSSL and config.SSLPort:
+                if config.SSLPort == 443:
+                    source = "https://{server}{uri}".format(server=config.ServerHostName, uri=uri)
+                else:
+                    source = "https://{server}:{port}{uri}".format(server=config.ServerHostName, port=config.SSLPort, uri=uri)
+            else:
+                if config.HTTPPort == 80:
+                    source = "https://{server}{uri}".format(server=config.ServerHostName, uri=uri)
+                else:
+                    source = "https://{server}:{port}{uri}".format(server=config.ServerHostName, port=config.HTTPPort, uri=uri)
+            vcard.addProperty(Property("SOURCE", source))
+
+            #  in 4.0 spec:
+            # 6.1.4.  KIND http://tools.ietf.org/html/rfc6350#section-6.1.4
+            #
+            # see also: http://www.iana.org/assignments/vcard-elements/vcard-elements.xml
+            #
+            vcard.addProperty(Property("KIND", self.kind))
+
+            # one more X- related to kind
+            if self.kind == "org":
+                vcard.addProperty(Property("X-ABShowAs", "COMPANY"))
+
+            return vcard
+
+        if not self._vCard:
+            self._vCard = generateVCard()
+
+        return self._vCard
+
+
+    def vCardText(self):
+        return str(self.vCard())
+
+
+    def uri(self):
+        return self.vCard().propertyValue("UID") + ".vcf"
+
+
+    def hRef(self, parentURI=None):
+        return davxml.HRef.fromString(joinURL(parentURI if parentURI else  self._directoryBackedAddressBook.uri, self.uri()))
+
+
+    def readProperty(self, property, request):
+
+        if type(property) is tuple:
+            qname = property
+        else:
+            qname = property.qname()
+        namespace, name = qname
+
+        if namespace == dav_namespace:
+            if name == "resourcetype":
+                result = davxml.ResourceType.empty #@UndefinedVariable
+                return result
+            elif name == "getetag":
+                result = davxml.GETETag(ETag(hashlib.md5(self.vCardText()).hexdigest()).generate())
+                return result
+            elif name == "getcontenttype":
+                mimeType = MimeType('text', 'vcard', {})
+                result = davxml.GETContentType(generateContentType(mimeType))
+                return result
+            elif name == "getcontentlength":
+                result = davxml.GETContentLength.fromString(str(len(self.vCardText())))
+                return result
+            elif name == "getlastmodified":
+                if self.vCard().hasProperty("REV"):
+                    modDatetime = parse_date(self.vCard().propertyValue("REV"))
+                else:
+                    # use creation date attribute if it exists
+                    creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
+                    if creationDateString:
+                        modDatetime = parse_date(creationDateString)
+                    else:
+                        modDatetime = datetime.datetime.utcnow()
+
+                #strip time zone because time zones are unimplemented in davxml.GETLastModified.fromDate
+                d = modDatetime.date()
+                t = modDatetime.time()
+                modDatetimeNoTZ = datetime.datetime(d.year, d.month, d.day, t.hour, t.minute, t.second, t.microsecond, None)
+                result = davxml.GETLastModified.fromDate(modDatetimeNoTZ)
+                return result
+            elif name == "creationdate":
+                creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
+                if creationDateString:
+                    creationDatetime = parse_date(creationDateString)
+                elif self.vCard().hasProperty("REV"):  # use modification date property if it exists
+                    creationDatetime = parse_date(self.vCard().propertyValue("REV"))
+                else:
+                    creationDatetime = datetime.datetime.utcnow()
+                result = davxml.CreationDate.fromDate(creationDatetime)
+                return result
+            elif name == "displayname":
+                # AddressBook.app uses N. Use FN or UID instead?
+                result = davxml.DisplayName.fromString(self.vCard().propertyValue("N"))
+                return result
+
+        elif namespace == twisted_dav_namespace:
+            return super(ABDirectoryQueryResult, self).readProperty(property, request)
+
+        return self._directoryBackedAddressBook.readProperty(property, request)
+
+
+    def listProperties(self, request):  # @UnusedVariable
+        qnames = set(self.liveProperties())
+
+        # Add dynamic live properties that exist
+        dynamicLiveProperties = (
+            (dav_namespace, "quota-available-bytes"),
+            (dav_namespace, "quota-used-bytes"),
+        )
+        for dqname in dynamicLiveProperties:
+            qnames.remove(dqname)
+
+        for qname in self.deadProperties().list():
+            if (qname not in qnames) and (qname[0] != twisted_private_namespace):
+                qnames.add(qname)
+
+        yield qnames
+
+    listProperties = deferredGenerator(listProperties)
+
+
+
+#remove illegal XML
+def removeControlChars(utf8String):
+    result = ''.join([c for c in utf8String if c not in "\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"])
+    return result

Modified: CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_addressbook_query.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_addressbook_query.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -21,26 +21,22 @@
 
 __all__ = ["report_urn_ietf_params_xml_ns_carddav_addressbook_query"]
 
-import urllib
-
-from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
-
 from twext.python.log import Logger
-from txweb2 import responsecode
-from txweb2.dav.http import ErrorResponse, MultiStatusResponse
-from txweb2.dav.method.report import NumberOfMatchesWithinLimits
-from txweb2.dav.util import joinURL
-from txweb2.http import HTTPError, StatusResponse
-
+from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
 from twistedcaldav import carddavxml
-from twistedcaldav.config import config
 from twistedcaldav.carddavxml import carddav_namespace, NResults
+from twistedcaldav.config import config
 from twistedcaldav.method import report_common
-
 from txdav.carddav.datastore.query.filter import Filter
 from txdav.common.icommondatastore import ConcurrentModification, \
     IndexedSearchException
 from txdav.xml import element as davxml
+from txweb2 import responsecode
+from txweb2.dav.http import ErrorResponse, MultiStatusResponse
+from txweb2.dav.method.report import NumberOfMatchesWithinLimits
+from txweb2.dav.util import joinURL
+from txweb2.http import HTTPError, StatusResponse
+import urllib
 
 log = Logger()
 
@@ -52,12 +48,12 @@
     """
     # Verify root element
     if addressbook_query.qname() != (carddav_namespace, "addressbook-query"):
-        raise ValueError("{CardDAV:}addressbook-query expected as root element, not %s." % (addressbook_query.sname(),))
+        raise ValueError("{CardDAV:}addressbook-query expected as root element, not {elementName}.".format(elementName=addressbook_query.sname()))
 
     if not self.isCollection():
         parent = (yield self.locateParent(request, request.uri))
         if not parent.isAddressBookCollection():
-            log.error("addressbook-query report is not allowed on a resource outside of an address book collection %s" % (self,))
+            log.error("addressbook-query report is not allowed on a resource outside of an address book collection {parent}", parent=self)
             raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Must be address book collection or address book resource"))
 
     responses = []
@@ -154,160 +150,127 @@
                     # of one of these resources in another request.  In this
                     # case, we ignore the now missing resource rather
                     # than raise an error for the entire report.
-                    log.error("Missing resource during sync: %s" % (href,))
+                    log.error("Missing resource during sync: {href}", href=href)
 
 
         @inlineCallbacks
         def queryDirectoryBackedAddressBook(directoryBackedAddressBook, addressBookFilter):
             """
             """
-            records, limited[0] = (yield directoryBackedAddressBook.directory.vCardRecordsForAddressBookQuery(addressBookFilter, query, max_number_of_results[0]))
-            for vCardRecord in records:
+            results, limited[0] = (yield directoryBackedAddressBook.doAddressBookDirectoryQuery(addressBookFilter, query, max_number_of_results[0]))
+            for vCardResult in results:
 
-                # match against original filter
-                if filter.match((yield vCardRecord.vCard())):
+                # match against original filter if different from addressBookFilter
+                if addressBookFilter is filter or filter.match((yield vCardResult.vCard())):
 
                     # Check size of results is within limit
                     checkMaxResults()
 
                     try:
-                        yield report_common.responseForHref(request, responses, vCardRecord.hRef(), vCardRecord, propertiesForResource, query, vcard=(yield vCardRecord.vCard()))
+                        yield report_common.responseForHref(request, responses, vCardResult.hRef(), vCardResult, propertiesForResource, query, vcard=(yield vCardResult.vCard()))
                     except ConcurrentModification:
                         # This can happen because of a race-condition between the
                         # time we determine which resources exist and the deletion
                         # of one of these resources in another request.  In this
                         # case, we ignore the now missing resource rather
                         # than raise an error for the entire report.
-                        log.error("Missing resource during sync: %s" % (vCardRecord.hRef(),))
+                        log.error("Missing resource during sync: {href}", href=vCardResult.hRef())
 
-        directoryAddressBookLock = None
-        try:
+        if not addrresource.isAddressBookCollection():
 
-            if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.cacheQuery:
+            #do UID lookup on last part of uri
+            resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+            if resource_name.endswith(".vcf") and len(resource_name) > 4:
 
-                directory = addrresource.directory
-                if directory.liveQuery:
-                    # if liveQuery and cacheQuery, get vCards into the directory address book on disk
-                    directoryAddressBookLock, limited[0] = (yield  directory.cacheVCardsForAddressBookQuery(filter, query, max_number_of_results[0]))
+                # see if parent is directory backed address book
+                parent = (yield  addrresource.locateParent(request, uri))
 
-                elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
-                    max_number_of_results[0] = directory.maxDSQueryRecords
+        # Check whether supplied resource is an address book or an address book object resource
+        if addrresource.isAddressBookCollection():
 
-            elif not addrresource.isAddressBookCollection():
+            if addrresource.isDirectoryBackedAddressBookCollection():
+                yield  maybeDeferred(queryDirectoryBackedAddressBook, addrresource, filter)
 
-                #do UID lookup on last part of uri
-                resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
-                if resource_name.endswith(".vcf") and len(resource_name) > 4:
+            else:
 
-                    # see if parent is directory backed address book
-                    parent = (yield  addrresource.locateParent(request, uri))
+                # Do some optimisation of access control calculation by determining any inherited ACLs outside of
+                # the child resource loop and supply those to the checkPrivileges on each child.
+                filteredaces = (yield addrresource.inheritedACEsforChildren(request))
 
-                    if parent.isDirectoryBackedAddressBookCollection() and parent.directory.cacheQuery:
+                # Check for disabled access
+                if filteredaces is not None:
+                    index_query_ok = True
+                    try:
+                        # Get list of children that match the search and have read access
+                        names = [name for name, ignore_uid in (yield addrresource.search(filter))] #@UnusedVariable
+                    except IndexedSearchException:
+                        names = yield addrresource.listChildren()
+                        index_query_ok = False
+                    if not names:
+                        return
 
-                        directory = parent.directory
-                        if directory.liveQuery:
-                            vCardFilter = carddavxml.Filter(*[carddavxml.PropertyFilter(
-                                                        carddavxml.TextMatch.fromString(resource_name[:-4]),
-                                                        name="UID", # attributes
-                                                        ), ])
-                            vCardFilter = Filter(vCardFilter)
+                    # Now determine which valid resources are readable and which are not
+                    ok_resources = []
+                    yield addrresource.findChildrenFaster(
+                        "1",
+                        request,
+                        lambda x, y: ok_resources.append((x, y)),
+                        None,
+                        None,
+                        None,
+                        names,
+                        (davxml.Read(),),
+                        inherited_aces=filteredaces
+                    )
+                    for child, child_uri in ok_resources:
+                        child_uri_name = child_uri[child_uri.rfind("/") + 1:]
 
-                            directoryAddressBookLock, limited[0] = (yield  directory.cacheVCardsForAddressBookQuery(vCardFilter, query, max_number_of_results[0]))
+                        if generate_address_data or not index_query_ok:
+                            vcard = yield child.vCard()
+                            assert vcard is not None, "vCard {name} is missing from address book collection {collection!r}".format(name=child_uri_name, collection=self)
+                        else:
+                            vcard = None
 
-                        elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
-                            max_number_of_results[0] = directory.maxDSQueryRecords
+                        yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok=index_query_ok)
 
-            # Check whether supplied resource is an address book or an address book object resource
-            if addrresource.isAddressBookCollection():
+        else:
 
-                if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.liveQuery and not addrresource.directory.cacheQuery:
-                    yield  maybeDeferred(queryDirectoryBackedAddressBook, addrresource, filter)
+            handled = False
+            resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+            if resource_name.endswith(".vcf") and len(resource_name) > 4:
 
-                else:
+                # see if parent is directory backed address book
+                parent = (yield  addrresource.locateParent(request, uri))
 
-                    # Do some optimisation of access control calculation by determining any inherited ACLs outside of
-                    # the child resource loop and supply those to the checkPrivileges on each child.
-                    filteredaces = (yield addrresource.inheritedACEsforChildren(request))
+                if parent.isDirectoryBackedAddressBookCollection():
 
-                    # Check for disabled access
-                    if filteredaces is not None:
-                        index_query_ok = True
-                        try:
-                            # Get list of children that match the search and have read access
-                            names = [name for name, ignore_uid in (yield addrresource.search(filter))] #@UnusedVariable
-                        except IndexedSearchException:
-                            names = yield addrresource.listChildren()
-                            index_query_ok = False
-                        if not names:
-                            return
+                    vCardFilter = carddavxml.Filter(*[carddavxml.PropertyFilter(
+                                                carddavxml.TextMatch.fromString(resource_name[:-4]),
+                                                name="UID", # attributes
+                                                ), ])
+                    vCardFilter = Filter(vCardFilter)
 
-                        # Now determine which valid resources are readable and which are not
-                        ok_resources = []
-                        yield addrresource.findChildrenFaster(
-                            "1",
-                            request,
-                            lambda x, y: ok_resources.append((x, y)),
-                            None,
-                            None,
-                            None,
-                            names,
-                            (davxml.Read(),),
-                            inherited_aces=filteredaces
-                        )
-                        for child, child_uri in ok_resources:
-                            child_uri_name = child_uri[child_uri.rfind("/") + 1:]
+                    yield  maybeDeferred(queryDirectoryBackedAddressBook, parent, vCardFilter)
+                    handled = True
 
-                            if generate_address_data or not index_query_ok:
-                                vcard = yield child.vCard()
-                                assert vcard is not None, "vCard %s is missing from address book collection %r" % (child_uri_name, self)
-                            else:
-                                vcard = None
+            if not handled:
+                vcard = yield addrresource.vCard()
+                yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
 
-                            yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok=index_query_ok)
+        if limited[0]:
+            raise NumberOfMatchesWithinLimits(matchcount[0])
 
-            else:
-
-                handled = False
-                resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
-                if resource_name.endswith(".vcf") and len(resource_name) > 4:
-
-                    # see if parent is directory backed address book
-                    parent = (yield  addrresource.locateParent(request, uri))
-
-                    if parent.isDirectoryBackedAddressBookCollection() and parent.directory.liveQuery and not parent.directory.cacheQuery:
-
-                        vCardFilter = carddavxml.Filter(*[carddavxml.PropertyFilter(
-                                                    carddavxml.TextMatch.fromString(resource_name[:-4]),
-                                                    name="UID", # attributes
-                                                    ), ])
-                        vCardFilter = Filter(vCardFilter)
-
-                        yield  maybeDeferred(queryDirectoryBackedAddressBook, parent, vCardFilter)
-                        handled = True
-
-                if not handled:
-                    vcard = yield addrresource.vCard()
-                    yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
-
-            if limited[0]:
-                raise NumberOfMatchesWithinLimits(matchcount[0])
-
-        finally:
-            if directoryAddressBookLock:
-                yield directoryAddressBookLock.release()
-
     # Run report taking depth into account
     try:
         depth = request.headers.getHeader("depth", "0")
         yield report_common.applyToAddressBookCollections(self, request, request.uri, depth, doQuery, (davxml.Read(),))
     except NumberOfMatchesWithinLimits, e:
-        self.log.info("Too many matching components in addressbook-query report. Limited to %d items" % e.maxLimit())
+        self.log.info("Too many matching components in addressbook-query report. Limited to {limit} items", limit=e.maxLimit())
         responses.append(davxml.StatusResponse(
                         davxml.HRef.fromString(request.uri),
                         davxml.Status.fromResponseCode(responsecode.INSUFFICIENT_STORAGE_SPACE),
                         davxml.Error(davxml.NumberOfMatchesWithinLimits()),
-                        #davxml.ResponseDescription("Results limited by %s at %d" % resultsWereLimited),
-                        davxml.ResponseDescription("Results limited to %d items" % e.maxLimit()),
+                        davxml.ResponseDescription("Results limited to {limit} items".format(limit=e.maxLimit())),
                     ))
 
     if not hasattr(request, "extendedLogItems"):

Modified: CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_multiget_common.py	2014-03-28 21:42:50 UTC (rev 13036)
+++ CalendarServer/branches/users/sagen/move2who-4/twistedcaldav/method/report_multiget_common.py	2014-03-28 21:51:51 UTC (rev 13037)
@@ -20,17 +20,8 @@
 
 __all__ = ["multiget_common"]
 
-from urllib import unquote
-
 from twext.python.log import Logger
-
-from txweb2 import responsecode
-from txweb2.dav.http import ErrorResponse, MultiStatusResponse
-from txweb2.dav.resource import AccessDeniedError
-from txweb2.http import HTTPError, StatusResponse
-
 from twisted.internet.defer import inlineCallbacks, returnValue
-
 from twistedcaldav import carddavxml
 from twistedcaldav.caldavxml import caldav_namespace
 from twistedcaldav.carddavxml import carddav_namespace
@@ -38,11 +29,15 @@
 from twistedcaldav.method import report_common
 from twistedcaldav.method.report_common import COLLECTION_TYPE_CALENDAR, \
     COLLECTION_TYPE_ADDRESSBOOK
-
 from txdav.carddav.datastore.query.filter import Filter
 from txdav.common.icommondatastore import ConcurrentModification
 from txdav.xml import element as davxml
 from txdav.xml.base import dav_namespace
+from txweb2 import responsecode
+from txweb2.dav.http import ErrorResponse, MultiStatusResponse
+from txweb2.dav.resource import AccessDeniedError
+from txweb2.http import HTTPError, StatusResponse
+from urllib import unquote
 
 log = Logger()
 
@@ -58,11 +53,11 @@
 
         if collection_type == COLLECTION_TYPE_CALENDAR:
             if not parent.isPseudoCalendarCollection():
-                log.error("calendar-multiget report is not allowed on a resource outside of a calendar collection %s" % (self,))
+                log.error("calendar-multiget report is not allowed on a resource outside of a calendar collection {res}", res=self)
                 raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Must be calendar resource"))
         elif collection_type == COLLECTION_TYPE_ADDRESSBOOK:
             if not parent.isAddressBookCollection():
-                log.error("addressbook-multiget report is not allowed on a resource outside of an address book collection %s" % (self,))
+                log.error("addressbook-multiget report is not allowed on a resource outside of an address book collection {res}", res=self)
                 raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Must be address book resource"))
 
     responses = []
@@ -106,7 +101,7 @@
 
     # Check size of results is within limit when data property requested
     if hasData and len(resources) > config.MaxMultigetWithDataHrefs:
-        log.error("Too many results in multiget report returning data: %d" % len(resources))
+        log.error("Too many resources in multiget report: {count}", count=len(resources))
         raise HTTPError(ErrorResponse(
             responsecode.FORBIDDEN,
             davxml.NumberOfMatchesWithinLimits(),
@@ -172,7 +167,7 @@
 
             # Special for addressbooks
             if collection_type == COLLECTION_TYPE_ADDRESSBOOK:
-                if self.isDirectoryBackedAddressBookCollection() and self.directory.liveQuery:
+                if self.isDirectoryBackedAddressBookCollection():
                     result = (yield doDirectoryAddressBookResponse())
                     returnValue(result)
 
@@ -214,8 +209,7 @@
                         isowner=isowner
                     )
                 except ValueError:
-                    log.error("Invalid calendar resource during multiget: %s" %
-                              (href,))
+                    log.error("Invalid calendar resource during multiget: {href}", href=href)
                     responses.append(davxml.StatusResponse(
                         davxml.HRef.fromString(href),
                         davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
@@ -225,7 +219,7 @@
                     # of one of these resources in another request.  In this
                     # case, return a 404 for the now missing resource rather
                     # than raise an error for the entire report.
-                    log.error("Missing resource during multiget: %s" % (href,))
+                    log.error("Missing resource during multiget: {href}", href=href)
                     responses.append(davxml.StatusResponse(
                         davxml.HRef.fromString(href),
                         davxml.Status.fromResponseCode(responsecode.NOT_FOUND)
@@ -255,11 +249,13 @@
                     resource_name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
                     if self._isChildURI(request, resource_uri) and resource_name.endswith(".vcf") and len(resource_name) > 4:
                         valid_hrefs.append(href)
+                        textMatchElement = carddavxml.TextMatch.fromString(resource_name[:-4])
+                        textMatchElement.attributes["match-type"] = "equals" # do equals compare. Default is "contains"
                         vCardFilters.append(carddavxml.PropertyFilter(
-                                                carddavxml.TextMatch.fromString(resource_name[:-4]),
+                                                textMatchElement,
                                                 name="UID", # attributes
                                             ))
-                    elif not self.directory.cacheQuery:
+                    else:
                         responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
 
                 # exit if not valid
@@ -268,40 +264,29 @@
 
                 addressBookFilter = carddavxml.Filter(*vCardFilters)
                 addressBookFilter = Filter(addressBookFilter)
-                if self.directory.cacheQuery:
-                    # add vcards to directory address book and run "normal case" below
-                    limit = config.DirectoryAddressBook.MaxQueryResults
-                    directoryAddressBookLock, limited = (yield  self.directory.cacheVCardsForAddressBookQuery(addressBookFilter, propertyreq, limit))
-                    if limited:
-                        log.error("Too many results in multiget report: %d" % len(resources))
-                        raise HTTPError(ErrorResponse(
-                            responsecode.FORBIDDEN,
-                            (dav_namespace, "number-of-matches-within-limits"),
-                            "Too many results",
-                        ))
-                else:
-                    #get vCards and filter
-                    limit = config.DirectoryAddressBook.MaxQueryResults
-                    vCardRecords, limited = (yield self.directory.vCardRecordsForAddressBookQuery(addressBookFilter, propertyreq, limit))
-                    if limited:
-                        log.error("Too many results in multiget report: %d" % len(resources))
-                        raise HTTPError(ErrorResponse(
-                            responsecode.FORBIDDEN,
-                            (dav_namespace, "number-of-matches-within-limits"),
-                            "Too many results",
-                        ))
 
-                    for href in valid_hrefs:
-                        matchingRecord = None
-                        for vCardRecord in vCardRecords:
-                            if href == vCardRecord.hRef(): # might need to compare urls instead - also case sens ok?
-                                matchingRecord = vCardRecord
-                                break
+                #get vCards and filter
+                limit = config.DirectoryAddressBook.MaxQueryResults
+                results, limited = (yield self.doAddressBookDirectoryQuery(addressBookFilter, propertyreq, limit))
+                if limited:
+                    log.error("Too many results in multiget report: {count}", count=len(resources))
+                    raise HTTPError(ErrorResponse(
+                        responsecode.FORBIDDEN,
+                        (dav_namespace, "number-of-matches-within-limits"),
+                        "Too many results",
+                    ))
 
-                        if matchingRecord:
-                            yield report_common.responseForHref(request, responses, href, matchingRecord, propertiesForResource, propertyreq, vcard=matchingRecord.vCard())
-                        else:
-                            responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+                for href in valid_hrefs:
+                    matchingResource = None
+                    for vCardResource in results:
+                        if href == vCardResource.hRef(): # might need to compare urls instead - also case sens ok?
+                            matchingResource = vCardResource
+                            break
+
+                    if matchingResource:
+                        yield report_common.responseForHref(request, responses, href, matchingResource, propertiesForResource, propertyreq, vcard=matchingResource.vCard())
+                    else:
+                        responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
             finally:
                 if directoryAddressBookLock:
                     yield directoryAddressBookLock.release()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140328/02c10b62/attachment-0001.html>


More information about the calendarserver-changes mailing list