[CalendarServer-changes] [9765] CalendarServer/branches/users/gaya/directorybacker
source_changes at macosforge.org
source_changes at macosforge.org
Fri Aug 31 17:56:45 PDT 2012
Revision: 9765
http://trac.macosforge.org/projects/calendarserver/changeset/9765
Author: gaya at apple.com
Date: 2012-08-31 17:56:43 -0700 (Fri, 31 Aug 2012)
Log Message:
-----------
merge in most of ldapdirectorybacker branch
Modified Paths:
--------------
CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py
CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml
CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py
Added Paths:
-----------
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py
CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py
Modified: CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/calendarserver/tap/util.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -34,6 +34,7 @@
from twext.python.log import Logger
from twext.web2.auth.basic import BasicCredentialFactory
from twext.web2.dav import auth
+from twext.web2.dav.util import joinURL
from twext.web2.http_headers import Headers
from twext.web2.static import File as FileResource
@@ -483,12 +484,12 @@
newStore,
)
- directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
if config.DirectoryAddressBook.Enabled and config.EnableSearchAddressBook:
log.info("Setting up directory address book: %r" % (directoryBackedAddressBookResourceClass,))
directoryBackedAddressBookCollection = directoryBackedAddressBookResourceClass(
- principalCollections=(principalCollection,)
+ principalCollections=(principalCollection,),
+ uri=joinURL("/", config.DirectoryAddressBook.name, "/")
)
if _reactor._started:
directoryBackedAddressBookCollection.provisionDirectory()
@@ -496,6 +497,7 @@
addSystemEventTrigger("after", "startup", directoryBackedAddressBookCollection.provisionDirectory)
else:
# remove /directory from previous runs that may have created it
+ directoryPath = os.path.join(config.DocumentRoot, config.DirectoryAddressBook.name)
try:
FilePath(directoryPath).remove()
log.info("Deleted: %s" % directoryPath)
Modified: CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/conf/auth/accounts-test.xml 2012-09-01 00:56:43 UTC (rev 9765)
@@ -106,6 +106,7 @@
<name>Public %02d</name>
<first-name>Public</first-name>
<last-name>%02d</last-name>
+ <email-address>public%02d at example.com</email-address>
</user>
<group>
<uid>group01</uid>
Modified: CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/conf/caldavd-test.plist 2012-09-01 00:56:43 UTC (rev 9765)
@@ -1015,6 +1015,30 @@
<string>English</string>
</dict>
-
+ <!--
+ Directory Address Book
+ -->
+
+ <!-- Disable Directory Address Book -->
+ <!--
+ <key>DirectoryAddressBook</key>
+ <false/>
+ -->
+
+ <!-- XML Directory-backed Directory Address Book -->
+ <key>EnableSearchAddressBook</key>
+ <true/>
+ <key>DirectoryAddressBook</key>
+ <dict>
+ <key>Enabled</key>
+ <true/>
+ <key>type</key>
+ <string>twistedcaldav.directory.xmldirectorybacker.XMLDirectoryBackingService</string>
+ <key>params</key>
+ <dict>
+ <key>xmlFile</key>
+ <string>./conf/auth/accounts-test.xml</string>
+ </dict>
+ </dict>
</dict>
</plist>
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectory.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -228,7 +228,7 @@
# config.
attrSet = set()
- if self.rdnSchema["guidAttr"]:
+ if self.rdnSchema.get("guidAttr"):
attrSet.add(self.rdnSchema["guidAttr"])
for recordType in self.recordTypes():
if self.rdnSchema[recordType]["attr"]:
@@ -255,7 +255,7 @@
attrSet.add(self.groupSchema["nestedGroupsAttr"])
if self.groupSchema["memberIdAttr"]:
attrSet.add(self.groupSchema["memberIdAttr"])
- if self.rdnSchema["users"]["loginEnabledAttr"]:
+ if self.rdnSchema.get("users") and self.rdnSchema["users"]["loginEnabledAttr"]:
attrSet.add(self.rdnSchema["users"]["loginEnabledAttr"])
if self.resourceSchema["resourceInfoAttr"]:
attrSet.add(self.resourceSchema["resourceInfoAttr"])
Added: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py (rev 0)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/ldapdirectorybacker.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -0,0 +1,376 @@
+##
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+"""
+Apple Open Directory directory service implementation for backing up directory-backed address books
+"""
+
+__all__ = [
+ "LdapDirectoryBackingService",
+]
+
+import traceback
+import ldap
+
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+
+from twistedcaldav.config import config
+from twistedcaldav.directory.ldapdirectory import LdapDirectoryService, normalizeDNstr
+from twistedcaldav.directory.opendirectorybacker import ABDirectoryQueryResult, dsFilterFromAddressBookFilter, propertiesInAddressBookQuery
+
+
+class LdapDirectoryBackingService(LdapDirectoryService):
+ """
+ Directory backer for L{LdapDirectoryService}.
+ """
+
+ def __init__(self, params):
+ self._actuallyConfigure(**params)
+
+ def _actuallyConfigure(self, **params):
+
+ self.log_debug("_actuallyConfigure: params=%s" % (params,))
+ defaults = {
+ "recordTypes": (), # for super
+ "rdnSchema": {
+ "base": "dc=example,dc=com",
+ "queries": (
+ { #people
+ "rdn":"ou=people",
+ "vcardPropToLdapAttrMap" : { # maps vCard properties to searchable ldap attributes
+ "FN" : "cn",
+ },
+ "ldapAttrToDSAttrMap" : { # maps ldap attributes to ds attribute types
+ "cn" : "dsAttrTypeStandard:RealName",
+ },
+ "additionalVCardProps":None,
+ },
+ ),
+
+ },
+ "removeDuplicateUIDs":True, # remove vCards with duplicate UIDs
+ "appleInternalServer":False, # does magic in ABDirectoryQueryResult
+ "maxQueryResults":0, # max records returned
+ "fakeETag":True, # eTag is fake, otherwise it is md5(all attributes)
+ }
+
+ #params = self.getParams(params, defaults, ignored)
+ def addDefaults(params, defaults, remove=None):
+ for key in defaults:
+ if not key in params:
+ params[key] = defaults[key]
+ return params
+
+ params = addDefaults(params, defaults)
+ self.log_debug("_actuallyConfigure after addDefaults: params=%s" % (params,))
+
+ # super does not like these extra params
+ directoryBackedAddressBook=params["directoryBackedAddressBook"]
+ del params["directoryBackedAddressBook"]
+ appleInternalServer=params["appleInternalServer"]
+ del params["appleInternalServer"]
+ maxQueryResults=params["maxQueryResults"]
+ del params["maxQueryResults"]
+ fakeETag=params["fakeETag"]
+ del params["fakeETag"]
+ removeDuplicateUIDs=params["removeDuplicateUIDs"]
+ del params["removeDuplicateUIDs"]
+
+
+ #standardize ds attributes type names
+ # or we could just require dsAttrTypeStandard: prefix in the plist
+ rdnSchema = params["rdnSchema"];
+ for query in rdnSchema["queries"]:
+ ldapAttrToDSAttrMap = query["ldapAttrToDSAttrMap"]
+ for ldapAttrName, dsAttrNames in ldapAttrToDSAttrMap.iteritems():
+ if not isinstance(dsAttrNames, list):
+ dsAttrNames = [dsAttrNames,]
+
+ normalizedDSAttrNames = []
+ for dsAttrName in dsAttrNames:
+ if not dsAttrName.startswith("dsAttrTypeStandard:") and not dsAttrName.startswith("dsAttrTypeNative:"):
+ normalizedDSAttrNames.append("dsAttrTypeStandard:" + dsAttrName)
+ else:
+ normalizedDSAttrNames.append(dsAttrName)
+
+ # not needed, but tests code paths
+ if len(normalizedDSAttrNames) > 1:
+ ldapAttrToDSAttrMap[ldapAttrName] = normalizedDSAttrNames
+ else:
+ ldapAttrToDSAttrMap[ldapAttrName] = normalizedDSAttrNames[0]
+
+
+ self.log_debug("_actuallyConfigure after clean: params=%s" % (params,))
+
+ assert directoryBackedAddressBook is not None
+ self.directoryBackedAddressBook = directoryBackedAddressBook
+
+ self.maxQueryResults = maxQueryResults
+
+ ### params for ABDirectoryQueryResult()
+ self.fakeETag = fakeETag
+ self.appleInternalServer = appleInternalServer
+ self.removeDuplicateUIDs = removeDuplicateUIDs
+
+ super(LdapDirectoryBackingService, self).__init__(params)
+
+
+ def createCache(self):
+ succeed(None)
+
+
+ @inlineCallbacks
+ def _getLdapQueryResults(self, base, queryStr, attributes=None, maxResults=0, ldapAttrToDSAttrMap=None, ldapAttrTransforms=None, additionalVCardProps=None, kind=None ):
+ """
+ Get a list of ABDirectoryQueryResult for the given query with the given attributes.
+ query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
+ """
+ limited = False
+ resultsDictionary = {}
+
+ # can't resist also using a timeout, 1 sec per request result for now
+ timeout = maxResults
+
+ self.log_debug("_getLdapQueryResults: LDAP query base=%s and filter=%s and attributes=%s timeout=%s resultLimit=%s" % (ldap.dn.dn2str(base), queryStr, attributes, timeout, maxResults))
+
+ ldapSearchResult = (yield self.timedSearch(ldap.dn.dn2str(base), ldap.SCOPE_SUBTREE, filterstr=queryStr, attrlist=attributes, timeoutSeconds=timeout, resultLimit=maxResults))
+ self.log_debug("_getLdapQueryResults: ldapSearchResult=%s" % (ldapSearchResult,))
+
+ if maxResults and len(ldapSearchResult) >= maxResults:
+ limited = True
+ self.log_debug("_getLdapQueryResults: limit (= %d) reached." % (maxResults, ))
+
+ for dn, ldapAttributes in ldapSearchResult:
+ #dn = normalizeDNstr(dn)
+ result = None
+ try:
+ if "dn" not in ldapAttributes:
+ ldapAttributes["dn"] = [normalizeDNstr(dn),]
+
+ # make a dsRecordAttributes dict from the ldap attributes
+ dsRecordAttributes = {}
+ for ldapAttributeName, ldapAttributeValues in ldapAttributes.iteritems():
+
+ #self.log_debug("inspecting ldapAttributeName %s with values %s" % (ldapAttributeName, ldapAttributeValues,))
+
+ # get rid of '' values
+ ldapAttributeValues = [attr for attr in ldapAttributeValues if len(attr)]
+
+ if len(ldapAttributeValues):
+
+
+ dsAttributeNames = ldapAttrToDSAttrMap.get(ldapAttributeName)
+ if dsAttributeNames:
+
+ if ldapAttrTransforms:
+
+ # do value transforms
+ # need to expand this to cover all cases
+ # All this does now is to pull part of an ldap string out
+ # e.g: uid=renuka,ou=People,o=apple.com,o=email -> renuka
+ transforms = ldapAttrTransforms.get(ldapAttributeName)
+ if transforms:
+ if not isinstance(transforms, list):
+ transforms = [transforms,]
+
+ transformedValues = []
+ for ldapAttributeValue in ldapAttributeValues:
+ transformedValue = ldapAttributeValue
+ for valuePart in normalizeDNstr(ldapAttributeValue).split(","):
+ kvPair = valuePart.split("=")
+ if len(kvPair) == 2:
+ for transform in transforms:
+ if transform.lower() == kvPair[0]:
+ transformedValue = kvPair[1]
+ break
+
+ transformedValues += [transformedValue,]
+
+ if (ldapAttributeValues != transformedValues):
+ self.log_debug("_getLdapQueryResults: %s %s transformed to %s" % (ldapAttributeName, ldapAttributeValues, transformedValues))
+ ldapAttributeValues = transformedValues
+
+
+
+ if not isinstance(dsAttributeNames, list):
+ dsAttributeNames = [dsAttributeNames,]
+
+ for dsAttributeName in dsAttributeNames:
+
+ # base64 encode binary attributes
+ if dsAttributeName in ABDirectoryQueryResult.binaryDSAttrNames:
+ ldapAttributeValues = [attr.encode('base64') for attr in ldapAttributeValues]
+
+ # add to dsRecordAttributes
+ if dsAttributeName not in dsRecordAttributes:
+ dsRecordAttributes[dsAttributeName] = list()
+
+ dsRecordAttributes[dsAttributeName] = list(set(dsRecordAttributes[dsAttributeName] + ldapAttributeValues))
+ self.log_debug("doAddressBookQuery: dsRecordAttributes[%s] = %s" % (dsAttributeName, dsRecordAttributes[dsAttributeName],))
+
+ # get a record for dsRecordAttributes
+ result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, kind=kind, additionalVCardProps=additionalVCardProps, appleInternalServer=self.appleInternalServer)
+ except:
+ traceback.print_exc()
+ self.log_info("Could not get vcard for %s" % (dn,))
+ else:
+ uid = result.vCard().propertyValue("UID")
+
+ if uid in resultsDictionary:
+ self.log_info("Record skipped due to duplicate UID: %s" % (dn,))
+ continue
+
+ self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+ resultsDictionary[uid] = result
+
+ self.log_debug("%s results (limited=%s)." % (len(resultsDictionary), limited))
+ returnValue((resultsDictionary, limited, ))
+
+ @inlineCallbacks
+ def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
+ """
+ Get vCards for a given addressBookFilter and addressBookQuery
+ """
+
+ results = {} if self.removeDuplicateUIDs else []
+ limited = False
+
+ #one ldap query for each rnd in queries
+ for queryMap in self.rdnSchema["queries"]:
+
+ rdn = queryMap["rdn"]
+ vcardPropToLdapAttrMap = queryMap["vcardPropToLdapAttrMap"]
+ ldapAttrToDSAttrMap = queryMap["ldapAttrToDSAttrMap"]
+ additionalVCardProps = queryMap.get("additionalVCardProps")
+ ldapAttrTransforms = queryMap.get("ldapAttrTransforms")
+ kind = queryMap.get("kind", "individual")
+
+ # add constants and KIND
+ constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+ if additionalVCardProps:
+ for key, value in additionalVCardProps.iteritems():
+ if key not in constantProperties:
+ constantProperties[key] = value
+
+ # add KIND as constant so that query can be skipped if addressBookFilter needs a different kind
+ constantProperties["KIND"] = kind
+
+
+ filterPropertyNames, dsFilter = dsFilterFromAddressBookFilter( addressBookFilter, vcardPropToLdapAttrMap, constantProperties=constantProperties );
+ self.log_debug("doAddressBookQuery: rdn=%s, query=%s, propertyNames=%s" % (rdn, dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames))
+
+ if dsFilter:
+ if dsFilter is True:
+ dsFilter = None
+
+ # calculate minimum attributes needed for this query
+ etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+
+ if (etagRequested and not self.fakeETag) or not queryPropNames:
+ queryAttributes = ldapAttrToDSAttrMap.keys()
+ elif queryPropNames:
+ '''
+ # To DO: Need mapping from properties to returned attributes
+ queryPropNames += filterPropertyNames
+ queryAttributes = []
+ for prop in queryPropNames:
+ attributes = ABDirectoryQueryResult.vcardPropToDSAttrMap.get(prop)
+ if attributes:
+ queryAttributes += attributes
+ '''
+
+ queryAttributes = ldapAttrToDSAttrMap.keys()
+
+ self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s, queryAttributes=%s" % (etagRequested, queryPropNames, queryAttributes,))
+
+ #get all ldap attributes -- for debug
+ if queryMap.get("getAllAttributes"):
+ queryAttributes = None
+
+ base = ldap.dn.str2dn(rdn) + self.base
+
+ queryStr = "(cn=*)" # all results query - should make a param
+ #add additional filter from config
+ queryFilter = queryMap.get("filter")
+ if dsFilter and queryFilter:
+ queryStr = "(&%s%s)" % (queryFilter, dsFilter.generate())
+ elif queryFilter:
+ queryStr = queryFilter
+ elif dsFilter:
+ queryStr = dsFilter.generate()
+
+
+ # keep trying ldap query till we get results based on filter. Especially when doing "all results" query
+ remainingMaxResults = maxResults - len(results) if maxResults else 0
+ maxLdapResults = int(remainingMaxResults * 1.2)
+
+ while True:
+ ldapQueryResultsDictionary, ldapQueryLimited = (yield self._getLdapQueryResults(base=base,
+ queryStr=queryStr,
+ attributes=queryAttributes,
+ maxResults=maxLdapResults,
+ kind=kind,
+ ldapAttrToDSAttrMap=ldapAttrToDSAttrMap,
+ ldapAttrTransforms=ldapAttrTransforms,
+ additionalVCardProps=additionalVCardProps))
+
+ for uid, ldapQueryResult in ldapQueryResultsDictionary.iteritems():
+
+ if self.removeDuplicateUIDs and uid in results:
+ self.log_info("Record skipped due to duplicate UID: %s" % (uid,))
+ continue
+
+ if not addressBookFilter.match(ldapQueryResult.vCard()):
+ self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (ldapQueryResult.vCard().propertyValue("FN"), uid,))
+ continue
+
+ if self.removeDuplicateUIDs:
+ results[uid] = ldapQueryResult
+ else:
+ results += [ldapQueryResult,]
+
+
+ #no more results
+ if not ldapQueryLimited:
+ break;
+
+ # more than requested results
+ if maxResults and len(results) >= maxResults:
+ break
+
+ # more than max report results
+ if len(results) >= config.MaxQueryWithDataResults:
+ break
+
+ # more than self limit
+ if self.maxQueryResults and maxLdapResults >= self.maxQueryResults:
+ break
+
+ # try again with 2x
+ maxLdapResults *= 2
+ if self.maxQueryResults and maxLdapResults > self.maxQueryResults:
+ maxLdapResults = self.maxQueryResults
+
+ if maxResults and len(results) >= maxResults:
+ break
+
+
+ limited = maxResults and len(results) >= maxResults
+
+ self.log_info("limited %s len(results) %s" % (limited,len(results),))
+ returnValue((results.values() if self.removeDuplicateUIDs else results, limited,))
+
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/opendirectorybacker.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -20,53 +20,45 @@
"""
__all__ = [
- "OpenDirectoryBackingService", "VCardRecord",
+ "OpenDirectoryBackingService", "ABDirectoryQueryResult",
]
import traceback
import hashlib
-
-import os
-import sys
import time
-
-from os import listdir
-from os.path import join, abspath
-from tempfile import mkstemp, gettempdir
from random import random
+from calendarserver.platform.darwin.od import dsattributes, dsquery
from pycalendar.n import N
from pycalendar.adr import Adr
from pycalendar.datetime import PyCalendarDateTime
-from socket import getfqdn
-
-from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue, deferredGenerator, succeed
-from twext.python.filepath import CachingFilePath as FilePath
+from twisted.python.reflect import namedModule
+
from txdav.xml import element as davxml
from txdav.xml.base import twisted_dav_namespace, dav_namespace, parse_date, twisted_private_namespace
+
+from twext.python.log import LoggingMixIn, Logger
from twext.web2.dav.resource import DAVPropertyMixIn
from twext.web2.dav.util import joinURL
from twext.web2.http_headers import MimeType, generateContentType, ETag
-
-from twistedcaldav import customxml, carddavxml
-from twistedcaldav.customxml import calendarserver_namespace
+from twistedcaldav import carddavxml
from twistedcaldav.config import config
-from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
-from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
+from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.query import addressbookqueryfilter
from twistedcaldav.vcard import Component, Property, vCardProductID
from xmlrpclib import datetime
-from calendarserver.platform.darwin.od import dsattributes, dsquery
-from twisted.python.reflect import namedModule
+log = Logger()
+addSourceProperty = False
+
class OpenDirectoryBackingService(DirectoryService):
"""
- Open Directory implementation of L{IDirectoryService}.
+ Directory backer for L{IDirectoryService}.
"""
baseGUID = "BF07A1A2-5BB5-4A4D-A59A-67260EA7E143"
@@ -82,24 +74,22 @@
peopleNode = "/Search/Contacts",
queryUserRecords=True,
userNode = "/Search",
+ queryGroupRecords=True,
+ groupNode = "/Search",
maxDSQueryRecords = 0, # maximum number of records requested for any ds query
- queryDSLocal = False, #query in DSLocal -- debug
+ queryDSLocal = False, #query in DSLocal -- debug
dsLocalCacheTimeout = 30,
ignoreSystemRecords = True,
- liveQuery = True, # query directory service as needed
- fakeETag = True, # eTag is not reliable if True
-
- cacheQuery = False,
- cacheTimeout=30, # cache timeout
-
- addDSAttrXProperties=False, # add dsattributes to vcards as "X-" attributes
- standardizeSyntheticUIDs = False, # use simple synthetic UIDs --- good for testing
+ fakeETag = True, # eTag is not reliable if True
+
+ addDSAttrXProperties=False, # add dsattributes to vcards as "X-" attributes
appleInternalServer=False,
- additionalAttributes=[],
- allowedAttributes=[],
+ additionalAttributes=None,
+ allowedAttributes=None,
+ searchAttributes=None,
directoryBackedAddressBook=None
):
"""
@@ -108,12 +98,7 @@
@maxDSQueryRecords: maximum number of (unfiltered) ds records retrieved before raising
NumberOfMatchesWithinLimits exception or returning results
@dsLocalCacheTimeout: how log to keep cache of DSLocal records
- @liveQuery: C{True} to query the directory as needed
@fakeETag: C{True} to use a fake eTag; allows ds queries with partial attributes
- @cacheQuery: C{True} to query the directory and cache results
- @cacheTimeout: if caching, the average cache timeout
- @standardizeSyntheticUIDs: C{True} when creating synthetic UID (==f(Node, Type, Record Name)),
- use a standard Node name. This allows testing with the same UID on different hosts
@allowedAttributes: list of DSAttributes that are used to create VCards
"""
@@ -125,425 +110,292 @@
self.userDirectory = None
self.userNode = None
- self.realmName = None # needed for super
- self.odModule = namedModule(config.OpenDirectoryModule)
+ # get node to record type map
+ def addNodesToNodeRecordTypeMap(nodeList, recordType):
+ for node in nodeList if isinstance(nodeList, list) else (nodeList,):
+ if not node in nodeRecordTypeMap:
+ nodeRecordTypeMap[node] = []
+ nodeRecordTypeMap[node] += [recordType,]
+ self.recordTypes += [recordType,]
+
+ nodeRecordTypeMap = {}
+ self.recordTypes = []
+ if queryPeopleRecords:
+ addNodesToNodeRecordTypeMap(peopleNode, dsattributes.kDSStdRecordTypePeople,)
+ if queryUserRecords:
+ addNodesToNodeRecordTypeMap(userNode, dsattributes.kDSStdRecordTypeUsers,)
+ if queryGroupRecords:
+ addNodesToNodeRecordTypeMap(groupNode, dsattributes.kDSStdRecordTypeGroups,)
- if queryPeopleRecords or not queryUserRecords:
- self.peopleNode = peopleNode
+ # get query info
+ nodeDirectoryRecordTypeMap = {}
+ self.odModule = namedModule(config.OpenDirectoryModule)
+ for node in nodeRecordTypeMap:
+ queryInfo = {"recordTypes":nodeRecordTypeMap[node],}
try:
- self.peopleDirectory = self.odModule.odInit(peopleNode)
+ queryInfo["directory"] = self.odModule.odInit(node)
except self.odModule.ODError, e:
- self.log_error("Open Directory (node=%s) Initialization error: %s" % (peopleNode, e))
+ self.log_error("Open Directory (node=%s) Initialization error: %s" % (node, e))
raise
- self.realmName = peopleNode
-
- if queryUserRecords:
- if self.peopleNode == userNode: # use sane directory and node if they are equal
- self.userNode = self.peopleNode
- self.userDirectory = self.peopleDirectory
- else:
- self.userNode = userNode
- try:
- self.userDirectory = self.odModule.odInit(userNode)
- except self.odModule.ODError, e:
- self.log_error("Open Directory (node=%s) Initialization error: %s" % (userNode, e))
- raise
- if self.realmName:
- self.realmName += "+" + userNode
- else:
- self.realmName = userNode
+
+ nodeDirectoryRecordTypeMap[node] = queryInfo
+ self.nodeDirectoryRecordTypeMap = nodeDirectoryRecordTypeMap
+
+ # calc realm name
+ self.realmName = "+".join(nodeDirectoryRecordTypeMap.keys())
+
+ self.queryPeopleRecords = queryPeopleRecords
+ self.queryUserRecords = queryUserRecords
+ self.queryGroupRecords = queryGroupRecords
self.maxDSQueryRecords = maxDSQueryRecords
self.ignoreSystemRecords = ignoreSystemRecords
self.queryDSLocal = queryDSLocal
self.dsLocalCacheTimeout = dsLocalCacheTimeout
- self.liveQuery = liveQuery or not cacheQuery
self.fakeETag = fakeETag
-
- self.cacheQuery = cacheQuery
-
- self.cacheTimeout = cacheTimeout if cacheTimeout > 0 else 30
-
+
self.addDSAttrXProperties = addDSAttrXProperties
- self.standardizeSyntheticUIDs = standardizeSyntheticUIDs
self.appleInternalServer = appleInternalServer
- self.additionalAttributes = additionalAttributes
- # filter allows attributes, but make sure there are a minimum of attributes for functionality
+
+ if searchAttributes is None:
+ # this is the intersection of ds default indexed attributes and ABDirectoryQueryResult.vcardPropToDSAttrMap.values()
+ # so, not all indexed attributes are below
+ searchAttributes = [
+ dsattributes.kDS1AttrGeneratedUID,
+ dsattributes.kDS1AttrDistinguishedName,
+ dsattributes.kDS1AttrFirstName,
+ dsattributes.kDS1AttrLastName,
+ dsattributes.kDSNAttrEMailAddress,
+ dsattributes.kDSNAttrPhoneNumber,
+ dsattributes.kDSNAttrMobileNumber,
+ dsattributes.kDSNAttrDepartment,
+ dsattributes.kDSNAttrCompany,
+ dsattributes.kDSNAttrStreet,
+ dsattributes.kDSNAttrState,
+ dsattributes.kDSNAttrCity,
+ dsattributes.kDSNAttrCountry,
+ ]
+ elif not searchAttributes:
+ # if search Attributes is [], don't restrict searching (but no binary)
+ searchAttributes = ABDirectoryQueryResult.stringDSAttrNames
+ self.log_debug("self.searchAttributes=%s" % (searchAttributes, ))
+
+ # calculate search map
+ vcardPropToSearchableDSAttrMap = {}
+ for prop, dsAttributeList in ABDirectoryQueryResult.vcardPropToDSAttrMap.iteritems():
+ dsIndexedAttributeList = [attr for attr in dsAttributeList if attr in searchAttributes]
+ if len(dsIndexedAttributeList):
+ vcardPropToSearchableDSAttrMap[prop] = dsIndexedAttributeList
+
+ self.vcardPropToSearchableDSAttrMap = vcardPropToSearchableDSAttrMap
+ self.log_debug("self.vcardPropToSearchableDSAttrMap=%s" % (self.vcardPropToSearchableDSAttrMap, ))
+
+ #get attributes required for needed for valid vCard
+ requiredAttributes = [attr for prop in ("UID", "FN", "N") for attr in ABDirectoryQueryResult.vcardPropToDSAttrMap[prop]]
+ requiredAttributes += [dsattributes.kDS1AttrModificationTimestamp, dsattributes.kDS1AttrCreationTimestamp,] # for VCardResult DAVPropertyMixIn
+ self.requiredAttributes = list(set(requiredAttributes))
+ self.log_debug("self.requiredAttributes=%s" % (self.requiredAttributes, ))
+
+ # get returned attributes
+ #allowedAttributes = [dsattributes.kDS1AttrUniqueID,]
if allowedAttributes:
- self.allowedDSQueryAttributes = sorted(list(set(
- [attr for attr in VCardRecord.allDSQueryAttributes
+
+ returnedAttributes = [attr for attr in ABDirectoryQueryResult.allDSQueryAttributes
if (isinstance(attr, str) and attr in allowedAttributes) or
- (isinstance(attr, tuple) and attr[0] in allowedAttributes)] +
- VCardRecord.dsqueryAttributesForProperty.get("X-INTERNAL-REQUIRED")
- )))
- if (self.allowedDSQueryAttributes != VCardRecord.allDSQueryAttributes):
- self.log_info("Allowed DS query attributes = %r" % (self.allowedDSQueryAttributes, ))
+ (isinstance(attr, tuple) and attr[0] in allowedAttributes)]
+ self.log_debug("allowedAttributes%s" % (allowedAttributes, ))
else:
- self.allowedDSQueryAttributes = VCardRecord.allDSQueryAttributes
-
- #self.returnedAttributes = VCardRecord.allDSQueryAttributes
- self.returnedAttributes = self.allowedDSQueryAttributes
-
+ returnedAttributes = ABDirectoryQueryResult.allDSQueryAttributes
+ # add required
+ returnedAttributes += self.requiredAttributes
+ if additionalAttributes:
+ returnedAttributes += additionalAttributes
- self._dsLocalRecords = []
- self._nextDSLocalQueryTime = 0
+ if ignoreSystemRecords:
+ returnedAttributes += [dsattributes.kDS1AttrUniqueID,]
+ if not queryDSLocal:
+ returnedAttributes += [dsattributes.kDSNAttrMetaNodeLocation,]
+ if queryGroupRecords:
+ returnedAttributes += [dsattributes.kDSNAttrGroupMembers,]
- # get this now once
- hostname = getfqdn()
- if hostname:
- self.defaultNodeName = "/LDAPv3/" + hostname
- else:
- self.defaultNodeName = None
-
- #cleanup
- self._cleanupTime = time.time()
-
- # file system locks
- self._initLockPath = join(config.DocumentRoot, ".directory_address_book_create_lock")
- self._createdLockPath = join(config.DocumentRoot, ".directory_address_book_created_lock")
- self._updateLockPath = join(config.DocumentRoot, ".directory_address_book_update_lock")
- self._tmpDirAddressBookLockPath = join(config.DocumentRoot, ".directory_address_book_tmpFolder_lock")
-
- self._updateLock = MemcacheLock("OpenDirectoryBacker", self._updateLockPath)
- self._tmpDirAddressBookLock = MemcacheLock("OpenDirectoryBacker", self._tmpDirAddressBookLockPath)
-
- # optimization so we don't have to always get create lock
- self._triedCreateLock = False
- self._created = False
+ #for debugging
+ returnedAttributes += [dsattributes.kDSNAttrRecordType,]
-
- def __cmp__(self, other):
- if not isinstance(other, DirectoryRecord):
- return super(DirectoryRecord, self).__eq__(other)
-
- for attr in ("directory", "node"):
- diff = cmp(getattr(self, attr), getattr(other, attr))
- if diff != 0:
- return diff
- return 0
-
- def __hash__(self):
- h = hash(self.__class__.__name__)
- for attr in ("node",):
- h = (h + hash(getattr(self, attr))) & sys.maxint
- return h
-
- @inlineCallbacks
- def available(self):
- if not self._triedCreateLock:
- returnValue( False )
- elif not self._created:
- createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
- self.log_debug("blocking on lock of: \"%s\")" % self._createdLockPath)
- self._created = (yield createdLock.locked())
+ self.returnedAttributes = list(set(returnedAttributes))
+ self.log_debug("self.returnedAttributes=%s" % (self.returnedAttributes, ))
+
- returnValue(self._created)
+ self._dsLocalResults = {}
+ self._nextDSLocalQueryTime = 0
-
- def updateLock(self):
- return self._updateLock
-
- @inlineCallbacks
def createCache(self):
- """
- If caching, create the cache for the first time.
- """
-
- if not self.liveQuery:
- self.log_info("loading directory address book")
-
- # get init lock
- initLock = MemcacheLock("OpenDirectoryBacker", self._initLockPath, timeout=0)
- self.log_debug("Attempt lock of: \"%s\")" % self._initLockPath)
- gotCreateLock = False
- try:
- yield initLock.acquire()
- gotCreateLock = True
- except MemcacheLockTimeoutError:
- pass
-
- self._triedCreateLock = True
-
- if gotCreateLock:
- self.log_debug("Got lock!")
- yield self._refreshCache( flushCache=False, creating=True )
- else:
- self.log_debug("Could not get lock - directory address book will be filled by peer")
-
-
+ succeed(None)
- @inlineCallbacks
- def _refreshCache(self, flushCache=False, creating=False, reschedule=True, query=None, attributes=None, keepLock=False, clear=False, maxRecords=0 ):
- """
- refresh the cache.
- """
-
- #print("_refreshCache:, flushCache=%s, creating=%s, reschedule=%s, query = %s" % (flushCache, creating, reschedule, "None" if query is None else query.generate(),))
-
- def refreshLater():
- #
- # Add jitter/fuzz factor to avoid stampede for large OD query
- #
- cacheTimeout = min(self.cacheTimeout, 60) * 60
- cacheTimeout = (cacheTimeout * random()) - (cacheTimeout / 2)
- cacheTimeout += self.cacheTimeout * 60
- reactor.callLater(cacheTimeout, self._refreshCache) #@UndefinedVariable
- self.log_info("Refresh directory address book in %d minutes %d seconds" % divmod(cacheTimeout, 60))
-
- def cleanupLater():
-
- # try to cancel previous call if last clean up was less than 15 minutes ago
- if (time.time() - self._cleanupTime) < 15*60:
- try:
- self._lastCleanupCall.cancel()
- except:
- pass
-
- #
- # Add jitter/fuzz factor
- #
- nom = 120
- later = nom* (random() + .5)
- self._lastCleanupCall = reactor.callLater(later, removeTmpAddressBooks) #@UndefinedVariable
- self.log_info("Remove temporary directory address books in %d minutes %d seconds" % divmod(later, 60))
-
-
- def getTmpDirAndTmpFilePrefixSuffix():
- # need to have temp file on same volumes as documents so that move works
- absDocPath = abspath(config.DocumentRoot)
- if absDocPath.startswith("/Volumes/"):
- tmpDir = absDocPath
- prefix = ".directoryAddressBook-"
- else:
- tmpDir = gettempdir()
- prefix = "directoryAddressBook-"
-
- return (tmpDir, prefix, ".tmp")
-
- def makeTmpFilename():
- tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
- fd, fname = mkstemp(suffix=suffix, prefix=prefix, dir=tmpDir)
- os.close(fd)
- os.remove(fname)
- return fname
+ def _isSystemRecord(self, recordShortName, recordAttributes):
- @inlineCallbacks
- def removeTmpAddressBooks():
- self.log_info("Checking for temporary directory address books")
- tmpDir, prefix, suffix = getTmpDirAndTmpFilePrefixSuffix()
-
- tmpDirLock = self._tmpDirAddressBookLock
- self.log_debug("blocking on lock of: \"%s\")" % self._tmpDirAddressBookLockPath)
- yield tmpDirLock.acquire()
-
- try:
- for name in listdir(tmpDir):
- if name.startswith(prefix) and name.endswith(suffix):
- try:
- path = join(tmpDir, name)
- self.log_info("Deleting temporary directory address book at: %s" % path)
- FilePath(path).remove()
- self.log_debug("Done deleting")
- except:
- self.log_info("Deletion failed")
- finally:
- self.log_debug("unlocking: \"%s\")" % self._tmpDirAddressBookLockPath)
- yield tmpDirLock.release()
-
- self._cleanupTime = time.time()
-
-
- updateLock = None
- limited = False
- try:
-
- try:
- # get the records
- if clear:
- records = {}
- else:
- records, limited = (yield self._getDirectoryRecords(query, attributes, maxRecords))
-
- # calculate the hash
- # simple for now, could use MD5 digest if too many collisions
- newAddressBookCTag = customxml.GETCTag(str(hash(self.baseGUID + ":" + self.realmName + ":" + "".join(str(hash(records[key])) for key in records.keys()))))
-
- # get the old hash
- oldAddressBookCTag = ""
- updateLock = self.updateLock()
- self.log_debug("blocking on lock of: \"%s\")" % self._updateLockPath)
- yield updateLock.acquire()
-
- if not flushCache:
- # get update lock
- try:
- oldAddressBookCTag = self.directoryBackedAddressBook.readDeadProperty((calendarserver_namespace, "getctag"))
- except:
- oldAddressBookCTag = ""
+ recordType = recordAttributes.get(dsattributes.kDSNAttrRecordType)
+ guid = recordAttributes.get(dsattributes.kDS1AttrGeneratedUID)
+ if guid and guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
+ self.log_info("Ignoring record %s (type %s) with %s %s" % (recordShortName, recordType, dsattributes.kDS1AttrGeneratedUID, guid,))
+ return True
- self.log_debug("Comparing {http://calendarserver.org/ns/}getctag: new = %s, old = %s" % (newAddressBookCTag, oldAddressBookCTag))
- if str(newAddressBookCTag) != str(oldAddressBookCTag):
-
- self.log_debug("unlocking: \"%s\")" % self._updateLockPath)
- yield updateLock.release()
- updateLock = None
-
+ uniqueID = recordAttributes.get(dsattributes.kDS1AttrUniqueID)
+ if uniqueID and (int(uniqueID) < 500 or (recordType == dsattributes.kDSStdRecordTypeUsers and int(uniqueID) == 1000)):
+ self.log_info("Ignoring record %s (type %s) with %s %s" % (recordShortName, recordType, dsattributes.kDS1AttrUniqueID, uniqueID,))
+ return True
- if not keepLock:
- self.log_debug("unlocking: \"%s\")" % self._updateLockPath)
- yield updateLock.release()
- updateLock = None
-
- except:
- cleanupLater()
- if reschedule:
- refreshLater()
- raise
-
- if creating:
- createdLock = MemcacheLock("OpenDirectoryBacker", self._createdLockPath)
- self.log_debug("blocking on lock of: \"%s\")" % self._createdLockPath)
- yield createdLock.acquire()
-
- cleanupLater()
- if reschedule:
- refreshLater()
+ if recordShortName.startswith("_"):
+ self.log_info("Ignoring record %s (type %s) with %s %s" % (recordShortName, recordType, dsattributes.kDSNAttrRecordName, recordShortName,))
+ return True
- except:
- if updateLock:
- yield updateLock.release()
- raise
+ return False
- returnValue( (updateLock, limited) )
-
-
-
- def _getDSLocalRecords(self):
-
- def generateDSLocalRecords():
+
+ def _getAllDSLocalResults(self):
+ """
+ Get a dictionary of ABDirectoryQueryResult by enumerating the local directory
+ """
+
+ def generateDSLocalResults():
- records = {}
+ resultsDictionary = {}
- recordTypes = [dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers, ]
try:
localNodeDirectory = self.odModule.odInit("/Local/Default")
self.log_debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r)" % (
"/DSLocal",
- recordTypes,
+ self.recordTypes,
self.returnedAttributes,
))
- results = list(self.odModule.listAllRecordsWithAttributes_list(
+ records = list(self.odModule.listAllRecordsWithAttributes_list(
localNodeDirectory,
- recordTypes,
+ self.recordTypes,
self.returnedAttributes,
))
except self.odModule.ODError, ex:
self.log_error("Open Directory (node=%s) error: %s" % ("/Local/Default", str(ex)))
raise
- self._dsLocalRecords = []
- for (recordShortName, value) in results: #@UnusedVariable
+ for (recordShortName, recordAttributes) in records: #@UnusedVariable
- record = VCardRecord(self, value, "/Local/Default")
+ try:
+ self.log_info("Inspecting record %s" % (recordAttributes,))
+ if self.ignoreSystemRecords:
+ if self._isSystemRecord(recordShortName, recordAttributes):
+ continue
- if self.ignoreSystemRecords:
- # remove system users and people
- if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
- self.log_info("Ignoring vcard for system record %s" % (record,))
+ result = ABDirectoryQueryResult(self.directoryBackedAddressBook, recordAttributes)
+
+ except:
+ traceback.print_exc()
+ self.log_info("Could not get vcard for record %s" % (recordShortName,))
+
+ else:
+ uid = result.vCard().propertyValue("UID")
+
+ if uid in resultsDictionary:
+ self.log_info("Record %s skipped due to duplicate UID: %s" % (recordShortName, uid,))
continue
+
+ self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+ resultsDictionary[uid] = result
- if record.guid in records:
- self.log_info("Record skipped due to conflict (duplicate uuid): %s" % (record,))
- else:
- try:
- vCardText = record.vCardText()
- except:
- traceback.print_exc()
- self.log_info("Could not get vcard for record %s" % (record,))
- else:
- self.log_debug("VCard text =\n%s" % (vCardText, ))
- records[record.guid] = record
- return records
+ return resultsDictionary
- if not self.liveQuery or not self.queryDSLocal:
+ if not self.queryDSLocal:
return {}
if time.time() > self._nextDSLocalQueryTime:
- self._dsLocalRecords = generateDSLocalRecords()
+ self._dsLocalResults = generateDSLocalResults()
# Add jitter/fuzz factor
self._nextDSLocalQueryTime = time.time() + self.dsLocalCacheTimeout * (random() + 0.5) * 60
- return self._dsLocalRecords
+ return self._dsLocalResults
@inlineCallbacks
- def _getDirectoryRecords(self, query=None, attributes=None, maxRecords=0 ):
+ def _getDirectoryQueryResults(self, query=None, attributes=None, maxRecords=0, allowedRecordTypes=None ):
"""
- Get a list of filtered VCardRecord for the given query with the given attributes.
- query == None gets all records. attribute == None gets VCardRecord.allDSQueryAttributes
+ Get a list of ABDirectoryQueryResult for the given query with the given attributes.
+ query == None gets all records. attribute == None gets ABDirectoryQueryResult.allDSQueryAttributes
"""
limited = False
- queryResults = (yield self._queryDirectory(query, attributes, maxRecords ))
- if maxRecords and len(queryResults) >= maxRecords:
+ records = (yield self._queryDirectory(query, attributes, maxRecords, allowedRecordTypes=allowedRecordTypes ))
+ if maxRecords and len(records) >= maxRecords:
limited = True
self.log_debug("Directory address book record limit (= %d) reached." % (maxRecords, ))
- self.log_debug("Query done. Inspecting %s results" % len(queryResults))
+ self.log_debug("Query done. Inspecting %s records" % (len(records),))
- records = self._getDSLocalRecords().copy()
- self.log_debug("Adding %s DSLocal results" % len(records.keys()))
+ resultsDictionary = self._getAllDSLocalResults().copy()
+ self.log_debug("Adding %s DSLocal results" % len(resultsDictionary.keys()))
- for (recordShortName, value) in queryResults: #@UnusedVariable
+ for (recordShortName, recordAttributes) in records: #@UnusedVariable
- record = VCardRecord(self, value, self.defaultNodeName)
+ try:
+ # fix ds strangeness
+ if recordAttributes.get(dsattributes.kDS1AttrLastName, "") == "99":
+ del recordAttributes[dsattributes.kDS1AttrLastName]
+
+ if self.ignoreSystemRecords:
+ if self._isSystemRecord(recordShortName, recordAttributes):
+ continue
+
+ if not self.queryDSLocal:
+ # skip records in local node which happens for non-complex od queries
+ node = recordAttributes.get(dsattributes.kDSNAttrMetaNodeLocation)
+ if node and node.startswith("/Local/"):
+ recordType = recordAttributes.get(dsattributes.kDSNAttrRecordType)
+ self.log_info("Ignoring record %s (type %s) with %s %s" % (recordShortName, recordType, dsattributes.kDSNAttrMetaNodeLocation, recordAttributes.get(dsattributes.kDSNAttrMetaNodeLocation),))
+ continue
- if self.ignoreSystemRecords:
- # remove system users and people
- if record.guid.startswith("FFFFEEEE-DDDD-CCCC-BBBB-AAAA"):
- self.log_info("Ignoring vcard for system record %s" % (record,))
+ result = ABDirectoryQueryResult(self.directoryBackedAddressBook, recordAttributes,
+ addDSAttrXProperties=self.addDSAttrXProperties,
+ appleInternalServer=self.appleInternalServer,
+ )
+ except:
+ traceback.print_exc()
+ self.log_info("Could not get vcard for record %s" % (recordShortName,))
+
+ else:
+ uid = result.vCard().propertyValue("UID")
+
+ if uid in resultsDictionary:
+ self.log_info("Record skipped due to duplicate UID: %s" % (recordShortName,))
continue
+
+ self.log_debug("VCard text =\n%s" % (result.vCardText(), ))
+ resultsDictionary[uid] = result
- if record.guid in records:
- self.log_info("Ignoring vcard for record due to conflict (duplicate uuid): %s" % (record,))
- else:
- records[record.guid] = record
-
- self.log_debug("After filtering, %s records (limited=%s)." % (len(records), limited))
- returnValue((records, limited, ))
+ self.log_debug("_getDirectoryQueryResults: %s results (limited=%s)." % (len(resultsDictionary), limited))
+ returnValue((resultsDictionary.values(), limited, ))
- def _queryDirectory(self, query=None, attributes=None, maxRecords=0 ):
+ def _queryDirectory(self, query=None, attributes=None, maxRecords=0, allowedRecordTypes=None ):
startTime = time.time()
-
-
if not attributes:
attributes = self.returnedAttributes
-
- attributes = list(set(attributes + self.additionalAttributes)) # remove duplicates
-
- directoryAndRecordTypes = []
- if self.peopleDirectory == self.userDirectory:
- # use single ds query if possible for best performance
- directoryAndRecordTypes.append( (self.peopleDirectory, self.peopleNode, (dsattributes.kDSStdRecordTypePeople, dsattributes.kDSStdRecordTypeUsers) ) )
- else:
- if self.peopleDirectory:
- directoryAndRecordTypes.append( (self.peopleDirectory, self.peopleNode, dsattributes.kDSStdRecordTypePeople) )
- if self.userDirectory:
- directoryAndRecordTypes.append( (self.userDirectory, self.userNode, dsattributes.kDSStdRecordTypeUsers) )
-
+
allResults = []
- for directory, node, recordType in directoryAndRecordTypes:
+ for node, queryInfo in self.nodeDirectoryRecordTypeMap.iteritems():
+ directory = queryInfo["directory"]
+ recordTypes = queryInfo["recordTypes"]
+ if not allowedRecordTypes is None:
+ recordTypes = list(set(recordTypes).intersection(set(allowedRecordTypes)))
+ if not recordTypes:
+ continue
+
try:
if query:
if isinstance(query, dsquery.match) and query.value is not "":
@@ -553,7 +405,7 @@
query.value,
query.matchType,
False,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
@@ -564,7 +416,7 @@
query.value,
query.matchType,
False,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
@@ -573,7 +425,7 @@
node,
query.generate(),
False,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
@@ -582,21 +434,21 @@
directory,
query.generate(),
False,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
else:
self.log_debug("opendirectory.listAllRecordsWithAttributes_list(%r,%r,%r,%r)" % (
node,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
results = list(
self.odModule.listAllRecordsWithAttributes_list(
directory,
- recordType,
+ recordTypes,
attributes,
maxRecords,
))
@@ -616,443 +468,443 @@
self.log_info("Timing: Directory query: %.1f ms (%d records, %.2f records/sec)" % (elaspedTime*1000, len(allResults), len(allResults)/elaspedTime))
return succeed(allResults)
- def _getDSFilter(self, addressBookFilter):
+
+ @inlineCallbacks
+ def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
"""
- Convert the supplied addressbook-query into an expression tree.
-
- @param filter: the L{Filter} for the addressbook-query to convert.
- @return: (needsAllRecords, espressionAttributes, expression) tuple
+ Get vCards for a given addressBookFilter and addressBookQuery
"""
- def propFilterListQuery(filterAllOf, propFilters):
+
+ def allowedRecordTypes():
+ constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+
+ # optimization: use KIND as constant to filter record type list
+ dsRecordTypeToKindMap = {
+ dsattributes.kDSStdRecordTypeGroups:"group",
+ dsattributes.kDSStdRecordTypeLocations:"location",
+ dsattributes.kDSStdRecordTypeResources:"device",
+ }
+
+ allowedRecordTypes = []
+ for recordType in set(self.recordTypes):
+ kind = dsRecordTypeToKindMap.get(recordType, "individual")
+ constantProperties["KIND"] = kind
+
+ filterPropertyNames, dsFilter = dsFilterFromAddressBookFilter( addressBookFilter,
+ self.vcardPropToSearchableDSAttrMap,
+ constantProperties=constantProperties );
+ if not dsFilter is False:
+ allowedRecordTypes += [recordType,]
+ return set(allowedRecordTypes)
+
- def propFilterExpression(filterAllOf, propFilter):
- #print("propFilterExpression")
- """
- Create an expression for a single prop-filter element.
+ filterPropertyNames, dsFilter = dsFilterFromAddressBookFilter( addressBookFilter,
+ self.vcardPropToSearchableDSAttrMap,
+ constantProperties=ABDirectoryQueryResult.constantProperties );
+ self.log_debug("doAddressBookQuery: query=%s, propertyNames=%s" % (dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames,))
+
+ results = []
+ limited = False
+ if dsFilter:
+
+ if dsFilter is True:
+ dsFilter = None # None means get all records hereafter
+
+ # calculate minimum attributes needed for this query
+ etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+
+ if (etagRequested and not self.fakeETag) or not queryPropNames:
+ queryAttributes = self.returnedAttributes
+ elif queryPropNames:
+ queryPropNames += filterPropertyNames
+ queryAttributes = []
+ for prop in queryPropNames:
+ attributes = ABDirectoryQueryResult.vcardPropToDSAttrMap.get(prop)
+ if attributes:
+ queryAttributes += attributes
+
+ queryAttributes = list(set(queryAttributes + self.requiredAttributes).intersection(self.returnedAttributes))
+
+ self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s, queryAttributes=%s" % (etagRequested, queryPropNames, queryAttributes,))
+
+ '''
+ # change query to ignore system records rather than post filtering
+ # but this is broken in open directory client
+ if self.ignoreSystemRecords:
+ ignoreExpression = dsquery.expression( dsquery.expression.NOT,
+ dsquery.match(dsattributes.kDS1AttrGeneratedUID, "FFFFEEEE-DDDD-CCCC-BBBB-AAAA", dsattributes.eDSStartsWith)
+ )
+ filterAttributes = list(set(filterAttributes).union(dsattributes.kDS1AttrGeneratedUID))
- @param propFilter: the L{PropertyFilter} element.
- @return: (needsAllRecords, espressionAttributes, expressions) tuple
- """
+ dsFilter = dsquery.expression( dsquery.expression.AND, (dsFilter, ignoreExpression,) ) if dsFilter else ignoreExpression
+ '''
+ maxRecords = int(maxResults * 1.2)
+
+ # keep trying query till we get results based on filter. Especially when doing "all results" query
+ while True:
+ dsQueryResults, dsQueryLimited = (yield self._getDirectoryQueryResults(dsFilter, queryAttributes, maxRecords, allowedRecordTypes=allowedRecordTypes()))
- def definedExpression( defined, allOf, filterName, constant, queryAttributes, allAttrStrings):
- if constant or filterName in ("N" , "FN", "UID", ):
- return (defined, [], []) # all records have this property so no records do not have it
+ filteredResults = []
+ for dsQueryResult in dsQueryResults:
+ if addressBookFilter.match(dsQueryResult.vCard()):
+ filteredResults.append(dsQueryResult)
else:
- matchList = list(set([dsquery.match(attrName, "", dsattributes.eDSStartsWith) for attrName in allAttrStrings]))
- if defined:
- return andOrExpression(allOf, queryAttributes, matchList)
- else:
- if len(matchList) > 1:
- expr = dsquery.expression( dsquery.expression.OR, matchList )
- else:
- expr = matchList
- return (False, queryAttributes, [dsquery.expression( dsquery.expression.NOT, expr),])
- #end isNotDefinedExpression()
+ self.log_debug("doAddressBookQuery: result did not match filter: %s (%s)" % (dsQueryResult.vCard().propertyValue("FN"), dsQueryResult.vCard().propertyValue("UID"),))
+
+ #no more results
+ if not dsQueryLimited:
+ break;
+
+ # more than requested results
+ if maxResults and len(filteredResults) >= maxResults:
+ break
+
+ # more than max report results
+ if len(filteredResults) >= config.MaxQueryWithDataResults:
+ break
+
+ # more than self limit
+ if self.maxDSQueryRecords and maxRecords >= self.maxDSQueryRecords:
+ break
+
+ # try again with 2x
+ maxRecords *= 2
+ if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
+ maxRecords = self.maxDSQueryRecords
+
+
+ results = filteredResults
+ limited = maxResults and len(results) >= maxResults
+
+ #if self.sortResults:
+ # results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+
+ self.log_debug("doAddressBookQuery: %s results (limited=%s)." % (len(results), limited))
+ returnValue((results, limited,))
+
+
+#utility
+def propertiesInAddressBookQuery( addressBookQuery ):
+ """
+ Get the vCard properties requested by a given query
+ """
+ etagRequested = False
+ propertyNames = []
+ if addressBookQuery.qname() == ("DAV:", "prop"):
- def andOrExpression(propFilterAllOf, queryAttributes, matchList):
- #print("andOrExpression(propFilterAllOf=%r, queryAttributes%r, matchList%r)" % (propFilterAllOf, queryAttributes, matchList))
- if propFilterAllOf and len(matchList):
- # add OR expression because parent will AND
- return (False, queryAttributes, [dsquery.expression( dsquery.expression.OR, matchList),])
- else:
- return (False, queryAttributes, matchList)
- #end andOrExpression()
+ for property in addressBookQuery.children:
+ if isinstance(property, carddavxml.AddressData):
+ for addressProperty in property.children:
+ if isinstance(addressProperty, carddavxml.Property):
+ propertyNames += [addressProperty.attributes["name"],]
+ elif property.qname() == ("DAV:", "getetag"):
+ # for a real etag == md5(vCard), we need all attributes
+ etagRequested = True
- # short circuit parameter filters
- def supportedParamter( filterName, paramFilters, propFilterAllOf ):
-
- def supported( paramFilterName, paramFilterDefined, params ):
- paramFilterName = paramFilterName.upper()
- if len(params.keys()) and ((paramFilterName in params.keys()) != paramFilterDefined):
- return False
- if len(params[paramFilterName]) and str(paramFilter.qualifier).upper() not in params[paramFilterName]:
- return False
- return True
- #end supported()
+ return (etagRequested, propertyNames if len(propertyNames) else None)
+
+
+def dsFilterFromAddressBookFilter(addressBookFilter, vcardPropToSearchableAttrMap, constantProperties={}):
+ """
+ Convert the supplied addressbook-query into a ds expression tree.
+
+ @param addressBookFilter: the L{Filter} for the addressbook-query to convert.
+ @param vcardPropToSearchableAttrMap: a mapping from vcard properties to searchable query attributes.
+ @param constantProperties: a mapping of constant properties. A query on a constant property will return all or None
+ @return: (filterProperyNames, expressions) tuple. expression==True means list all results, expression==False means no results
+ """
+ def propFilterListQuery(filterAllOf, propFilters):
+
+ def combineExpressionLists(expressionList, allOf, addedExpressions):
+ """
+ deal with the 4-state logic
+ addedExpressions=None means ignore
+ addedExpressions=True means all records
+ addedExpressions=False means no records
+ addedExpressions=[expressionlist] add to expression list
+ """
+ #def explen(exp): return len(exp) if isinstance(exp, list) else 0
+ #log.debug("propFilterListQuery(): allOf=%s, expressionList=%s (%s), addedExpressions=%s (%s)" % (allOf, expressionList, explen(expressionList), addedExpressions, explen(addedExpressions)))
+ if expressionList is None:
+ expressionList = addedExpressions
+ elif addedExpressions is not None:
+ if addedExpressions is True:
+ if not allOf:
+ expressionList = True # expressionList or True is True
+ #else expressionList and True is expressionList
+ elif addedExpressions is False:
+ if allOf:
+ expressionList = False # expressionList and False is False
+ #else expressionList or False is expressionList
+ else:
+ if expressionList is False:
+ if not allOf:
+ expressionList = addedExpressions # False or addedExpressions is addedExpressions
+ #else False and addedExpressions is False
+ elif expressionList is True:
+ if allOf:
+ expressionList = addedExpressions # False or addedExpressions is addedExpressions
+ #else False and addedExpressions is False
+ else:
+ expressionList += addedExpressions
+ #log.debug("propFilterListQuery(): out expressionList=%s (%s)" % (expressionList, explen(expressionList)))
+ return expressionList
+
+
+ def propFilterExpression(filterAllOf, propFilter):
+ """
+ Create an expression for a single prop-filter element.
+
+ @param propFilter: the L{PropertyFilter} element.
+ @return: (filterProperyNames, expressions) tuple. expression==True means list all results, expression==False means no results
+ """
+
+ def definedExpression( defined, allOf ):
+ if constant or propFilter.filter_name in ("N" , "FN", "UID", "SOURCE",):
+ return defined # all records have this property so no records do not have it
+ else:
+ matchList = [dsquery.match(attrName, "", dsattributes.eDSStartsWith) for attrName in searchablePropFilterAttrNames]
+ if defined:
+ return andOrExpression(allOf, matchList)
+ else:
+ if len(matchList) > 1:
+ expr = dsquery.expression( dsquery.expression.OR, matchList )
+ else:
+ expr = matchList[0]
+ return [dsquery.expression( dsquery.expression.NOT, expr),]
+ #end definedExpression()
+
+
+ def andOrExpression(propFilterAllOf, matchList):
+ if propFilterAllOf and len(matchList) > 1:
+ # add OR expression because parent will AND
+ return [dsquery.expression( dsquery.expression.OR, matchList),]
+ else:
+ return matchList
+ #end andOrExpression()
+
+ def paramFilterElementExpression(propFilterAllOf, paramFilterElement):
+
+ params = ABDirectoryQueryResult.vcardPropToParamMap.get(propFilter.filter_name.upper())
+ defined = params and paramFilterElement.filter_name.upper() in params
+
+ #defined test
+ if defined != paramFilterElement.defined:
+ return False
+
+ #parameter value text match
+ if defined and paramFilterElement.filters:
+ paramValues = params[paramFilterElement.filter_name.upper()]
+ if paramValues and paramFilterElement.filters[0].text.upper() not in paramValues:
+ return False
+
+ return True
+
+
+ def textMatchElementExpression( propFilterAllOf, textMatchElement ):
+
+ # pre process text match strings for ds query
+ def getMatchStrings( propFilter, matchString ):
+
+ if propFilter.filter_name in ("REV" , "BDAY", ):
+ rawString = matchString
+ matchString = ""
+ for c in rawString:
+ if not c in "TZ-:":
+ matchString += c
+ elif propFilter.filter_name == "GEO":
+ matchString = ",".join(matchString.split(";"))
- oneSupported = False
- for paramFilter in paramFilters:
- if filterName == "PHOTO":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "ENCODING": ["B",], "TYPE": ["JPEG",], }):
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "ADR":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": ["WORK", "PREF", "POSTAL", "PARCEL",], }):
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "LABEL":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": ["POSTAL", "PARCEL",]}):
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "TEL":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": [], }): # has params derived from ds attributes
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "EMAIL":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "TYPE": [], }): # has params derived from ds attributes
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "URL":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, {}):
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif filterName == "KEY":
- if propFilterAllOf != supported( paramFilter.filter_name, paramFilter.defined, { "ENCODING": ["B",], "TYPE": ["PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE",] }):
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
- elif not filterName.startswith("X-"): #X- IMHandles X-ABRELATEDNAMES excepted, no other params are used
- if propFilterAllOf == paramFilter.defined:
- return not propFilterAllOf
- oneSupported |= propFilterAllOf
+ if propFilter.filter_name in ("N" , "ADR", "ORG", ):
+ # for structured properties, change into multiple strings for ds query
+ if propFilter.filter_name == "ADR":
+ #split by newline and comma
+ rawStrings = ",".join( matchString.split("\n") ).split(",")
+ else:
+ #split by space
+ rawStrings = matchString.split(" ")
+
+ # remove empty strings
+ matchStrings = []
+ for oneString in rawStrings:
+ if len(oneString):
+ matchStrings += [oneString,]
+ return matchStrings
- if propFilterAllOf:
- return True
+ elif len(matchString):
+ return [matchString,]
else:
- return oneSupported
- #end supportedParamter()
-
-
- def textMatchElementExpression( propFilterAllOf, textMatchElement ):
-
- # pre process text match strings for ds query
- def getMatchStrings( propFilter, matchString ):
-
- if propFilter.filter_name in ("REV" , "BDAY", ):
- rawString = matchString
- matchString = ""
- for c in rawString:
- if not c in "TZ-:":
- matchString += c
- elif propFilter.filter_name == "GEO":
- matchString = ",".join(matchString.split(";"))
-
- if propFilter.filter_name in ("N" , "ADR", "ORG", ):
- # for structured properties, change into multiple strings for ds query
- if propFilter.filter_name == "ADR":
- #split by newline and comma
- rawStrings = ",".join( matchString.split("\n") ).split(",")
- else:
- #split by space
- rawStrings = matchString.split(" ")
-
- # remove empty strings
- matchStrings = []
- for oneString in rawStrings:
- if len(oneString):
- matchStrings += [oneString,]
- return matchStrings
-
- elif len(matchString):
- return [matchString,]
- else:
- return []
- # end getMatchStrings
-
- if constant:
- # do the match right now! Return either all or none.
- return( textMatchElement.test([constant,]), [], [] )
- else:
+ return []
+ # end getMatchStrings
- matchStrings = getMatchStrings(propFilter, textMatchElement.text)
+ if constant:
+ #FIXME: match is not implemented in twisteddaldav.query.addressbookqueryfilter.TextMatch so use _match for now
+ return textMatchElement._match([constant,])
+ else:
- if not len(matchStrings) or binaryAttrStrs:
- # no searching text in binary ds attributes, so change to defined/not defined case
- if textMatchElement.negate:
- return definedExpression(False, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
- # else fall through to attribute exists case below
- else:
-
- # special case UID's formed from node and record name
- if propFilter.filter_name == "UID":
- matchString = matchStrings[0]
- seperatorIndex = matchString.find(VCardRecord.peopleUIDSeparator)
- if seperatorIndex > 1:
- recordNameStart = seperatorIndex + len(VCardRecord.peopleUIDSeparator)
- else:
- seperatorIndex = matchString.find(VCardRecord.userUIDSeparator)
- if seperatorIndex > 1:
- recordNameStart = seperatorIndex + len(VCardRecord.userUIDSeparator)
- else:
- recordNameStart = sys.maxint
-
+ matchStrings = getMatchStrings(propFilter, textMatchElement.text)
+
+ if not len(matchStrings):
+ # no searching text in binary ds attributes, so change to defined/not defined case
+ if textMatchElement.negate:
+ return definedExpression(False, propFilterAllOf)
+ # else fall through to attribute exists case below
+ else:
+
+ # special case UID's formed from node and record name
+ if propFilter.filter_name == "UID":
+ matchString = matchStrings[0]
+ seperatorIndex = matchString.find(ABDirectoryQueryResult.uidSeparator)
+ if seperatorIndex > 1:
+ recordNameStart = seperatorIndex + len(ABDirectoryQueryResult.uidSeparator)
+
if recordNameStart < len(matchString)-1:
try:
recordNameQualifier = matchString[recordNameStart:].decode("base64").decode("utf8")
except Exception, e:
- self.log_debug("Could not decode UID string %r in %r: %r" % (matchString[recordNameStart:], matchString, e,))
+ log.debug("Could not decode UID string %r in %r: %r" % (matchString[recordNameStart:], matchString, e,))
else:
if textMatchElement.negate:
- return (False, queryAttributes,
- [dsquery.expression(dsquery.expression.NOT, dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact)),]
- )
+ return [dsquery.expression(dsquery.expression.NOT, dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact)),]
else:
- return (False, queryAttributes,
- [dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact),]
- )
-
- # use match_type where possible depending on property/attribute mapping
- # Note that case sensitive negate will not work
- # Should return all records in that case
- matchType = dsattributes.eDSContains
- if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV", "LABEL", ):
- if textMatchElement.match_type == "equals":
- matchType = dsattributes.eDSExact
- elif textMatchElement.match_type == "starts-with":
- matchType = dsattributes.eDSStartsWith
- elif textMatchElement.match_type == "ends-with":
- matchType = dsattributes.eDSEndsWith
-
- matchList = []
- for matchString in matchStrings:
- matchList += [dsquery.match(attrName, matchString, matchType) for attrName in stringAttrStrs]
-
- matchList = list(set(matchList))
-
- if textMatchElement.negate:
- if len(matchList) > 1:
- expr = dsquery.expression( dsquery.expression.OR, matchList )
- else:
- expr = matchList
- return (False, queryAttributes, [dsquery.expression( dsquery.expression.NOT, expr),])
+ return [dsquery.match(dsattributes.kDSNAttrRecordName, recordNameQualifier, dsattributes.eDSExact),]
+
+ # use match_type where possible depending on property/attribute mapping
+ # FIXME: case-sensitive negate will not work. This should return all all records in that case
+ matchType = dsattributes.eDSContains
+ if propFilter.filter_name in ("NICKNAME" , "TITLE" , "NOTE" , "UID", "URL", "N", "ADR", "ORG", "REV", "LABEL", ):
+ if textMatchElement.match_type == "equals":
+ matchType = dsattributes.eDSExact
+ elif textMatchElement.match_type == "starts-with":
+ matchType = dsattributes.eDSStartsWith
+ elif textMatchElement.match_type == "ends-with":
+ matchType = dsattributes.eDSEndsWith
+
+ matchList = []
+ for matchString in matchStrings:
+ matchList += [dsquery.match(attrName, matchString, matchType) for attrName in searchableAttributes]
+
+ matchList = list(set(matchList))
+
+ if textMatchElement.negate:
+ if len(matchList) > 1:
+ expr = dsquery.expression( dsquery.expression.OR, matchList )
else:
- return andOrExpression(propFilterAllOf, queryAttributes, matchList)
-
- # attribute exists search
- return definedExpression(True, propFilterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
- #end textMatchElementExpression()
-
-
- # get attribute strings from dsqueryAttributesForProperty list
- queryAttributes = list(set(VCardRecord.dsqueryAttributesForProperty.get(propFilter.filter_name, [])).intersection(set(self.allowedDSQueryAttributes)))
-
- binaryAttrStrs = []
- stringAttrStrs = []
- for attr in queryAttributes:
- if isinstance(attr, tuple):
- binaryAttrStrs.append(attr[0])
- else:
- stringAttrStrs.append(attr)
- allAttrStrings = stringAttrStrs + binaryAttrStrs
-
- constant = VCardRecord.constantProperties.get(propFilter.filter_name)
- if not constant and not allAttrStrings:
- return (False, [], [])
-
- if propFilter.qualifier and isinstance(propFilter.qualifier, addressbookqueryfilter.IsNotDefined):
- return definedExpression(False, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
-
- paramFilterElements = [paramFilterElement for paramFilterElement in propFilter.filters if isinstance(paramFilterElement, addressbookqueryfilter.ParameterFilter)]
- textMatchElements = [textMatchElement for textMatchElement in propFilter.filters if isinstance(textMatchElement, addressbookqueryfilter.TextMatch)]
- propFilterAllOf = propFilter.propfilter_test == "allof"
-
- # handle parameter filter elements
- if len(paramFilterElements) > 0:
- if supportedParamter(propFilter.filter_name, paramFilterElements, propFilterAllOf ):
- if len(textMatchElements) == 0:
- return definedExpression(True, filterAllOf, propFilter.filter_name, constant, queryAttributes, allAttrStrings)
- else:
- if propFilterAllOf:
- return (False, [], [])
-
- # handle text match elements
- propFilterNeedsAllRecords = propFilterAllOf
- propFilterAttributes = []
- propFilterExpressionList = []
- for textMatchElement in textMatchElements:
-
- textMatchNeedsAllRecords, textMatchExpressionAttributes, textMatchExpression = textMatchElementExpression(propFilterAllOf, textMatchElement)
- if propFilterAllOf:
- propFilterNeedsAllRecords &= textMatchNeedsAllRecords
- else:
- propFilterNeedsAllRecords |= textMatchNeedsAllRecords
- propFilterAttributes += textMatchExpressionAttributes
- propFilterExpressionList += textMatchExpression
-
+ expr = matchList[0]
+ return [dsquery.expression( dsquery.expression.NOT, expr),]
+ else:
+ return andOrExpression(propFilterAllOf, matchList)
- if (len(propFilterExpressionList) > 1) and (filterAllOf != propFilterAllOf):
- propFilterExpressions = [dsquery.expression(dsquery.expression.AND if propFilterAllOf else dsquery.expression.OR , list(set(propFilterExpressionList)))] # remove duplicates
- else:
- propFilterExpressions = list(set(propFilterExpressionList))
+ # attribute exists search
+ return definedExpression(True, propFilterAllOf)
+ #end textMatchElementExpression()
- return (propFilterNeedsAllRecords, propFilterAttributes, propFilterExpressions)
- #end propFilterExpression
- #print("propFilterListQuery: filterAllOf=%r, propFilters=%r" % (filterAllOf, propFilters,))
- """
- Create an expression for a list of prop-filter elements.
+ # searchablePropFilterAttrNames are attributes to be used by this propfilter's expression
+ searchableAttributes = vcardPropToSearchableAttrMap.get(propFilter.filter_name, [])
+ if isinstance(searchableAttributes, str):
+ searchableAttributes = [searchableAttributes,]
+ searchablePropFilterAttrNames = list(searchableAttributes)
- @param filterAllOf: the C{True} if parent filter test is "allof"
- @param propFilters: the C{list} of L{ComponentFilter} elements.
- @return: (needsAllRecords, espressionAttributes, expression) tuple
- """
- needsAllRecords = filterAllOf
- attributes = []
- expressions = []
- for propFilter in propFilters:
+ constant = constantProperties.get(propFilter.filter_name)
+ if not searchablePropFilterAttrNames and not constant:
+ # not allAttrNames means propFilter.filter_name is not mapped
+ # return None to try to match all items if this is the only property filter
+ return None
+
+ #create a textMatchElement for the IsNotDefined qualifier
+ if isinstance(propFilter.qualifier, addressbookqueryfilter.IsNotDefined):
+ textMatchElement = addressbookqueryfilter.TextMatch(carddavxml.TextMatch.fromString(""))
+ textMatchElement.negate = True
+ propFilter.filters.append(textMatchElement)
+
+ # if only one propFilter, then use filterAllOf as propFilterAllOf to reduce subexpressions and simplify generated query string
+ if len(propFilter.filters) == 1:
+ propFilterAllOf = filterAllOf
+ else:
+ propFilterAllOf = propFilter.propfilter_test == "allof"
+
+ propFilterExpressions = None
+ for propFilterElement in propFilter.filters:
+ propFilterExpression = None
+ if isinstance(propFilterElement, addressbookqueryfilter.ParameterFilter):
+ propFilterExpression = paramFilterElementExpression(propFilterAllOf, propFilterElement)
+ elif isinstance(propFilterElement, addressbookqueryfilter.TextMatch):
+ propFilterExpression = textMatchElementExpression(propFilterAllOf, propFilterElement)
+ propFilterExpressions = combineExpressionLists(propFilterExpressions, propFilterAllOf, propFilterExpression)
+ if isinstance(propFilterExpressions, bool) and propFilterAllOf != propFilterExpression:
+ break
- propNeedsAllRecords, propExpressionAttributes, propExpression = propFilterExpression(filterAllOf, propFilter)
- if filterAllOf:
- needsAllRecords &= propNeedsAllRecords
- else:
- needsAllRecords |= propNeedsAllRecords
- attributes += propExpressionAttributes
- expressions += propExpression
+ if isinstance(propFilterExpressions, list):
+ propFilterExpressions = list(set(propFilterExpressions))
+ if propFilterExpressions and (filterAllOf != propFilterAllOf):
+ propFilterExpressions = [dsquery.expression(dsquery.expression.AND if propFilterAllOf else dsquery.expression.OR , propFilterExpressions)]
+
+ return propFilterExpressions
+ #end propFilterExpression
+ """
+ Create an expression for a list of prop-filter elements.
+
+ @param filterAllOf: the C{True} if parent filter test is "allof"
+ @param propFilters: the C{list} of L{ComponentFilter} elements.
+ @return: (filterProperyNames, expressions) tuple. expression==True means list all results, expression==False means no results
+ """
+ expressions = None
+ for propFilter in propFilters:
+
+ propExpressions = propFilterExpression(filterAllOf, propFilter)
+ expressions = combineExpressionLists(expressions, filterAllOf, propExpressions)
+
+ # early loop exit
+ if isinstance(expressions, bool) and filterAllOf != expressions:
+ break
+
+ # convert to needsAllRecords to return
+ if isinstance(expressions, list):
+ expressions = list(set(expressions))
if len(expressions) > 1:
- expr = dsquery.expression(dsquery.expression.AND if filterAllOf else dsquery.expression.OR , list(set(expressions))) # remove duplicates
+ expr = dsquery.expression(dsquery.expression.AND if filterAllOf else dsquery.expression.OR , expressions)
elif len(expressions):
expr = expressions[0]
else:
- expr = None
-
- return (needsAllRecords, attributes, expr)
-
-
- #print("_getDSFilter")
- # Lets assume we have a valid filter from the outset
-
- # Top-level filter contains zero or more prop-filters
- if addressBookFilter:
- filterAllOf = addressBookFilter.filter_test == "allof"
- if len(addressBookFilter.children) > 0:
- return propFilterListQuery(filterAllOf, addressBookFilter.children)
- else:
- return (filterAllOf, [], [])
+ expr = not filterAllOf # empty expression list. should not happen
+ elif expressions is None:
+ expr = expr = not filterAllOf
else:
- return (False, [], [])
-
-
+ # True or False
+ expr = expressions
+
+ properties = [propFilter.filter_name for propFilter in propFilters]
- def _attributesForAddressBookQuery(self, addressBookQuery ):
-
- propertyNames = []
- #print( "addressBookQuery.qname=%r" % addressBookQuery.qname)
- if addressBookQuery.qname() == ("DAV:", "prop"):
-
- for property in addressBookQuery.children:
- #print("property = %r" % property )
- if isinstance(property, carddavxml.AddressData):
- for addressProperty in property.children:
- #print("addressProperty = %r" % addressProperty )
- if isinstance(addressProperty, carddavxml.Property):
- #print("Adding property %r", addressProperty.attributes["name"])
- propertyNames.append(addressProperty.attributes["name"])
-
- elif not self.fakeETag and property.qname() == ("DAV:", "getetag"):
- # for a real etag == md5(vCard), we need all attributes
- propertyNames = None
- break;
-
-
- if not len(propertyNames):
- #print("using all attributes")
- return self.returnedAttributes
-
- else:
- propertyNames.append("X-INTERNAL-MINIMUM-VCARD-PROPERTIES") # these properties are required to make a vCard
- queryAttributes = []
- for prop in propertyNames:
- if prop in VCardRecord.dsqueryAttributesForProperty:
- #print("adding attributes %r" % VCardRecord.dsqueryAttributesForProperty.get(prop))
- queryAttributes += VCardRecord.dsqueryAttributesForProperty.get(prop)
-
- return list(set(queryAttributes).intersection(set(self.returnedAttributes)))
-
+ return (list(set(properties)), expr)
- @inlineCallbacks
- def cacheVCardsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
- """
- Cache the vCards for a given addressBookFilder and addressBookQuery
- """
- startTime = time.time()
- #print("Timing: cacheVCardsForAddressBookQuery.starttime=%f" % startTime)
-
+ # Lets assume we have a valid filter from the outset
- allRecords, filterAttributes, dsFilter = self._getDSFilter( addressBookFilter );
- #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-
- if allRecords:
- dsFilter = None # None expression == all Records
- clear = not allRecords and not dsFilter
-
- #get unique list of requested attributes
- if clear:
- attributes = None
+ # Top-level filter contains zero or more prop-filters
+ if addressBookFilter:
+ filterAllOf = addressBookFilter.filter_test == "allof"
+ if len(addressBookFilter.children):
+ return propFilterListQuery(filterAllOf, addressBookFilter.children)
else:
- queryAttributes = self._attributesForAddressBookQuery( addressBookQuery )
- attributes = filterAttributes + queryAttributes
-
- #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
- maxRecords = int(maxResults * 1.2)
- if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
- maxRecords = self.maxDSQueryRecords
-
- updateLock, limited = (yield self._refreshCache(reschedule=False, query=dsFilter, attributes=attributes, keepLock=True, clear=clear, maxRecords=maxRecords ))
-
- elaspedTime = time.time()-startTime
- self.log_info("Timing: Cache fill: %.1f ms" % (elaspedTime*1000,))
-
-
- returnValue((updateLock, limited))
-
-
- @inlineCallbacks
- def vCardRecordsForAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
- """
- Get vCards for a given addressBookFilder and addressBookQuery
- """
+ return ([], not filterAllOf)
+ else:
+ return ([], False)
- allRecords, filterAttributes, dsFilter = self._getDSFilter( addressBookFilter );
- #print("allRecords = %s, query = %s" % (allRecords, "None" if dsFilter is None else dsFilter.generate(),))
-
- # testing:
- # allRecords = True
-
- if allRecords:
- dsFilter = None # None expression == all Records
- clear = not allRecords and not dsFilter
-
- queryRecords = []
- limited = False
-
- if not clear:
- queryAttributes = self._attributesForAddressBookQuery( addressBookQuery )
- attributes = filterAttributes + queryAttributes
-
- #calc maxRecords from passed in maxResults allowing extra for second stage filtering in caller
- maxRecords = int(maxResults * 1.2)
- if self.maxDSQueryRecords and maxRecords > self.maxDSQueryRecords:
- maxRecords = self.maxDSQueryRecords
-
- records, limited = (yield self._getDirectoryRecords(dsFilter, attributes, maxRecords))
-
- #filter out bad records --- should only happen during development
- for record in records.values():
- try:
- vCardText = record.vCardText()
- except:
- traceback.print_exc()
- self.log_info("Could not get vcard for record %s" % (record,))
- else:
- if not record.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation).startswith("/Local"):
- self.log_debug("VCard text =\n%s" % (vCardText, ))
- queryRecords.append(record)
- returnValue((queryRecords, limited,))
-
-class VCardRecord(DirectoryRecord, DAVPropertyMixIn):
+class ABDirectoryQueryResult(DAVPropertyMixIn, LoggingMixIn):
"""
- Open Directory implementation of L{IDirectoryRecord}.
+ Result from ab query report or multiget on directory
"""
# od attributes that may contribute to vcard properties
# will be used to translate vCard queries to od queries
- dsqueryAttributesForProperty = {
+ vcardPropToDSAttrMap = {
"FN" : [
dsattributes.kDS1AttrFirstName,
@@ -1130,10 +982,7 @@
],
"UID" : [
dsattributes.kDS1AttrGeneratedUID,
- # special cased
- #dsattributes.kDSNAttrMetaNodeLocation,
- #dsattributes.kDSNAttrRecordName,
- #dsattributes.kDS1AttrDistinguishedName,
+ dsattributes.kDSNAttrRecordName,
],
"URL" : [
dsattributes.kDS1AttrWeblogURI,
@@ -1146,66 +995,44 @@
(dsattributes.kDS1AttrUserPKCS12Data, "base64"),
(dsattributes.kDS1AttrUserSMIMECertificate, "base64"),
],
- # too bad this is not one X-Attribute with params. Would make searching easier
- "X-AIM" : [
+ "IMPP" : [
dsattributes.kDSNAttrIMHandle,
],
- "X-JABBER" : [
- dsattributes.kDSNAttrIMHandle,
- ],
- "X-MSN" : [
- dsattributes.kDSNAttrIMHandle,
- ],
- "X-YAHOO" : [
- dsattributes.kDSNAttrIMHandle,
- ],
- "X-ICQ" : [
- dsattributes.kDSNAttrIMHandle,
- ],
"X-ABRELATEDNAMES" : [
dsattributes.kDSNAttrRelationships,
- ],
- "X-INTERNAL-MINIMUM-VCARD-PROPERTIES" : [
+ ],
+ "SOURCE" : [
dsattributes.kDS1AttrGeneratedUID,
- dsattributes.kDSNAttrMetaNodeLocation,
- dsattributes.kDS1AttrFirstName,
- dsattributes.kDS1AttrLastName,
- dsattributes.kDS1AttrMiddleName,
- dsattributes.kDSNAttrNamePrefix,
- dsattributes.kDSNAttrNameSuffix,
- dsattributes.kDS1AttrDistinguishedName,
dsattributes.kDSNAttrRecordName,
- dsattributes.kDSNAttrRecordType,
- dsattributes.kDS1AttrModificationTimestamp,
- dsattributes.kDS1AttrCreationTimestamp,
],
- "X-INTERNAL-REQUIRED" : [
- dsattributes.kDS1AttrGeneratedUID,
- dsattributes.kDSNAttrMetaNodeLocation,
- dsattributes.kDS1AttrDistinguishedName,
- dsattributes.kDSNAttrRecordName,
- dsattributes.kDS1AttrFirstName,
- dsattributes.kDS1AttrLastName,
- dsattributes.kDSNAttrRecordType,
- ],
-
}
-
- allDSQueryAttributes = sorted(list(set([attr for lookupAttributes in dsqueryAttributesForProperty.values()
- for attr in lookupAttributes])))
-
- binaryDSAttributeStrs = [attr[0] for attr in allDSQueryAttributes
+ allDSQueryAttributes = list(set([attr for lookupAttributes in vcardPropToDSAttrMap.values()
+ for attr in lookupAttributes]))
+ binaryDSAttrNames = [attr[0] for attr in allDSQueryAttributes
if isinstance(attr, tuple) ]
-
- stringDSAttributeStrs = [attr for attr in allDSQueryAttributes
+ stringDSAttrNames = [attr for attr in allDSQueryAttributes
if isinstance(attr, str) ]
+ allDSAttrNames = stringDSAttrNames + binaryDSAttrNames
+
+ # all possible generated parameters.
+ vcardPropToParamMap = {
+ "PHOTO": { "ENCODING": ("B",), "TYPE": ("JPEG",), },
+ "ADR": { "TYPE": ("WORK", "PREF", "POSTAL", "PARCEL",), },
+ "LABEL": { "TYPE": ("POSTAL", "PARCEL",)},
+ "TEL": { "TYPE": None, }, # None means param can contain can be anything
+ "EMAIL": { "TYPE": None, },
+ "KEY": { "ENCODING": ("B",), "TYPE": ("PGPPUBILICKEY", "USERCERTIFICATE", "USERPKCS12DATA", "USERSMIMECERTIFICATE",) },
+ "URL": { "TYPE": ("WEBLOG", "HOMEPAGE",) },
+ "IMPP": { "TYPE": ("PREF",), "X-SERVICE-TYPE": None, },
+ "X-ABRELATEDNAMES" : { "TYPE":None, },
+ "X-AIM": { "TYPE": ("PREF",), },
+ "X-JABBER": { "TYPE": ("PREF",), },
+ "X-MSN": { "TYPE": ("PREF",), },
+ "X-ICQ": { "TYPE": ("PREF",), },
+ }
- allDSAttributeStrs = stringDSAttributeStrs + binaryDSAttributeStrs
-
- #peopleUIDSeparator = "-" + OpenDirectoryBackingService.baseGUID + "-"
- userUIDSeparator = "-bf07a1a2-"
- peopleUIDSeparator = "-cf07a1a2-"
+ uidSeparator = "-cf07a1a2-"
constantProperties = {
@@ -1216,93 +1043,75 @@
}
- def __init__(self, service, recordAttributes, defaultNodeName=None):
+ def __init__(self, directoryBackedAddressBook, recordAttributes,
+ kind=None,
+ additionalVCardProps=None,
+ addDSAttrXProperties=False,
+ appleInternalServer=False,
+ ):
+
+ self.log_debug("directoryBackedAddressBook=%s, attributes=%s, additionalVCardProps=%s" % (directoryBackedAddressBook, recordAttributes, additionalVCardProps,))
+ constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+ if additionalVCardProps:
+ for key, value in additionalVCardProps.iteritems():
+ if key not in constantProperties:
+ constantProperties[key] = value
+ self.constantProperties = constantProperties
+ self.log_debug("directoryBackedAddressBook=%s, attributes=%s, self.constantProperties=%s" % (directoryBackedAddressBook, recordAttributes, self.constantProperties,))
- self.log_debug("service=%s, attributes=%s" % (service, recordAttributes))
-
#save off for debugging
- if service.addDSAttrXProperties:
+ self.addDSAttrXProperties = addDSAttrXProperties;
+ if addDSAttrXProperties:
self.originalAttributes = recordAttributes.copy()
+ self.appleInternalServer = appleInternalServer
- self.directoryBackedAddressBook = service.directoryBackedAddressBook
+ self._directoryBackedAddressBook = directoryBackedAddressBook
self._vCard = None
- self._vCardText = None
- self._uriName = None
- self._hRef = None
+ #clean attributes
self.attributes = {}
for key, values in recordAttributes.items():
- if key in VCardRecord.stringDSAttributeStrs:
+ if key in ABDirectoryQueryResult.stringDSAttrNames:
if isinstance(values, list):
self.attributes[key] = [removeControlChars(val).decode("utf8") for val in values]
else:
self.attributes[key] = removeControlChars(values).decode("utf8")
else:
self.attributes[key] = values
-
- # fill in missing essential attributes used for filtering
- fullName = self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)
- if not fullName:
- fullName = self.firstValueForAttribute(dsattributes.kDSNAttrRecordName)
- self.attributes[dsattributes.kDS1AttrDistinguishedName] = fullName
-
- node = self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation)
-
- # use a better node name -- makes better synthetic GUIDS
- if not node or node == "/LDAPv3/127.0.0.1":
- node = defaultNodeName if defaultNodeName else service.realmName
- self.attributes[dsattributes.kDSNAttrMetaNodeLocation] = node
-
+
+ # find or create guid
guid = self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)
if not guid:
- if service.standardizeSyntheticUIDs:
- nodeUUIDStr = "00000000"
- else:
- nodeUUIDStr = "%x" % abs(hash(node))
- nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("utf8").encode("base64").split("\n"))
- if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
- guid = VCardRecord.userUIDSeparator.join([nodeUUIDStr, nameUUIDStr,])
- else:
- guid = VCardRecord.peopleUIDSeparator.join([nodeUUIDStr, nameUUIDStr,])
-
+ nameUUIDStr = "".join(self.firstValueForAttribute(dsattributes.kDSNAttrRecordName).encode("base64").split("\n"))
+ guid = ABDirectoryQueryResult.uidSeparator.join(["00000000", nameUUIDStr,])
+ #guid = ABDirectoryQueryResult.uidSeparator.join(["d9a8e41b", nameUUIDStr,])
- # since guid is used as file name, normalize so uid uniqueness == fine name uniqueness
- #guid = "/".join(guid.split(":")).upper()
- self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
+ self.attributes[dsattributes.kDS1AttrGeneratedUID] = guid
- if self.firstValueForAttribute(dsattributes.kDS1AttrLastName) == "99":
- del self.attributes[dsattributes.kDS1AttrLastName]
+ if not kind:
+ dsRecordTypeToKindMap = {
+ #dsattributes.kDSStdRecordTypePeople:"individual",
+ #dsattributes.kDSStdRecordTypeUsers:"individual",
+ dsattributes.kDSStdRecordTypeGroups:"group",
+ dsattributes.kDSStdRecordTypeLocations:"location",
+ dsattributes.kDSStdRecordTypeResources:"device",
+ }
+ recordType = self.firstValueForAttribute(dsattributes.kDSNAttrRecordType)
+ kind = dsRecordTypeToKindMap.get(recordType, "individual")
+ self.kind = kind.lower()
+
+
+ #generate a vCard here. May throw an exception
+ self.vCard()
- if self.firstValueForAttribute(dsattributes.kDSNAttrRecordType) != dsattributes.kDSStdRecordTypePeople:
- recordType = DirectoryService.recordType_users
- else:
- recordType = DirectoryService.recordType_people
-
- super(VCardRecord, self).__init__(
- service = service,
- recordType = recordType,
- guid = guid,
- shortNames = tuple(self.valuesForAttribute(dsattributes.kDSNAttrRecordName)),
- fullName = fullName,
- firstName = self.firstValueForAttribute(dsattributes.kDS1AttrFirstName, None),
- lastName = self.firstValueForAttribute(dsattributes.kDS1AttrLastName, None),
- emailAddresses = (),
- calendarUserAddresses = (),
- autoSchedule = False,
- enabledForCalendaring = False,
- )
-
def __repr__(self):
- return "<%s[%s(%s)] %s(%s) %r>" % (
+ return "<%s[%s(%s)]>" % (
self.__class__.__name__,
- self.firstValueForAttribute(dsattributes.kDSNAttrRecordType),
- self.firstValueForAttribute(dsattributes.kDSNAttrMetaNodeLocation),
- self.guid,
- self.shortNames,
- self.fullName
+ self.vCard().propertyValue("FN"),
+ self.vCard().propertyValue("UID")
)
def __hash__(self):
@@ -1312,12 +1121,6 @@
])
return hash(s)
- """
- def nextFileName(self):
- self.renameCounter += 1
- self.fileName = self.baseFileName + "-" + str(self.renameCounter)
- self.fileNameLower = self.fileName.lower()
- """
def hasAttribute(self, attributeName ):
return self.valuesForAttribute(attributeName, None) is not None
@@ -1399,15 +1202,17 @@
vcard.addProperty(Property("X-ABLabel", label, group=groupPrefix))
# for attributes of the form param:value
- def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, defaultLabel, nolabelParamTypes, labelMap, attrType):
+ def addPropertiesAndLabelsForPrefixedAttribute(groupCount, propertyPrefix, propertyName, attrType, defaultLabel, nolabelParamTypes=(), labelMap={}, specialParamType=None):
preferred = True
for attrValue in self.valuesForAttribute(attrType):
try:
# special case for Apple
- if self.service.appleInternalServer and attrType == dsattributes.kDSNAttrIMHandle:
+ if self.appleInternalServer and attrType == dsattributes.kDSNAttrIMHandle:
splitValue = attrValue.split("|")
if len (splitValue) > 1:
attrValue = splitValue[0]
+ if splitValue[1].upper() in nolabelParamTypes:
+ defaultLabel = splitValue[1]
colonIndex = attrValue.find(":")
if (colonIndex > len(attrValue)-2):
@@ -1417,11 +1222,16 @@
labelString = attrValue[:colonIndex] if colonIndex > 0 else defaultLabel
paramTypeString = labelString.upper()
- # add PREF to first prop's parameters
- paramTypeStrings = [paramTypeString,]
- if preferred and "PREF" != paramTypeString:
- paramTypeStrings += ["PREF",]
- parameters = { "TYPE": paramTypeStrings, }
+ if specialParamType:
+ parameters = { specialParamType: (paramTypeString,) }
+ if preferred:
+ parameters["TYPE"] = ("PREF",)
+ else:
+ # add PREF to first prop's parameters
+ paramTypeStrings = [paramTypeString,]
+ if preferred and "PREF" != paramTypeString:
+ paramTypeStrings += ["PREF",]
+ parameters = { "TYPE": paramTypeStrings, }
#special case for IMHandles which the param is the last part of the property like X-AIM or X-JABBER
if propertyPrefix:
@@ -1432,8 +1242,8 @@
addUniqueProperty(vcard, Property(propertyName, attrValue[colonIndex+1:], params=parameters), None, attrValue, attrType)
else:
# use special localizable addressbook labels where possible
- abLabelString = labelMap.get(labelString, labelString)
- addPropertyAndLabel(groupCount, abLabelString, propertyName, propertyValue, parameters)
+ localizedABLabelString = labelMap.get(labelString, labelString)
+ addPropertyAndLabel(groupCount, localizedABLabelString, propertyName, propertyValue, parameters)
preferred = False
except Exception, e:
@@ -1442,15 +1252,14 @@
self.log_error("addPropertiesAndLabelsForPrefixedAttribute(): Trouble parsing attribute %s, with value \"%s\". Error = %s" % (attrType, attrValue, e,))
- #print("VCardRecord.vCard")
# create vCard
vcard = Component("VCARD")
groupCount = [0]
# add constant properties - properties that are the same regardless of the record attributes
- for key, value in VCardRecord.constantProperties.items():
+ for key, value in self.constantProperties.items():
vcard.addProperty(Property(key, value))
-
+
# 3.1 IDENTIFICATION TYPES http://tools.ietf.org/html/rfc2426#section-3.1
# 3.1.1 FN Type Definition
# dsattributes.kDS1AttrDistinguishedName, # Users distinguished or real name
@@ -1459,14 +1268,14 @@
#vcard.addProperty(Property("FN", self.firstValueForAttribute(dsattributes.kDS1AttrDistinguishedName)))
# 3.1.2 N Type Definition
- # dsattributes.kDS1AttrFirstName, # Used for first name of user or person record.
+ # dsattributes.kDS1AttrFirstName, # Used for first name of user or person record.
# dsattributes.kDS1AttrLastName, # Used for the last name of user or person record.
- # dsattributes.kDS1AttrMiddleName, # Used for the middle name of user or person record.
- # dsattributes.kDSNAttrNameSuffix, # Represents the name suffix of a user or person.
+ # dsattributes.kDS1AttrMiddleName, #Used for the middle name of user or person record.
+ # dsattributes.kDSNAttrNameSuffix, # Represents the name suffix of a user or person.
# ie. Jr., Sr., etc.
# Usually found in user or people records (kDSStdRecordTypeUsers or
# dsattributes.kDSStdRecordTypePeople).
- # dsattributes.kDSNAttrNamePrefix, # Represents the title prefix of a user or person.
+ # dsattributes.kDSNAttrNamePrefix, # Represents the title prefix of a user or person.
# ie. Mr., Ms., Mrs., Dr., etc.
# Usually found in user or people records (kDSStdRecordTypeUsers or
# dsattributes.kDSStdRecordTypePeople).
@@ -1500,12 +1309,13 @@
addUniqueProperty(vcard, Property("NICKNAME", nickname), None, dsattributes.kDSNAttrNickName, nickname)
# 3.1.4 PHOTO Type Definition
- # dsattributes.kDSNAttrJPEGPhoto, # Used to store binary picture data in JPEG format.
+ # dsattributes.kDSNAttrJPEGPhoto, # Used to store binary picture data in JPEG format.
# Usually found in user, people or group records (kDSStdRecordTypeUsers,
# dsattributes.kDSStdRecordTypePeople,dsattributes.kDSStdRecordTypeGroups).
# pyOpenDirectory always returns binary-encoded string
for photo in self.valuesForAttribute(dsattributes.kDSNAttrJPEGPhoto):
+ photo = "".join("".join(photo.split("\r")).split("\n")) #get rid of line folding: for PHOTO
addUniqueProperty(vcard, Property("PHOTO", photo, params={"ENCODING": ["b",], "TYPE": ["JPEG",],}), None, dsattributes.kDSNAttrJPEGPhoto, photo)
@@ -1549,8 +1359,8 @@
# 3.2.2 LABEL Type Definition
- # dsattributes.kDSNAttrPostalAddress, # The postal address usually excluding postal code.
- # dsattributes.kDSNAttrPostalAddressContacts, # multi-valued attribute that defines a record's alternate postal addresses .
+ # dsattributes.kDSNAttrPostalAddress, # The postal address usually excluding postal code.
+ # dsattributes.kDSNAttrPostalAddressContacts, # multi-valued attribute that defines a record's alternate postal addresses .
# found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
# dsattributes.kDSNAttrAddressLine1, # Line one of multiple lines of address data for a user.
# dsattributes.kDSNAttrAddressLine2, # Line two of multiple lines of address data for a user.
@@ -1577,18 +1387,18 @@
# 3.3.1 TEL Type Definition
# TEL;TYPE=work,voice,pref,msg:+1-213-555-1234
- # dsattributes.kDSNAttrPhoneNumber, # Telephone number of a user.
+ # dsattributes.kDSNAttrPhoneNumber, # Telephone number of a user.
# dsattributes.kDSNAttrMobileNumber, # Represents the mobile numbers of a user or person.
# Usually found in user or people records (kDSStdRecordTypeUsers or
# dsattributes.kDSStdRecordTypePeople).
- # dsattributes.kDSNAttrFaxNumber, # Represents the FAX numbers of a user or person.
+ # dsattributes.kDSNAttrFaxNumber, # Represents the FAX numbers of a user or person.
# Usually found in user or people records (kDSStdRecordTypeUsers or
# kDSStdRecordTypePeople).
- # dsattributes.kDSNAttrPagerNumber, # Represents the pager numbers of a user or person.
+ # dsattributes.kDSNAttrPagerNumber, # Represents the pager numbers of a user or person.
# Usually found in user or people records (kDSStdRecordTypeUsers or
# dsattributes.kDSStdRecordTypePeople).
- # dsattributes.kDSNAttrHomePhoneNumber, # Home telephone number of a user or person.
- # dsattributes.kDSNAttrPhoneContacts, # multi-valued attribute that defines a record's custom phone numbers .
+ # dsattributes.kDSNAttrHomePhoneNumber, # Home telephone number of a user or person.
+ # dsattributes.kDSNAttrPhoneContacts, # multi-valued attribute that defines a record's custom phone numbers .
# found in user records (kDSStdRecordTypeUsers).
# Example: home fax:408-555-4444
@@ -1617,9 +1427,9 @@
addUniqueProperty(vcard, Property("TEL", phone, params=params), (("TYPE", "PREF"),), phone, dsattributes.kDSNAttrHomePhoneNumber)
params = { "TYPE": ["HOME", "VOICE",], }
- addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "TEL", "work",
- ["VOICE", "CELL", "FAX", "PAGER",], {},
- dsattributes.kDSNAttrPhoneContacts, )
+ addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="TEL", defaultLabel="work",
+ nolabelParamTypes=("VOICE", "CELL", "FAX", "PAGER",),
+ attrType=dsattributes.kDSNAttrPhoneContacts, )
"""
# EXTEND: Use this attribute
@@ -1637,14 +1447,14 @@
addUniqueProperty(vcard, Property("EMAIL", emailAddress, params=params), (("TYPE", "PREF"),), emailAddress, dsattributes.kDSNAttrEMailAddress)
params = workParams
- # dsattributes.kDSNAttrEMailContacts, # multi-valued attribute that defines a record's custom email addresses .
+ # dsattributes.kDSNAttrEMailContacts, # multi-valued attribute that defines a record's custom email addresses .
# found in user records (kDSStdRecordTypeUsers).
# Example: home:johndoe at mymail.com
# check to see if parameters type are open ended. Could be any string
- addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "EMAIL", "work",
- ["WORK", "HOME",], {},
- dsattributes.kDSNAttrEMailContacts, )
+ addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="EMAIL", defaultLabel="work",
+ nolabelParamTypes=("WORK", "HOME",),
+ attrType=dsattributes.kDSNAttrEMailContacts, )
"""
# UNIMPLEMENTED:
@@ -1656,7 +1466,7 @@
# 3.4.1 TZ Type Definition
"""
# 3.4.2 GEO Type Definition
- #dsattributes.kDSNAttrMapCoordinates, # attribute that defines coordinates for a user's location .
+ #dsattributes.kDSNAttrMapCoordinates, # attribute that defines coordinates for a user's location .
# Found in user records (kDSStdRecordTypeUsers) and resource records (kDSStdRecordTypeResources).
# Example: 7.7,10.6
for coordinate in self.valuesForAttribute(dsattributes.kDSNAttrMapCoordinates):
@@ -1693,14 +1503,12 @@
# 3.6.1 CATEGORIES Type Definition
"""
# 3.6.2 NOTE Type Definition
- # dsattributes.kDS1AttrComment, # Attribute used for unformatted comment.
+ # dsattributes.kDS1AttrComment, # Attribute used for unformatted comment.
# dsattributes.kDS1AttrNote, # Note attribute. Commonly used in printer records.
- for comment in self.valuesForAttribute(dsattributes.kDS1AttrComment):
- addUniqueProperty(vcard, Property("NOTE", comment), None, dsattributes.kDS1AttrComment, comment)
+ notes = self.valuesForAttribute(dsattributes.kDS1AttrComment, []) + self.valuesForAttribute(dsattributes.kDS1AttrNote, []);
+ if len(notes):
+ vcard.addProperty(Property("NOTE", "\n".join(notes),))
- for note in self.valuesForAttribute(dsattributes.kDS1AttrNote):
- addUniqueProperty(vcard, Property("NOTE", note), None, dsattributes.kDS1AttrNote, note)
-
# 3.6.3 PRODID Type Definition
#vcard.addProperty(Property("PRODID", vCardProductID + "//BUILD %s" % twistedcaldav.__version__))
#vcard.addProperty(Property("PRODID", vCardProductID))
@@ -1720,19 +1528,20 @@
# dsattributes.kDS1AttrGeneratedUID, # Used for 36 character (128 bit) unique ID. Usually found in user,
# group, and computer records. An example value is "A579E95E-CDFE-4EBC-B7E7-F2158562170F".
# The standard format contains 32 hex characters and four hyphen characters.
- # !! don't use self.guid which is URL encoded
+
vcard.addProperty(Property("UID", self.firstValueForAttribute(dsattributes.kDS1AttrGeneratedUID)))
+
# 3.6.8 URL Type Definition
- # dsattributes.kDSNAttrURL, # List of URLs.
- # dsattributes.kDS1AttrWeblogURI, # Single-valued attribute that defines the URI of a user's weblog.
+ # dsattributes.kDSNAttrURL, # List of URLs.
+ # dsattributes.kDS1AttrWeblogURI, # Single-valued attribute that defines the URI of a user's weblog.
# Usually found in user records (kDSStdRecordTypeUsers).
# Example: http://example.com/blog/jsmith
for url in self.valuesForAttribute(dsattributes.kDS1AttrWeblogURI):
- addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters = {"TYPE": ["Weblog",]})
+ addPropertyAndLabel(groupCount, "weblog", "URL", url, parameters = {"TYPE": ["WEBLOG",]})
for url in self.valuesForAttribute(dsattributes.kDSNAttrURL):
- addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters = {"TYPE": ["Homepage",]})
+ addPropertyAndLabel(groupCount, "_$!<HomePage>!$_", "URL", url, parameters = {"TYPE": ["HOMEPAGE",]})
# 3.6.9 VERSION Type Definition
@@ -1745,11 +1554,11 @@
# 3.7.2 KEY Type Definition
# dsattributes.kDSNAttrPGPPublicKey, # Pretty Good Privacy public encryption key.
- # dsattributes.kDS1AttrUserCertificate, # Attribute containing the binary of the user's certificate.
+ # dsattributes.kDS1AttrUserCertificate, # Attribute containing the binary of the user's certificate.
# Usually found in user records. The certificate is data which identifies a user.
# This data is attested to by a known party, and can be independently verified
# by a third party.
- # dsattributes.kDS1AttrUserPKCS12Data, # Attribute containing binary data in PKCS #12 format.
+ # dsattributes.kDS1AttrUserPKCS12Data, # Attribute containing binary data in PKCS #12 format.
# Usually found in user records. The value can contain keys, certificates,
# and other related information and is encrypted with a passphrase.
# dsattributes.kDS1AttrUserSMIMECertificate,# Attribute containing the binary of the user's SMIME certificate.
@@ -1779,19 +1588,27 @@
# Values should be prefixed with the appropriate IM type
# ie. AIM:, Jabber:, MSN:, Yahoo:, or ICQ:
# Usually found in user records (kDSStdRecordTypeUsers).
-
- addPropertiesAndLabelsForPrefixedAttribute(groupCount, "X-", None, "aim",
- ["AIM", "JABBER", "MSN", "YAHOO", "ICQ"],
- {},
- dsattributes.kDSNAttrIMHandle,)
+ imNolabelParamTypes=("AIM", "FACEBOOK", "GAGU-GAGU", "GOOGLE TALK", "ICQ", "JABBER", "MSN", "QQ", "SKYPE", "YAHOO",)
+ addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix="X-", propertyName=None, defaultLabel="aim",
+ nolabelParamTypes=imNolabelParamTypes,
+ attrType=dsattributes.kDSNAttrIMHandle,)
+
+
+
+ # IMPP
+ # Address Book's implementation of http://tools.ietf.org/html/rfc6350#section-6.4.3
+ # adding IMPP property allows ab query report search on one property
+ addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="IMPP", defaultLabel="aim",
+ specialParamType = "X-SERVICE-TYPE",
+ nolabelParamTypes=imNolabelParamTypes,
+ attrType=dsattributes.kDSNAttrIMHandle,)
# X-ABRELATEDNAMES
- # dsattributes.kDSNAttrRelationships, # multi-valued attribute that defines the relationship to the record type .
+ # dsattributes.kDSNAttrRelationships, # multi-valued attribute that defines the relationship to the record type .
# found in user records (kDSStdRecordTypeUsers).
# Example: brother:John
- addPropertiesAndLabelsForPrefixedAttribute(groupCount, None, "X-ABRELATEDNAMES", "friend",
- [],
- { "FATHER":"_$!<Father>!$_",
+ addPropertiesAndLabelsForPrefixedAttribute(groupCount=groupCount, propertyPrefix=None, propertyName="X-ABRELATEDNAMES", defaultLabel="friend",
+ labelMap={ "FATHER":"_$!<Father>!$_",
"MOTHER":"_$!<Mother>!$_",
"PARENT":"_$!<Parent>!$_",
"BROTHER":"_$!<Brother>!$_",
@@ -1802,11 +1619,11 @@
"PARTNER":"_$!<Partner>!$_",
"ASSISTANT":"_$!<Assistant>!$_",
"MANAGER":"_$!<Manager>!$_", },
- dsattributes.kDSNAttrRelationships, )
+ attrType=dsattributes.kDSNAttrRelationships, )
# special case for Apple
- if self.service.appleInternalServer:
+ if self.appleInternalServer:
for manager in self.valuesForAttribute("dsAttrTypeNative:appleManager"):
splitManager = manager.split("|")
if len(splitManager) >= 4:
@@ -1815,8 +1632,17 @@
managerValue = "%s %s" % (splitManager[0], splitManager[1])
else:
managerValue = manager
- addPropertyAndLabel( groupCount, "_$!<Manager>!$_", "X-ABRELATEDNAMES", managerValue, parameters={ "TYPE": ["Manager",]} )
+ addPropertyAndLabel( groupCount, "_$!<Manager>!$_", "X-ABRELATEDNAMES", managerValue, parameters={ "TYPE": ["MANAGER",]} )
+
+ # add apple-defined group vcard properties if record type is group
+ if self.kind == "group":
+ vcard.addProperty(Property("X-ADDRESSBOOKSERVER-KIND", "group"))
+
+ # add members
+ for memberguid in self.valuesForAttribute(dsattributes.kDSNAttrGroupMembers):
+ vcard.addProperty(Property("X-ADDRESSBOOKSERVER-MEMBER", "urn:uuid:" + memberguid))
+
"""
# UNIMPLEMENTED: X- attributes
@@ -1828,18 +1654,51 @@
sattributes.kDS1AttrPicture, # Represents the path of the picture for each user displayed in the login window.
# Found in user records (kDSStdRecordTypeUsers).
- dsattributes.kDS1AttrMapGUID, # Represents the GUID for a record's map.
+ dsattributes.kDS1AttrMapGUID, # Represents the GUID for a record's map.
dsattributes.kDSNAttrMapURI, # attribute that defines the URI of a user's location.
- dsattributes.kDSNAttrOrganizationInfo, # Usually the organization info of a user.
- dsattributes.kDSNAttrAreaCode, # Area code of a user's phone number.
+ dsattributes.kDSNAttrOrganizationInfo, # Usually the organization info of a user.
+ dsattributes.kDSNAttrAreaCode, # Area code of a user's phone number.
- dsattributes.kDSNAttrMIME, # Data contained in this attribute type is a fully qualified MIME Type.
+ dsattributes.kDSNAttrMIME, # Data contained in this attribute type is a fully qualified MIME Type.
"""
- # debug, create x attributes for all ds attributes
- if self.service.addDSAttrXProperties:
+ # 2.1.4 SOURCE Type http://tools.ietf.org/html/rfc2426#section-2.1.4
+ # If the SOURCE type is present, then its value provides information
+ # how to find the source for the vCard.
+
+ # add the source, so that if the SOURCE is copied out and preserved, the client can refresh information
+ # However, client should really do a ab-query report matching UID on /directory/ not a multiget.
+ uri = joinURL(self._directoryBackedAddressBook.uri, vcard.propertyValue("UID") + ".vcf")
+
+ # seems like this should be in some standard place.
+ if config.EnableSSL and config.SSLPort:
+ if config.SSLPort == 443:
+ source = "https://%s%s" % (config.ServerHostName, uri)
+ else:
+ source = "https://%s:%s%s" % (config.ServerHostName, config.SSLPort, uri)
+ elif config.HTTPPort:
+ if config.HTTPPort == 80:
+ source = "http://%s%s" % (config.ServerHostName, uri)
+ else:
+ source = "http://%s:%s%s" % (config.ServerHostName, config.HTTPPort, uri)
+ vcard.addProperty(Property("SOURCE", source))
+
+ # in 4.0 spec:
+ # 6.1.4. KIND http://tools.ietf.org/html/rfc6350#section-6.1.4
+ #
+ # see also: http://www.iana.org/assignments/vcard-elements/vcard-elements.xml
+ #
+ vcard.addProperty(Property("KIND", self.kind))
+
+ # one more X- related to kind
+ if self.kind == "org":
+ vcard.addProperty(Property("X-ABShowAs", "COMPANY"))
+
+
+ # debug, create X-attributes for all ds attributes
+ if self.addDSAttrXProperties:
for attribute in self.originalAttributes:
for value in self.valuesForAttribute(attribute):
vcard.addProperty(Property("X-"+"-".join(attribute.split(":")), removeControlChars(value)))
@@ -1853,53 +1712,36 @@
return self._vCard
def vCardText(self):
- if not self._vCardText:
- self._vCardText = str(self.vCard())
-
- return self._vCardText
-
- def uriName(self):
- if not self._uriName:
- self._uriName = self.vCard().getProperty("UID").value() + ".vcf"
- #print("uriName():self._uriName=%s" % self._uriName)
- return self._uriName
-
+ return str(self.vCard())
- def hRef(self, parentURI="/directory/"):
- if not self._hRef:
- self._hRef = davxml.HRef.fromString(joinURL(parentURI, self.uriName()))
-
- return self._hRef
-
-
+ def uri(self):
+ return self.vCard().propertyValue("UID") + ".vcf"
+
+ def hRef(self, parentURI=None):
+ return davxml.HRef.fromString(joinURL(parentURI if parentURI else self._directoryBackedAddressBook.uri, self.uri()))
+
+
def readProperty(self, property, request):
if type(property) is tuple:
qname = property
else:
qname = property.qname()
-
namespace, name = qname
-
- #print("VCardResource.readProperty: qname = %s" % (qname, ))
-
+
if namespace == dav_namespace:
if name == "resourcetype":
result = davxml.ResourceType.empty #@UndefinedVariable
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getetag":
result = davxml.GETETag( ETag(hashlib.md5(self.vCardText()).hexdigest()).generate() )
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getcontenttype":
mimeType = MimeType('text', 'vcard', {})
result = davxml.GETContentType(generateContentType(mimeType))
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getcontentlength":
result = davxml.GETContentLength.fromString(str(len(self.vCardText())))
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getlastmodified":
if self.vCard().hasProperty("REV"):
@@ -1917,7 +1759,6 @@
t = modDatetime.time()
modDatetimeNoTZ = datetime.datetime(d.year, d.month, d.day, t.hour, t.minute, t.second, t.microsecond, None)
result = davxml.GETLastModified.fromDate(modDatetimeNoTZ)
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "creationdate":
creationDateString = self.isoDateStringForDateAttribute(dsattributes.kDS1AttrCreationTimestamp)
@@ -1928,22 +1769,18 @@
else:
creationDatetime = datetime.datetime.utcnow()
result = davxml.CreationDate.fromDate(creationDatetime)
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "displayname":
# AddressBook.app uses N. Use FN or UID instead?
result = davxml.DisplayName.fromString(self.vCard().propertyValue("N"))
- #print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif namespace == twisted_dav_namespace:
- return super(VCardRecord, self).readProperty(property, request)
- #return DAVPropertyMixIn.readProperty(self, property, request)
+ return super(ABDirectoryQueryResult, self).readProperty(property, request)
- return self.directoryBackedAddressBook.readProperty(property, request)
+ return self._directoryBackedAddressBook.readProperty(property, request)
def listProperties(self, request):
- #print("VCardResource.listProperties()")
qnames = set(self.liveProperties())
# Add dynamic live properties that exist
@@ -1952,33 +1789,20 @@
(dav_namespace, "quota-used-bytes" ),
)
for dqname in dynamicLiveProperties:
- #print("VCardResource.listProperties: removing dqname=%s" % (dqname,))
qnames.remove(dqname)
for qname in self.deadProperties().list():
if (qname not in qnames) and (qname[0] != twisted_private_namespace):
- #print("listProperties: adding qname=%s" % (qname,))
qnames.add(qname)
- #for qn in qnames: print("VCardResource.listProperties: qn=%s" % (qn,))
-
yield qnames
listProperties = deferredGenerator(listProperties)
# utility
-#remove control characters because vCard does not support them
+#remove illegal XML
def removeControlChars( utf8String ):
- result = utf8String
- for a in utf8String:
- if '\x00' <= a <= '\x1F':
- result = ""
- for c in utf8String:
- if '\x00' <= c <= '\x1F':
- pass
- else:
- result += c
- #if utf8String != result: print ("changed %r to %r" % (utf8String, result))
+ result = ''.join([c for c in utf8String if c not in "\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"])
return result
Added: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py (rev 0)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directory/xmldirectorybacker.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -0,0 +1,308 @@
+##
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+"""
+Apple Open Directory directory service implementation for backing up directory-backed address books
+"""
+
+__all__ = [
+ "XMLDirectoryBackingService",
+]
+
+import traceback
+
+from calendarserver.platform.darwin.od import dsattributes, dsquery
+
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twistedcaldav.directory.xmlfile import XMLDirectoryService
+from twistedcaldav.directory.opendirectorybacker import ABDirectoryQueryResult, dsFilterFromAddressBookFilter, propertiesInAddressBookQuery
+
+
+class XMLDirectoryBackingService(XMLDirectoryService):
+ """
+ Directory backer for L{XMLDirectoryService}.
+ """
+
+ def __init__(self, params):
+ self._actuallyConfigure(**params)
+
+ def _actuallyConfigure(self, **params):
+
+ self.log_debug("_actuallyConfigure: params=%s" % (params,))
+ defaults = {
+ "recordTypes": (self.recordType_users, self.recordType_groups, ),
+ "rdnSchema": {
+ self.recordType_users : {
+ "vcardPropToDirRecordAttrMap" : {
+ "FN" : (
+ "fullName",
+ "shortNames",
+ "firstName",
+ "lastName",
+ ),
+ "N" : (
+ "fullName",
+ "shortNames",
+ "firstName",
+ "lastName",
+ ),
+ "EMAIL" : "emailAddresses",
+ "UID" : "guid",
+ },
+ "dirRecordAttrToDSAttrMap" : {
+ "guid" : dsattributes.kDS1AttrGeneratedUID,
+ "fullName" : dsattributes.kDS1AttrDistinguishedName,
+ "firstName" : dsattributes.kDS1AttrFirstName,
+ "lastName" : dsattributes.kDS1AttrLastName,
+ "emailAddresses" : dsattributes.kDSNAttrEMailAddress,
+ },
+ },
+ self.recordType_groups : {
+ "vcardPropToDirRecordAttrMap" : {
+ "FN" : (
+ "fullName",
+ "shortNames",
+ "firstName",
+ "lastName",
+ ),
+ "N" : (
+ "fullName",
+ "shortNames",
+ "firstName",
+ "lastName",
+ ),
+ "EMAIL" : "emailAddresses",
+ "UID" : "guid",
+ "X-ADDRESSBOOKSERVER-MEMBER" : "members",
+ },
+ "dirRecordAttrToDSAttrMap" : {
+ "guid" : dsattributes.kDS1AttrGeneratedUID,
+ "fullName" : dsattributes.kDS1AttrDistinguishedName,
+ "firstName" : dsattributes.kDS1AttrFirstName,
+ "lastName" : dsattributes.kDS1AttrLastName,
+ "emailAddresses" : dsattributes.kDSNAttrEMailAddress,
+ "members" : dsattributes.kDSNAttrGroupMembers,
+ },
+ },
+ },
+ "maxQueryResults":0, # max records returned
+ "sortResults":True, # sort results by UID
+ "implementNot":True, # implement Not query by listing all records and subtracting
+ }
+
+ #params = self.getParams(params, defaults, ignored)
+ def addDefaults(params, defaults, remove=None):
+ for key in defaults:
+ if not key in params:
+ params[key] = defaults[key]
+ return params
+
+ params = addDefaults(params, defaults)
+ self.log_debug("_actuallyConfigure after addDefaults: params=%s" % (params,))
+
+ # super does not like these extra params
+ directoryBackedAddressBook=params["directoryBackedAddressBook"]
+ #del params["directoryBackedAddressBook"]
+ rdnSchema=params["rdnSchema"]
+ del params["rdnSchema"]
+ maxQueryResults=params["maxQueryResults"]
+ del params["maxQueryResults"]
+ sortResults=params["sortResults"]
+ del params["sortResults"]
+ implementNot=params["implementNot"]
+ del params["implementNot"]
+
+
+ assert directoryBackedAddressBook is not None
+ self.directoryBackedAddressBook = directoryBackedAddressBook
+
+ self.maxQueryResults = maxQueryResults
+ self.sortResults = sortResults
+ self.implementNot = implementNot
+ self.rdnSchema = rdnSchema
+
+
+ super(XMLDirectoryBackingService, self).__init__(params)
+
+
+ def createCache(self):
+ succeed(None)
+
+
+ @inlineCallbacks
+ def doAddressBookQuery(self, addressBookFilter, addressBookQuery, maxResults ):
+ """
+ Get vCards for a given addressBookFilter and addressBookQuery
+ """
+
+ results = []
+ limited = False
+
+ for recordType in self.recordTypes():
+
+ queryMap = self.rdnSchema[recordType]
+ vcardPropToDirRecordAttrMap = queryMap["vcardPropToDirRecordAttrMap"]
+ dirRecordAttrToDSAttrMap = queryMap["dirRecordAttrToDSAttrMap"]
+
+ kind = {self.recordType_groups:"group",
+ self.recordType_locations:"location",
+ self.recordType_resources:"calendarresource",
+ }.get(recordType, "individual")
+
+ constantProperties = ABDirectoryQueryResult.constantProperties.copy()
+ constantProperties["KIND"] = kind
+ # add KIND as constant so that query can be skipped if addressBookFilter needs a different kind
+
+ filterPropertyNames, dsFilter = dsFilterFromAddressBookFilter( addressBookFilter, vcardPropToDirRecordAttrMap, constantProperties=constantProperties );
+ self.log_debug("doAddressBookQuery: rdn=%s, query=%s, propertyNames=%s" % (recordType, dsFilter if isinstance(dsFilter, bool) else dsFilter.generate(), filterPropertyNames))
+
+ if dsFilter:
+
+ @inlineCallbacks
+ def recordsForDSFilter(dsFilter, recordType):
+
+ """
+ Athough recordsForDSFilter() exercises the dsFilter expression tree and recordsMatchingFields(),
+ it make little difference to the result of a addressbook query because of filtering.
+ """
+
+ if not isinstance(dsFilter, dsquery.expression):
+ #change match list into an expression and recurse
+ returnValue((yield recordsForDSFilter(dsquery.expression( dsquery.expression.OR, (dsFilter,)), recordType)))
+
+ else:
+ #self.log_debug("recordsForDSFilter: dsFilter=%s" % (dsFilter.generate(), ))
+ dsFilterSubexpressions = dsFilter.subexpressions if isinstance(dsFilter.subexpressions, list) else (dsFilter.subexpressions,)
+ #self.log_debug("recordsForDSFilter: #subs %s" % (len(dsFilterSubexpressions), ))
+
+ # evaluate matches
+ matches = [match for match in dsFilterSubexpressions if isinstance(match, dsquery.match)]
+ fields = []
+ for match in matches:
+ #self.log_debug("recordsForDSFilter: match=%s" % (match.generate(), ))
+ xmlMatchType = {
+ dsattributes.eDSExact : "exact",
+ dsattributes.eDSStartsWith : "starts-with",
+ dsattributes.eDSContains : "contains",
+ }.get(match.matchType)
+ if not xmlMatchType:
+ self.log_debug("recordsForDSFilter: match type=%s match not supported" % (match.generate(), ))
+ returnValue(None) # match type not supported by recordsMatchingFields()
+
+ fields += ((match.attribute, match.value, True, xmlMatchType,),)
+ #self.log_debug("recordsForDSFilter: fields=%s" % (fields,))
+
+ # if there were matches, call get records that match
+ result = None
+ if len(fields):
+ operand = "and" if dsFilter.operator == dsquery.expression.AND else "or"
+ #self.log_debug("recordsForDSFilter: recordsMatchingFields(fields=%s, operand=%s, recordType=%s)" % (fields, operand, recordType,))
+ result = set((yield self.recordsMatchingFields(fields, operand=operand, recordType=recordType)))
+ #self.log_debug("recordsForDSFilter: result=%s" % (result,))
+ if dsFilter.operator == dsquery.expression.NOT:
+ if self.implementNot:
+ result = (yield self.listRecords(recordType)).difference(result)
+ else:
+ self.log_debug("recordsForDSFilter: NOT expression not supported" % (match.generate(), ))
+ returnValue(None)
+
+
+ # evaluate subexpressions
+ subexpressions = [subexpression for subexpression in dsFilterSubexpressions if isinstance(subexpression, dsquery.expression)]
+ for subexpression in subexpressions:
+ #self.log_debug("recordsForDSFilter: subexpression=%s" % (subexpression.generate(), ))
+ subresult = (yield recordsForDSFilter(subexpression, recordType))
+ #self.log_debug("recordsForDSFilter: subresult=%s" % (subresult,))
+ if subresult is None:
+ returnValue(None)
+
+ if dsFilter.operator == dsquery.expression.NOT:
+ if self.implementNot:
+ result = (yield self.listRecords(recordType)).difference(subresult)
+ else:
+ self.log_debug("recordsForDSFilter: NOT expression not supported" % (match.generate(), ))
+ returnValue(None)
+ elif result is None:
+ result = subresult
+ elif dsFilter.operator == dsquery.expression.OR:
+ result = result.union(subresult)
+ else:
+ result = result.intersection(subresult)
+
+ #self.log_debug("recordsForDSFilter: dsFilter=%s returning %s" % (dsFilter.generate(), result, ))
+ returnValue(result)
+
+ # calculate minimum attributes needed for this query: results unused
+ etagRequested, queryPropNames = propertiesInAddressBookQuery( addressBookQuery )
+ self.log_debug("doAddressBookQuery: etagRequested=%s, queryPropNames=%s" % (etagRequested, queryPropNames,))
+
+ # walk the expression tree
+ if dsFilter is True:
+ xmlDirectoryRecords = None
+ else:
+ xmlDirectoryRecords = (yield recordsForDSFilter(dsFilter, recordType))
+ self.log_debug("doAddressBookQuery: #xmlDirectoryRecords %s" % (len(xmlDirectoryRecords) if xmlDirectoryRecords is not None else xmlDirectoryRecords, ))
+
+ if xmlDirectoryRecords is None:
+ xmlDirectoryRecords = (yield self.listRecords(recordType))
+ self.log_debug("doAddressBookQuery: all #xmlDirectoryRecords %s" % (len(xmlDirectoryRecords), ))
+
+
+ for xmlDirectoryRecord in xmlDirectoryRecords:
+
+ def dsRecordAttributesFromDirectoryRecord( xmlDirectoryRecord ):
+ #FIXME should filter based on request
+ dsRecordAttributes = {}
+ for attr in dirRecordAttrToDSAttrMap:
+ try:
+ if attr == "members":
+ value = [member.guid for member in xmlDirectoryRecord.members()]
+ else:
+ value = getattr(xmlDirectoryRecord, attr)
+ if value:
+ dsRecordAttributes[dirRecordAttrToDSAttrMap[attr]] = value
+ except AttributeError:
+ self.log_info("Could not get attribute %s from record %s" % (attr, xmlDirectoryRecord,))
+ pass
+ return dsRecordAttributes
+
+ result = None
+ dsRecordAttributes = dsRecordAttributesFromDirectoryRecord( xmlDirectoryRecord )
+ try:
+ result = ABDirectoryQueryResult(self.directoryBackedAddressBook, dsRecordAttributes, kind=kind)
+ except:
+ traceback.print_exc()
+ self.log_info("Could not get vcard for %s" % (xmlDirectoryRecord,))
+ else:
+ self.log_debug("doAddressBookQuery: VCard text =\n%s" % (result.vCard(),))
+ if addressBookFilter.match(result.vCard()):
+ results.append(result)
+ else:
+ # should also filter for duplicate UIDs
+ self.log_debug("doAddressBookQuery did not match filter: %s (%s)" % (result.vCard().propertyValue("FN"), result.vCard().propertyValue("UID"),))
+
+ if len(results) >= maxResults:
+ limited = True
+ break
+
+ #sort results so that CalDAVTester can have consistent results when it uses limits
+ if self.sortResults:
+ results = sorted(list(results), key=lambda result:result.vCard().propertyValue("UID"))
+
+ self.log_info("limited %s len(results) %s" % (limited,len(results),))
+ returnValue((results, limited,))
+
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/directorybackedaddressbook.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2008-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2008-2012 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -45,10 +45,11 @@
Directory-backed address book
"""
- def __init__(self, principalCollections):
+ def __init__(self, principalCollections, uri):
CalDAVResource.__init__(self, principalCollections=principalCollections)
-
+
+ self.uri = uri
self.directory = None # creates directory attribute
# create with permissions, similar to CardDAVOptions in tap.py
@@ -93,14 +94,11 @@
return succeed(None)
return self.directory.createCache()
-
- #print ("DirectoryBackedAddressBookResource.provisionDirectory: provisioned")
-
+
return succeed(None)
def defaultAccessControlList(self):
- #print( "DirectoryBackedAddressBookResource.defaultAccessControlList" )
if config.AnonymousDirectoryAddressBookAccess:
# DAV:Read for all principals (includes anonymous)
accessPrincipal = davxml.All()
@@ -141,7 +139,6 @@
return True
def isAddressBookCollection(self):
- #print( "DirectoryBackedAddressBookResource.isAddressBookCollection: return True" )
return True
def isCollection(self):
@@ -155,22 +152,6 @@
def renderHTTP(self, request):
if not self.directory:
raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,"Service is starting up" ))
- elif self.directory.liveQuery:
- response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
- returnValue(response)
- else:
- available = (yield maybeDeferred(self.directory.available, ))
-
- if not available:
- raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE,"Service is starting up" ))
- else:
- updateLock = self.directory.updateLock()
- yield updateLock.acquire()
- try:
- response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
-
- finally:
- yield updateLock.release()
-
- returnValue(response)
+ response = (yield maybeDeferred(super(DirectoryBackedAddressBookResource, self).renderHTTP, request))
+ returnValue(response)
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_addressbook_query.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -118,7 +118,7 @@
@param addrresource: the L{CalDAVResource} for an address book collection.
@param uri: the uri for the address book collecton resource.
"""
-
+
def checkMaxResults():
matchcount[0] += 1
if max_number_of_results[0] is not None and matchcount[0] > max_number_of_results[0]:
@@ -159,142 +159,109 @@
def queryDirectoryBackedAddressBook(directoryBackedAddressBook, addressBookFilter):
"""
"""
- records, limited[0] = (yield directoryBackedAddressBook.directory.vCardRecordsForAddressBookQuery( addressBookFilter, query, max_number_of_results[0] ))
- for vCardRecord in records:
+ results, limited[0] = (yield directoryBackedAddressBook.directory.doAddressBookQuery( addressBookFilter, query, max_number_of_results[0] ))
+ for vCardResult in results:
- # match against original filter
- if filter.match((yield vCardRecord.vCard())):
+ # match against original filter if different from addressBookFilter
+ if addressBookFilter is filter or filter.match((yield vCardResult.vCard())):
# Check size of results is within limit
checkMaxResults()
try:
- yield report_common.responseForHref(request, responses, vCardRecord.hRef(), vCardRecord, propertiesForResource, query, vcard=(yield vCardRecord.vCard()))
+ yield report_common.responseForHref(request, responses, vCardResult.hRef(), vCardResult, propertiesForResource, query, vcard=(yield vCardResult.vCard()))
except ConcurrentModification:
# This can happen because of a race-condition between the
# time we determine which resources exist and the deletion
# of one of these resources in another request. In this
# case, we ignore the now missing resource rather
# than raise an error for the entire report.
- log.err("Missing resource during sync: %s" % (vCardRecord.hRef(),))
+ log.err("Missing resource during sync: %s" % (vCardResult.hRef(),))
- directoryAddressBookLock = None
- try:
- if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.cacheQuery:
-
- directory = addrresource.directory
- if directory.liveQuery:
- # if liveQuery and cacheQuery, get vCards into the directory address book on disk
- directoryAddressBookLock, limited[0] = (yield directory.cacheVCardsForAddressBookQuery( filter, query, max_number_of_results[0] ) )
-
- elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
- max_number_of_results[0] = directory.maxDSQueryRecords
-
-
- elif not addrresource.isAddressBookCollection():
-
- #do UID lookup on last part of uri
- resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
- if resource_name.endswith(".vcf") and len(resource_name) > 4:
-
- # see if parent is directory backed address book
- parent = (yield addrresource.locateParent( request, uri ) )
-
- if parent.isDirectoryBackedAddressBookCollection() and parent.directory.cacheQuery:
-
- directory = parent.directory
- if directory.liveQuery:
- vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
- carddavxml.TextMatch.fromString(resource_name[:-4]),
- name="UID", # attributes
- ),] )
- vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
-
- directoryAddressBookLock, limited[0] = (yield directory.cacheVCardsForAddressBookQuery( vCardFilter, query, max_number_of_results[0] ) )
+ if not addrresource.isAddressBookCollection():
- elif directory.maxDSQueryRecords and directory.maxDSQueryRecords < max_number_of_results[0]:
- max_number_of_results[0] = directory.maxDSQueryRecords
-
+ #do UID lookup on last part of uri
+ resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+ if resource_name.endswith(".vcf") and len(resource_name) > 4:
+
+ # see if parent is directory backed address book
+ parent = (yield addrresource.locateParent( request, uri ) )
-
- # Check whether supplied resource is an address book or an address book object resource
- if addrresource.isAddressBookCollection():
- if addrresource.isDirectoryBackedAddressBookCollection() and addrresource.directory.liveQuery and not addrresource.directory.cacheQuery:
- yield maybeDeferred( queryDirectoryBackedAddressBook, addrresource, filter )
-
- else:
+ # Check whether supplied resource is an address book or an address book object resource
+ if addrresource.isAddressBookCollection():
- # Do some optimisation of access control calculation by determining any inherited ACLs outside of
- # the child resource loop and supply those to the checkPrivileges on each child.
- filteredaces = (yield addrresource.inheritedACEsforChildren(request))
+ if addrresource.isDirectoryBackedAddressBookCollection():
+ yield maybeDeferred( queryDirectoryBackedAddressBook, addrresource, filter )
+
+ else:
+
+ # Do some optimisation of access control calculation by determining any inherited ACLs outside of
+ # the child resource loop and supply those to the checkPrivileges on each child.
+ filteredaces = (yield addrresource.inheritedACEsforChildren(request))
+
+ # Check for disabled access
+ if filteredaces is not None:
+ # See whether the filter is valid for an index only query
+ index_query_ok = addrresource.index().searchValid(filter)
- # Check for disabled access
- if filteredaces is not None:
- # See whether the filter is valid for an index only query
- index_query_ok = addrresource.index().searchValid(filter)
-
- # Get list of children that match the search and have read access
- names = [name for name, ignore_uid in (yield addrresource.index().search(filter))] #@UnusedVariable
- if not names:
- return
-
- # Now determine which valid resources are readable and which are not
- ok_resources = []
- yield addrresource.findChildrenFaster(
- "1",
- request,
- lambda x, y: ok_resources.append((x, y)),
- None,
- None,
- names,
- (davxml.Read(),),
- inherited_aces=filteredaces
- )
- for child, child_uri in ok_resources:
- child_uri_name = child_uri[child_uri.rfind("/") + 1:]
-
- if generate_address_data or not index_query_ok:
- vcard = yield child.vCard()
- assert vcard is not None, "vCard %s is missing from address book collection %r" % (child_uri_name, self)
- else:
- vcard = None
-
- yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok = index_query_ok)
+ # Get list of children that match the search and have read access
+ names = [name for name, ignore_uid in (yield addrresource.index().search(filter))] #@UnusedVariable
+ if not names:
+ return
+
+ # Now determine which valid resources are readable and which are not
+ ok_resources = []
+ yield addrresource.findChildrenFaster(
+ "1",
+ request,
+ lambda x, y: ok_resources.append((x, y)),
+ None,
+ None,
+ names,
+ (davxml.Read(),),
+ inherited_aces=filteredaces
+ )
+ for child, child_uri in ok_resources:
+ child_uri_name = child_uri[child_uri.rfind("/") + 1:]
- else:
+ if generate_address_data or not index_query_ok:
+ vcard = yield child.vCard()
+ assert vcard is not None, "vCard %s is missing from address book collection %r" % (child_uri_name, self)
+ else:
+ vcard = None
+
+ yield queryAddressBookObjectResource(child, uri, child_uri_name, vcard, query_ok = index_query_ok)
+
+ else:
+
+ handled = False;
+ resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
+ if resource_name.endswith(".vcf") and len(resource_name) > 4:
- handled = False;
- resource_name = urllib.unquote(uri[uri.rfind("/") + 1:])
- if resource_name.endswith(".vcf") and len(resource_name) > 4:
+ # see if parent is directory backed address book
+ parent = (yield addrresource.locateParent( request, uri ) )
+
+ if parent.isDirectoryBackedAddressBookCollection():
+
+ vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
+ carddavxml.TextMatch.fromString(resource_name[:-4]),
+ name="UID", # attributes
+ ),] )
+ vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
- # see if parent is directory backed address book
- parent = (yield addrresource.locateParent( request, uri ) )
+ yield maybeDeferred( queryDirectoryBackedAddressBook, parent, vCardFilter )
+ handled = True
+
+ if not handled:
+ vcard = yield addrresource.vCard()
+ yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
- if parent.isDirectoryBackedAddressBookCollection() and parent.directory.liveQuery and not parent.directory.cacheQuery:
-
- vCardFilter = carddavxml.Filter( *[carddavxml.PropertyFilter(
- carddavxml.TextMatch.fromString(resource_name[:-4]),
- name="UID", # attributes
- ),] )
- vCardFilter = addressbookqueryfilter.Filter(vCardFilter)
-
- yield maybeDeferred( queryDirectoryBackedAddressBook, parent, vCardFilter )
- handled = True
+ if limited[0]:
+ raise NumberOfMatchesWithinLimits(matchcount[0])
- if not handled:
- vcard = yield addrresource.vCard()
- yield queryAddressBookObjectResource(addrresource, uri, None, vcard)
-
- if limited[0]:
- raise NumberOfMatchesWithinLimits(matchcount[0])
-
- finally:
- if directoryAddressBookLock:
- yield directoryAddressBookLock.release()
-
# Run report taking depth into account
try:
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/method/report_multiget_common.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2012 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -170,7 +170,7 @@
# Special for addressbooks
if collection_type == COLLECTION_TYPE_ADDRESSBOOK:
- if self.isDirectoryBackedAddressBookCollection() and self.directory.liveQuery:
+ if self.isDirectoryBackedAddressBookCollection():
result = (yield doDirectoryAddressBookResponse())
returnValue(result)
@@ -249,11 +249,13 @@
resource_name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
if self._isChildURI(request, resource_uri) and resource_name.endswith(".vcf") and len(resource_name) > 4:
valid_hrefs.append(href)
+ textMatchElement = carddavxml.TextMatch.fromString(resource_name[:-4])
+ textMatchElement.attributes["match-type"] = "equals" # do equals compare. Default is "contains"
vCardFilters.append(carddavxml.PropertyFilter(
- carddavxml.TextMatch.fromString(resource_name[:-4]),
+ textMatchElement,
name="UID", # attributes
))
- elif not self.directory.cacheQuery:
+ else:
responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
# exit if not valid
@@ -262,40 +264,29 @@
addressBookFilter = carddavxml.Filter( *vCardFilters )
addressBookFilter = addressbookqueryfilter.Filter(addressBookFilter)
- if self.directory.cacheQuery:
- # add vcards to directory address book and run "normal case" below
- limit = config.DirectoryAddressBook.MaxQueryResults
- directoryAddressBookLock, limited = (yield self.directory.cacheVCardsForAddressBookQuery(addressBookFilter, propertyreq, limit) )
- if limited:
- log.err("Too many results in multiget report: %d" % len(resources))
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (dav_namespace, "number-of-matches-within-limits"),
- "Too many results",
- ))
- else:
- #get vCards and filter
- limit = config.DirectoryAddressBook.MaxQueryResults
- vCardRecords, limited = (yield self.directory.vCardRecordsForAddressBookQuery( addressBookFilter, propertyreq, limit ))
- if limited:
- log.err("Too many results in multiget report: %d" % len(resources))
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (dav_namespace, "number-of-matches-within-limits"),
- "Too many results",
- ))
-
- for href in valid_hrefs:
- matchingRecord = None
- for vCardRecord in vCardRecords:
- if href == vCardRecord.hRef(): # might need to compare urls instead - also case sens ok?
- matchingRecord = vCardRecord
- break;
+
+ #get vCards and filter
+ limit = config.DirectoryAddressBook.MaxQueryResults
+ results, limited = (yield self.directory.doAddressBookQuery( addressBookFilter, propertyreq, limit ))
+ if limited:
+ log.err("Too many results in multiget report: %d" % len(resources))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (dav_namespace, "number-of-matches-within-limits"),
+ "Too many results",
+ ))
+
+ for href in valid_hrefs:
+ matchingResource = None
+ for vCardResource in results:
+ if href == vCardResource.hRef(): # might need to compare urls instead - also case sens ok?
+ matchingResource = vCardResource
+ break;
- if matchingRecord:
- yield report_common.responseForHref(request, responses, href, matchingRecord, propertiesForResource, propertyreq, vcard=matchingRecord.vCard())
- else:
- responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+ if matchingResource:
+ yield report_common.responseForHref(request, responses, href, matchingResource, propertiesForResource, propertyreq, vcard=matchingResource.vCard())
+ else:
+ responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
finally:
if directoryAddressBookLock:
yield directoryAddressBookLock.release()
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/query/addressbookqueryfilter.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2011 Apple Inc. All rights reserved.
+# Copyright (c) 2011-2012 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -73,7 +73,7 @@
return not allof
return allof
else:
- return True
+ return not allof
def valid(self):
"""
@@ -120,9 +120,6 @@
else:
raise ValueError("Unknown child element: %s" % (qname,))
- if qualifier and isinstance(qualifier, IsNotDefined) and (len(filters) != 0):
- raise ValueError("No other tests allowed when CardDAV:is-not-defined is present")
-
if xml_element.qname() == (carddav_namespace, "prop-filter"):
propfilter_test = xml_element.attributes.get("test", "anyof")
if propfilter_test not in ("anyof", "allof"):
@@ -130,13 +127,16 @@
else:
propfilter_test = "anyof"
+ if qualifier and isinstance(qualifier, IsNotDefined) and (len(filters) != 0) and propfilter_test == "allof":
+ raise ValueError("When test is allof, no other tests allowed when CardDAV:is-not-defined is present")
+
self.propfilter_test = propfilter_test
self.qualifier = qualifier
self.filters = filters
self.filter_name = xml_element.attributes["name"]
if isinstance(self.filter_name, unicode):
self.filter_name = self.filter_name.encode("utf-8")
- self.defined = not self.qualifier or not isinstance(qualifier, IsNotDefined)
+ self.defined = not self.qualifier or not isinstance(qualifier, IsNotDefined) or len(filters)
def match(self, item):
"""
@@ -144,20 +144,17 @@
matches this filter, False otherwise.
"""
- # Always return True for the is-not-defined case as the result of this will
- # be negated by the caller
- if not self.defined: return True
+ allof = self.propfilter_test == "allof"
+ if self.qualifier and allof != self.qualifier.match(item):
+ return not allof
- if self.qualifier and not self.qualifier.match(item): return False
-
if len(self.filters) > 0:
- allof = self.propfilter_test == "allof"
for filter in self.filters:
if allof != filter._match(item):
return not allof
return allof
else:
- return True
+ return not allof
class PropertyFilter (FilterChildBase):
"""
Modified: CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py 2012-08-31 22:40:26 UTC (rev 9764)
+++ CalendarServer/branches/users/gaya/directorybacker/twistedcaldav/stdconfig.py 2012-09-01 00:56:43 UTC (rev 9765)
@@ -207,28 +207,64 @@
directoryAddressBookBackingServiceDefaultParams = {
- "twistedcaldav.directory.xmlfile.XMLDirectoryService": {
- "xmlFile": "/etc/carddavd/accounts.xml",
+ "twistedcaldav.directory.xmldirectorybacker.XMLDirectoryBackingService": {
+ "xmlFile": "accounts.xml",
+ "recordTypes": ("users", "groups"),
+ "statSeconds" : 15,
},
"twistedcaldav.directory.opendirectorybacker.OpenDirectoryBackingService": {
"queryPeopleRecords": True,
"peopleNode": "/Search/Contacts",
"queryUserRecords": True,
- "userNode": "/Search/Contacts",
+ "userNode": "/Search",
+ "queryGroupRecords": True,
+ "groupNode": "/Search",
"maxDSQueryRecords":0,
"queryDSLocal": False,
"ignoreSystemRecords": True,
"dsLocalCacheTimeout":30,
- "liveQuery": True,
"fakeETag": True,
- "cacheQuery": False,
- "cacheTimeout": 30,
- "standardizeSyntheticUIDs": False,
"addDSAttrXProperties": False,
"appleInternalServer": False,
- "additionalAttributes" : [],
- "allowedAttributes" : [],
+ "additionalAttributes" : None,
+ "allowedAttributes" : None,
+ "searchAttributes" : None,
},
+ "twistedcaldav.directory.ldapdirectorybacker.LdapDirectoryBackingService": {
+ "appleInternalServer": False,
+ "warningThresholdSeconds": 3,
+ "uri": "ldap://localhost/",
+ "tls": False,
+ "tlsCACertFile": None,
+ "tlsCACertDir": None,
+ "tlsRequireCert": None, # never, allow, try, demand, hard
+ "credentials": {
+ "dn": None,
+ "password": None,
+ },
+ "authMethod": "LDAP",
+ "rdnSchema": {
+ "base": "dc=example,dc=com",
+ "queryTypes": ("people", ),
+ "people": {
+ "rdn": "ou=People",
+ "attr": "uid", # used only to synthesize email address
+ "filter": None, # additional filter for this type
+ "vcardPropToLdapAttrMap" : { # maps address book query vCard properties to ldap attributes
+ "givenName" : "FirstName",
+ "sn" : "LastName",
+ "cn" : "RealName",
+ "uid" : "GeneratedUID",
+ "mail" : "EMailAddress",
+ },
+ "ldapAttrToDSAttrMap" : { # maps ldap attributes to ds record attributes
+ "FN" : "cn",
+ "EMAIL" : "mail",
+ "UID" : "uid",
+ },
+ },
+ },
+ },
}
DEFAULT_CONFIG = {
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120831/87743a7e/attachment-0001.html>
More information about the calendarserver-changes
mailing list