[CalendarServer-changes] [8564] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jan 19 17:53:19 PST 2012
Revision: 8564
http://trac.macosforge.org/projects/calendarserver/changeset/8564
Author: sagen at apple.com
Date: 2012-01-19 17:53:18 -0800 (Thu, 19 Jan 2012)
Log Message:
-----------
Make GroupCacher more efficient when talking to OD -- only fetch the relevant groups (i.e. the ones that have been delegated-to)
Modified Paths:
--------------
CalendarServer/trunk/conf/caldavd-apple.plist
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py
CalendarServer/trunk/twistedcaldav/directory/util.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/conf/caldavd-apple.plist 2012-01-20 01:53:18 UTC (rev 8564)
@@ -186,8 +186,6 @@
<dict>
<key>node</key>
<string>/Search</string>
- <key>cacheTimeout</key>
- <integer>1</integer>
</dict>
</dict>
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2012-01-20 01:53:18 UTC (rev 8564)
@@ -207,7 +207,7 @@
<key>node</key>
<string>/Search</string>
<key>cacheTimeout</key>
- <integer>1</integer>
+ <integer>10</integer>
</dict>
</dict>
-->
Modified: CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/directory/appleopendirectory.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -28,7 +28,7 @@
import time
from uuid import UUID
-from twisted.internet.defer import succeed
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twisted.cred.credentials import UsernamePassword
from twext.web2.auth.digest import DigestedCredentials
@@ -37,6 +37,7 @@
CachingDirectoryRecord
from twistedcaldav.directory.directory import DirectoryService, DirectoryRecord
from twistedcaldav.directory.directory import DirectoryError, UnknownRecordTypeError
+from twistedcaldav.directory.util import splitIntoBatches
from twistedcaldav.directory.principal import cuAddressConverter
from calendarserver.platform.darwin.od import dsattributes, dsquery
@@ -72,6 +73,7 @@
'restrictEnabledRecords' : False,
'restrictToGroup' : '',
'cacheTimeout' : 1, # Minutes
+ 'batchSize' : 100, # for splitting up large queries
'negativeCaching' : False,
'recordTypes' : (
self.recordType_users,
@@ -103,6 +105,7 @@
self.node = params['node']
self.restrictEnabledRecords = params['restrictEnabledRecords']
self.restrictToGroup = params['restrictToGroup']
+ self.batchSize = params['batchSize']
try:
UUID(self.restrictToGroup)
except:
@@ -269,6 +272,10 @@
for key, value in results:
recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
+ if not recordGUID:
+ self.log_warn("Ignoring record missing GUID: %s %s" %
+ (key, value,))
+ continue
# Skip if group restriction is in place and guid is not
# a member (but don't skip any groups)
@@ -324,8 +331,11 @@
if type(nestedGUIDs) is str:
nestedGUIDs = (nestedGUIDs,)
memberGUIDs += tuple(nestedGUIDs)
+ else:
+ nestedGUIDs = ()
else:
memberGUIDs = ()
+ nestedGUIDs = ()
record = OpenDirectoryRecord(
service = self,
@@ -339,6 +349,7 @@
lastName = "",
emailAddresses = "",
memberGUIDs = memberGUIDs,
+ nestedGUIDs = nestedGUIDs,
extProxies = proxyGUIDs,
extReadOnlyProxies = readOnlyProxyGUIDs,
)
@@ -565,6 +576,24 @@
value.get(dsattributes.kDSNAttrEMailAddress),
lower=True)
+ # Special case for groups, which have members.
+ if recordType == self.recordType_groups:
+ memberGUIDs = value.get(dsattributes.kDSNAttrGroupMembers)
+ if memberGUIDs is None:
+ memberGUIDs = ()
+ elif type(memberGUIDs) is str:
+ memberGUIDs = (memberGUIDs,)
+ nestedGUIDs = value.get(dsattributes.kDSNAttrNestedGroups)
+ if nestedGUIDs:
+ if type(nestedGUIDs) is str:
+ nestedGUIDs = (nestedGUIDs,)
+ memberGUIDs += tuple(nestedGUIDs)
+ else:
+ nestedGUIDs = ()
+ else:
+ nestedGUIDs = ()
+ memberGUIDs = ()
+
# Create records but don't store them in our index or
# send them to memcached, because these are transient,
# existing only so we can create principal resource
@@ -581,7 +610,8 @@
firstName = recordFirstName,
lastName = recordLastName,
emailAddresses = recordEmailAddresses,
- memberGUIDs = (),
+ memberGUIDs = memberGUIDs,
+ nestedGUIDs = nestedGUIDs,
extProxies = (),
extReadOnlyProxies = (),
)
@@ -685,6 +715,8 @@
dsattributes.kDS1AttrLastName,
dsattributes.kDSNAttrEMailAddress,
dsattributes.kDSNAttrMetaNodeLocation,
+ dsattributes.kDSNAttrGroupMembers,
+ dsattributes.kDSNAttrNestedGroups,
],
operand
)
@@ -855,8 +887,11 @@
if type(nestedGUIDs) is str:
nestedGUIDs = (nestedGUIDs,)
memberGUIDs += tuple(nestedGUIDs)
+ else:
+ nestedGUIDs = ()
else:
memberGUIDs = ()
+ nestedGUIDs = ()
# Special case for resources and locations
autoSchedule = False
@@ -888,6 +923,7 @@
lastName = recordLastName,
emailAddresses = recordEmailAddresses,
memberGUIDs = memberGUIDs,
+ nestedGUIDs = nestedGUIDs,
extProxies = proxyGUIDs,
extReadOnlyProxies = readOnlyProxyGUIDs,
)
@@ -998,7 +1034,54 @@
return True
+ @inlineCallbacks
+ def getGroups(self, guids):
+ """
+ Returns a set of group records for the list of guids passed in. For
+ any group that also contains subgroups, those subgroups' records are
+ also returned, and so on.
+ """
+ recordsByGUID = {}
+ valuesToFetch = guids
+
+ loop = 1
+ while valuesToFetch:
+ self.log_info("getGroups loop %d" % (loop,))
+
+ results = []
+
+ for batch in splitIntoBatches(valuesToFetch, self.batchSize):
+ fields = []
+ for value in batch:
+ fields.append(["guid", value, False, "equals"])
+ self.log_info("getGroups fetching batch of %d" %
+ (len(fields),))
+ result = list((yield self.recordsMatchingFields(fields,
+ recordType=self.recordType_groups)))
+ results.extend(result)
+ self.log_info("getGroups got back batch of %d for subtotal of %d" %
+ (len(result), len(results)))
+
+ # Reset values for next iteration
+ valuesToFetch = set()
+
+ for record in results:
+ guid = record.guid
+ if guid not in recordsByGUID:
+ recordsByGUID[guid] = record
+
+ # record.nestedGUIDs() contains the sub groups of this group
+ for memberGUID in record.nestedGUIDs():
+ if memberGUID not in recordsByGUID:
+ self.log_info("getGroups group %s contains group %s" %
+ (record.guid, memberGUID))
+ valuesToFetch.add(memberGUID)
+
+ loop += 1
+
+ returnValue(recordsByGUID.values())
+
def buildQueries(recordTypes, fields, mapping):
"""
Determine how many queries need to be performed in order to work around opendirectory
@@ -1025,7 +1108,7 @@
"""
def __init__(
self, service, recordType, guid, nodeName, shortNames, authIDs,
- fullName, firstName, lastName, emailAddresses, memberGUIDs,
+ fullName, firstName, lastName, emailAddresses, memberGUIDs, nestedGUIDs,
extProxies, extReadOnlyProxies,
):
super(OpenDirectoryRecord, self).__init__(
@@ -1044,6 +1127,7 @@
self.nodeName = nodeName
self._memberGUIDs = tuple(memberGUIDs)
+ self._nestedGUIDs = tuple(nestedGUIDs)
self._groupMembershipGUIDs = None
@@ -1084,6 +1168,9 @@
def memberGUIDs(self):
return set(self._memberGUIDs)
+ def nestedGUIDs(self):
+ return set(self._nestedGUIDs)
+
def verifyCredentials(self, credentials):
if isinstance(credentials, UsernamePassword):
# Check cached password
Modified: CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -56,6 +56,7 @@
CachingDirectoryRecord)
from twistedcaldav.directory.directory import DirectoryConfigurationError
from twistedcaldav.directory.augment import AugmentRecord
+from twistedcaldav.directory.util import splitIntoBatches
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twext.web2.http import HTTPError, StatusResponse
from twext.web2 import responsecode
@@ -1127,20 +1128,8 @@
return filterstr
-def splitIntoBatches(data, size):
- """
- Return a generator of sets consisting of the contents of the data set
- split into parts no larger than size.
- """
- if not data:
- yield set([])
- data = list(data)
- while data:
- yield set(data[:size])
- del data[:size]
-
class LdapDirectoryRecord(CachingDirectoryRecord):
"""
LDAP implementation of L{IDirectoryRecord}.
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -18,8 +18,9 @@
from twistedcaldav.directory.ldapdirectory import (
buildFilter, LdapDirectoryService,
MissingGuidException, MissingRecordNameException,
- splitIntoBatches, normalizeDNstr, dnContainedIn
+ normalizeDNstr, dnContainedIn
)
+ from twistedcaldav.directory.util import splitIntoBatches
from twistedcaldav.test.util import proxiesFile
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
from twistedcaldav.directory import calendaruserproxy
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_opendirectory.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -81,6 +81,7 @@
lastName = "User",
emailAddresses = set(("someuser at example.com",)),
memberGUIDs = [],
+ nestedGUIDs = [],
extProxies = [],
extReadOnlyProxies = [],
)
@@ -99,6 +100,7 @@
lastName = "User",
emailAddresses = set(("someuser at example.com",)),
memberGUIDs = [],
+ nestedGUIDs = [],
extProxies = [],
extReadOnlyProxies = [],
)
@@ -121,6 +123,7 @@
lastName = "User",
emailAddresses = set(("someuser at example.com",)),
memberGUIDs = [],
+ nestedGUIDs = [],
extProxies = [],
extReadOnlyProxies = [],
)
Modified: CalendarServer/trunk/twistedcaldav/directory/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/util.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/directory/util.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -96,3 +96,15 @@
return transaction
+def splitIntoBatches(data, size):
+ """
+ Return a generator of sets consisting of the contents of the data set
+ split into parts no larger than size.
+ """
+ if not data:
+ yield set([])
+ data = list(data)
+ while data:
+ yield set(data[:size])
+ del data[:size]
+
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -53,14 +53,15 @@
},
"twistedcaldav.directory.appleopendirectory.OpenDirectoryService": {
"node": "/Search",
- "cacheTimeout": 1, # Minutes
+ "cacheTimeout": 10, # Minutes
+ "batchSize": 100, # for splitting up large queries
"negativeCaching": False,
"restrictEnabledRecords": False,
"restrictToGroup": "",
"recordTypes": ("users", "groups"),
},
"twistedcaldav.directory.ldapdirectory.LdapDirectoryService": {
- "cacheTimeout": 1, # Minutes
+ "cacheTimeout": 10, # Minutes
"negativeCaching": False,
"warningThresholdSeconds": 3,
"batchSize": 500, # for splitting up large queries
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2012-01-20 01:40:15 UTC (rev 8563)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2012-01-20 01:53:18 UTC (rev 8564)
@@ -875,6 +875,7 @@
# the values in augments
augmentService = directory.augmentService
if augmentService:
+ log.warn("Migrating auto-schedule settings")
augmentRecords = []
dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
if os.path.exists(dbPath):
@@ -890,6 +891,7 @@
augmentRecords.append(augmentRecord)
yield augmentService.addAugmentRecords(augmentRecords)
+ log.warn("Migrated auto-schedule settings")
class UpgradeFileSystemFormatService(Service, object):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120119/f5f125b7/attachment-0001.html>
More information about the calendarserver-changes
mailing list