[CalendarServer-changes] [8060] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Fri Sep 9 13:53:02 PDT 2011
Revision: 8060
http://trac.macosforge.org/projects/calendarserver/changeset/8060
Author: sagen at apple.com
Date: 2011-09-09 13:52:59 -0700 (Fri, 09 Sep 2011)
Log Message:
-----------
Batch large LDAP queries
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
Modified: CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py 2011-09-09 20:52:59 UTC (rev 8060)
@@ -84,6 +84,7 @@
"cacheTimeout": 1, # Minutes
"negativeCaching": False,
"warningThresholdSeconds": 3,
+ "batchSize": 500, # for splitting up large queries
"queryLocationsImplicitly": True,
"restrictEnabledRecords": False,
"restrictToGroup": "",
@@ -192,6 +193,7 @@
params["negativeCaching"])
self.warningThresholdSeconds = params["warningThresholdSeconds"]
+ self.batchSize = params["batchSize"]
self.queryLocationsImplicitly = params["queryLocationsImplicitly"]
self.augmentService = params["augmentService"]
self.groupMembershipCache = params["groupMembershipCache"]
@@ -931,22 +933,24 @@
valuesToFetch = guids
while valuesToFetch:
+ results = []
if attributeToSearch == "dn":
# Since DN can't be searched on in a filter we have to call
# recordsMatchingFields for *each* DN.
- results = []
for value in valuesToFetch:
fields = [["dn", value, False, "equals"]]
result = (yield self.recordsMatchingFields(fields,
recordType=self.recordType_groups))
results.extend(result)
else:
- fields = []
- for value in valuesToFetch:
- fields.append([attributeToSearch, value, False, "equals"])
- results = (yield self.recordsMatchingFields(fields,
- recordType=self.recordType_groups))
+ for batch in splitIntoBatches(valuesToFetch, self.batchSize):
+ fields = []
+ for value in batch:
+ fields.append([attributeToSearch, value, False, "equals"])
+ result = (yield self.recordsMatchingFields(fields,
+ recordType=self.recordType_groups))
+ results.extend(result)
# Reset values for next iteration
valuesToFetch = set()
@@ -1030,6 +1034,20 @@
return filterstr
+def splitIntoBatches(data, size):
+ """
+ Return a generator of sets consisting of the contents of the data set
+ split into parts no larger than size.
+ """
+ if not data:
+ yield set([])
+ data = list(data)
+ while data:
+ yield set(data[:size])
+ del data[:size]
+
+
+
class LdapDirectoryRecord(CachingDirectoryRecord):
"""
LDAP implementation of L{IDirectoryRecord}.
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py 2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py 2011-09-09 20:52:59 UTC (rev 8060)
@@ -16,7 +16,8 @@
try:
from twistedcaldav.directory.ldapdirectory import (
- buildFilter, LdapDirectoryService, MissingGuidException
+ buildFilter, LdapDirectoryService, MissingGuidException,
+ splitIntoBatches
)
from twistedcaldav.test.util import proxiesFile
from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
@@ -132,6 +133,7 @@
"cacheTimeout": 1, # Minutes
"negativeCaching": False,
"warningThresholdSeconds": 3,
+ "batchSize": 500,
"queryLocationsImplicitly": True,
"restrictEnabledRecords": False,
"restrictToGroup": "",
@@ -493,9 +495,9 @@
updater = GroupMembershipCacheUpdater(calendaruserproxy.ProxyDBService,
self.service, 30, cache=cache, useExternalProxies=False)
- # Fake LDAP results for the group listRecords performed within updateCache()
-
- # Also include recursive groups to make sure we handle that situation
+ # Fake LDAP results for the getGroups() call performed within
+ # updateCache(). Also include recursive groups to make sure we
+ # handle that situation.
self.service.ldap.addTestResults([
(
"cn=recursive1_coasts,cn=groups,dc=example,dc=com",
@@ -616,3 +618,19 @@
record = self.service.recordWithShortName(users, shortName)
self.assertEquals(groups, (yield record.cachedGroups()))
+
+
+ def test_splitIntoBatches(self):
+ # Data is perfect multiple of size
+ results = list(splitIntoBatches(set(range(12)), 4))
+ self.assertEquals(results,
+ [set([0, 1, 2, 3]), set([4, 5, 6, 7]), set([8, 9, 10, 11])])
+
+ # Some left overs
+ results = list(splitIntoBatches(set(range(12)), 5))
+ self.assertEquals(results,
+ [set([0, 1, 2, 3, 4]), set([8, 9, 5, 6, 7]), set([10, 11])])
+
+ # Empty
+ results = list(splitIntoBatches(set([]), 5)) # empty data
+ self.assertEquals(results, [set([])])
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2011-09-09 20:52:59 UTC (rev 8060)
@@ -58,6 +58,7 @@
"cacheTimeout": 1, # Minutes
"negativeCaching": False,
"warningThresholdSeconds": 3,
+ "batchSize": 500, # for splitting up large queries
"queryLocationsImplicitly": True,
"restrictEnabledRecords": False,
"restrictToGroup": "",
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110909/31a51a15/attachment-0001.html>
More information about the calendarserver-changes
mailing list