[CalendarServer-changes] [8060] CalendarServer/trunk/twistedcaldav

[source_changes at macosforge.org]

Revision: 8060
          http://trac.macosforge.org/projects/calendarserver/changeset/8060
Author:   sagen at apple.com
Date:     2011-09-09 13:52:59 -0700 (Fri, 09 Sep 2011)
Log Message:
-----------
Batch large LDAP queries

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
    CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py

Modified: CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py	2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/directory/ldapdirectory.py	2011-09-09 20:52:59 UTC (rev 8060)
@@ -84,6 +84,7 @@
             "cacheTimeout": 1, # Minutes
             "negativeCaching": False,
             "warningThresholdSeconds": 3,
+            "batchSize": 500, # for splitting up large queries
             "queryLocationsImplicitly": True,
             "restrictEnabledRecords": False,
             "restrictToGroup": "",
@@ -192,6 +193,7 @@
                                                    params["negativeCaching"])
 
         self.warningThresholdSeconds = params["warningThresholdSeconds"]
+        self.batchSize = params["batchSize"]
         self.queryLocationsImplicitly = params["queryLocationsImplicitly"]
         self.augmentService = params["augmentService"]
         self.groupMembershipCache = params["groupMembershipCache"]
@@ -931,22 +933,24 @@
         valuesToFetch = guids
 
         while valuesToFetch:
+            results = []
 
             if attributeToSearch == "dn":
                 # Since DN can't be searched on in a filter we have to call
                 # recordsMatchingFields for *each* DN.
-                results = []
                 for value in valuesToFetch:
                     fields = [["dn", value, False, "equals"]]
                     result = (yield self.recordsMatchingFields(fields,
                         recordType=self.recordType_groups))
                     results.extend(result)
             else:
-                fields = []
-                for value in valuesToFetch:
-                    fields.append([attributeToSearch, value, False, "equals"])
-                results = (yield self.recordsMatchingFields(fields,
-                    recordType=self.recordType_groups))
+                for batch in splitIntoBatches(valuesToFetch, self.batchSize):
+                    fields = []
+                    for value in batch:
+                        fields.append([attributeToSearch, value, False, "equals"])
+                    result = (yield self.recordsMatchingFields(fields,
+                        recordType=self.recordType_groups))
+                    results.extend(result)
 
             # Reset values for next iteration
             valuesToFetch = set()
@@ -1030,6 +1034,20 @@
     return filterstr
 
 
+def splitIntoBatches(data, size):
+    """
+    Return a generator of sets consisting of the contents of the data set
+    split into parts no larger than size.
+    """
+    if not data:
+        yield set([])
+    data = list(data)
+    while data:
+        yield set(data[:size])
+        del data[:size]
+
+
+
 class LdapDirectoryRecord(CachingDirectoryRecord):
     """
     LDAP implementation of L{IDirectoryRecord}.

Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py	2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_ldapdirectory.py	2011-09-09 20:52:59 UTC (rev 8060)
@@ -16,7 +16,8 @@
 
 try:
     from twistedcaldav.directory.ldapdirectory import (
-        buildFilter, LdapDirectoryService, MissingGuidException
+        buildFilter, LdapDirectoryService, MissingGuidException,
+        splitIntoBatches
     )
     from twistedcaldav.test.util import proxiesFile
     from twistedcaldav.directory.calendaruserproxyloader import XMLCalendarUserProxyLoader
@@ -132,6 +133,7 @@
                 "cacheTimeout": 1, # Minutes
                 "negativeCaching": False,
                 "warningThresholdSeconds": 3,
+                "batchSize": 500,
                 "queryLocationsImplicitly": True,
                 "restrictEnabledRecords": False,
                 "restrictToGroup": "",
@@ -493,9 +495,9 @@
             updater = GroupMembershipCacheUpdater(calendaruserproxy.ProxyDBService,
                 self.service, 30, cache=cache, useExternalProxies=False)
 
-            # Fake LDAP results for the group listRecords performed within updateCache()
-
-            # Also include recursive groups to make sure we handle that situation
+            # Fake LDAP results for the getGroups() call performed within
+            # updateCache().  Also include recursive groups to make sure we
+            # handle that situation.
             self.service.ldap.addTestResults([
                 (
                     "cn=recursive1_coasts,cn=groups,dc=example,dc=com",
@@ -616,3 +618,19 @@
 
                 record = self.service.recordWithShortName(users, shortName)
                 self.assertEquals(groups, (yield record.cachedGroups()))
+
+
+        def test_splitIntoBatches(self):
+            # Data is perfect multiple of size
+            results = list(splitIntoBatches(set(range(12)), 4))
+            self.assertEquals(results,
+                [set([0, 1, 2, 3]), set([4, 5, 6, 7]), set([8, 9, 10, 11])])
+
+            # Some left overs
+            results = list(splitIntoBatches(set(range(12)), 5))
+            self.assertEquals(results,
+                [set([0, 1, 2, 3, 4]), set([8, 9, 5, 6, 7]), set([10, 11])])
+
+            # Empty
+            results = list(splitIntoBatches(set([]), 5)) # empty data
+            self.assertEquals(results, [set([])])

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-09-09 20:37:00 UTC (rev 8059)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-09-09 20:52:59 UTC (rev 8060)
@@ -58,6 +58,7 @@
         "cacheTimeout": 1, # Minutes
         "negativeCaching": False,
         "warningThresholdSeconds": 3,
+        "batchSize": 500, # for splitting up large queries
         "queryLocationsImplicitly": True,
         "restrictEnabledRecords": False,
         "restrictToGroup": "",
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110909/31a51a15/attachment-0001.html>