[CalendarServer-changes] [15633] twext/trunk/twext/who/ldap/_service.py

source_changes at macosforge.org source_changes at macosforge.org
Mon May 23 12:36:46 PDT 2016


Revision: 15633
          http://trac.calendarserver.org//changeset/15633
Author:   sagen at apple.com
Date:     2016-05-23 12:36:46 -0700 (Mon, 23 May 2016)
Log Message:
-----------
Split large members( ) queries into batches; log an error if an LDAP query exceeds a time threshold

Modified Paths:
--------------
    twext/trunk/twext/who/ldap/_service.py

Modified: twext/trunk/twext/who/ldap/_service.py
===================================================================
--- twext/trunk/twext/who/ldap/_service.py	2016-05-23 17:23:34 UTC (rev 15632)
+++ twext/trunk/twext/who/ldap/_service.py	2016-05-23 19:36:46 UTC (rev 15633)
@@ -27,6 +27,7 @@
 
 import collections
 import ldap.async
+import time
 
 from twisted.python.constants import Names, NamedConstant
 from twisted.internet.defer import succeed, inlineCallbacks, returnValue
@@ -237,6 +238,7 @@
         threadPoolMax=10,
         connectionMax=10,
         tries=3,
+        warningThresholdSeconds=5,
         _debug=False,
     ):
         """
@@ -281,6 +283,7 @@
         self._timeout = timeout
         self._extraFilters = extraFilters
         self._tries = tries
+        self._warningThresholdSeconds = warningThresholdSeconds
 
         if tlsCACertificateFile is None:
             self._tlsCACertificateFile = None
@@ -682,6 +685,8 @@
                             ),
                         )
                         try:
+                            startTime = time.time()
+
                             s = ldap.async.List(connection)
                             s.startSearch(
                                 ldap.dn.dn2str(rdn),
@@ -699,6 +704,7 @@
                             )
                             s.processResults()
 
+
                         except ldap.SIZELIMIT_EXCEEDED as e:
                             self.log.debug(
                                 "LDAP result limit exceeded: {limit}",
@@ -748,6 +754,16 @@
                             in s.allResults
                         ]
 
+                        totalTime = time.time() - startTime
+                        if totalTime > self._warningThresholdSeconds:
+                            if filteredQuery and len(filteredQuery) > 500:
+                                filteredQuery = "%s..." % (filteredQuery[:500],)
+                            self.log.error(
+                                "LDAP query exceeded threshold: {totalTime:.2f} seconds for {rdn} {query} (#results={resultCount})",
+                                totalTime=totalTime, rdn=rdn,
+                                query=filteredQuery, resultCount=len(reply)
+                            )
+
                         newRecords = self._recordsFromReply(
                             reply, recordType=recordType
                         )
@@ -1067,7 +1083,20 @@
     #     return succeed(None)
 
 
+def splitIntoBatches(data, size):
+    """
+    Return a generator of lists consisting of the contents of the data set
+    split into parts no larger than size.
+    """
+    if not data:
+        yield []
+    data = list(data)
+    while data:
+        yield data[:size]
+        del data[:size]
 
+
+
 @implementer(IPlaintextPasswordVerifier)
 class DirectoryRecord(BaseDirectoryRecord):
     """
@@ -1127,15 +1156,18 @@
                             matchType=MatchType.equals
                         )
                     )
-            expression = CompoundExpression(
-                matchExpressions,
-                Operand.OR
-            )
-            for record in (yield self.service.recordsFromCompoundExpression(
-                expression, recordTypes=[recordType]
-            )):
-                members.add(record)
 
+                # split into batches of 500
+                for batch in splitIntoBatches(matchExpressions, 500):
+                    expression = CompoundExpression(
+                        batch,
+                        Operand.OR
+                    )
+                    for record in (yield self.service.recordsFromCompoundExpression(
+                        expression, recordTypes=[recordType]
+                    )):
+                        members.add(record)
+
         for dnStr in faultByDN:
             record = yield self.service._recordWithDN(dnStr.replace("+", "\+"))
             members.add(record)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160523/0170e9a0/attachment.html>


More information about the calendarserver-changes mailing list