[CalendarServer-changes] [15633] twext/trunk/twext/who/ldap/_service.py
source_changes at macosforge.org
source_changes at macosforge.org
Mon May 23 12:36:46 PDT 2016
Revision: 15633
http://trac.calendarserver.org//changeset/15633
Author: sagen at apple.com
Date: 2016-05-23 12:36:46 -0700 (Mon, 23 May 2016)
Log Message:
-----------
Split large members( ) queries into batches; log an error if an LDAP query exceeds a time threshold
Modified Paths:
--------------
twext/trunk/twext/who/ldap/_service.py
Modified: twext/trunk/twext/who/ldap/_service.py
===================================================================
--- twext/trunk/twext/who/ldap/_service.py 2016-05-23 17:23:34 UTC (rev 15632)
+++ twext/trunk/twext/who/ldap/_service.py 2016-05-23 19:36:46 UTC (rev 15633)
@@ -27,6 +27,7 @@
import collections
import ldap.async
+import time
from twisted.python.constants import Names, NamedConstant
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
@@ -237,6 +238,7 @@
threadPoolMax=10,
connectionMax=10,
tries=3,
+ warningThresholdSeconds=5,
_debug=False,
):
"""
@@ -281,6 +283,7 @@
self._timeout = timeout
self._extraFilters = extraFilters
self._tries = tries
+ self._warningThresholdSeconds = warningThresholdSeconds
if tlsCACertificateFile is None:
self._tlsCACertificateFile = None
@@ -682,6 +685,8 @@
),
)
try:
+ startTime = time.time()
+
s = ldap.async.List(connection)
s.startSearch(
ldap.dn.dn2str(rdn),
@@ -699,6 +704,7 @@
)
s.processResults()
+
except ldap.SIZELIMIT_EXCEEDED as e:
self.log.debug(
"LDAP result limit exceeded: {limit}",
@@ -748,6 +754,16 @@
in s.allResults
]
+ totalTime = time.time() - startTime
+ if totalTime > self._warningThresholdSeconds:
+ if filteredQuery and len(filteredQuery) > 500:
+ filteredQuery = "%s..." % (filteredQuery[:500],)
+ self.log.error(
+ "LDAP query exceeded threshold: {totalTime:.2f} seconds for {rdn} {query} (#results={resultCount})",
+ totalTime=totalTime, rdn=rdn,
+ query=filteredQuery, resultCount=len(reply)
+ )
+
newRecords = self._recordsFromReply(
reply, recordType=recordType
)
@@ -1067,7 +1083,20 @@
# return succeed(None)
+def splitIntoBatches(data, size):
+ """
+ Return a generator of lists consisting of the contents of the data set
+ split into parts no larger than size.
+ """
+ if not data:
+ yield []
+ data = list(data)
+ while data:
+ yield data[:size]
+ del data[:size]
+
+
@implementer(IPlaintextPasswordVerifier)
class DirectoryRecord(BaseDirectoryRecord):
"""
@@ -1127,15 +1156,18 @@
matchType=MatchType.equals
)
)
- expression = CompoundExpression(
- matchExpressions,
- Operand.OR
- )
- for record in (yield self.service.recordsFromCompoundExpression(
- expression, recordTypes=[recordType]
- )):
- members.add(record)
+ # split into batches of 500
+ for batch in splitIntoBatches(matchExpressions, 500):
+ expression = CompoundExpression(
+ batch,
+ Operand.OR
+ )
+ for record in (yield self.service.recordsFromCompoundExpression(
+ expression, recordTypes=[recordType]
+ )):
+ members.add(record)
+
for dnStr in faultByDN:
record = yield self.service._recordWithDN(dnStr.replace("+", "\+"))
members.add(record)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160523/0170e9a0/attachment.html>
More information about the calendarserver-changes
mailing list