<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[13605] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/13605">13605</a></dd>
<dt>Author</dt> <dd>sagen@apple.com</dd>
<dt>Date</dt> <dd>2014-06-03 21:28:02 -0700 (Tue, 03 Jun 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Handle AMP size limits</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkrequirementsstabletxt">CalendarServer/trunk/requirements-stable.txt</a></li>
<li><a href="#CalendarServertrunktxdavdpsclientpy">CalendarServer/trunk/txdav/dps/client.py</a></li>
<li><a href="#CalendarServertrunktxdavdpscommandspy">CalendarServer/trunk/txdav/dps/commands.py</a></li>
<li><a href="#CalendarServertrunktxdavdpsserverpy">CalendarServer/trunk/txdav/dps/server.py</a></li>
<li><a href="#CalendarServertrunktxdavdpstesttest_clientpy">CalendarServer/trunk/txdav/dps/test/test_client.py</a></li>
<li><a href="#CalendarServertrunktxdavwhotestsupportpy">CalendarServer/trunk/txdav/who/test/support.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkrequirementsstabletxt"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/requirements-stable.txt (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/requirements-stable.txt 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/requirements-stable.txt 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -5,7 +5,7 @@
</span><span class="cx"> # For CalendarServer development, don't try to get these projects from PyPI; use svn.
</span><span class="cx">
</span><span class="cx"> -e .
</span><del>--e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@13602#egg=twextpy
</del><ins>+-e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@13604#egg=twextpy
</ins><span class="cx"> -e svn+http://svn.calendarserver.org/repository/calendarserver/PyKerberos/trunk@13420#egg=kerberos
</span><span class="cx"> -e svn+http://svn.calendarserver.org/repository/calendarserver/PyCalendar/trunk@13576#egg=pycalendar
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpsclientpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/client.py (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/client.py 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/txdav/dps/client.py 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -39,7 +39,7 @@
</span><span class="cx"> RecordsMatchingTokensCommand, RecordsMatchingFieldsCommand,
</span><span class="cx"> MembersCommand, GroupsCommand, SetMembersCommand,
</span><span class="cx"> VerifyPlaintextPasswordCommand, VerifyHTTPDigestCommand,
</span><del>- WikiAccessForUID
</del><ins>+ WikiAccessForUID, ContinuationCommand
</ins><span class="cx"> )
</span><span class="cx"> from txdav.who.directory import (
</span><span class="cx"> CalendarDirectoryRecordMixin, CalendarDirectoryServiceMixin
</span><span class="lines">@@ -127,12 +127,24 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def _processSingleRecord(self, result):
</span><ins>+ """
+ Takes a dictionary with a "fields" key whose value is a pickled
+ dictionary of a record's fields, and returns a record.
+ """
</ins><span class="cx"> serializedFields = pickle.loads(result['fields'])
</span><span class="cx"> return self._dictToRecord(serializedFields)
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def _processMultipleRecords(self, result):
</span><del>- serializedFieldsList = pickle.loads(result['fieldsList'])
</del><ins>+ """
+ Takes a dictionary with a "fieldsList" key whose value is an iterable
+ of pickled dictionaries (of records' fields), and returns a list of
+ records.
+ """
+ serializedFieldsList = []
+ for fields in result["fieldsList"]:
+ fields = pickle.loads(fields)
+ serializedFieldsList.append(fields)
</ins><span class="cx"> results = []
</span><span class="cx"> for serializedFields in serializedFieldsList:
</span><span class="cx"> record = self._dictToRecord(serializedFields)
</span><span class="lines">@@ -158,6 +170,26 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def _sendCommand(self, command, **kwds):
+ """
+ Execute a remote AMP command, first making the connection to the peer.
+ Any kwds are passed on to the AMP command.
+
+ @param command: the AMP command to call
+ @type command: L{twisted.protocols.amp.Command}
+ """
+ ampProto = (yield self._getConnection())
+ try:
+ results = (yield ampProto.callRemote(command, **kwds))
+ except Exception, e:
+ log.error("Failed AMP command", error=e)
+ # FIXME: is there a way to hook into ConnectionLost?
+ self._connection = None
+ raise
+ returnValue(results)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def _call(self, command, postProcess, **kwds):
</span><span class="cx"> """
</span><span class="cx"> Execute a remote AMP command, first making the connection to the peer,
</span><span class="lines">@@ -172,14 +204,27 @@
</span><span class="cx"> L{Deferred} which fires with the post-processed results
</span><span class="cx"> @type postProcess: callable
</span><span class="cx"> """
</span><del>- ampProto = (yield self._getConnection())
- try:
- results = (yield ampProto.callRemote(command, **kwds))
- except Exception, e:
- log.error("Failed AMP command", error=e)
- # FIXME: is there a way to hook into ConnectionLost?
- self._connection = None
- raise
</del><ins>+ results = yield self._sendCommand(command, **kwds)
+ if results.get("continuation", None) is None:
+ # We have all the results
+ returnValue(postProcess(results))
+
+ # There are more results to fetch, so loop until the continuation
+ # keyword we get back is None
+
+ multi = [results]
+
+ while results.get("continuation", None) is not None:
+ results = yield self._sendCommand(
+ ContinuationCommand,
+ continuation=results["continuation"]
+ )
+ multi.append(results)
+
+ results = {"fieldsList": []}
+ for result in multi:
+ results["fieldsList"].extend(result["fieldsList"])
+
</ins><span class="cx"> returnValue(postProcess(results))
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpscommandspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/commands.py (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/commands.py 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/txdav/dps/commands.py 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -53,7 +53,8 @@
</span><span class="cx"> ('recordType', amp.String()),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -63,18 +64,31 @@
</span><span class="cx"> ('emailAddress', amp.String()),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+class ContinuationCommand(amp.Command):
+ arguments = [
+ ('continuation', amp.String(optional=True)),
+ ]
+ response = [
+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
+ ]
+
+
+
</ins><span class="cx"> class RecordsMatchingTokensCommand(amp.Command):
</span><span class="cx"> arguments = [
</span><span class="cx"> ('tokens', amp.ListOf(amp.String())),
</span><span class="cx"> ('context', amp.String(optional=True)),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -86,14 +100,15 @@
</span><span class="cx"> ('recordType', amp.String(optional=True)),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> class UpdateRecordsCommand(amp.Command):
</span><span class="cx"> arguments = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
</ins><span class="cx"> ('create', amp.Boolean(optional=True)),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><span class="lines">@@ -117,7 +132,8 @@
</span><span class="cx"> ('uid', amp.String()),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
+ ('continuation', amp.String(optional=True)),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -127,7 +143,7 @@
</span><span class="cx"> ('uid', amp.String()),
</span><span class="cx"> ]
</span><span class="cx"> response = [
</span><del>- ('fieldsList', amp.String()),
</del><ins>+ ('fieldsList', amp.ListOf(amp.String())),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpsserverpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/server.py (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/server.py 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/txdav/dps/server.py 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -15,6 +15,7 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx"> import cPickle as pickle
</span><ins>+import datetime
</ins><span class="cx"> import uuid
</span><span class="cx">
</span><span class="cx"> from twext.python.log import Logger
</span><span class="lines">@@ -35,7 +36,7 @@
</span><span class="cx"> RecordsMatchingTokensCommand, RecordsMatchingFieldsCommand,
</span><span class="cx"> MembersCommand, GroupsCommand, SetMembersCommand,
</span><span class="cx"> VerifyPlaintextPasswordCommand, VerifyHTTPDigestCommand,
</span><del>- WikiAccessForUID
</del><ins>+ WikiAccessForUID, ContinuationCommand
</ins><span class="cx"> # UpdateRecordsCommand, RemoveRecordsCommand
</span><span class="cx"> )
</span><span class="cx"> from txdav.who.util import directoryFromConfig
</span><span class="lines">@@ -62,7 +63,104 @@
</span><span class="cx"> amp.AMP.__init__(self)
</span><span class="cx"> self._directory = directory
</span><span class="cx">
</span><ins>+ # How to large we let an AMP response get before breaking it up
+ self._maxSize = 60000
</ins><span class="cx">
</span><ins>+ # The cache of results we have not fully responded with. A dictionary
+ # whose keys are "continuation tokens" and whose values are tuples of
+ # (timestamp, list-of-records). When a response does not fit within
+ # AMP size limits, the remaining records are stored in this dictionary
+ # keyed by an opaque token we generate to return to the client so that
+ # it can ask for the remaining results later.
+ self._continuations = {}
+
+
+ def _storeContinuation(self, records):
+ """
+ Store an iterable of records and generate an opaque token we can
+ give back to the client so they can later retrieve these remaining
+ results that did not fit in the previous AMP response.
+
+ @param records: an iterable of records
+ @return: a C{str} token
+ """
+ token = str(uuid.uuid4())
+ # FIXME: I included a timestamp just in case we want to have code that
+ # looks for stale continuations to expire them.
+ self._continuations[token] = (datetime.datetime.now(), records)
+ return token
+
+
+ def _retrieveContinuation(self, token):
+ """
+ Retrieve the previously stored iterable of records associated with
+ the token, and remove the token.
+
+ @param token: a C{str} token previously returned by _storeContinuation
+ @return: an iterable of records, or None if the token does not exist
+ """
+ if token in self._continuations:
+ timestamp, records = self._continuations[token]
+ del self._continuations[token]
+ else:
+ records = None
+ return records
+
+
+ @ContinuationCommand.responder
+ def continuation(self, continuation):
+ """
+ The client calls this command in order to retrieve records that did
+ not fit into an earlier response.
+
+ @param continuation: the token returned via the "continuation" key
+ in the previous response.
+ """
+ log.debug("Continuation: {c}", c=continuation)
+ records = self._retrieveContinuation(continuation)
+ response = self._recordsToResponse(records)
+ log.debug("Responding with: {response}", response=response)
+ return response
+
+
+ def _recordsToResponse(self, records):
+ """
+ Craft an AMP response containing as many records as will fit within
+ the size limit. Remaining records are stored as a "continuation",
+ identified by a token that is returned to the client to fetch later
+ via the ContinuationCommand.
+
+ @param records: an iterable of records
+ @return: the response dictionary, with a list of pickled records
+ stored in the "fieldsList" key, and if there are leftover
+ records that did not fit, there will be a "continuation" key
+ containing the token the client must send via ContinuationCommand.
+ """
+ fieldsList = []
+ count = 0
+ if records:
+ size = 0
+ while size < self._maxSize:
+ try:
+ record = records.pop()
+ except (KeyError, IndexError):
+ # We're done.
+ # Note: because records is an iterable (list or set)
+ # we're catching both KeyError and IndexError.
+ break
+ pickled = pickle.dumps(self.recordToDict(record))
+ size = size + len(pickled)
+ fieldsList.append(pickled)
+ count += 1
+
+ response = {"fieldsList": fieldsList}
+
+ if records:
+ response["continuation"] = self._storeContinuation(records)
+
+ return response
+
+
</ins><span class="cx"> def recordToDict(self, record):
</span><span class="cx"> """
</span><span class="cx"> Turn a record in a dictionary of fields which can be reconstituted
</span><span class="lines">@@ -140,12 +238,7 @@
</span><span class="cx"> records = (yield self._directory.recordsWithRecordType(
</span><span class="cx"> self._directory.recordType.lookupByName(recordType))
</span><span class="cx"> )
</span><del>- fieldsList = []
- for record in records:
- fieldsList.append(self.recordToDict(record))
- response = {
- "fieldsList": pickle.dumps(fieldsList),
- }
</del><ins>+ response = self._recordsToResponse(records)
</ins><span class="cx"> log.debug("Responding with: {response}", response=response)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span><span class="lines">@@ -156,16 +249,13 @@
</span><span class="cx"> emailAddress = emailAddress.decode("utf-8")
</span><span class="cx"> log.debug("RecordsWithEmailAddress: {e}", e=emailAddress)
</span><span class="cx"> records = (yield self._directory.recordsWithEmailAddress(emailAddress))
</span><del>- fieldsList = []
- for record in records:
- fieldsList.append(self.recordToDict(record))
- response = {
- "fieldsList": pickle.dumps(fieldsList),
- }
</del><ins>+ response = self._recordsToResponse(records)
</ins><span class="cx"> log.debug("Responding with: {response}", response=response)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> @RecordsMatchingTokensCommand.responder
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def recordsMatchingTokens(self, tokens, context=None):
</span><span class="lines">@@ -174,16 +264,12 @@
</span><span class="cx"> records = yield self._directory.recordsMatchingTokens(
</span><span class="cx"> tokens, context=context
</span><span class="cx"> )
</span><del>- fieldsList = []
- for record in records:
- fieldsList.append(self.recordToDict(record))
- response = {
- "fieldsList": pickle.dumps(fieldsList),
- }
</del><ins>+ response = self._recordsToResponse(records)
</ins><span class="cx"> log.debug("Responding with: {response}", response=response)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @RecordsMatchingFieldsCommand.responder
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def recordsMatchingFields(self, fields, operand="OR", recordType=None):
</span><span class="lines">@@ -209,12 +295,7 @@
</span><span class="cx"> records = yield self._directory.recordsMatchingFields(
</span><span class="cx"> newFields, operand=operand, recordType=recordType
</span><span class="cx"> )
</span><del>- fieldsList = []
- for record in records:
- fieldsList.append(self.recordToDict(record))
- response = {
- "fieldsList": pickle.dumps(fieldsList),
- }
</del><ins>+ response = self._recordsToResponse(records)
</ins><span class="cx"> log.debug("Responding with: {response}", response=response)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span><span class="lines">@@ -230,13 +311,12 @@
</span><span class="cx"> log.error("Failed in members", error=e)
</span><span class="cx"> record = None
</span><span class="cx">
</span><del>- fieldsList = []
</del><ins>+ records = []
</ins><span class="cx"> if record is not None:
</span><span class="cx"> for member in (yield record.members()):
</span><del>- fieldsList.append(self.recordToDict(member))
- response = {
- "fieldsList": pickle.dumps(fieldsList),
- }
</del><ins>+ records.append(member)
+
+ response = self._recordsToResponse(records)
</ins><span class="cx"> log.debug("Responding with: {response}", response=response)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpstesttest_clientpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/test/test_client.py (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/test/test_client.py 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/txdav/dps/test/test_client.py 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -30,6 +30,9 @@
</span><span class="cx"> from txdav.dps.client import DirectoryService
</span><span class="cx"> from txdav.dps.server import DirectoryProxyAMPProtocol
</span><span class="cx"> from txdav.who.directory import CalendarDirectoryServiceMixin
</span><ins>+from txdav.who.test.support import (
+ TestRecord, CalendarInMemoryDirectoryService
+)
</ins><span class="cx"> from twistedcaldav.test.util import StoreTestCase
</span><span class="cx"> from twistedcaldav.config import config
</span><span class="cx">
</span><span class="lines">@@ -590,3 +593,109 @@
</span><span class="cx"> )
</span><span class="cx"> )
</span><span class="cx"> self.assertEquals(authenticated, answer)
</span><ins>+
+
+class DPSClientLargeResultsTest(unittest.TestCase):
+ """
+ Tests the client against a single directory service (as opposed to the
+ augmented, aggregated structure you get from directoryFromConfig(), which
+ is tested in the class below)
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+
+ self.numUsers = 1000
+
+ # The "local" directory service
+ self.directory = DirectoryService(None)
+
+ # The "remote" directory service
+ remoteDirectory = CalendarInMemoryDirectoryService(None)
+
+ # Add users
+ records = []
+ fieldName = remoteDirectory.fieldName
+ for i in xrange(self.numUsers):
+ records.append(
+ TestRecord(
+ remoteDirectory,
+ {
+ fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
+ fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
+ fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
+ fieldName.recordType: RecordType.user,
+ }
+ )
+ )
+
+ # Add a big group
+ records.append(
+ TestRecord(
+ remoteDirectory,
+ {
+ fieldName.uid: u"bigGroup",
+ fieldName.recordType: RecordType.group,
+ }
+ )
+ )
+
+ yield remoteDirectory.updateRecords(records, create=True)
+
+ group = yield remoteDirectory.recordWithUID(u"bigGroup")
+ members = yield remoteDirectory.recordsWithRecordType(RecordType.user)
+ yield group.setMembers(members)
+
+ # Connect the two services directly via an IOPump
+ client = AMP()
+ server = DirectoryProxyAMPProtocol(remoteDirectory)
+ pump = returnConnected(server, client)
+
+ # Replace the normal _getConnection method with one that bypasses any
+ # actual networking
+ self.patch(self.directory, "_getConnection", lambda: succeed(client))
+
+ # Wrap the normal _call method with one that flushes the IOPump
+ # afterwards
+ origCall = self.directory._call
+
+ def newCall(*args, **kwds):
+ d = origCall(*args, **kwds)
+ pump.flush()
+ return d
+
+ self.patch(self.directory, "_call", newCall)
+
+
+ @inlineCallbacks
+ def test_tooBigResults(self):
+ """
+ The AMP protocol limits values to 65,535 bytes, so the DPS server
+ breaks up the responses to fit. This test uses 1000 records to verify
+ the various methods work seamlessly in the face of large results.
+ Normally only a couple hundred records would fit in a single response.
+ """
+
+ # recordsMatchingTokens
+ records = yield self.directory.recordsMatchingTokens([u"foo"])
+ self.assertEquals(len(records), self.numUsers)
+
+ # recordsMatchingFields
+ fields = (
+ (u"fullNames", "foo", MatchFlags.caseInsensitive, MatchType.contains),
+ )
+ records = yield self.directory.recordsMatchingFields(
+ fields, operand=Operand.OR, recordType=RecordType.user
+ )
+ self.assertEquals(len(records), self.numUsers)
+
+ # recordsWithRecordType
+ records = yield self.directory.recordsWithRecordType(
+ RecordType.user
+ )
+ self.assertEquals(len(records), self.numUsers)
+
+ # members()
+ group = yield self.directory.recordWithUID(u"bigGroup")
+ members = yield group.members()
+ self.assertEquals(len(members), self.numUsers)
</ins></span></pre></div>
<a id="CalendarServertrunktxdavwhotestsupportpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/who/test/support.py (13604 => 13605)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/who/test/support.py 2014-06-04 04:27:23 UTC (rev 13604)
+++ CalendarServer/trunk/txdav/who/test/support.py 2014-06-04 04:28:02 UTC (rev 13605)
</span><span class="lines">@@ -18,15 +18,24 @@
</span><span class="cx"> RecordType,
</span><span class="cx"> NoSuchRecordError
</span><span class="cx"> )
</span><del>-from twext.who.index import DirectoryService as IndexDirectoryService
</del><ins>+from twext.who.index import (
+ DirectoryService as IndexDirectoryService,
+ DirectoryRecord as IndexedDirectoryRecord
+)
</ins><span class="cx"> from twext.who.util import ConstantsContainer
</span><ins>+from twisted.internet.defer import succeed, inlineCallbacks
+from txdav.who.directory import (
+ CalendarDirectoryRecordMixin, CalendarDirectoryServiceMixin
+)
</ins><span class="cx"> from txdav.who.idirectory import (
</span><span class="cx"> RecordType as CalRecordType
</span><span class="cx"> )
</span><del>-from twisted.internet.defer import succeed, inlineCallbacks
</del><span class="cx">
</span><span class="cx">
</span><ins>+class TestRecord(IndexedDirectoryRecord, CalendarDirectoryRecordMixin):
+ pass
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> class InMemoryDirectoryService(IndexDirectoryService):
</span><span class="cx"> """
</span><span class="cx"> An in-memory IDirectoryService. You must call updateRecords( ) if you want
</span><span class="lines">@@ -69,3 +78,11 @@
</span><span class="cx"> if record.uid in uids:
</span><span class="cx"> recordSet.remove(record)
</span><span class="cx"> return succeed(None)
</span><ins>+
+
+
+class CalendarInMemoryDirectoryService(
+ InMemoryDirectoryService,
+ CalendarDirectoryServiceMixin
+):
+ pass
</ins></span></pre>
</div>
</div>
</body>
</html>