[CalendarServer-changes] [5636] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Thu May 20 18:06:38 PDT 2010
Revision: 5636
http://trac.macosforge.org/projects/calendarserver/changeset/5636
Author: sagen at apple.com
Date: 2010-05-20 18:06:36 -0700 (Thu, 20 May 2010)
Log Message:
-----------
Splits memcacheprops gets_multi and set_multi into chunks of 250 keys to reduce memcacheclient timeouts.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/memcacheprops.py
CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py
Modified: CalendarServer/trunk/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacheprops.py 2010-05-20 20:03:34 UTC (rev 5635)
+++ CalendarServer/trunk/twistedcaldav/memcacheprops.py 2010-05-21 01:06:36 UTC (rev 5636)
@@ -133,7 +133,8 @@
for childName in childNames
))
- result = client.gets_multi((key for key, name in keys))
+ result = self._split_gets_multi((key for key, name in keys),
+ client.gets_multi)
if self.logger.willLogAtLevel("debug"):
if abortIfMissing:
@@ -162,6 +163,49 @@
return result
+
+ def _split_gets_multi(self, keys, func, chunksize=250):
+ """
+ Splits gets_multi into chunks to avoid a memcacheclient timeout due
+ of a large number of keys. Consolidates and returns results.
+ Takes a function parameter for easier unit testing.
+ """
+
+ results = {}
+ count = 0
+ subset = []
+ for key in keys:
+ if count == 0:
+ subset = []
+ subset.append(key)
+ count += 1
+ if count == chunksize:
+ results.update(func(subset))
+ count = 0
+ if count:
+ results.update(func(subset))
+ return results
+
+ def _split_set_multi(self, values, func, time=0, chunksize=250):
+ """
+ Splits set_multi into chunks to avoid a memcacheclient timeout due
+ of a large number of keys.
+ Takes a function parameter for easier unit testing.
+ """
+ count = 0
+ subset = {}
+ for key, value in values.iteritems():
+ if count == 0:
+ subset.clear()
+ subset[key] = value
+ count += 1
+ if count == chunksize:
+ func(subset, time=time)
+ count = 0
+ if count:
+ func(subset, time=time)
+
+
def _storeCache(self, cache):
self.log_debug("Storing cache for %s" % (self.collection,))
@@ -173,7 +217,8 @@
client = self.memcacheClient()
if client is not None:
- client.set_multi(values, time=self.cacheTimeout)
+ self._split_set_multi(values, client.set_multi,
+ time=self.cacheTimeout)
def _buildCache(self, childNames=None):
if childNames is None:
Modified: CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py 2010-05-20 20:03:34 UTC (rev 5635)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py 2010-05-21 01:06:36 UTC (rev 5636)
@@ -280,3 +280,80 @@
"val0%s" % (uid if uid else "",))
self.assertEquals(child2.deadProperties().get(("ns1:", "prop3"), uid=uid).value,
"val0%s" % (uid if uid else "",))
+
+
+ def _stub_set_multi(self, values, time=None):
+
+ self.callCount += 1
+ for key, value in values.iteritems():
+ self.results[key] = value
+
+ def test_splitSetMulti(self):
+
+ self.callCount = 0
+ self.results = {}
+
+ mpc = MemcachePropertyCollection(None)
+ values = {}
+ for i in xrange(600):
+ values["key%d" % (i,)] = "value%d" % (i,)
+
+ mpc._split_set_multi(values, self._stub_set_multi)
+
+ self.assertEquals(self.callCount, 3)
+ self.assertEquals(self.results, values)
+
+
+ def test_splitSetMultiWithChunksize(self):
+
+ self.callCount = 0
+ self.results = {}
+
+ mpc = MemcachePropertyCollection(None)
+ values = {}
+ for i in xrange(13):
+ values["key%d" % (i,)] = "value%d" % (i,)
+
+ mpc._split_set_multi(values, self._stub_set_multi, chunksize=3)
+
+ self.assertEquals(self.callCount, 5)
+ self.assertEquals(self.results, values)
+
+
+ def _stub_gets_multi(self, keys):
+
+ self.callCount += 1
+ result = {}
+ for key in keys:
+ result[key] = self.expected[key]
+ return result
+
+ def test_splitGetsMulti(self):
+
+ self.callCount = 0
+ self.expected = {}
+ keys = []
+ for i in xrange(600):
+ keys.append("key%d" % (i,))
+ self.expected["key%d" % (i,)] = "value%d" % (i,)
+
+ mpc = MemcachePropertyCollection(None)
+ result = mpc._split_gets_multi(keys, self._stub_gets_multi)
+
+ self.assertEquals(self.callCount, 3)
+ self.assertEquals(self.expected, result)
+
+ def test_splitGetsMultiWithChunksize(self):
+
+ self.callCount = 0
+ self.expected = {}
+ keys = []
+ for i in xrange(600):
+ keys.append("key%d" % (i,))
+ self.expected["key%d" % (i,)] = "value%d" % (i,)
+
+ mpc = MemcachePropertyCollection(None)
+ result = mpc._split_gets_multi(keys, self._stub_gets_multi, chunksize=12)
+
+ self.assertEquals(self.callCount, 50)
+ self.assertEquals(self.expected, result)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100520/5e7b9435/attachment.html>
More information about the calendarserver-changes
mailing list