[CalendarServer-changes] [5636] CalendarServer/trunk/twistedcaldav

source_changes at macosforge.org source_changes at macosforge.org
Thu May 20 18:06:38 PDT 2010


Revision: 5636
          http://trac.macosforge.org/projects/calendarserver/changeset/5636
Author:   sagen at apple.com
Date:     2010-05-20 18:06:36 -0700 (Thu, 20 May 2010)
Log Message:
-----------
Splits memcacheprops gets_multi and set_multi into chunks of 250 keys to reduce memcacheclient timeouts.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/memcacheprops.py
    CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py

Modified: CalendarServer/trunk/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacheprops.py	2010-05-20 20:03:34 UTC (rev 5635)
+++ CalendarServer/trunk/twistedcaldav/memcacheprops.py	2010-05-21 01:06:36 UTC (rev 5636)
@@ -133,7 +133,8 @@
             for childName in childNames
         ))
 
-        result = client.gets_multi((key for key, name in keys))
+        result = self._split_gets_multi((key for key, name in keys),
+            client.gets_multi)
 
         if self.logger.willLogAtLevel("debug"):
             if abortIfMissing:
@@ -162,6 +163,49 @@
 
         return result
 
+
+    def _split_gets_multi(self, keys, func, chunksize=250):
+        """
+        Splits gets_multi into chunks to avoid a memcacheclient timeout due
+        of a large number of keys.  Consolidates and returns results.
+        Takes a function parameter for easier unit testing.
+        """
+
+        results = {}
+        count = 0
+        subset = []
+        for key in keys:
+            if count == 0:
+                subset = []
+            subset.append(key)
+            count += 1
+            if count == chunksize:
+                results.update(func(subset))
+                count = 0
+        if count:
+            results.update(func(subset))
+        return results
+
+    def _split_set_multi(self, values, func, time=0, chunksize=250):
+        """
+        Splits set_multi into chunks to avoid a memcacheclient timeout due
+        of a large number of keys.
+        Takes a function parameter for easier unit testing.
+        """
+        count = 0
+        subset = {}
+        for key, value in values.iteritems():
+            if count == 0:
+                subset.clear()
+            subset[key] = value
+            count += 1
+            if count == chunksize:
+                func(subset, time=time)
+                count = 0
+        if count:
+            func(subset, time=time)
+
+
     def _storeCache(self, cache):
         self.log_debug("Storing cache for %s" % (self.collection,))
 
@@ -173,7 +217,8 @@
 
         client = self.memcacheClient()
         if client is not None:
-            client.set_multi(values, time=self.cacheTimeout)
+            self._split_set_multi(values, client.set_multi,
+                time=self.cacheTimeout)
 
     def _buildCache(self, childNames=None):
         if childNames is None:

Modified: CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py	2010-05-20 20:03:34 UTC (rev 5635)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcacheprops.py	2010-05-21 01:06:36 UTC (rev 5636)
@@ -280,3 +280,80 @@
                 "val0%s" % (uid if uid else "",))
             self.assertEquals(child2.deadProperties().get(("ns1:", "prop3"), uid=uid).value,
                 "val0%s" % (uid if uid else "",))
+
+
+    def _stub_set_multi(self, values, time=None):
+
+        self.callCount += 1
+        for key, value in values.iteritems():
+            self.results[key] = value
+
+    def test_splitSetMulti(self):
+
+        self.callCount = 0
+        self.results = {}
+
+        mpc = MemcachePropertyCollection(None)
+        values = {}
+        for i in xrange(600):
+            values["key%d" % (i,)] = "value%d" % (i,)
+
+        mpc._split_set_multi(values, self._stub_set_multi)
+
+        self.assertEquals(self.callCount, 3)
+        self.assertEquals(self.results, values)
+
+
+    def test_splitSetMultiWithChunksize(self):
+
+        self.callCount = 0
+        self.results = {}
+
+        mpc = MemcachePropertyCollection(None)
+        values = {}
+        for i in xrange(13):
+            values["key%d" % (i,)] = "value%d" % (i,)
+
+        mpc._split_set_multi(values, self._stub_set_multi, chunksize=3)
+
+        self.assertEquals(self.callCount, 5)
+        self.assertEquals(self.results, values)
+
+
+    def _stub_gets_multi(self, keys):
+
+        self.callCount += 1
+        result = {}
+        for key in keys:
+            result[key] = self.expected[key]
+        return result
+
+    def test_splitGetsMulti(self):
+
+        self.callCount = 0
+        self.expected = {}
+        keys = []
+        for i in xrange(600):
+            keys.append("key%d" % (i,))
+            self.expected["key%d" % (i,)] = "value%d" % (i,)
+
+        mpc = MemcachePropertyCollection(None)
+        result = mpc._split_gets_multi(keys, self._stub_gets_multi)
+
+        self.assertEquals(self.callCount, 3)
+        self.assertEquals(self.expected, result)
+
+    def test_splitGetsMultiWithChunksize(self):
+
+        self.callCount = 0
+        self.expected = {}
+        keys = []
+        for i in xrange(600):
+            keys.append("key%d" % (i,))
+            self.expected["key%d" % (i,)] = "value%d" % (i,)
+
+        mpc = MemcachePropertyCollection(None)
+        result = mpc._split_gets_multi(keys, self._stub_gets_multi, chunksize=12)
+
+        self.assertEquals(self.callCount, 50)
+        self.assertEquals(self.expected, result)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100520/5e7b9435/attachment.html>


More information about the calendarserver-changes mailing list