[CalendarServer-changes] [11500] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jul 10 12:56:02 PDT 2013
Revision: 11500
http://trac.calendarserver.org//changeset/11500
Author: cdaboo at apple.com
Date: 2013-07-10 12:56:02 -0700 (Wed, 10 Jul 2013)
Log Message:
-----------
Add test to verify that failure to store memcache item due to an over size value does not prevent properties
from working correctly.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/memcacher.py
CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
CalendarServer/trunk/txdav/base/propertystore/test/test_sql.py
Modified: CalendarServer/trunk/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacher.py 2013-07-10 18:50:26 UTC (rev 11499)
+++ CalendarServer/trunk/twistedcaldav/memcacher.py 2013-07-10 19:56:02 UTC (rev 11500)
@@ -28,13 +28,14 @@
class Memcacher(CachePoolUserMixIn):
log = Logger()
- MEMCACHE_KEY_LIMIT = 250 # the memcached key length limit
- NAMESPACE_MAX_LENGTH = 32 # max size of namespace we will allow
- HASH_LENGTH = 32 # length of hash we will generate
+ MEMCACHE_KEY_LIMIT = 250 # the memcached key length limit
+ NAMESPACE_MAX_LENGTH = 32 # max size of namespace we will allow
+ HASH_LENGTH = 32 # length of hash we will generate
TRUNCATED_KEY_LENGTH = MEMCACHE_KEY_LIMIT - NAMESPACE_MAX_LENGTH - HASH_LENGTH - 2 # 2 accounts for delimiters
+ MEMCACHE_VALUE_LIMIT = 1024 * 1024 # the memcached default value length limit
# Translation table: all ctrls (0x00 - 0x1F) and space and 0x7F mapped to _
- keyNormalizeTranslateTable = string.maketrans("".join([chr(i) for i in range(33)]) + chr(0x7F), "_"*33 + "_")
+ keyNormalizeTranslateTable = string.maketrans("".join([chr(i) for i in range(33)]) + chr(0x7F), "_" * 33 + "_")
allowTestCache = False
memoryCacheInstance = None
@@ -52,6 +53,8 @@
self._clock = 0
def add(self, key, value, expireTime=0):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
+ return succeed(False)
if key not in self._cache:
if not expireTime:
expireTime = 99999
@@ -61,9 +64,11 @@
return succeed(False)
def set(self, key, value, expireTime=0):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
+ return succeed(False)
if not expireTime:
expireTime = 99999
- if self._cache.has_key(key):
+ if key in self._cache:
identifier = self._cache[key][2]
identifier += 1
else:
@@ -72,9 +77,11 @@
return succeed(True)
def checkAndSet(self, key, value, cas, flags=0, expireTime=0):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
+ return succeed(False)
if not expireTime:
expireTime = 99999
- if self._cache.has_key(key):
+ if key in self._cache:
identifier = self._cache[key][2]
if cas != str(identifier):
return succeed(False)
@@ -85,16 +92,22 @@
return succeed(True)
def get(self, key, withIdentifier=False):
- value, expires, identifier = self._cache.get(key, (None, 0, ""))
- if self._clock >= expires:
- value = None
- identifier = ""
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
+ value, expires, identifier = (None, 0, "")
+ else:
+ value, expires, identifier = self._cache.get(key, (None, 0, ""))
+ if self._clock >= expires:
+ value = None
+ identifier = ""
+
if withIdentifier:
return succeed((0, value, str(identifier)))
else:
return succeed((0, value,))
def delete(self, key):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
+ return succeed(False)
try:
del self._cache[key]
return succeed(True)
@@ -102,6 +115,8 @@
return succeed(False)
def incr(self, key, delta=1):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
+ return succeed(False)
value = self._cache.get(key, None)
if value is not None:
value, expire, identifier = value
@@ -115,6 +130,8 @@
return succeed(value)
def decr(self, key, delta=1):
+ if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
+ return succeed(False)
value = self._cache.get(key, None)
if value is not None:
value, expire, identifier = value
@@ -135,7 +152,8 @@
def advanceClock(self, seconds):
self._clock += seconds
-
+
+
#TODO: an sqlite based cacher that can be used for multiple instance servers
# in the absence of memcached. This is not ideal and we may want to not implement
# this, but it is being documented for completeness.
@@ -171,6 +189,7 @@
def flushAll(self):
return succeed(True)
+
def __init__(self, namespace, pickle=False, no_invalidation=False, key_normalization=True):
"""
@param namespace: a unique namespace for this cache's keys
@@ -187,7 +206,7 @@
work is done to truncate and append a hash.
@type key_normalization: C{bool}
"""
-
+
assert len(namespace) <= Memcacher.NAMESPACE_MAX_LENGTH, "Memcacher namespace must be less than or equal to %s characters long" % (Memcacher.NAMESPACE_MAX_LENGTH,)
self._memcacheProtocol = None
self._cachePoolHandle = namespace
@@ -220,7 +239,7 @@
def _normalizeKey(self, key):
-
+
if isinstance(key, unicode):
key = key.encode("utf-8")
assert isinstance(key, str), "Key must be a str."
@@ -232,8 +251,9 @@
else:
return key
+
def add(self, key, value, expireTime=0):
-
+
proto = self._getMemcacheProtocol()
my_value = value
@@ -242,8 +262,9 @@
self.log.debug("Adding Cache Token for %r" % (key,))
return proto.add('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expireTime)
+
def set(self, key, value, expireTime=0):
-
+
proto = self._getMemcacheProtocol()
my_value = value
@@ -252,6 +273,7 @@
self.log.debug("Setting Cache Token for %r" % (key,))
return proto.set('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expireTime)
+
def checkAndSet(self, key, value, cas, flags=0, expireTime=0):
proto = self._getMemcacheProtocol()
@@ -262,6 +284,7 @@
self.log.debug("Setting Cache Token for %r" % (key,))
return proto.checkAndSet('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, cas, expireTime=expireTime)
+
def get(self, key, withIdentifier=False):
def _gotit(result, withIdentifier):
if withIdentifier:
@@ -279,18 +302,22 @@
d.addCallback(_gotit, withIdentifier)
return d
+
def delete(self, key):
self.log.debug("Deleting Cache Token for %r" % (key,))
return self._getMemcacheProtocol().delete('%s:%s' % (self._namespace, self._normalizeKey(key)))
+
def incr(self, key, delta=1):
self.log.debug("Incrementing Cache Token for %r" % (key,))
return self._getMemcacheProtocol().incr('%s:%s' % (self._namespace, self._normalizeKey(key)), delta)
+
def decr(self, key, delta=1):
self.log.debug("Decrementing Cache Token for %r" % (key,))
return self._getMemcacheProtocol().incr('%s:%s' % (self._namespace, self._normalizeKey(key)), delta)
+
def flushAll(self):
self.log.debug("Flushing All Cache Tokens")
return self._getMemcacheProtocol().flushAll()
Modified: CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2013-07-10 18:50:26 UTC (rev 11499)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2013-07-10 19:56:02 UTC (rev 11500)
@@ -182,3 +182,23 @@
self.assertFalse((yield cacher.checkAndSet("akey", "yetanother", "0")))
# Should work because identifier does match:
self.assertTrue((yield cacher.checkAndSet("akey", "yetanother", "1")))
+
+
+ @inlineCallbacks
+ def test_keyValueLimits(self):
+
+ config.ProcessType = "Single"
+ cacher = Memcacher("testing", key_normalization=False)
+
+ result = yield cacher.set("*", "*")
+ self.assertTrue(result)
+
+ # Key limits
+ result = yield cacher.set("*" * (Memcacher.MEMCACHE_KEY_LIMIT + 10), "*")
+ self.assertFalse(result)
+ value = yield cacher.get("*" * (Memcacher.MEMCACHE_KEY_LIMIT + 10), "*")
+ self.assertEquals(value, (None, "",))
+
+ # Value limits
+ result = yield cacher.set("*", "*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10))
+ self.assertFalse(result)
Modified: CalendarServer/trunk/txdav/base/propertystore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/base/propertystore/test/test_sql.py 2013-07-10 18:50:26 UTC (rev 11499)
+++ CalendarServer/trunk/txdav/base/propertystore/test/test_sql.py 2013-07-10 19:56:02 UTC (rev 11500)
@@ -21,6 +21,8 @@
from twisted.internet.defer import inlineCallbacks, returnValue
+from twistedcaldav.memcacher import Memcacher
+
from txdav.common.datastore.test.util import buildStore, StubNotifierFactory
from txdav.base.propertystore.test.base import (
@@ -209,5 +211,63 @@
yield store1_user1.__setitem__(pname, pvalue)
self.assertEqual(store1_user1[pname], pvalue)
+
+ @inlineCallbacks
+ def test_cacher_failure(self):
+ """
+ Test that properties can still be read and written even when they are too larger for the
+ cacher to handle.
+ """
+
+ # Existing store - add a normal property
+ self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
+ store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
+ self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
+
+ pname1 = propertyName("dummy1")
+ pvalue1 = propertyValue("*")
+
+ yield store1_user1.__setitem__(pname1, pvalue1)
+ self.assertEqual(store1_user1[pname1], pvalue1)
+
+ self.assertEqual(len(store1_user1._cached), 1)
+
+ yield self._txn.commit()
+
+ # Existing store - add a large property
+ self._txn = self.store.newTransaction()
+ self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
+ store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
+ self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
+
+ pname2 = propertyName("dummy2")
+ pvalue2 = propertyValue("*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10))
+
+ yield store1_user1.__setitem__(pname2, pvalue2)
+ self.assertEqual(store1_user1[pname2], pvalue2)
+
+ self.assertEqual(len(store1_user1._cached), 2)
+
+ yield self._txn.commit()
+
+ # Try again - the cacher will fail large values
+ self._txn = self.store.newTransaction()
+ self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
+ store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
+ self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
+
+ self.assertEqual(store1_user1[pname1], pvalue1)
+ self.assertEqual(store1_user1[pname2], pvalue2)
+ self.assertEqual(len(store1_user1._cached), 2)
+
+ yield store1_user1.__delitem__(pname1)
+ self.assertTrue(pname1 not in store1_user1)
+
+ yield store1_user1.__delitem__(pname2)
+ self.assertTrue(pname2 not in store1_user1)
+
+ self.assertEqual(len(store1_user1._cached), 0)
+ self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
+
if PropertyStore is None:
PropertyStoreTest.skip = importErrorMessage
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130710/39acf29f/attachment-0001.html>
More information about the calendarserver-changes
mailing list