[CalendarServer-changes] [14111] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Oct 27 19:23:28 PDT 2014
Revision: 14111
http://trac.calendarserver.org//changeset/14111
Author: cdaboo at apple.com
Date: 2014-10-27 19:23:27 -0700 (Mon, 27 Oct 2014)
Log Message:
-----------
Make sure delegate cache values are str's not unicode's. Fix in-memory cache to reject attempts to use
unicode for the key or value to match what the real memcacheclient does.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
CalendarServer/trunk/twistedcaldav/memcacher.py
CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
CalendarServer/trunk/twistedcaldav/test/util.py
CalendarServer/trunk/txdav/caldav/datastore/scheduling/test/test_processing.py
CalendarServer/trunk/txdav/who/delegates.py
Modified: CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -99,7 +99,7 @@
config.Memcached.Pools.Default.ServerEnabled = False
ClientFactory.allowTestCache = True
memcacher.Memcacher.allowTestCache = True
- memcacher.Memcacher.memoryCacheInstance = None
+ memcacher.Memcacher.reset()
config.DirectoryAddressBook.Enabled = False
config.UsePackageTimezones = True
Modified: CalendarServer/trunk/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacher.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/twistedcaldav/memcacher.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -38,7 +38,10 @@
keyNormalizeTranslateTable = string.maketrans("".join([chr(i) for i in range(33)]) + chr(0x7F), "_" * 33 + "_")
allowTestCache = False
- memoryCacheInstance = None
+ memoryCacheInstance = {
+ True: None,
+ False: None,
+ }
class memoryCacher():
"""
@@ -48,11 +51,23 @@
memcached may not be running.
"""
- def __init__(self):
+ def __init__(self, pickle=False):
self._cache = {} # (value, expireTime, check-and-set identifier)
self._clock = 0
+ self._pickle = pickle
+ def _check_key(self, key):
+ if not isinstance(key, str):
+ raise ValueError("memcache keys must be str type")
+
+ def _check_value(self, value):
+ if not self._pickle and not isinstance(value, str):
+ raise ValueError("memcache values must be str type")
+
def add(self, key, value, expireTime=0):
+ self._check_key(key)
+ self._check_value(value)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
return succeed(False)
if key not in self._cache:
@@ -64,6 +79,9 @@
return succeed(False)
def set(self, key, value, expireTime=0):
+ self._check_key(key)
+ self._check_value(value)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
return succeed(False)
if not expireTime:
@@ -77,6 +95,9 @@
return succeed(True)
def checkAndSet(self, key, value, cas, flags=0, expireTime=0):
+ self._check_key(key)
+ self._check_value(value)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len(str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT:
return succeed(False)
if not expireTime:
@@ -92,6 +113,8 @@
return succeed(True)
def get(self, key, withIdentifier=False):
+ self._check_key(key)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
value, expires, identifier = (None, 0, "")
else:
@@ -106,6 +129,8 @@
return succeed((0, value,))
def delete(self, key):
+ self._check_key(key)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
return succeed(False)
try:
@@ -115,6 +140,8 @@
return succeed(False)
def incr(self, key, delta=1):
+ self._check_key(key)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
return succeed(False)
value = self._cache.get(key, None)
@@ -130,6 +157,8 @@
return succeed(value)
def decr(self, key, delta=1):
+ self._check_key(key)
+
if len(key) > Memcacher.MEMCACHE_KEY_LIMIT:
return succeed(False)
value = self._cache.get(key, None)
@@ -224,11 +253,11 @@
self._memcacheProtocol = self.getCachePool()
elif config.ProcessType == "Single" or self._noInvalidation or self.allowTestCache:
- # NB no need to pickle the memory cacher as it handles python types natively
- if Memcacher.memoryCacheInstance is None:
- Memcacher.memoryCacheInstance = Memcacher.memoryCacher()
- self._memcacheProtocol = Memcacher.memoryCacheInstance
- self._pickle = False
+ # The memory cacher handles python types natively, but we need to treat non-str types as an error
+ # if pickling is off, so we use two global memory cachers for each pickle state
+ if Memcacher.memoryCacheInstance[self._pickle] is None:
+ Memcacher.memoryCacheInstance[self._pickle] = Memcacher.memoryCacher(self._pickle)
+ self._memcacheProtocol = Memcacher.memoryCacheInstance[self._pickle]
else:
# NB no need to pickle the null cacher as it handles python types natively
@@ -321,3 +350,11 @@
def flushAll(self):
self.log.debug("Flushing All Cache Tokens")
return self._getMemcacheProtocol().flushAll()
+
+
+ @classmethod
+ def reset(cls):
+ """
+ Reset the memory cachers
+ """
+ cls.memoryCacheInstance = {True: None, False: None}
Modified: CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -100,7 +100,7 @@
for processType in ("Single", "Combined",):
config.ProcessType = processType
- cacher = Memcacher("testing", no_invalidation=True)
+ cacher = Memcacher("testing", pickle=True, no_invalidation=True)
result = yield cacher.set("akey", ["1", "2", "3", ])
self.assertTrue(result)
@@ -132,6 +132,15 @@
self.assertTrue("\r" not in key)
+ def test_key_value_str(self):
+
+ config.ProcessType = "Single"
+
+ cacher = Memcacher("testing", pickle=False)
+ self.failUnlessRaises(ValueError, cacher.set, "akey", ["1", "2", "3", ])
+ self.failUnlessRaises(ValueError, cacher.set, "akey", u"abc")
+
+
@inlineCallbacks
def test_expiration(self):
Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/twistedcaldav/test/util.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -136,7 +136,7 @@
config.Memcached.Pools.Default.ServerEnabled = False
ClientFactory.allowTestCache = True
memcacher.Memcacher.allowTestCache = True
- memcacher.Memcacher.memoryCacheInstance = None
+ memcacher.Memcacher.reset()
config.DirectoryAddressBook.Enabled = False
config.UsePackageTimezones = True
@@ -352,7 +352,7 @@
config.Memcached.Pools.Default.ServerEnabled = False
ClientFactory.allowTestCache = True
memcacher.Memcacher.allowTestCache = True
- memcacher.Memcacher.memoryCacheInstance = None
+ memcacher.Memcacher.reset()
config.DirectoryAddressBook.Enabled = False
config.UsePackageTimezones = True
Modified: CalendarServer/trunk/txdav/caldav/datastore/scheduling/test/test_processing.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/test/test_processing.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/test/test_processing.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -83,7 +83,7 @@
config.Memcached.Pools.Default.ClientEnabled = False
config.Memcached.Pools.Default.ServerEnabled = False
memcacher.Memcacher.allowTestCache = True
- memcacher.Memcacher.memoryCacheInstance = None
+ memcacher.Memcacher.reset()
@inlineCallbacks
Modified: CalendarServer/trunk/txdav/who/delegates.py
===================================================================
--- CalendarServer/trunk/txdav/who/delegates.py 2014-10-27 19:08:31 UTC (rev 14110)
+++ CalendarServer/trunk/txdav/who/delegates.py 2014-10-28 02:23:27 UTC (rev 14111)
@@ -242,7 +242,7 @@
return "{}{}:{}#{}".format(
keyname,
"-expanded" if expanded else "",
- uid,
+ uid.encode("utf-8"),
"write" if readWrite else "read",
)
@@ -255,19 +255,19 @@
def setMembers(self, uid, readWrite, members, expanded):
return self.set(
self._membersKey(uid, readWrite, expanded),
- ",".join(members),
+ ",".join(members).encode("utf-8"),
)
def setMemberships(self, uid, readWrite, memberships):
return self.set(
self._membershipsKey(uid, readWrite),
- ",".join(memberships),
+ ",".join(memberships).encode("utf-8"),
)
@staticmethod
def _value_decode(value):
if value:
- return set(value.split(","))
+ return set(value.decode("utf-8").split(","))
elif value is None:
return None
else:
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20141027/dac752f9/attachment-0001.html>
More information about the calendarserver-changes
mailing list