[CalendarServer-changes] [4197] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed May 6 20:13:19 PDT 2009
Revision: 4197
http://trac.macosforge.org/projects/calendarserver/changeset/4197
Author: cdaboo at apple.com
Date: 2009-05-06 20:13:18 -0700 (Wed, 06 May 2009)
Log Message:
-----------
Protect against over long memcache keys and those using invalid characters.
Modified Paths:
--------------
CalendarServer/trunk/run
CalendarServer/trunk/twistedcaldav/memcachelock.py
CalendarServer/trunk/twistedcaldav/memcacher.py
CalendarServer/trunk/twistedcaldav/resource.py
CalendarServer/trunk/twistedcaldav/test/test_memcachelock.py
CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
Modified: CalendarServer/trunk/run
===================================================================
--- CalendarServer/trunk/run 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/run 2009-05-07 03:13:18 UTC (rev 4197)
@@ -727,7 +727,7 @@
caldavtester="${top}/CalDAVTester";
-svn_get "CalDAVTester" "${caldavtester}" "${svn_uri_base}/CalDAVTester/trunk" 4193;
+svn_get "CalDAVTester" "${caldavtester}" "${svn_uri_base}/CalDAVTester/trunk" 4196;
#
# PyFlakes
Modified: CalendarServer/trunk/twistedcaldav/memcachelock.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcachelock.py 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/twistedcaldav/memcachelock.py 2009-05-07 03:13:18 UTC (rev 4197)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2008 Apple Inc. All rights reserved.
+# Copyright (c) 2008-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@
from twisted.internet.defer import inlineCallbacks, Deferred, returnValue,\
succeed
from twisted.internet import reactor
-import hashlib
import time
class MemcacheLock(Memcacher):
@@ -39,10 +38,7 @@
"""
super(MemcacheLock, self).__init__(namespace)
- if isinstance(locktoken, unicode):
- locktoken = locktoken.encode("utf-8")
- assert isinstance(locktoken, str), "Lock token must be a str."
- self._locktoken = hashlib.md5(locktoken).hexdigest()
+ self._locktoken = locktoken
self._timeout = timeout
self._retry_interval = retry_interval
self._expire_time = expire_time
Modified: CalendarServer/trunk/twistedcaldav/memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/memcacher.py 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/twistedcaldav/memcacher.py 2009-05-07 03:13:18 UTC (rev 4197)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2008 Apple Inc. All rights reserved.
+# Copyright (c) 2008-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,10 +19,20 @@
from twistedcaldav.log import LoggingMixIn
from twistedcaldav.memcachepool import CachePoolUserMixIn
from twistedcaldav.config import config
+import hashlib
import cPickle
+import string
class Memcacher(LoggingMixIn, CachePoolUserMixIn):
+ MEMCACHE_KEY_LIMIT = 250 # the memcached key length limit
+ NAMESPACE_MAX_LENGTH = 32 # max size of namespace we will allow
+ HASH_LENGTH = 32 # length of hash we will generate
+ TRUNCATED_KEY_LENGTH = MEMCACHE_KEY_LIMIT - NAMESPACE_MAX_LENGTH - HASH_LENGTH - 2 # 2 accounts for delimiters
+
+ # Translation table: all ctrls (0x00 - 0x1F) and space and 0x7F mapped to _
+ keyNormalizeTranslateTable = string.maketrans("".join([chr(i) for i in range(33)]) + chr(0x7F), "_"*33 + "_")
+
class memoryCacher():
"""
A class implementing the memcache client API we care about but
@@ -92,6 +102,8 @@
nullCacher will be used for the multi-instance case when memcached is not configured.
@type no_invalidation: C{bool}
"""
+
+ assert len(namespace) <= Memcacher.NAMESPACE_MAX_LENGTH, "Memcacher namespace must be less than or equal to %s characters long" % (Memcacher.NAMESPACE_MAX_LENGTH,)
self._memcacheProtocol = None
self._namespace = namespace
self._pickle = pickle
@@ -118,6 +130,16 @@
return self._memcacheProtocol
+ def _normalizeKey(self, key):
+
+ if isinstance(key, unicode):
+ key = key.encode("utf-8")
+ assert isinstance(key, str), "Key must be a str."
+
+ hash = hashlib.md5(key).hexdigest()
+ key = key[:Memcacher.TRUNCATED_KEY_LENGTH]
+ return "%s-%s" % (key.translate(Memcacher.keyNormalizeTranslateTable), hash,)
+
def add(self, key, value, expire_time=0):
proto = self._getMemcacheProtocol()
@@ -126,7 +148,7 @@
if self._pickle:
my_value = cPickle.dumps(value)
self.log_debug("Adding Cache Token for %r" % (key,))
- return proto.add('%s:%s' % (self._namespace, key), my_value, expireTime=expire_time)
+ return proto.add('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expire_time)
def set(self, key, value, expire_time=0):
@@ -136,7 +158,7 @@
if self._pickle:
my_value = cPickle.dumps(value)
self.log_debug("Setting Cache Token for %r" % (key,))
- return proto.set('%s:%s' % (self._namespace, key), my_value, expireTime=expire_time)
+ return proto.set('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expire_time)
def get(self, key):
def _gotit(result):
@@ -146,11 +168,11 @@
return value
self.log_debug("Getting Cache Token for %r" % (key,))
- d = self._getMemcacheProtocol().get('%s:%s' % (self._namespace, key))
+ d = self._getMemcacheProtocol().get('%s:%s' % (self._namespace, self._normalizeKey(key)))
d.addCallback(_gotit)
return d
def delete(self, key):
self.log_debug("Deleting Cache Token for %r" % (key,))
- return self._getMemcacheProtocol().delete('%s:%s' % (self._namespace, key))
+ return self._getMemcacheProtocol().delete('%s:%s' % (self._namespace, self._normalizeKey(key)))
Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/twistedcaldav/resource.py 2009-05-07 03:13:18 UTC (rev 4197)
@@ -27,7 +27,6 @@
"isPseudoCalendarCollectionResource",
]
-import os
import urllib
from zope.interface import implements
@@ -61,8 +60,6 @@
from twistedcaldav.ical import allowedComponents
from twistedcaldav.icaldav import ICalDAVResource, ICalendarPrincipalResource
from twistedcaldav.log import LoggingMixIn
-from twistedcaldav.memcacher import Memcacher
-from twistedcaldav.sql import AbstractSQLDatabase, db_prefix
from urlparse import urlsplit
Modified: CalendarServer/trunk/twistedcaldav/test/test_memcachelock.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcachelock.py 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcachelock.py 2009-05-07 03:13:18 UTC (rev 4197)
@@ -91,8 +91,8 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking")
return self._test(
lock.get("foo"),
- "get lock:foo\r\n",
- "VALUE lock:foo 0 3\r\nbar\r\nEND\r\n",
+ "get lock:foo-acbd18db4cc2f85cedef654fccc4a4d8\r\n",
+ "VALUE lock:foo-acbd18db4cc2f85cedef654fccc4a4d8 0 3\r\nbar\r\nEND\r\n",
"bar"
)
@@ -105,7 +105,7 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking")
return self._test(
lock.set("foo", "bar"),
- "set lock:foo 0 0 3\r\nbar\r\n",
+ "set lock:foo-acbd18db4cc2f85cedef654fccc4a4d8 0 0 3\r\nbar\r\n",
"STORED\r\n",
True
)
@@ -120,7 +120,7 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking")
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"STORED\r\n",
True
)
@@ -136,7 +136,7 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking", timeout=0)
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"STORED\r\n",
True
)
@@ -153,7 +153,7 @@
try:
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"NOT_STORED\r\n",
True
)
@@ -175,14 +175,14 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking")
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"STORED\r\n",
True
)
self.assertTrue(lock._hasLock)
yield self._test(
lock.release(),
- "delete lock:559159aa00cc525bfe5c4b34cf16cccb\r\n",
+ "delete lock:locking-559159aa00cc525bfe5c4b34cf16cccb\r\n",
"DELETED\r\n",
True
)
@@ -198,13 +198,13 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", "locking")
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"STORED\r\n",
True
)
yield self._test(
lock.clean(),
- "delete lock:559159aa00cc525bfe5c4b34cf16cccb\r\n",
+ "delete lock:locking-559159aa00cc525bfe5c4b34cf16cccb\r\n",
"DELETED\r\n",
True
)
@@ -219,19 +219,34 @@
lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", u"locking")
yield self._test(
lock.acquire(),
- "add lock:559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
+ "add lock:locking-559159aa00cc525bfe5c4b34cf16cccb 0 0 1\r\n1\r\n",
"STORED\r\n",
True
)
self.assertTrue(lock._hasLock)
- def test_acquire_invalid_token(self):
+ @inlineCallbacks
+ def test_acquire_invalid_token1(self):
"""
L{MemCacheProtocol.get} should return a L{Deferred} which is
called back with the value and the flag associated with the given key
if the server returns a successful result.
"""
- self.assertRaises(AssertionError, MemCacheTestCase.FakedMemcacheLock, *(self.proto, "lock", 1))
- self.assertRaises(AssertionError, MemCacheTestCase.FakedMemcacheLock, *(self.proto, "lock", ("abc",)))
+ try:
+ lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", 1)
+ yield lock.acquire()
+ self.fail("AssertionError not raised")
+ except AssertionError:
+ pass
+ except:
+ self.fail("AssertionError not raised")
+ try:
+ lock = MemCacheTestCase.FakedMemcacheLock(self.proto, "lock", ("abc",))
+ yield lock.acquire()
+ self.fail("AssertionError not raised")
+ except AssertionError:
+ pass
+ except:
+ self.fail("AssertionError not raised")
Modified: CalendarServer/trunk/twistedcaldav/test/test_memcacher.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2009-05-07 02:40:53 UTC (rev 4196)
+++ CalendarServer/trunk/twistedcaldav/test/test_memcacher.py 2009-05-07 03:13:18 UTC (rev 4197)
@@ -110,3 +110,18 @@
result = yield cacher.get("akey")
self.assertEquals(None, result)
+ def test_keynormalization(self):
+
+ for processType in ("Single", "Combined",):
+ config.ProcessType = processType
+
+ cacher = Memcacher("testing")
+
+ self.assertTrue(len(cacher._normalizeKey("A" * 100)) <= 250)
+ self.assertTrue(len(cacher._normalizeKey("A" * 512)) <= 250)
+
+ key = cacher._normalizeKey(" \n\t\r" * 20)
+ self.assertTrue(" " not in key)
+ self.assertTrue("\n" not in key)
+ self.assertTrue("\t" not in key)
+ self.assertTrue("\r" not in key)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090506/4cf88560/attachment-0001.html>
More information about the calendarserver-changes
mailing list