[CalendarServer-changes] [4016] CalendarServer/branches/users/sagen/mo-cache-4014
source_changes at macosforge.org
source_changes at macosforge.org
Mon Apr 13 18:54:45 PDT 2009
Revision: 4016
http://trac.macosforge.org/projects/calendarserver/changeset/4016
Author: sagen at apple.com
Date: 2009-04-13 18:54:44 -0700 (Mon, 13 Apr 2009)
Log Message:
-----------
trunk changes merged into branch
Modified Paths:
--------------
CalendarServer/branches/users/sagen/mo-cache-4014/calendarserver/provision/root.py
CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/extensions.py
CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/static.py
Added Paths:
-----------
CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/memcacheprops.py
Property Changed:
----------------
CalendarServer/branches/users/sagen/mo-cache-4014/
CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-privatecomments.txt
CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-privatecomments.xml
CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-schedulingchanges.txt
CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-schedulingchanges.xml
Property changes on: CalendarServer/branches/users/sagen/mo-cache-4014
___________________________________________________________________
Added: svn:mergeinfo
+ /CalendarServer/branches/users/wsanchez/mo-cache:3979-4015
Modified: CalendarServer/branches/users/sagen/mo-cache-4014/calendarserver/provision/root.py
===================================================================
--- CalendarServer/branches/users/sagen/mo-cache-4014/calendarserver/provision/root.py 2009-04-14 01:52:24 UTC (rev 4015)
+++ CalendarServer/branches/users/sagen/mo-cache-4014/calendarserver/provision/root.py 2009-04-14 01:54:44 UTC (rev 4016)
@@ -28,9 +28,10 @@
from twisted.web2.auth.wrapper import UnauthorizedResponse
from twisted.web.xmlrpc import Proxy
-from twistedcaldav.extensions import DAVFile, CachingXattrPropertyStore
+from twistedcaldav.extensions import DAVFile, CachingPropertyStore
from twistedcaldav.extensions import DirectoryPrincipalPropertySearchMixIn
from twistedcaldav.extensions import ReadOnlyResourceMixIn
+from twistedcaldav.memcacheprops import MemcachePropertyStore
from twistedcaldav.config import config
from twistedcaldav.log import Logger
from twistedcaldav.cache import _CachedResponseResource
@@ -81,11 +82,17 @@
return response
self.contentFilters.append((addConnectionClose, True))
-
def deadProperties(self):
+ # FIXME: Same as in static.py's CalDAVFile
if not hasattr(self, "_dead_properties"):
- self._dead_properties = CachingXattrPropertyStore(self)
+ # Get the property store from super
+ deadProperties = super(RootResource, self).deadProperties()
+ # Wrap the property store in a memory store
+ deadProperties = CachingPropertyStore(deadProperties)
+
+ self._dead_properties = deadProperties
+
return self._dead_properties
def defaultAccessControlList(self):
Property changes on: CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-privatecomments.txt
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-privatecomments-00.txt:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-privatecomments-00.txt:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-privatecomments.txt:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-privatecomments-00.txt:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-privatecomments.txt:3574-3581
+ /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-privatecomments-00.txt:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-privatecomments-00.txt:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-privatecomments.txt:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-privatecomments-00.txt:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-privatecomments.txt:3574-3581
/CalendarServer/branches/users/wsanchez/mo-cache/doc/Extensions/caldav-privatecomments.txt:3979-4015
Property changes on: CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-privatecomments.xml
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-privatecomments-00.xml:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-privatecomments-00.xml:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-privatecomments.xml:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-privatecomments-00.xml:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-privatecomments.xml:3574-3581
+ /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-privatecomments-00.xml:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-privatecomments-00.xml:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-privatecomments.xml:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-privatecomments-00.xml:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-privatecomments.xml:3574-3581
/CalendarServer/branches/users/wsanchez/mo-cache/doc/Extensions/caldav-privatecomments.xml:3979-4015
Property changes on: CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-schedulingchanges.txt
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-schedulingchanges-01.txt:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-schedulingchanges-01.txt:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-schedulingchanges.txt:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-schedulingchanges-01.txt:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-schedulingchanges.txt:3574-3581
+ /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-schedulingchanges-01.txt:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-schedulingchanges-01.txt:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-schedulingchanges.txt:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-schedulingchanges-01.txt:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-schedulingchanges.txt:3574-3581
/CalendarServer/branches/users/wsanchez/mo-cache/doc/Extensions/caldav-schedulingchanges.txt:3979-4015
Property changes on: CalendarServer/branches/users/sagen/mo-cache-4014/doc/Extensions/caldav-schedulingchanges.xml
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-schedulingchanges-01.xml:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-schedulingchanges-01.xml:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-schedulingchanges.xml:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-schedulingchanges-01.xml:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-schedulingchanges.xml:3574-3581
+ /CalendarServer/branches/users/cdaboo/attendee-comments-2886/doc/Extensions/caldav-schedulingchanges-01.xml:2887-2910
/CalendarServer/branches/users/cdaboo/byebye-serviceslocator-2937/doc/Extensions/caldav-schedulingchanges-01.xml:2938-3097
/CalendarServer/branches/users/cdaboo/implicit-if-match-3306/doc/Extensions/caldav-schedulingchanges.xml:3307-3349
/CalendarServer/branches/users/cdaboo/implicitauto-2947/doc/Extensions/caldav-schedulingchanges-01.xml:2948-2989
/CalendarServer/branches/users/cdaboo/location-partial-accept-3573/doc/Extensions/caldav-schedulingchanges.xml:3574-3581
/CalendarServer/branches/users/wsanchez/mo-cache/doc/Extensions/caldav-schedulingchanges.xml:3979-4015
Modified: CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/extensions.py
===================================================================
--- CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/extensions.py 2009-04-14 01:52:24 UTC (rev 4015)
+++ CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/extensions.py 2009-04-14 01:54:44 UTC (rev 4016)
@@ -51,7 +51,6 @@
from twisted.web2.dav.resource import DAVResource as SuperDAVResource
from twisted.web2.dav.resource import DAVPrincipalResource as SuperDAVPrincipalResource
from twisted.web2.dav.util import joinURL
-from twisted.web2.dav.xattrprops import xattrPropertyStore
from twisted.web2.dav.method import prop_common
from twisted.web2.dav.method.report import max_number_of_matches
@@ -469,7 +468,6 @@
"""
Extended L{twisted.web2.dav.resource.DAVResource} implementation.
"""
-
def renderHTTP(self, request):
log.info("%s %s %s" % (request.method, urllib.unquote(request.uri), "HTTP/%s.%s" % request.clientproto))
return super(DAVResource, self).renderHTTP(request)
@@ -1052,19 +1050,21 @@
HTTPError.__init__(self,
StatusResponse(
responsecode.NOT_FOUND,
- "No such property: %s" % (qname,)
+ "No such property: {%s}%s" % qname
)
)
-class CachingXattrPropertyStore(xattrPropertyStore, LoggingMixIn):
+class CachingPropertyStore (LoggingMixIn):
"""
- A Property Store that caches attributes from the xattrs.
+ DAV property store using a dict in memory on top of another
+ property store implementation.
"""
- def __init__(self, resource):
- super(CachingXattrPropertyStore, self).__init__(resource)
+ def __init__(self, propertyStore):
+ self.propertyStore = propertyStore
+ self.resource = propertyStore.resource
def get(self, qname):
- self.log_debug("Get: %r, %r" % (self.resource.fp.path, qname))
+ #self.log_debug("Get: %r, %r" % (self.resource.fp.path, qname))
cache = self._cache()
@@ -1073,30 +1073,27 @@
if property is None:
self.log_debug("Cache miss: %r, %r, %r" % (self, self.resource.fp.path, qname))
try:
- property = super(CachingXattrPropertyStore, self).get(qname)
+ property = self.propertyStore.get(qname)
except HTTPError:
- self.log_debug("Cache double miss: %r, %r, %r" % (self, self.resource.fp.path, qname))
del cache[qname]
raise PropertyNotFoundError(qname)
cache[qname] = property
- else:
- self.log_debug("Cache hit: %r, %r, %r" % (self, self.resource.fp.path, qname))
return property
else:
raise PropertyNotFoundError(qname)
def set(self, property):
- self.log_debug("Set: %r, %r" % (self.resource.fp.path, property))
+ #self.log_debug("Set: %r, %r" % (self.resource.fp.path, property))
cache = self._cache()
cache[property.qname()] = None
- super(CachingXattrPropertyStore, self).set(property)
+ self.propertyStore.set(property)
cache[property.qname()] = property
def contains(self, qname):
- self.log_debug("Contains: %r, %r" % (self.resource.fp.path, qname))
+ #self.log_debug("Contains: %r, %r" % (self.resource.fp.path, qname))
try:
cache = self._cache()
@@ -1107,28 +1104,29 @@
raise
if qname in cache:
- self.log_debug("Contains cache hit: %r, %r, %r" % (self, self.resource.fp.path, qname))
+ #self.log_debug("Contains cache hit: %r, %r, %r" % (self, self.resource.fp.path, qname))
return True
else:
return False
def delete(self, qname):
- self.log_debug("Delete: %r, %r" % (self.resource.fp.path, qname))
+ #self.log_debug("Delete: %r, %r" % (self.resource.fp.path, qname))
if self._data is not None and qname in self._data:
del self._data[qname]
- super(CachingXattrPropertyStore, self).delete(qname)
+ self.propertyStore.delete(qname)
def list(self):
- self.log_debug("List: %r" % (self.resource.fp.path,))
+ #self.log_debug("List: %r" % (self.resource.fp.path,))
return self._cache().iterkeys()
def _cache(self):
if not hasattr(self, "_data"):
+ #self.log_debug("Cache init: %r" % (self.resource.fp.path,))
self._data = dict(
(name, None)
- for name in super(CachingXattrPropertyStore, self).list()
+ for name in self.propertyStore.list()
)
return self._data
Copied: CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/memcacheprops.py (from rev 4015, CalendarServer/branches/users/wsanchez/mo-cache/twistedcaldav/memcacheprops.py)
===================================================================
--- CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/memcacheprops.py (rev 0)
+++ CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/memcacheprops.py 2009-04-14 01:54:44 UTC (rev 4016)
@@ -0,0 +1,460 @@
+##
+# Copyright (c) 2009 Apple Computer, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+##
+
+"""
+DAV Property store using memcache on top of another property store
+implementation.
+"""
+
+__all__ = ["MemcachePropertyStore", "MemcachePropertyCollection"]
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import new as md5
+
+from memcache import ClientFactory as MemcacheClientFactory, MemcacheError
+
+from twisted.python.filepath import FilePath
+from twisted.web2 import responsecode
+from twisted.web2.http import HTTPError, StatusResponse
+
+from twistedcaldav.config import config
+from twistedcaldav.log import LoggingMixIn, Logger
+
+log = Logger()
+
+NoValue = ""
+
+class MemcachePropertyStore (LoggingMixIn):
+ """
+ DAV property store using memcache on top of another property store
+ implementation.
+ """
+ def __init__(self, propertyStore, cacheTimeout=0):
+ self.propertyStore = propertyStore
+ self.resource = propertyStore.resource
+ self.cacheTimeout = cacheTimeout
+
+ def _getMemcacheClient(self, refresh=False):
+ raise NotImplementedError()
+ if not config.Memcached.ClientEnabled:
+ # Not allowed to switch form caching to not caching
+ assert not hasattr(self, self._memcacheClient)
+ return None
+
+ if refresh or not hasattr(self, "_memcacheClient"):
+ self._memcacheClient = MemcacheClient(
+ ["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
+ debug=0,
+ pickleProtocol=2,
+ )
+ assert self._memcacheClient is not None
+ return self._memcacheClient
+
+ def keyForQname(self, qname):
+ # FIXME: This only works for file resources
+ key = "|".join((
+ self.__class__.__name__,
+ self.propertyStore.resource.fp.path,
+ "{%s}%s" % qname,
+ ))
+ return md5(key).hexdigest()
+
+ def get(self, qname):
+ #self.log_debug("MemcachePropertyStore read for %s on %s"
+ # % (qname, self.propertyStore.resource.fp.path))
+
+ #
+ # First, check to see if we've cached the property already.
+ #
+ key = self.keyForQname(qname)
+ client = self._getMemcacheClient()
+
+ if client is not None:
+ try:
+ property = client.get(key)
+ if property is not None:
+ if property is "":
+ # Negative cache hit
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+ else:
+ # Cache hit
+ return property
+ except MemcacheError, e:
+ self.log_error("Error from memcache: %s" % (e,))
+ pass
+
+ #
+ # No luck, check in the property store
+ #
+ self.log_debug("Cache miss for %s on %s"
+ % (qname, self.propertyStore.resource.fp.path))
+
+ if client is None:
+ return self.propertyStore.get(qname)
+
+ try:
+ property = self.propertyStore.get(qname)
+ assert property is not None
+
+ # Cache the result
+ client.set(key, property, time=self.cacheTimeout)
+ except HTTPError, e:
+ if e.response.code == responsecode.NOT_FOUND:
+ # Cache the non-result
+ client.set(key, NoValue, time=self.cacheTimeout)
+ raise
+
+ return property
+
+ def set(self, property):
+ #self.log_debug("Write for %s on %s"
+ # % (property.qname(), self.propertyStore.resource.fp.path))
+
+ client = self._getMemcacheClient()
+ if client is not None:
+ key = self.keyForQname(property.qname())
+
+ if client.set(key, property, time=self.cacheTimeout):
+ return
+
+ # Refresh the memcache connection and try again
+ client = self._getMemcacheClient(refresh=True)
+ if client.set(key, property, time=self.cacheTimeout):
+ self.log_error("Temporary write failure for %s on %s"
+ % (property.qname(), self.propertyStore.resource.fp.path))
+ return
+
+ message = (
+ "Write failure for %s on %s"
+ % (property.qname(), self.propertyStore.resource.fp.path)
+ )
+ self.log_error(message)
+ raise MemcacheError(message)
+
+ self.propertyStore.set(property)
+
+ def delete(self, qname):
+ #self.log_debug("Delete for %s on %s"
+ # % (qname, self.propertyStore.resource.fp.path))
+ client = self._getMemcacheClient()
+ if client is not None:
+ key = self.keyForQname(property.qname())
+ # Flush the cache
+ client.delete(key)
+
+ self.propertyStore.delete(qname)
+
+ def contains(self, qname):
+ #self.log_debug("Contains for %s"
+ # % (self.propertyStore.resource.fp.path,))
+ client = self._getMemcacheClient()
+ if client is not None:
+ key = self.keyForQname(qname)
+ property = client.get(key)
+ if property is not None:
+ if property is "":
+ # Negative cache hit
+ return False
+ else:
+ # Cache hit
+ return True
+
+ return self.propertyStore.contains(qname)
+
+ def list(self):
+ #self.log_debug("List for %s"
+ # % (self.propertyStore.resource.fp.path,))
+ return self.propertyStore.list()
+
+
+class MemcachePropertyCollection (LoggingMixIn):
+ """
+ Manages a single property store for all resources in a collection.
+ """
+ def __init__(self, collection, cacheTimeout=0):
+ self.collection = collection
+ self.cacheTimeout = cacheTimeout
+
+ @classmethod
+ def memcacheClient(cls, refresh=False):
+ if not hasattr(MemcachePropertyCollection, "_memcacheClient"):
+
+ # if not config.Memcached.ClientEnabled:
+ # return None
+
+ log.info("Instantiating memcache connection for MemcachePropertyCollection")
+ MemcachePropertyCollection._memcacheClient = MemcacheClientFactory.getClient(["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
+ debug=0,
+ pickleProtocol=2,
+ )
+ assert MemcachePropertyCollection._memcacheClient is not None
+
+ return MemcachePropertyCollection._memcacheClient
+
+ def propertyCache(self):
+ # The property cache has this format:
+ # {
+ # "/path/to/resource/file":
+ # (
+ # {
+ # (namespace, name): property,
+ # ...,
+ # },
+ # memcache_token,
+ # ),
+ # ...,
+ # }
+ if not hasattr(self, "_propertyCache"):
+ self._propertyCache = self._loadCache()
+ return self._propertyCache
+
+ def childCache(self, child):
+ path = child.fp.path
+ key = self._keyForPath(path)
+ propertyCache = self.propertyCache()
+
+ try:
+ childCache, token = propertyCache["key"]
+ except KeyError:
+ self.log_debug("No child property cache for %s" % (child,))
+ childCache, token = ({}, None)
+
+ #message = "No child property cache for %s" % (child,)
+ #log.error(message)
+ #raise AssertionError(message)
+
+ return propertyCache, key, childCache, token
+
+ def _keyForPath(self, path):
+ key = "|".join((
+ self.__class__.__name__,
+ path
+ ))
+ return md5(key).hexdigest()
+
+ def _loadCache(self, childNames=None):
+ if childNames is None:
+ abortIfMissing = False
+ childNames = self.collection.listChildren()
+ else:
+ if childNames:
+ abortIfMissing = True
+ else:
+ return {}
+
+ self.log_debug("Loading cache for %s" % (self.collection,))
+
+ client = self.memcacheClient()
+ assert client is not None, "OMG no cache!"
+ if client is None:
+ return None
+
+ keys = tuple((
+ (self._keyForPath(self.collection.fp.child(childName).path), childName)
+ for childName in childNames
+ ))
+
+ result = client.gets_multi((key for key, name in keys))
+
+ if self.logger.willLogAtLevel("debug"):
+ if abortIfMissing:
+ missing = "missing "
+ else:
+ missing = ""
+ self.log_debug("Loaded keys for %schildren of %s: %s" % (
+ missing,
+ self.collection,
+ [name for key, name in keys],
+ ))
+
+ missing = tuple((
+ name for key, name in keys
+ if key not in result
+ ))
+
+ if missing:
+ if abortIfMissing:
+ raise MemcacheError("Unable to fully load cache for %s" % (self.collection,))
+
+ loaded = self._buildCache(childNames=missing)
+ loaded = self._loadCache(childNames=(FilePath(name).basename() for name in loaded.iterkeys()))
+
+ result.update(loaded.iteritems())
+
+ return result
+
+ def _storeCache(self, cache):
+ self.log_debug("Storing cache for %s" % (self.collection,))
+
+ values = dict((
+ (self._keyForPath(path), props)
+ for path, props
+ in cache.iteritems()
+ ))
+
+ client = self.memcacheClient()
+ if client is not None:
+ client.set_multi(values, time=self.cacheTimeout)
+
+ def _buildCache(self, childNames=None):
+ if childNames is None:
+ childNames = self.collection.listChildren()
+ elif not childNames:
+ return {}
+
+ self.log_debug("Building cache for %s" % (self.collection,))
+
+ cache = {}
+
+ for childName in childNames:
+ child = self.collection.getChild(childName)
+ propertyStore = child.deadProperties()
+
+ props = {}
+ for qname in propertyStore.list(cache=False):
+ props[qname] = propertyStore.get(qname, cache=False)
+
+ cache[child.fp.path] = props
+
+ self._storeCache(cache)
+
+ return cache
+
+ def setProperty(self, child, property):
+ propertyCache, key, childCache, token = self.childCache(child)
+
+ if childCache.get(property.qname(), None) == property:
+ # No changes
+ return
+
+ childCache[property.qname()] = property
+
+ client = self.memcacheClient()
+ if client is not None:
+ result = client.set(key, childCache, time=self.cacheTimeout, token=token)
+ if not result:
+ delattr(self, "_propertyCache")
+ raise MemcacheError("Unable to set property %s on %s"
+ % (property.sname(), child))
+
+ loaded = self._loadCache(childNames=(child.fp.basename(),))
+ propertyCache.update(loaded.iteritems())
+
+ def flushCache(self, child):
+ path = child.fp.path
+ key = self._keyForPath(path)
+ propertyCache = self.propertyCache()
+
+ if key in propertyCache:
+ del propertyCache[key]
+
+ client = self.memcacheClient()
+ if client is not None:
+ result = client.delete(key)
+ if not result:
+ raise MemcacheError("Unable to flush cache on %s" % (child,))
+
+ def deleteProperty(self, child, qname):
+ propertyCache, key, childCache, token = self.childCache(child)
+
+ del childCache[qname]
+
+ client = self.memcacheClient()
+ if client is not None:
+ result = client.set(key, childCache, time=self.cacheTimeout, token=token)
+ if not result:
+ delattr(self, "_propertyCache")
+ raise MemcacheError("Unable to delete property {%s}%s on %s"
+ % (qname[0], qname[1], child))
+
+ loaded = self._loadCache(childNames=(child.fp.basename(),))
+ propertyCache.update(loaded.iteritems())
+
+ def propertyStoreForChild(self, child, childPropertyStore):
+ return self.ChildPropertyStore(self, child, childPropertyStore)
+
+ class ChildPropertyStore (LoggingMixIn):
+ def __init__(self, parentPropertyCollection, child, childPropertyStore):
+ self.parentPropertyCollection = parentPropertyCollection
+ self.child = child
+ self.childPropertyStore = childPropertyStore
+
+ def propertyCache(self):
+ path = self.child.fp.path
+ key = self.parentPropertyCollection._keyForPath(path)
+ parentPropertyCache = self.parentPropertyCollection.propertyCache()
+ return parentPropertyCache.get(key, ({}, None))[0]
+
+ def flushCache(self):
+ self.parentPropertyCollection.flushCache(self.child)
+
+ def get(self, qname, cache=True):
+ if cache:
+ propertyCache = self.propertyCache()
+ if qname in propertyCache:
+ return propertyCache[qname]
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+
+ self.log_debug("Read for %s on %s"
+ % (qname, self.childPropertyStore.resource.fp.path))
+ return self.childPropertyStore.get(qname)
+
+ def set(self, property):
+ self.log_debug("Write for %s on %s"
+ % (property.qname(), self.childPropertyStore.resource.fp.path))
+
+ self.parentPropertyCollection.setProperty(self.child, property)
+ self.childPropertyStore.set(property)
+
+ def delete(self, qname):
+ self.log_debug("Delete for %s on %s"
+ % (qname, self.childPropertyStore.resource.fp.path))
+
+ self.parentPropertyCollection.deleteProperty(self.child, qname)
+ self.childPropertyStore.delete(qname)
+
+ def contains(self, qname, cache=True):
+ if cache:
+ propertyCache = self.propertyCache()
+ return qname in propertyCache
+
+ self.log_debug("Contains for %s"
+ % (self.childPropertyStore.resource.fp.path,))
+ return self.childPropertyStore.contains(qname)
+
+ def list(self, cache=True):
+ if cache:
+ propertyCache = self.propertyCache()
+ return propertyCache.iterkeys()
+
+ self.log_debug("List for %s"
+ % (self.childPropertyStore.resource.fp.path,))
+ return self.childPropertyStore.list()
Modified: CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/static.py
===================================================================
--- CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/static.py 2009-04-14 01:52:24 UTC (rev 4015)
+++ CalendarServer/branches/users/sagen/mo-cache-4014/twistedcaldav/static.py 2009-04-14 01:54:44 UTC (rev 4016)
@@ -39,8 +39,7 @@
import errno
from urlparse import urlsplit
-from twisted.internet.defer import fail, succeed, inlineCallbacks, returnValue,\
- maybeDeferred
+from twisted.internet.defer import fail, succeed, inlineCallbacks, returnValue, maybeDeferred
from twisted.python.failure import Failure
from twisted.web2 import responsecode, http, http_headers
from twisted.web2.http import HTTPError, StatusResponse
@@ -57,9 +56,9 @@
from twistedcaldav import customxml
from twistedcaldav.caldavxml import caldav_namespace
from twistedcaldav.config import config
-from twistedcaldav.customxml import TwistedCalendarAccessProperty,\
- TwistedScheduleMatchETags
-from twistedcaldav.extensions import DAVFile, CachingXattrPropertyStore
+from twistedcaldav.customxml import TwistedCalendarAccessProperty, TwistedScheduleMatchETags
+from twistedcaldav.extensions import DAVFile, CachingPropertyStore
+from twistedcaldav.memcacheprops import MemcachePropertyStore, MemcachePropertyCollection
from twistedcaldav.freebusyurl import FreeBusyURLResource
from twistedcaldav.ical import Component as iComponent
from twistedcaldav.ical import Property as iProperty
@@ -142,10 +141,17 @@
return super(CalDAVFile, self).checkPreconditions(request)
- def deadProperties(self):
+ def deadProperties(self, caching=True):
if not hasattr(self, "_dead_properties"):
- self._dead_properties = CachingXattrPropertyStore(self)
+ # Get the property store from super
+ deadProperties = super(CalDAVFile, self).deadProperties()
+ if caching:
+ # Wrap the property store in a memory store
+ deadProperties = CachingPropertyStore(deadProperties)
+
+ self._dead_properties = deadProperties
+
return self._dead_properties
##
@@ -375,6 +381,53 @@
if not child.startswith(".")
]
+ def propertyCollection(self):
+ if not hasattr(self, "_propertyCollection"):
+ self._propertyCollection = MemcachePropertyCollection(self)
+ return self._propertyCollection
+
+ def createSimilarFile(self, path):
+ if path == self.fp.path:
+ return self
+
+ similar = super(CalDAVFile, self).createSimilarFile(path)
+
+ if isCalendarCollectionResource(self):
+ #
+ # Override the dead property store
+ #
+ superDeadProperties = similar.deadProperties
+
+ def deadProperties():
+ if not hasattr(similar, "_dead_properties"):
+ similar._dead_properties = self.propertyCollection().propertyStoreForChild(
+ similar,
+ superDeadProperties(caching=False)
+ )
+ return similar._dead_properties
+
+ similar.deadProperties = deadProperties
+
+ #
+ # Override DELETE, MOVE
+ #
+ for method in ("DELETE", "MOVE"):
+ method = "http_" + method
+ original = getattr(similar, method)
+
+ def override(request, original=original):
+ # Call original method
+ response = original(request)
+
+ # Wipe the cache
+ similar.deadProperties().flushCache()
+
+ return response
+
+ setattr(similar, method, override)
+
+ return similar
+
def updateCTag(self):
assert self.isCollection()
try:
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090413/3d691d39/attachment-0001.html>
More information about the calendarserver-changes
mailing list