[CalendarServer-changes] [4962] CalendarServer/branches/users/wsanchez/deployment-fileprops/ twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jan 25 17:25:49 PST 2010
Revision: 4962
http://trac.macosforge.org/projects/calendarserver/changeset/4962
Author: wsanchez at apple.com
Date: 2010-01-25 17:25:49 -0800 (Mon, 25 Jan 2010)
Log Message:
-----------
Store properties in a file per calendar
Modified Paths:
--------------
CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/memcacheprops.py
CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/static.py
Added Paths:
-----------
CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py
Added: CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py (rev 0)
+++ CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py 2010-01-26 01:25:49 UTC (rev 4962)
@@ -0,0 +1,332 @@
+##
+# Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+##
+
+__all__ = ["PropertyCollection"]
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import new as md5
+
+from cPickle import dumps as pickle, loads as unpickle, UnpicklingError
+
+from memcacheclient import ClientFactory as MemcacheClientFactory, MemcacheError, TokenMismatchError
+
+from twisted.python.filepath import FilePath
+from twisted.web2 import responsecode
+from twisted.web2.http import HTTPError, StatusResponse
+
+from twistedcaldav.config import config
+from twistedcaldav.log import LoggingMixIn, Logger
+
+log = Logger()
+
+
+class PropertyCollection (LoggingMixIn):
+ """
+ Manages a single property store for all resources in a collection.
+ """
+ def __init__(self, collection, cacheTimeout=0):
+ self.collection = collection
+ self.cacheTimeout = cacheTimeout
+ self._dirty = False
+
+ @classmethod
+ def memcacheClient(cls, refresh=False):
+ if not hasattr(PropertyCollection, "_memcacheClient"):
+ log.info("Instantiating memcache connection for PropertyCollection")
+
+ PropertyCollection._memcacheClient = MemcacheClientFactory.getClient(["%s:%s" % (config.Memcached.BindAddress, config.Memcached.Port)],
+ debug=0,
+ pickleProtocol=2,
+ )
+ assert PropertyCollection._memcacheClient is not None
+
+ return PropertyCollection._memcacheClient
+
+ def propertyCache(self):
+ # The property cache has this format:
+ # {
+ # "/path/to/resource/file":
+ # (
+ # {
+ # (namespace, name): property,
+ # ...,
+ # },
+ # memcache_token,
+ # ),
+ # ...,
+ # }
+ if not hasattr(self, "_propertyCache"):
+ self._propertyCache = self._loadCache()
+ return self._propertyCache
+
+ def childCache(self, child):
+ path = child.fp.path
+ key = self._keyForPath(path)
+ propertyCache = self.propertyCache()
+
+ try:
+ childCache, token = propertyCache[key]
+ except KeyError:
+ self.log_debug("No child property cache for %s" % (child,))
+ childCache, token = ({}, None)
+
+ #message = "No child property cache for %s" % (child,)
+ #log.error(message)
+ #raise AssertionError(message)
+
+ return propertyCache, key, childCache, token
+
+ def _keyForPath(self, path):
+ key = "|".join((
+ self.__class__.__name__,
+ path
+ ))
+ return md5(key).hexdigest()
+
+ def _loadCache(self, childNames=None):
+ if childNames is None:
+ childNames = self.collection.listChildren()
+ elif not childNames:
+ return {}
+
+ self.log_debug("Loading cache for %s" % (self.collection,))
+
+ client = self.memcacheClient()
+ assert client is not None, "OMG no cache!"
+ if client is None:
+ return None
+
+ keys = tuple((
+ (self._keyForPath(self.collection.fp.child(childName).path), childName)
+ for childName in childNames
+ ))
+
+ result = client.gets_multi((key for key, name in keys))
+
+ if self.logger.willLogAtLevel("debug"):
+ self.log_debug("Loaded keys for children of %s: %s" % (
+ self.collection,
+ [name for key, name in keys],
+ ))
+
+ return result
+
+ def _storeCache(self, cache):
+ self.log_debug("Storing cache for %s" % (self.collection,))
+
+ values = dict((
+ (self._keyForPath(path), props)
+ for path, props
+ in cache.iteritems()
+ ))
+
+ client = self.memcacheClient()
+ if client is not None:
+ client.set_multi(values, time=self.cacheTimeout)
+
+ def _cacheFilePath(self):
+ return self.collection.fp.child(".davprops.xml")
+
+ def _buildCache(self):
+ def argh(what):
+ msg = ("Unable to %s property collection store: %s" % (what, cacheFilePath.fp))
+ log.err(msg)
+ raise HTTPError(StatusResponse(responsecode.INTERNAL_SERVER_ERROR, msg))
+
+ cacheFilePath = self._cacheFilePath()
+ try:
+ cacheFile = cacheFilePath.open()
+ except (OSError, IOError):
+ if cacheFilePath.exists():
+ raise
+ else:
+ # No cache file: build from old prop store
+ cache = self._buildCacheOldSchool()
+ self._dirty = True
+ return cache
+
+ try:
+ data = cacheFile.read()
+ except (OSError, IOError):
+ argh("read")
+ finally:
+ cacheFile.close()
+
+ try:
+ return unpickle(data)
+ except UnpicklingError:
+ argh("parse")
+
+ def _buildCacheOldSchool(self):
+ self.log_info("Building property file from xattrs for %s" % (self.collection,))
+
+ cache = {}
+
+ for childName in self.collection.listChildren():
+ child = self.collection.getChild(childName)
+ if child is None:
+ continue
+
+ # Find the original property store
+ propertyStore = child.deadProperties().oldPropertyStore
+
+ props = {}
+ for qname in propertyStore.list():
+ props[qname] = propertyStore.get(qname)
+
+ cache[child.fp.path] = props
+
+ self._storeCache(cache)
+
+ self.log_info("Done building property file from xattrs for %s" % (self.collection,))
+
+ return cache
+
+ def setProperty(self, child, property, delete=False):
+ propertyCache, key, childCache, token = self.childCache(child)
+
+ self._dirty = True
+
+ if delete:
+ qname = property
+ if childCache.has_key(qname):
+ del childCache[qname]
+ else:
+ qname = property.qname()
+ childCache[qname] = property
+
+ client = self.memcacheClient()
+
+ if client is not None:
+ retries = 10
+ while retries:
+ try:
+ if client.set(key, childCache, time=self.cacheTimeout,
+ token=token):
+ # Success
+ break
+
+ except TokenMismatchError:
+ # The value in memcache has changed since we last
+ # fetched it
+ log.debug("memcacheprops setProperty TokenMismatchError; retrying...")
+
+ finally:
+ # Re-fetch the properties for this child
+ loaded = self._loadCache(childNames=(child.fp.basename(),))
+ propertyCache.update(loaded.iteritems())
+
+ retries -= 1
+
+ propertyCache, key, childCache, token = self.childCache(child)
+
+ if delete:
+ if childCache.has_key(qname):
+ del childCache[qname]
+ else:
+ childCache[qname] = property
+
+ else:
+ log.error("memcacheprops setProperty had too many failures")
+ delattr(self, "_propertyCache")
+ raise MemcacheError("Unable to %s property {%s}%s on %s"
+ % ("delete" if delete else "set",
+ qname[0], qname[1], child))
+
+ def flush(self):
+ if self._dirty:
+ def argh(what):
+ msg = ("Unable to %s property collection store: %s" % (what, cacheFilePath.fp))
+ log.err(msg)
+ raise HTTPError(StatusResponse(responsecode.INTERNAL_SERVER_ERROR, msg))
+
+ cacheFilePath = self._cacheFilePath()
+ try:
+ cacheFile = cacheFilePath.open("w")
+ except (OSError, IOError):
+ argh("open (for writing)")
+
+ try:
+ cacheFile.write(pickle(self.propertyCache()))
+ except (OSError, IOError):
+ argh("write")
+ finally:
+ cacheFile.close()
+
+ def deleteProperty(self, child, qname):
+ return self.setProperty(child, qname, delete=True)
+
+ def deleteAll(self, child):
+ path = child.fp.path
+ key = self._keyForPath(path)
+ propertyCache = self.propertyCache()
+
+ if key in propertyCache:
+ del propertyCache[key]
+
+ client = self.memcacheClient()
+ if client is not None:
+ result = client.delete(key)
+ if not result:
+ raise MemcacheError("Unable to flush cache on %s" % (child,))
+
+ def propertyStoreForChild(self, child, oldPropertyStore):
+ return self.ChildPropertyStore(self, child, oldPropertyStore)
+
+ class ChildPropertyStore (LoggingMixIn):
+ def __init__(self, parentPropertyCollection, child, oldPropertyStore):
+ self.parentPropertyCollection = parentPropertyCollection
+ self.child = child
+ self.oldPropertyStore = oldPropertyStore
+
+ def propertyCache(self):
+ path = self.child.fp.path
+ key = self.parentPropertyCollection._keyForPath(path)
+ parentPropertyCache = self.parentPropertyCollection.propertyCache()
+ return parentPropertyCache.get(key, ({}, None))[0]
+
+ def get(self, qname):
+ propertyCache = self.propertyCache()
+ if qname in propertyCache:
+ return propertyCache[qname]
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+
+ def deleteAll(self):
+ self.parentPropertyCollection.deleteAll(self.child)
+
+ def set(self, property):
+ self.parentPropertyCollection.setProperty(self.child, property)
+
+ def delete(self, qname):
+ self.parentPropertyCollection.deleteProperty(self.child, qname)
+
+ def contains(self, qname):
+ return qname in self.propertyCache()
+
+ def list(self,):
+ return self.propertyCache().iterkeys()
Modified: CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/memcacheprops.py 2010-01-26 01:25:21 UTC (rev 4961)
+++ CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/memcacheprops.py 2010-01-26 01:25:49 UTC (rev 4962)
@@ -53,6 +53,7 @@
def __init__(self, collection, cacheTimeout=0):
self.collection = collection
self.cacheTimeout = cacheTimeout
+ self._dirty = False
@classmethod
def memcacheClient(cls, refresh=False):
@@ -202,6 +203,8 @@
def setProperty(self, child, property, delete=False):
propertyCache, key, childCache, token = self.childCache(child)
+ self._dirty = True
+
if delete:
qname = property
if childCache.has_key(qname):
@@ -251,7 +254,7 @@
def deleteProperty(self, child, qname):
return self.setProperty(child, qname, delete=True)
- def flushCache(self, child):
+ def deleteAll(self, child):
path = child.fp.path
key = self._keyForPath(path)
propertyCache = self.propertyCache()
@@ -265,6 +268,9 @@
if not result:
raise MemcacheError("Unable to flush cache on %s" % (child,))
+ def flush(self):
+ pass
+
def propertyStoreForChild(self, child, childPropertyStore):
return self.ChildPropertyStore(self, child, childPropertyStore)
@@ -280,9 +286,6 @@
parentPropertyCache = self.parentPropertyCollection.propertyCache()
return parentPropertyCache.get(key, ({}, None))[0]
- def flushCache(self):
- self.parentPropertyCollection.flushCache(self.child)
-
def get(self, qname, cache=True):
if cache:
propertyCache = self.propertyCache()
@@ -298,6 +301,9 @@
% (qname, self.childPropertyStore.resource.fp.path))
return self.childPropertyStore.get(qname)
+ def deleteAll(self):
+ self.parentPropertyCollection.deleteAll(self.child)
+
def set(self, property):
self.log_debug("Write for %s on %s"
% (property.qname(), self.childPropertyStore.resource.fp.path))
Modified: CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/static.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/static.py 2010-01-26 01:25:21 UTC (rev 4961)
+++ CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/static.py 2010-01-26 01:25:49 UTC (rev 4962)
@@ -58,7 +58,7 @@
from twistedcaldav.customxml import TwistedCalendarAccessProperty
from twistedcaldav.extensions import DAVFile
from twistedcaldav.extensions import CachingPropertyStore
-from twistedcaldav.memcacheprops import MemcachePropertyCollection
+from twistedcaldav.fileprops import PropertyCollection
from twistedcaldav.ical import Component as iComponent
from twistedcaldav.ical import Property as iProperty
from twistedcaldav.index import Index, IndexSchedule
@@ -330,7 +330,7 @@
def propertyCollection(self):
if not hasattr(self, "_propertyCollection"):
- self._propertyCollection = MemcachePropertyCollection(self)
+ self._propertyCollection = PropertyCollection(self)
return self._propertyCollection
def createSimilarFile(self, path):
@@ -341,7 +341,7 @@
if isCalendarCollectionResource(self):
- # Short-circuit stat with information we know to be true at this point
+ # Short-circuit stat with information we know to be true at this point
if isinstance(path, FilePath) and hasattr(self, "knownChildren"):
if os.path.basename(path.path) in self.knownChildren:
path.existsCached = True
@@ -376,12 +376,25 @@
response = (yield original(request))
# Wipe the cache
- similar.deadProperties().flushCache()
+ similar.deadProperties().deleteAll()
returnValue(response)
setattr(similar, method, override)
+ #
+ # Flush properties at end of request
+ #
+ def renderHTTP(request, original=similar.renderHTTP):
+ def flushProperties(request, response):
+ self.propertyCollection().flush()
+ return response
+ request.addResponseFilter(flushProperties)
+
+ return original(request)
+
+ similar.renderHTTP = renderHTTP
+
return similar
def updateCTag(self):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100125/5045e7e0/attachment-0001.html>
More information about the calendarserver-changes
mailing list